public LogParser createLogParser(File logFile) throws LogParserException, IOException {
BufferedReader reader = null;
try {
reader = new BufferedReader(new FileReader(logFile));
String line = reader.readLine();
LogFormat format = detectLogFormat(line);
if (DEBUG) {
System.out.println("Log format=" + format.name() + " for line:" + line);
}
switch (format) {
case UNIFORM_LOG_FORMAT:
return new UniformLogParser(logFile.getName());
case ODL_LOG_FORMAT:
return new ODLLogParser(logFile.getName());
default:
return new RawLogParser(logFile.getName());
}
} finally {
if (reader != null) {
reader.close();
}
}
}
/** Main method of this example. */
@Override
public void runExample(ExampleDriver driver) throws Exception {
long startMs = System.currentTimeMillis();
CommandLine cli = driver.getCommandLine();
// Size of index batch requests to Solr
// int batchSize = Integer.parseInt(cli.getOptionValue("batchSize", "500"));
// Get a connection to Solr cloud using Zookeeper
String zkHost = cli.getOptionValue("zkhost", ZK_HOST);
String collectionName = cli.getOptionValue("collection", COLLECTION);
int zkClientTimeout = Integer.parseInt(cli.getOptionValue("zkClientTimeout", "15000"));
CloudSolrServer solr = new CloudSolrServer(zkHost);
solr.setDefaultCollection(collectionName);
solr.setZkClientTimeout(zkClientTimeout);
solr.connect();
int numSent = 0;
int numSkipped = 0;
int lineNum = 0;
SolrInputDocument doc = null;
String line = null;
// read file line-by-line
BufferedReader reader = new BufferedReader(driver.readFile("log"));
driver.rememberCloseable(reader);
LogFormat fmt = LogFormat.valueOf(cli.getOptionValue("format", "solr"));
// process each sighting as a document
while ((line = reader.readLine()) != null) {
doc = parseNextDoc(line, ++lineNum, fmt);
if (doc != null) {
addDocWithRetry(solr, doc, 10);
++numSent;
} else {
++numSkipped;
continue;
}
if (lineNum % 1000 == 0) log.info(String.format("Processed %d lines.", lineNum));
}
// hard commit all docs sent
solr.commit(true, true);
float tookSecs = Math.round(((System.currentTimeMillis() - startMs) / 1000f) * 100f) / 100f;
log.info(
String.format(
"Sent %d log messages (skipped %d) took %f seconds", numSent, numSkipped, tookSecs));
// queries to demonstrate results of indexing
SolrQuery solrQuery = new SolrQuery("*:*");
solrQuery.setRows(0);
QueryResponse resp = solr.query(solrQuery);
SolrDocumentList hits = resp.getResults();
log.info("Match all docs distributed query found " + hits.getNumFound() + " docs.");
solrQuery.set("shards", "shard1");
resp = solr.query(solrQuery);
hits = resp.getResults();
log.info(
"Match all docs non-distributed query to shard1 found " + hits.getNumFound() + " docs.");
solrQuery.set("shards", "shard2");
resp = solr.query(solrQuery);
hits = resp.getResults();
log.info(
"Match all docs non-distributed query to shard2 found " + hits.getNumFound() + " docs.");
solr.shutdown();
}
