@Override
public void store(String url, Status status, Metadata metadata, Date nextFetch) throws Exception {
XContentBuilder builder = jsonBuilder().startObject();
builder.field("url", url);
// TODO update the status e.g more than N fetcherrors => error
builder.field("status", status);
// check that we don't overwrite an existing entry
// When create is used, the index operation will fail if a document
// by that id already exists in the index.
boolean create = status.equals(Status.DISCOVERED);
builder.startObject("metadata");
Iterator<String> mdKeys = metadata.keySet().iterator();
while (mdKeys.hasNext()) {
String mdKey = mdKeys.next();
String[] values = metadata.getValues(mdKey);
builder.array(mdKey, values);
}
builder.endObject();
builder.field("nextFetchDate", nextFetch);
builder.endObject();
IndexRequestBuilder request =
connection
.getClient()
.prepareIndex(indexName, docType)
.setSource(builder)
.setCreate(create)
.setId(url);
if (StringUtils.isNotBlank(metadataRouting)) {
String valueForRouting = metadata.getFirstValue(metadataRouting);
if (StringUtils.isNotBlank(valueForRouting)) {
request.setRouting(valueForRouting);
}
}
connection.getProcessor().add(request.request());
}
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
super.prepare(stormConf, context, collector);
indexName =
ConfUtils.getString(stormConf, StatusUpdaterBolt.ESStatusIndexNameParamName, "status");
docType = ConfUtils.getString(stormConf, StatusUpdaterBolt.ESStatusDocTypeParamName, "status");
metadataRouting = ConfUtils.getString(stormConf, StatusUpdaterBolt.ESStatusRoutingParamName);
try {
connection = ElasticSearchConnection.getConnection(stormConf, ESBoltType);
} catch (Exception e1) {
LOG.error("Can't connect to ElasticSearch", e1);
throw new RuntimeException(e1);
}
}
@Override
public void cleanup() {
if (connection != null) connection.close();
}