private String fetchBatches(
DBCollection collection, Set<String> loadedUris, String lastDCIdFetched) {
BulkWriteOperation builder = collection.initializeUnorderedBulkOperation();
String prevDcId = lastDCIdFetched;
DCQueryParameters params;
params =
lastDCIdFetched == null
? new DCQueryParameters(searchCronField, DCOrdering.DESCENDING)
: new DCQueryParameters(
searchCronField,
DCOrdering.DESCENDING,
DCOperator.LT,
"\"" + lastDCIdFetched + "\"");
// (LT & descending order to leave possible null geo:name values at the end rather than having
// to skip them)
logger.debug("Querying the Data Core");
long queryStart = System.currentTimeMillis();
List<DCResource> resources = datacore.findResources(project, areaModel, params, 0, batchSize);
long queryEnd = System.currentTimeMillis();
logger.debug("Fetched {} resources in {} ms", resources.size(), queryEnd - queryStart);
boolean hasOne = false;
for (DCResource res : resources) {
String name = null;
for (Languages language : Languages.values()) {
GeographicalArea area = geographicalDAO.toGeographicalArea(res, language.getLanguage());
if (area == null) {
continue;
}
if (!loadedUris.contains(language.getLanguage() + area.getUri())) {
hasOne = true;
if (name == null) {
name = area.getName();
// logger.debug("{} - {}", name, area.getUri());
}
DBObject dbObject = new BasicDBObject();
mappingMongoConverter.write(area, dbObject);
builder.insert(dbObject);
loadedUris.add(language.getLanguage() + area.getUri());
} else {
if (name == null) {
name = area.getName();
logger.debug(
"Area {} already inserted for language {}, skipping",
area.getName(),
language.getLanguage());
}
}
}
String id =
res
.getUri(); // ID resource in DC is always encoded, so to match values we need to
// encoded as well
if (id != null) {
lastDCIdFetched = id;
}
}
if (hasOne) {
long st = System.currentTimeMillis();
builder.execute();
long durationSave = System.currentTimeMillis() - st;
logger.debug(
"Saved resources; total save time={} ms (avg = {} ms)",
durationSave,
durationSave / resources.size());
}
if ((prevDcId != null && prevDcId.equals(lastDCIdFetched)) || resources.size() < batchSize) {
return null;
} else return lastDCIdFetched;
}