/**
* Queries the {@link #rdfEndpoint(String)} with each of the {@link #rdfQueries} and harvests the
* results of the query.
*/
private void harvestFromEndpoint() {
Query query;
QueryExecution qExec;
for (String rdfQuery : rdfQueries) {
if (closed) break;
logger.info(
"Harvesting with query: [{}] on index [{}] and type [{}]", rdfQuery, indexName, typeName);
try {
query = QueryFactory.create(rdfQuery);
} catch (QueryParseException qpe) {
logger.error("Could not parse [{}]. Please provide a relevant query. {}", rdfQuery, qpe);
continue;
}
qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
try {
harvest(qExec);
} catch (Exception e) {
logger.error("Exception [{}] occurred while harvesting", e.getLocalizedMessage());
} finally {
qExec.close();
}
}
}
public List<ResourceName> queryMDS(PackageName packageName) {
MDSInstance mdsInstance;
List<ResourceName> resources = new ArrayList<ResourceName>();
try {
mdsInstance = storeInitializer.getMDSInstance();
packageName = (packageName == null) ? PackageName.createPackageName("/") : packageName;
NameCondition condition =
ConditionFactory.createNameCondition(packageName.getAbsoluteName(), "%");
ResourceQuery query = QueryFactory.createResourceQuery(mdsInstance, condition);
Iterator<QueryResult> contents = query.execute();
if (contents == null) {
return resources;
}
while (contents.hasNext()) {
QueryResult result = contents.next();
if (result.getResultType() == QueryResult.ResultType.PACKAGE_RESULT) {
PackageResult pack = (PackageResult) result;
resources.add(pack.getPackageName());
} else {
DocumentResult doc = (DocumentResult) result;
resources.add(doc.getDocumentName());
}
}
return resources;
} catch (MDSAccessException e) {
throw new IllegalArgumentException("Could not connect to MDS, check login data", e);
} catch (InvalidReferenceException e) {
throw new IllegalArgumentException("Could not find/read " + packageName, e);
} catch (InvalidReferenceTypeException e) {
throw new IllegalArgumentException("No correct type!", e);
}
}
protected void execute(String queryString, HttpAction action) {
String queryStringLog = ServletOps.formatForLog(queryString);
if (action.verbose) action.log.info(format("[%d] Query = \n%s", action.id, queryString));
else action.log.info(format("[%d] Query = %s", action.id, queryStringLog));
Query query = null;
try {
// NB syntax is ARQ (a superset of SPARQL)
query = QueryFactory.create(queryString, QueryParseBase, Syntax.syntaxARQ);
queryStringLog = formatForLog(query);
validateQuery(action, query);
} catch (ActionErrorException ex) {
throw ex;
} catch (QueryParseException ex) {
ServletOps.errorBadRequest(
"Parse error: \n" + queryString + "\n\r" + messageForQueryException(ex));
}
// Should not happen.
catch (QueryException ex) {
ServletOps.errorBadRequest("Error: \n" + queryString + "\n\r" + ex.getMessage());
}
// Assumes finished whole thing by end of sendResult.
try {
action.beginRead();
Dataset dataset = decideDataset(action, query, queryStringLog);
try (QueryExecution qExec = createQueryExecution(query, dataset); ) {
SPARQLResult result = executeQuery(action, qExec, query, queryStringLog);
// Deals with exceptions itself.
sendResults(action, result, query.getPrologue());
}
} catch (QueryParseException ex) {
// Late stage static error (e.g. bad fixed Lucene query string).
ServletOps.errorBadRequest(
"Query parse error: \n" + queryString + "\n\r" + messageForQueryException(ex));
} catch (QueryCancelledException ex) {
// Additional counter information.
incCounter(action.getEndpoint().getCounters(), QueryTimeouts);
throw ex;
} finally {
action.endRead();
}
}
/**
* Returns the string value of the first of the properties in the uriDescriptionList for the given
* resource (as an URI). In case the resource does not have any of the properties mentioned, its
* URI is returned. The value is obtained by querying the endpoint and the endpoint is queried
* repeatedly until it gives a response (value or the lack of it)
*
* <p>It is highly recommended that the list contains properties like labels or titles, with test
* values.
*
* @param uri - the URI for which a label is required
* @return a String value, either a label for the parameter or its value if no label is obtained
* from the endpoint
*/
private String getLabelForUri(String uri) {
String result;
if (uriLabelCache.containsKey(uri)) {
return uriLabelCache.get(uri);
}
for (String prop : uriDescriptionList) {
String innerQuery = "SELECT ?r WHERE {<" + uri + "> <" + prop + "> ?r } LIMIT 1";
try {
Query query = QueryFactory.create(innerQuery);
QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
boolean keepTrying = true;
while (keepTrying) {
keepTrying = false;
try {
ResultSet results = qExec.execSelect();
if (results.hasNext()) {
QuerySolution sol = results.nextSolution();
result = EEASettings.parseForJson(sol.getLiteral("r").getLexicalForm());
if (!result.isEmpty()) {
uriLabelCache.put(uri, result);
return result;
}
}
} catch (Exception e) {
keepTrying = true;
logger.warn("Could not get label for uri {}. Retrying.", uri);
} finally {
qExec.close();
}
}
} catch (QueryParseException qpe) {
logger.error("Exception for query {}. The label cannot be obtained", innerQuery);
}
}
return uri;
}
/**
* Get a set of unique queryObjName returned from a select query
*
* <p>Used to retrieve sets of modified objects used in sync
*
* @param rdfQuery query to execute
* @param queryObjName name of the object returned
* @return set of values for queryObjectName in the rdfQuery result
*/
HashSet<String> executeSyncQuery(String rdfQuery, String queryObjName) {
HashSet<String> rdfUrls = new HashSet<String>();
Query query;
try {
query = QueryFactory.create(rdfQuery);
} catch (QueryParseException qpe) {
logger.warn(
"Could not parse [{}]. Please provide a relevant query. {}",
rdfQuery,
qpe.getLocalizedMessage());
return null;
}
QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
try {
ResultSet results = qExec.execSelect();
while (results.hasNext()) {
QuerySolution sol = results.nextSolution();
try {
String value = sol.getResource(queryObjName).toString();
rdfUrls.add(value);
} catch (NoSuchElementException e) {
logger.error("Encountered a NoSuchElementException: " + e.getLocalizedMessage());
return null;
}
}
} catch (Exception e) {
logger.error(
"Encountered a [{}] while querying the endpoint for sync", e.getLocalizedMessage());
return null;
} finally {
qExec.close();
}
return rdfUrls;
}
/**
* Starts a harvester with predefined queries to synchronize with the changes from the SPARQL
* endpoint
*/
public boolean sync() {
logger.info("Sync resources newer than {}", startTime);
String rdfQueryTemplate =
"PREFIX xsd:<http://www.w3.org/2001/XMLSchema#> "
+ "SELECT DISTINCT ?resource WHERE { "
+ " GRAPH ?graph { %s }"
+ " ?graph <%s> ?time . %s "
+ " FILTER (?time > xsd:dateTime(\"%s\")) }";
String queryStr =
String.format(
rdfQueryTemplate, syncConditions, syncTimeProp, graphSyncConditions, startTime);
Set<String> syncUris = executeSyncQuery(queryStr, "resource");
if (syncUris == null) {
logger.error("Errors occurred during sync procedure. Aborting!");
return false;
}
/**
* If desired, query for old data that has the sync conditions modified
*
* <p>This option is useful in the case in which the application indexes resources that match
* some conditions. In this case, if they are modified and no longer match the initial
* conditions, they will not be synchronized. When syncOldData is True, the modified resources
* that no longer match the conditions are deleted.
*/
int deleted = 0;
int count = 0;
if (this.syncOldData) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
queryStr =
String.format(
rdfQueryTemplate,
syncConditions,
syncTimeProp,
graphSyncConditions,
sdf.format(new Date(0)));
HashSet<String> allIndexURIs = executeSyncQuery(queryStr, "resource");
if (allIndexURIs == null) {
logger.error("Errors occurred during modified content sync query. Aborting!");
return false;
}
deleted = removeMissingUris(allIndexURIs);
}
/* Prepare a series of bulk uris to be described so we can make
* a smaller number of calls to the SPARQL endpoint. */
ArrayList<ArrayList<String>> bulks = new ArrayList<ArrayList<String>>();
ArrayList<String> currentBulk = new ArrayList<String>();
for (String uri : syncUris) {
currentBulk.add(uri);
if (currentBulk.size() == EEASettings.DEFAULT_BULK_SIZE) {
bulks.add(currentBulk);
currentBulk = new ArrayList<String>();
}
}
if (currentBulk.size() > 0) {
bulks.add(currentBulk);
}
/* Execute RDF queries for the resources in each bulk */
for (ArrayList<String> bulk : bulks) {
String syncQuery = getSyncQueryStr(bulk);
try {
Query query = QueryFactory.create(syncQuery);
QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
try {
Model constructModel = ModelFactory.createDefaultModel();
qExec.execConstruct(constructModel);
BulkRequestBuilder bulkRequest = client.prepareBulk();
/**
* When adding the model to ES do not use toDescribeURIs as the query already returned the
* correct labels.
*/
addModelToES(constructModel, bulkRequest, false);
count += bulk.size();
} catch (Exception e) {
logger.error("Error while querying for modified content. {}", e.getLocalizedMessage());
return false;
} finally {
qExec.close();
}
} catch (QueryParseException qpe) {
logger.warn(
"Could not parse Sync query. Please provide a relevant query. {}",
qpe.getLocalizedMessage());
return false;
}
}
logger.info(
"Finished synchronisation: Deleted {}, Updated {}/{}", deleted, count, syncUris.size());
return true;
}
