private void assertScore(boolean isScoresTest, SolrDocument doc) {
if (isScoresTest) {
assertThat(
"score join doesn't return 1.0", doc.getFirstValue("score").toString(), not("1.0"));
} else {
assertEquals("Solr join has constant score", "1.0", doc.getFirstValue("score").toString());
}
}
private static DcsManifestation getManifestation(SolrDocument doc) {
DcsManifestation man = new DcsManifestation();
man.setId(get(doc, EntityField.ID));
if (has(doc, ManifestationField.DELIVERABLE_UNIT)) {
man.setDeliverableUnit(get(doc, ManifestationField.DELIVERABLE_UNIT));
}
man.setMetadata(getMetadataSet(doc));
man.setMetadataRef(getMetadataRefSet(doc, EntityField.METADATA_REF));
man.setManifestationFiles(getManifestationFileSet(doc));
if (has(doc, CoreMetadataField.TYPE)) {
man.setType(get(doc, CoreMetadataField.TYPE));
}
man.setTechnicalEnvironment(getStringSet(doc, ManifestationField.TECH));
if (has(doc, ManifestationField.DATE_CREATED)) {
man.setDateCreated(
DateUtility.toIso8601(
((Date) doc.getFirstValue(ManifestationField.DATE_CREATED.solrName())).getTime()));
}
man.setAlternateIds(getResourceIdentifierSet(doc));
return man;
}
private static SeadEvent getEvent(SolrDocument doc) {
SeadEvent event = new SeadEvent();
event.setId(get(doc, EntityField.ID));
Date date = (Date) doc.getFirstValue(EventField.DATE.solrName());
if (date != null) {
event.setDate(DateUtil.getThreadLocalDateFormat().format(date));
}
event.setDetail(getFirst(doc, EventField.DETAIL));
event.setOutcome(getFirst(doc, EventField.OUTCOME));
event.setEventType(getFirst(doc, EventField.TYPE));
event.setTargets(getEntityRefSet(doc, EventField.TARGET));
event.setAlternateIds(getResourceIdentifierSet(doc));
SeadLogDetail logDetail = new SeadLogDetail();
// if (has(doc, DetailLogField.IPADDRESS)) {
logDetail.setIpAddress(
(String) doc.getFieldValue(SeadSolrField.DetailLogField.IPADDRESS.solrName()));
// }
if (has(doc, SeadSolrField.DetailLogField.USERAGENT)) {
logDetail.setUserAgent(
(String) doc.getFieldValue(SeadSolrField.DetailLogField.USERAGENT.solrName()));
}
if (has(doc, SeadSolrField.DetailLogField.SUBJECT)) {
logDetail.setSubject(
(String) doc.getFieldValue(SeadSolrField.DetailLogField.SUBJECT.solrName()));
}
if (has(doc, SeadSolrField.DetailLogField.NODEIDENTIFIER)) {
logDetail.setNodeIdentifier(
(String) doc.getFieldValue(SeadSolrField.DetailLogField.NODEIDENTIFIER.solrName()));
}
event.setLogDetail(logDetail);
return event;
}
private void testJoins(String toColl, String fromColl, Integer toDocId, boolean isScoresTest)
throws SolrServerException, IOException {
// verify the join with fromIndex works
final String fromQ = "match_s:c^2";
CloudSolrClient client = cluster.getSolrClient();
{
final String joinQ =
"{!join "
+ anyScoreMode(isScoresTest)
+ "from=join_s fromIndex="
+ fromColl
+ " to=join_s}"
+ fromQ;
QueryRequest qr =
new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
QueryResponse rsp = new QueryResponse(client.request(qr), client);
SolrDocumentList hits = rsp.getResults();
assertTrue("Expected 1 doc, got " + hits, hits.getNumFound() == 1);
SolrDocument doc = hits.get(0);
assertEquals(toDocId, doc.getFirstValue("id"));
assertEquals("b", doc.getFirstValue("get_s"));
assertScore(isScoresTest, doc);
}
// negative test before creating an alias
checkAbsentFromIndex(fromColl, toColl, isScoresTest);
// create an alias for the fromIndex and then query through the alias
String alias = fromColl + "Alias";
CollectionAdminRequest.CreateAlias request =
CollectionAdminRequest.createAlias(alias, fromColl);
request.process(client);
{
final String joinQ =
"{!join "
+ anyScoreMode(isScoresTest)
+ "from=join_s fromIndex="
+ alias
+ " to=join_s}"
+ fromQ;
final QueryRequest qr =
new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
final QueryResponse rsp = new QueryResponse(client.request(qr), client);
final SolrDocumentList hits = rsp.getResults();
assertTrue("Expected 1 doc", hits.getNumFound() == 1);
SolrDocument doc = hits.get(0);
assertEquals(toDocId, doc.getFirstValue("id"));
assertEquals("b", doc.getFirstValue("get_s"));
assertScore(isScoresTest, doc);
}
// negative test after creating an alias
checkAbsentFromIndex(fromColl, toColl, isScoresTest);
{
// verify join doesn't work if no match in the "from" index
final String joinQ =
"{!join "
+ (anyScoreMode(isScoresTest))
+ "from=join_s fromIndex="
+ fromColl
+ " to=join_s}match_s:d";
final QueryRequest qr =
new QueryRequest(params("collection", toColl, "q", joinQ, "fl", "id,get_s,score"));
final QueryResponse rsp = new QueryResponse(client.request(qr), client);
final SolrDocumentList hits = rsp.getResults();
assertTrue("Expected no hits", hits.getNumFound() == 0);
}
}
@Override
// Returned Map will have suffixes in the key names - client is responsible for handling them
public List<Map<String, Object>> get(String type, String cql) throws PersistenceException {
if (StringUtils.isBlank(type)) {
throw new PersistenceException(
"The type of object(s) to retrieve must be non-null and not blank, e.g., notification, metacard, etc.");
}
List<Map<String, Object>> results = new ArrayList<>();
// Set Solr Core name to type and create/connect to Solr Core
SolrServer coreSolrServer = getSolrCore(type);
if (coreSolrServer == null) {
return results;
}
SolrQueryFilterVisitor visitor = new SolrQueryFilterVisitor(coreSolrServer, type);
try {
SolrQuery solrQuery;
// If not cql specified, then return all items
if (StringUtils.isBlank(cql)) {
solrQuery = new SolrQuery("*:*");
} else {
Filter filter = CQL.toFilter(cql);
solrQuery = (SolrQuery) filter.accept(visitor, null);
}
QueryResponse solrResponse = coreSolrServer.query(solrQuery, METHOD.POST);
long numResults = solrResponse.getResults().getNumFound();
LOGGER.debug("numResults = {}", numResults);
SolrDocumentList docs = solrResponse.getResults();
for (SolrDocument doc : docs) {
PersistentItem result = new PersistentItem();
Collection<String> fieldNames = doc.getFieldNames();
for (String name : fieldNames) {
LOGGER.debug("field name = {} has value = {}", name, doc.getFieldValue(name));
if (name.endsWith(PersistentItem.TEXT_SUFFIX) && doc.getFieldValues(name).size() > 1) {
result.addProperty(
name,
doc.getFieldValues(name)
.stream()
.filter(s -> s instanceof String)
.map(s -> (String) s)
.collect(Collectors.toSet()));
} else if (name.endsWith(PersistentItem.XML_SUFFIX)) {
result.addXmlProperty(name, (String) doc.getFirstValue(name));
} else if (name.endsWith(PersistentItem.TEXT_SUFFIX)) {
result.addProperty(name, (String) doc.getFirstValue(name));
} else if (name.endsWith(PersistentItem.LONG_SUFFIX)) {
result.addProperty(name, (Long) doc.getFirstValue(name));
} else if (name.endsWith(PersistentItem.INT_SUFFIX)) {
result.addProperty(name, (Integer) doc.getFirstValue(name));
} else if (name.endsWith(PersistentItem.DATE_SUFFIX)) {
result.addProperty(name, (Date) doc.getFirstValue(name));
} else {
LOGGER.info("Not adding field {} because it has invalid suffix", name);
}
}
results.add(result);
}
} catch (CQLException e) {
throw new PersistenceException(
"CQLException while getting Solr data with cql statement " + cql, e);
} catch (SolrServerException e) {
throw new PersistenceException(
"SolrServerException while getting Solr data with cql statement " + cql, e);
}
return results;
}
private static DcsEntity getDeliverableUnit(SolrDocument doc) {
SeadDeliverableUnit du = new SeadDeliverableUnit();
du.setId(get(doc, EntityField.ID));
if (has(doc, CoreMetadataField.RIGHTS)) {
du.setRights(get(doc, CoreMetadataField.RIGHTS));
}
du.setCollections(getCollectionRefSet(doc, DeliverableUnitField.COLLECTIONS));
du.setFormerExternalRefs(getStringSet(doc, DeliverableUnitField.FORMER_REFS));
if (has(doc, CoreMetadataField.TITLE)) {
du.setTitle(getFirst(doc, CoreMetadataField.TITLE));
}
if (has(doc, SeadSolrField.EntityField.ABSTRACT)) {
String abstrct = get(doc, SeadSolrField.EntityField.ABSTRACT);
abstrct = StringEscapeUtils.unescapeHtml(abstrct);
du.setAbstrct(abstrct);
}
if (has(doc, SeadSolrField.EntityField.PUBDATE)) {
String pubdate = getFirst(doc, SeadSolrField.EntityField.PUBDATE);
du.setPubdate(pubdate);
}
if (has(doc, SeadSolrField.EntityField.MDUPDATE_DATE)) {
Date date = (Date) doc.getFirstValue(SeadSolrField.EntityField.MDUPDATE_DATE.solrName());
if (date != null) {
du.setMetadataUpdateDate(DateUtil.getThreadLocalDateFormat().format(date));
}
}
// if (has(doc, DeliverableUnitField.LOCATION)) {
// String location = getFirst(doc, DeliverableUnitField.LOCATION);
// du.addSite(location);
// }
du.setSites(getStringSet(doc, SeadSolrField.DeliverableUnitField.LOCATION));
if (has(doc, SeadSolrField.DeliverableUnitField.SIZEBYTES)) {
long size = (Long) doc.getFirstValue(SeadSolrField.DeliverableUnitField.SIZEBYTES.solrName());
if (size > 0) {
du.setSizeBytes(size);
}
}
if (has(doc, SeadSolrField.DeliverableUnitField.FILENO)) {
long fileNo = (Long) doc.getFirstValue(SeadSolrField.DeliverableUnitField.FILENO.solrName());
if (fileNo > 0) {
du.setFileNo(fileNo);
}
}
if (has(doc, SeadSolrField.CoreMetadataField.CONTACT)) {
String contact = getFirst(doc, SeadSolrField.CoreMetadataField.CONTACT);
du.setContact(contact);
}
du.setCollections(getCollectionRefSet(doc, DeliverableUnitField.COLLECTIONS));
du.setFormerExternalRefs(getStringSet(doc, DeliverableUnitField.FORMER_REFS));
if (has(doc, DeliverableUnitField.DIGITAL_SURROGATE)) {
du.setDigitalSurrogate(
(Boolean) doc.getFieldValue(DeliverableUnitField.DIGITAL_SURROGATE.solrName()));
}
du.setMetadataRef(getMetadataRefSet(doc, EntityField.METADATA_REF));
du.setMetadata(getMetadataSet(doc));
du.setRelations(getRelations(doc));
du.setParents(getDeliverableUnitRefSet(doc, DeliverableUnitField.PARENT));
du.setSubjects(getStringSet(doc, CoreMetadataField.SUBJECT));
du.setDataContributors(getCreatorSet(doc));
du.setSubmitter(getSubmitter(doc));
du.setAlternateIds(getResourceIdentifierSet(doc));
SeadDataLocation primaryDataLocation = getPrimaryDataLocation(doc);
du.setPrimaryLocation(primaryDataLocation);
du.setSecondaryDataLocations(getSecondaryDataLocationSet(doc));
if (has(doc, CoreMetadataField.TYPE)) {
du.setType(getFirst(doc, CoreMetadataField.TYPE));
}
return du;
}
private static DcsFile getFile(SolrDocument doc) {
SeadFile file = new SeadFile();
file.setId(get(doc, EntityField.ID));
file.setFixity(getFixitySet(doc));
file.setFormats(getFormatSet(doc));
file.setExtant((Boolean) doc.getFieldValue(FileField.EXTANT.solrName()));
if (has(doc, FileField.VALID)) {
file.setValid((Boolean) doc.getFieldValue(FileField.VALID.solrName()));
}
if (has(doc, FileField.NAME)) {
file.setName(getFirst(doc, FileField.NAME));
}
if (has(doc, FileField.SOURCE)) {
file.setSource(getFirst(doc, FileField.SOURCE));
}
if (has(doc, SeadSolrField.EntityField.IMMEDIATEANCESTRY)) {
file.setParent(getFirst(doc, SeadSolrField.EntityField.IMMEDIATEANCESTRY));
}
if (has(doc, SeadSolrField.EntityField.MDUPDATE_DATE)) {
Object object = doc.getFirstValue(SeadSolrField.EntityField.MDUPDATE_DATE.solrName());
if (object instanceof Date) {
Date date = (Date) object;
if (date != null) {
file.setMetadataUpdateDate(DateUtil.getThreadLocalDateFormat().format(date));
}
} else {
if (object != null) {
file.setMetadataUpdateDate(object.toString());
}
}
} else {
file.setMetadataUpdateDate("2012-10-27T22:05:20.809Z");
}
if (has(doc, SeadSolrField.FileField.DEPOSITDATE)) {
Object object = doc.getFirstValue(SeadSolrField.FileField.DEPOSITDATE.solrName());
if (object instanceof Date) {
Date date = (Date) object; // doc.getFirstValue(FileField.DEPOSITDATE.solrName());
if (date != null) {
file.setDepositDate(DateUtil.getThreadLocalDateFormat().format(date));
}
} else {
if (object != null) {
file.setDepositDate(object.toString());
}
}
} else {
file.setDepositDate("2012-10-27T22:05:20.809Z");
}
long size = (Long) doc.getFirstValue(FileField.SIZE.solrName());
if (size > 0) {
file.setSizeBytes(size);
}
file.setMetadata(getMetadataSet(doc));
file.setMetadataRef(getMetadataRefSet(doc, EntityField.METADATA_REF));
file.setAlternateIds(getResourceIdentifierSet(doc));
SeadDataLocation primaryDataLocation = getPrimaryDataLocation(doc);
file.setPrimaryLocation(primaryDataLocation);
file.setSecondaryDataLocations(getSecondaryDataLocationSet(doc));
return file;
}
private static String getFirst(SolrDocument doc, SolrName field) {
return (String) doc.getFirstValue(field.solrName());
}
