// private void multiThreadedMarshal(PrintWriter writer, List<Result> results,
private String multiThreadedMarshal(
List<Result> results, String recordSchema, final Map<String, Serializable> arguments)
throws CatalogTransformerException {
CompletionService<BinaryContent> completionService =
new ExecutorCompletionService<>(queryExecutor);
try {
final MetacardTransformer transformer =
metacardTransformerManager.getTransformerBySchema(recordSchema);
if (transformer == null) {
throw new CatalogTransformerException(
"Cannot find transformer for schema: " + recordSchema);
}
Map<Future<BinaryContent>, Result> futures = new HashMap<>(results.size());
for (Result result : results) {
final Metacard mc = result.getMetacard();
// the "current" thread will run submitted task when queueSize exceeded; effectively
// blocking enqueue of more tasks.
futures.put(
completionService.submit(
() -> {
BinaryContent content = transformer.transform(mc, arguments);
return content;
}),
result);
}
// TODO - really? I can't use a list of some sort?
InputStream[] contents = new InputStream[results.size()];
while (!futures.isEmpty()) {
Future<BinaryContent> completedFuture = completionService.take();
int index = results.indexOf(futures.get(completedFuture));
contents[index] = completedFuture.get().getInputStream();
futures.remove(completedFuture);
}
CharArrayWriter accum = new CharArrayWriter(ACCUM_INITIAL_SIZE);
for (InputStream is : contents) {
IOUtils.copy(is, accum);
}
return accum.toString();
} catch (IOException | InterruptedException | ExecutionException xe) {
throw new CatalogTransformerException(xe);
}
} // end multiThreadedMarshal()
@Override
public QueryResponse processPostQuery(QueryResponse input) throws StopProcessingException {
if (input.getRequest() == null || input.getRequest().getProperties() == null) {
throw new StopProcessingException(
"Unable to filter contents of current message, no user Subject available.");
}
Subject subject = getSubject(input);
List<Result> results = input.getResults();
List<Result> newResults = new ArrayList<>(results.size());
Metacard metacard;
KeyValueCollectionPermission securityPermission =
new KeyValueCollectionPermission(CollectionPermission.READ_ACTION);
int filteredMetacards = 0;
for (Result result : results) {
metacard = result.getMetacard();
Attribute attr = metacard.getAttribute(Metacard.SECURITY);
if (!checkPermissions(attr, securityPermission, subject, CollectionPermission.READ_ACTION)) {
for (FilterStrategy filterStrategy : filterStrategies.values()) {
FilterResult filterResult = filterStrategy.process(input, metacard);
if (filterResult.processed()) {
if (filterResult.metacard() != null) {
newResults.add(new ResultImpl(filterResult.metacard()));
}
break;
// returned responses are ignored for queries
}
}
filteredMetacards++;
} else {
newResults.add(result);
}
}
LOGGER.info("Filtered {} metacards, returned {}", filteredMetacards, newResults.size());
SecurityLogger.logInfo(
"Filtered " + filteredMetacards + " metacards, returned " + newResults.size());
input.getResults().clear();
input.getResults().addAll(newResults);
newResults.clear();
return input;
}
/**
* Replaces the site name(s) of {@link FederatedSource}s in the {@link QueryResponse} with the
* fanout's site name to keep info about the {@link FederatedSource}s hidden from the external
* client.
*
* @param queryResponse the original {@link QueryResponse} from the query request
* @return the updated {@link QueryResponse} with all site names replaced with fanout's site name
*/
public QueryResponse replaceSourceId(QueryResponse queryResponse) {
LOGGER.trace("ENTERING: replaceSourceId()");
List<Result> results = queryResponse.getResults();
QueryResponseImpl newResponse =
new QueryResponseImpl(queryResponse.getRequest(), queryResponse.getProperties());
for (Result result : results) {
MetacardImpl newMetacard = new MetacardImpl(result.getMetacard());
newMetacard.setSourceId(this.getId());
ResultImpl newResult = new ResultImpl(newMetacard);
// Copy over scores
newResult.setDistanceInMeters(result.getDistanceInMeters());
newResult.setRelevanceScore(result.getRelevanceScore());
newResponse.addResult(newResult, false);
}
newResponse.setHits(queryResponse.getHits());
newResponse.closeResultQueue();
LOGGER.trace("EXITING: replaceSourceId()");
return newResponse;
}
private QueryResponse populateQueryResponsePolicyMap(QueryResponse queryResponse)
throws FederationException {
HashMap<String, Set<String>> responsePolicyMap = new HashMap<>();
Map<String, Serializable> unmodifiableProperties =
Collections.unmodifiableMap(queryResponse.getProperties());
for (Result result : queryResponse.getResults()) {
HashMap<String, Set<String>> itemPolicyMap = new HashMap<>();
for (PolicyPlugin plugin : frameworkProperties.getPolicyPlugins()) {
try {
PolicyResponse policyResponse = plugin.processPostQuery(result, unmodifiableProperties);
opsSecuritySupport.buildPolicyMap(itemPolicyMap, policyResponse.itemPolicy().entrySet());
opsSecuritySupport.buildPolicyMap(
responsePolicyMap, policyResponse.operationPolicy().entrySet());
} catch (StopProcessingException e) {
throw new FederationException("Query could not be executed.", e);
}
}
result.getMetacard().setAttribute(new AttributeImpl(Metacard.SECURITY, itemPolicyMap));
}
queryResponse.getProperties().put(PolicyPlugin.OPERATION_SECURITY, responsePolicyMap);
return queryResponse;
}
@Override
public PolicyResponse processPostQuery(Result input, Map<String, Serializable> properties)
throws StopProcessingException {
HashMap<String, Set<String>> itemPolicy = new HashMap<>();
Metacard metacard = input.getMetacard();
if (metacard.getTags().contains(RegistryConstants.REGISTRY_TAG)) {
if ((whiteList && !registryEntryIds.contains(metacard.getId()))
|| (!whiteList && registryEntryIds.contains(metacard.getId()))) {
itemPolicy.putAll(bypassAccessPolicy);
} else {
itemPolicy.putAll(readAccessPolicy);
}
}
return new PolicyResponseImpl(new HashMap<>(), itemPolicy);
}
@Override
protected Object doExecute() throws Exception {
String formatString =
"%1$-33s %2$-26s %3$-" + TITLE_MAX_LENGTH + "s %4$-" + EXCERPT_MAX_LENGTH + "s%n";
CatalogFacade catalogProvider = getCatalog();
Filter filter = null;
if (cqlFilter != null) {
filter = CQL.toFilter(cqlFilter);
} else {
if (searchPhrase == null) {
searchPhrase = "*";
}
if (caseSensitive) {
filter =
getFilterBuilder()
.attribute(Metacard.ANY_TEXT)
.is()
.like()
.caseSensitiveText(searchPhrase);
} else {
filter = getFilterBuilder().attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase);
}
}
QueryImpl query = new QueryImpl(filter);
query.setRequestsTotalResultsCount(true);
if (numberOfItems > -1) {
query.setPageSize(numberOfItems);
}
long start = System.currentTimeMillis();
SourceResponse response = catalogProvider.query(new QueryRequestImpl(query));
long end = System.currentTimeMillis();
int size = 0;
if (response.getResults() != null) {
size = response.getResults().size();
}
console.println();
console.printf(
" %d result(s) out of %s%d%s in %3.3f seconds",
(size),
Ansi.ansi().fg(Ansi.Color.CYAN).toString(),
response.getHits(),
Ansi.ansi().reset().toString(),
(end - start) / MILLISECONDS_PER_SECOND);
console.printf(formatString, "", "", "", "");
printHeaderMessage(String.format(formatString, ID, DATE, TITLE, EXCERPT));
for (Result result : response.getResults()) {
Metacard metacard = result.getMetacard();
String title = (metacard.getTitle() != null ? metacard.getTitle() : "N/A");
String excerpt = "N/A";
String modifiedDate = "";
if (searchPhrase != null) {
if (metacard.getMetadata() != null) {
XPathHelper helper = new XPathHelper(metacard.getMetadata());
String indexedText = helper.getDocument().getDocumentElement().getTextContent();
indexedText = indexedText.replaceAll("\\r\\n|\\r|\\n", " ");
String normalizedSearchPhrase = searchPhrase.replaceAll("\\*", "");
int index = -1;
if (caseSensitive) {
index = indexedText.indexOf(normalizedSearchPhrase);
} else {
index = indexedText.toLowerCase().indexOf(normalizedSearchPhrase.toLowerCase());
}
if (index != -1) {
int contextLength = (EXCERPT_MAX_LENGTH - normalizedSearchPhrase.length() - 8) / 2;
excerpt = "..." + indexedText.substring(Math.max(index - contextLength, 0), index);
excerpt = excerpt + Ansi.ansi().fg(Ansi.Color.GREEN).toString();
excerpt =
excerpt + indexedText.substring(index, index + normalizedSearchPhrase.length());
excerpt = excerpt + Ansi.ansi().reset().toString();
excerpt =
excerpt
+ indexedText.substring(
index + normalizedSearchPhrase.length(),
Math.min(
indexedText.length(),
index + normalizedSearchPhrase.length() + contextLength))
+ "...";
}
}
}
if (metacard.getModifiedDate() != null) {
modifiedDate =
new DateTime(metacard.getModifiedDate().getTime()).toString(DATETIME_FORMATTER);
}
console.printf(
formatString,
metacard.getId(),
modifiedDate,
title.substring(0, Math.min(title.length(), TITLE_MAX_LENGTH)),
excerpt);
}
return null;
}
@Override
public PolicyResponse processPostQuery(Result input, Map<String, Serializable> properties)
throws StopProcessingException {
return new PolicyResponseImpl(null, buildSecurityMap(input.getMetacard()));
}