private void handleException(Exception e) {
if (INTERRUPTED_EXCEPTION_TYPES.contains(e.getClass())) {
LOG.info("Changes feed was interrupted");
} else {
LOG.error("Caught exception while listening to changes feed:", e);
}
}
public void run() {
// each file is processed into a local hash table and then merged with the global results
// this will cause much less contention on the global table, but still avoids a sequential
// update
Hashtable<String, Integer> local_results =
new Hashtable<String, Integer>(WordCountJ.HASH_SIZE, WordCountJ.LF);
// grab a file to work on
String cf;
while ((cf = files.poll()) != null) {
try {
BufferedReader input = new BufferedReader(new FileReader(cf));
String text;
// well go line-by-line... maybe this is not the fastest
while ((text = input.readLine()) != null) {
// parse words
Matcher matcher = pattern.matcher(text);
while (matcher.find()) {
String word = matcher.group(1);
if (local_results.containsKey(word)) {
local_results.put(word, 1 + local_results.get(word));
} else {
local_results.put(word, 1);
}
}
}
input.close();
} catch (Exception e) {
System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
return;
}
// merge local hashmap with shared one,could have a
// seperate thread do this but that might be cheating
Iterator<Map.Entry<String, Integer>> updates = local_results.entrySet().iterator();
while (updates.hasNext()) {
Map.Entry<String, Integer> kv = updates.next();
String k = kv.getKey();
Integer v = kv.getValue();
synchronized (results) {
if (results.containsKey(k)) {
results.put(k, v + results.get(k));
} else {
results.put(k, v);
}
}
}
local_results.clear();
}
}
@Override
public void handle(HttpExchange httpExchange) throws IOException {
// Set common response headers
httpExchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
Future<String> json =
corenlpExecutor.submit(
() -> {
try {
// Get the document
Properties props =
new Properties() {
{
setProperty("annotators", "tokenize,ssplit,pos,lemma,ner,depparse");
}
};
Annotation doc = getDocument(props, httpExchange);
if (!doc.containsKey(CoreAnnotations.SentencesAnnotation.class)) {
StanfordCoreNLP pipeline = mkStanfordCoreNLP(props);
pipeline.annotate(doc);
}
// Construct the matcher
Map<String, String> params = getURLParams(httpExchange.getRequestURI());
// (get the pattern)
if (!params.containsKey("pattern")) {
respondError("Missing required parameter 'pattern'", httpExchange);
return "";
}
String pattern = params.get("pattern");
// (get whether to filter / find)
String filterStr = params.getOrDefault("filter", "false");
final boolean filter =
filterStr.trim().isEmpty()
|| "true".equalsIgnoreCase(filterStr.toLowerCase());
// (create the matcher)
final SemgrexPattern regex = SemgrexPattern.compile(pattern);
// Run TokensRegex
return JSONOutputter.JSONWriter.objectToJSON(
(docWriter) -> {
if (filter) {
// Case: just filter sentences
docWriter.set(
"sentences",
doc.get(CoreAnnotations.SentencesAnnotation.class)
.stream()
.map(
sentence ->
regex
.matcher(
sentence.get(
SemanticGraphCoreAnnotations
.CollapsedCCProcessedDependenciesAnnotation
.class))
.matches())
.collect(Collectors.toList()));
} else {
// Case: find matches
docWriter.set(
"sentences",
doc.get(CoreAnnotations.SentencesAnnotation.class)
.stream()
.map(
sentence ->
(Consumer<JSONOutputter.Writer>)
(JSONOutputter.Writer sentWriter) -> {
SemgrexMatcher matcher =
regex.matcher(
sentence.get(
SemanticGraphCoreAnnotations
.CollapsedCCProcessedDependenciesAnnotation
.class));
int i = 0;
while (matcher.find()) {
sentWriter.set(
Integer.toString(i),
(Consumer<JSONOutputter.Writer>)
(JSONOutputter.Writer matchWriter) -> {
IndexedWord match = matcher.getMatch();
matchWriter.set("text", match.word());
matchWriter.set(
"begin", match.index() - 1);
matchWriter.set("end", match.index());
for (String capture :
matcher.getNodeNames()) {
matchWriter.set(
"$" + capture,
(Consumer<JSONOutputter.Writer>)
groupWriter -> {
IndexedWord node =
matcher.getNode(
capture);
groupWriter.set(
"text", node.word());
groupWriter.set(
"begin",
node.index() - 1);
groupWriter.set(
"end", node.index());
});
}
});
i += 1;
}
sentWriter.set("length", i);
}));
}
});
} catch (Exception e) {
e.printStackTrace();
try {
respondError(e.getClass().getName() + ": " + e.getMessage(), httpExchange);
} catch (IOException ignored) {
}
}
return "";
});
// Send response
byte[] response = new byte[0];
try {
response = json.get(5, TimeUnit.SECONDS).getBytes();
} catch (InterruptedException | ExecutionException | TimeoutException e) {
respondError("Timeout when executing Semgrex query", httpExchange);
}
if (response.length > 0) {
httpExchange.getResponseHeaders().add("Content-Type", "text/json");
httpExchange.getResponseHeaders().add("Content-Length", Integer.toString(response.length));
httpExchange.sendResponseHeaders(HTTP_OK, response.length);
httpExchange.getResponseBody().write(response);
httpExchange.close();
}
}
@Override
public void handle(HttpExchange httpExchange) throws IOException {
// Set common response headers
httpExchange.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
// Get sentence.
Properties props;
Annotation ann;
StanfordCoreNLP.OutputFormat of;
log("[" + httpExchange.getRemoteAddress() + "] Received message");
try {
props = getProperties(httpExchange);
ann = getDocument(props, httpExchange);
of =
StanfordCoreNLP.OutputFormat.valueOf(
props.getProperty("outputFormat", "json").toUpperCase());
// Handle direct browser connections (i.e., not a POST request).
if (ann.get(CoreAnnotations.TextAnnotation.class).length() == 0) {
log("[" + httpExchange.getRemoteAddress() + "] Interactive connection");
staticPageHandle.handle(httpExchange);
return;
}
log("[" + httpExchange.getRemoteAddress() + "] API call");
} catch (Exception e) {
// Return error message.
e.printStackTrace();
String response = e.getMessage();
httpExchange.getResponseHeaders().add("Content-Type", "text/plain");
httpExchange.sendResponseHeaders(HTTP_BAD_INPUT, response.length());
httpExchange.getResponseBody().write(response.getBytes());
httpExchange.close();
return;
}
try {
// Annotate
StanfordCoreNLP pipeline = mkStanfordCoreNLP(props);
Future<Annotation> completedAnnotationFuture =
corenlpExecutor.submit(
() -> {
pipeline.annotate(ann);
return ann;
});
Annotation completedAnnotation = completedAnnotationFuture.get(5, TimeUnit.SECONDS);
// Get output
ByteArrayOutputStream os = new ByteArrayOutputStream();
StanfordCoreNLP.createOutputter(props, AnnotationOutputter.getOptions(pipeline))
.accept(completedAnnotation, os);
os.close();
byte[] response = os.toByteArray();
httpExchange.getResponseHeaders().add("Content-Type", getContentType(props, of));
httpExchange.getResponseHeaders().add("Content-Length", Integer.toString(response.length));
httpExchange.sendResponseHeaders(HTTP_OK, response.length);
httpExchange.getResponseBody().write(response);
httpExchange.close();
} catch (TimeoutException e) {
respondError("CoreNLP request timed out", httpExchange);
} catch (Exception e) {
// Return error message.
respondError(e.getClass().getName() + ": " + e.getMessage(), httpExchange);
}
}
@Override
public void onChange(String resource) {
try {
LOG.debug("Reading " + resource);
HashMap<CommandExecutor.Method, Stage> newexecutorsMap =
new HashMap<CommandExecutor.Method, Stage>();
Document document = getResourceLoader().getDocument(resource);
Map<Stage, Integer> totals = new EnumMap<Stage, Integer>(Stage.class);
if (document != null) {
org.w3c.dom.NodeList ellist = document.getDocumentElement().getChildNodes();
Stage prevStage = Stage.RECOGNIZER;
for (int i = 0; i <= ellist.getLength(); i++) {
if (ellist.item(i) instanceof Element) {
Element el = (Element) ellist.item(i);
if (el.getTagName().equals("localhost")) {
int max = Integer.parseInt(el.getAttribute("max_simultaneous_transcoders"));
Stage s = Stage.valueOf(el.getAttribute("stage").toUpperCase());
Integer t = totals.get(s);
if (t == null) t = 0;
t += max;
totals.put(s, t);
for (int j = 1; j <= max; j++) {
newexecutorsMap.put(new CommandExecutor.Method(), s);
}
} else if (el.getTagName().equals("server")) {
int max = Integer.parseInt(el.getAttribute("max_simultaneous_transcoders"));
Stage s = Stage.valueOf(el.getAttribute("stage").toUpperCase());
Integer t = totals.get(s);
if (t == null) t = 0;
t += max;
totals.put(s, t);
String host = el.getAttribute("host");
int port = Integer.parseInt(el.getAttribute("port"));
for (int j = 1; j <= max; j++) {
newexecutorsMap.put(new CommandExecutor.Method(host, port), s);
}
}
}
}
for (Map.Entry<Stage, Integer> e : totals.entrySet()) {
threadPools.get(e.getKey()).setCorePoolSize(e.getValue());
threadPools.get(e.getKey()).setMaximumPoolSize(e.getValue());
}
} else {
LOG.warn("No " + resource);
}
synchronized (executorsMap) {
executorsMap.clear();
executorsMap.putAll(newexecutorsMap);
}
LOG.service(
"Reading of configuration file "
+ resource
+ " successfull. Executors "
+ executorsMap
+ ". Max simultaneous transcoders: "
+ totals);
} catch (Exception e) {
LOG.error(
e.getClass()
+ " "
+ e.getMessage()
+ " In "
+ resource
+ " Executors now "
+ executorsMap
+ " (not changed)",
e);
}
}