@Override
public void process(
String outputName,
InputSupplier<? extends InputStream> in,
OutputSupplier<? extends OutputStream> out,
boolean closeAtFinish)
throws IOException {
process(outputName, in.getInput(), out.getOutput(), closeAtFinish);
}
public static void encode(JvmOptions jvmOptions, OutputSupplier<? extends Writer> writerSupplier)
throws IOException {
Writer writer = writerSupplier.getOutput();
try {
GSON.toJson(jvmOptions, writer);
} finally {
writer.close();
}
}
/**
* Reads conll sentences, parses them, and writes the json-serialized results.
*
* @param inputSupplier where to read conll sentences from
* @param outputSupplier where to write the results to
* @param numThreads the number of threads to use
* @throws IOException
* @throws InterruptedException
*/
public void runParser(
final InputSupplier<? extends Readable> inputSupplier,
final OutputSupplier<? extends Writer> outputSupplier,
final int numThreads)
throws IOException, InterruptedException {
// use the producer-worker-consumer pattern to parse all sentences in multiple threads, while
// keeping
// output in order.
final BlockingQueue<Future<Optional<SemaforParseResult>>> results =
Queues.newLinkedBlockingDeque(5 * numThreads);
final ExecutorService workerThreadPool = newFixedThreadPool(numThreads);
// try to shutdown gracefully. don't worry too much if it doesn't work
Runtime.getRuntime()
.addShutdownHook(
new Thread(
new Runnable() {
@Override
public void run() {
try {
workerThreadPool.shutdown();
workerThreadPool.awaitTermination(5, TimeUnit.SECONDS);
} catch (InterruptedException ignored) {
}
}
}));
final PrintWriter output = new PrintWriter(outputSupplier.getOutput());
try {
// Start thread to fetch computed results and write to file
final Thread consumer =
new Thread(
new Runnable() {
@Override
public void run() {
while (!Thread.currentThread().isInterrupted()) {
try {
final Optional<SemaforParseResult> oResult = results.take().get();
if (!oResult.isPresent()) break; // got poison pill. we're done
output.println(oResult.get().toJson());
output.flush();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}
});
consumer.start();
// in main thread, put placeholders on results queue (so results stay in order), then
// tell a worker thread to fill up the placeholder
final SentenceCodec.SentenceIterator sentences =
ConllCodec.readInput(inputSupplier.getInput());
try {
int i = 0;
while (sentences.hasNext()) {
final Sentence sentence = sentences.next();
final int sentenceId = i;
results.put(
workerThreadPool.submit(
new Callable<Optional<SemaforParseResult>>() {
@Override
public Optional<SemaforParseResult> call() throws Exception {
final long start = System.currentTimeMillis();
try {
final SemaforParseResult result = parseSentence(sentence);
final long end = System.currentTimeMillis();
System.err.printf(
"parsed sentence %d in %d millis.%n", sentenceId, end - start);
return Optional.of(result);
} catch (Exception e) {
e.printStackTrace();
throw e;
}
}
}));
i++;
}
// put a poison pill on the queue to signal that we're done
results.put(
workerThreadPool.submit(
new Callable<Optional<SemaforParseResult>>() {
@Override
public Optional<SemaforParseResult> call() throws Exception {
return Optional.absent();
}
}));
workerThreadPool.shutdown();
} finally {
closeQuietly(sentences);
}
// wait for consumer to finish
consumer.join();
} finally {
closeQuietly(output);
}
System.err.println("Done.");
}
public static OutputStreamWriter writerForFile(String filename, boolean append)
throws IOException {
OutputSupplier<OutputStreamWriter> os =
Files.newWriterSupplier(new File(filename), CHARSET, append);
return os.getOutput();
}