@Override
 public void process(
     String outputName,
     InputSupplier<? extends InputStream> in,
     OutputSupplier<? extends OutputStream> out,
     boolean closeAtFinish)
     throws IOException {
   process(outputName, in.getInput(), out.getOutput(), closeAtFinish);
 }
 public static void encode(JvmOptions jvmOptions, OutputSupplier<? extends Writer> writerSupplier)
     throws IOException {
   Writer writer = writerSupplier.getOutput();
   try {
     GSON.toJson(jvmOptions, writer);
   } finally {
     writer.close();
   }
 }
Beispiel #3
0
  /**
   * Reads conll sentences, parses them, and writes the json-serialized results.
   *
   * @param inputSupplier where to read conll sentences from
   * @param outputSupplier where to write the results to
   * @param numThreads the number of threads to use
   * @throws IOException
   * @throws InterruptedException
   */
  public void runParser(
      final InputSupplier<? extends Readable> inputSupplier,
      final OutputSupplier<? extends Writer> outputSupplier,
      final int numThreads)
      throws IOException, InterruptedException {
    // use the producer-worker-consumer pattern to parse all sentences in multiple threads, while
    // keeping
    // output in order.
    final BlockingQueue<Future<Optional<SemaforParseResult>>> results =
        Queues.newLinkedBlockingDeque(5 * numThreads);
    final ExecutorService workerThreadPool = newFixedThreadPool(numThreads);
    // try to shutdown gracefully. don't worry too much if it doesn't work
    Runtime.getRuntime()
        .addShutdownHook(
            new Thread(
                new Runnable() {
                  @Override
                  public void run() {
                    try {
                      workerThreadPool.shutdown();
                      workerThreadPool.awaitTermination(5, TimeUnit.SECONDS);
                    } catch (InterruptedException ignored) {
                    }
                  }
                }));

    final PrintWriter output = new PrintWriter(outputSupplier.getOutput());
    try {
      // Start thread to fetch computed results and write to file
      final Thread consumer =
          new Thread(
              new Runnable() {
                @Override
                public void run() {
                  while (!Thread.currentThread().isInterrupted()) {
                    try {
                      final Optional<SemaforParseResult> oResult = results.take().get();
                      if (!oResult.isPresent()) break; // got poison pill. we're done
                      output.println(oResult.get().toJson());
                      output.flush();
                    } catch (Exception e) {
                      e.printStackTrace();
                      throw new RuntimeException(e);
                    }
                  }
                }
              });
      consumer.start();
      // in main thread, put placeholders on results queue (so results stay in order), then
      // tell a worker thread to fill up the placeholder
      final SentenceCodec.SentenceIterator sentences =
          ConllCodec.readInput(inputSupplier.getInput());
      try {
        int i = 0;
        while (sentences.hasNext()) {
          final Sentence sentence = sentences.next();
          final int sentenceId = i;
          results.put(
              workerThreadPool.submit(
                  new Callable<Optional<SemaforParseResult>>() {
                    @Override
                    public Optional<SemaforParseResult> call() throws Exception {
                      final long start = System.currentTimeMillis();
                      try {
                        final SemaforParseResult result = parseSentence(sentence);
                        final long end = System.currentTimeMillis();
                        System.err.printf(
                            "parsed sentence %d in %d millis.%n", sentenceId, end - start);
                        return Optional.of(result);
                      } catch (Exception e) {
                        e.printStackTrace();
                        throw e;
                      }
                    }
                  }));
          i++;
        }
        // put a poison pill on the queue to signal that we're done
        results.put(
            workerThreadPool.submit(
                new Callable<Optional<SemaforParseResult>>() {
                  @Override
                  public Optional<SemaforParseResult> call() throws Exception {
                    return Optional.absent();
                  }
                }));
        workerThreadPool.shutdown();
      } finally {
        closeQuietly(sentences);
      }
      // wait for consumer to finish
      consumer.join();
    } finally {
      closeQuietly(output);
    }
    System.err.println("Done.");
  }
Beispiel #4
0
 public static OutputStreamWriter writerForFile(String filename, boolean append)
     throws IOException {
   OutputSupplier<OutputStreamWriter> os =
       Files.newWriterSupplier(new File(filename), CHARSET, append);
   return os.getOutput();
 }