Ejemplo n.º 1
0
 @Override
 public AtomicLong next() {
   if (done) {
     return fileCounter;
   }
   try {
     URI uri = input.poll();
     if (uri != null) {
       push(uri);
     } else {
       done = true;
     }
     fileCounter.incrementAndGet();
   } catch (Exception e) {
     logger.error(e.getMessage(), e);
     done = true;
   }
   return fileCounter;
 }
Ejemplo n.º 2
0
  public static void main(String[] args) {
    int exitcode = 0;
    try {
      OptionParser parser =
          new OptionParser() {
            {
              accepts("elasticsearch").withRequiredArg().ofType(String.class).required();
              accepts("index").withRequiredArg().ofType(String.class).required();
              accepts("type").withRequiredArg().ofType(String.class).required();
              accepts("maxbulkactions").withRequiredArg().ofType(Integer.class).defaultsTo(1000);
              accepts("maxconcurrentbulkrequests")
                  .withRequiredArg()
                  .ofType(Integer.class)
                  .defaultsTo(4 * Runtime.getRuntime().availableProcessors());
              accepts("mock").withOptionalArg().ofType(Boolean.class).defaultsTo(Boolean.FALSE);
              accepts("path").withRequiredArg().ofType(String.class).required();
              accepts("pattern")
                  .withRequiredArg()
                  .ofType(String.class)
                  .required()
                  .defaultsTo("*.txt");
              accepts("threads").withRequiredArg().ofType(Integer.class).defaultsTo(1);
              accepts("help");
            }
          };
      final OptionSet options = parser.parse(args);
      if (options.hasArgument("help")) {
        System.err.println(
            "Help for "
                + Medline.class.getCanonicalName()
                + lf
                + " --help                 print this help message"
                + lf
                + " --elasticsearch <uri>  Elasticesearch URI"
                + lf
                + " --index <index>        Elasticsearch index name"
                + lf
                + " --type <type>          Elasticsearch type name"
                + lf
                + " --maxbulkactions <n>   the number of bulk actions per request (optional, default: 1000)"
                + " --maxconcurrentbulkrequests <n>the number of concurrent bulk requests (optional, default: 4 * cpu cores)"
                + " --path <path>          a file path from where the input files are recursively collected (required)"
                + lf
                + " --pattern <pattern>    a regex for selecting matching file names for input (default: *.txt)"
                + lf
                + " --threads <n>          the number of threads (optional, default: <num-of=cpus)");
        System.exit(1);
      }
      input =
          new Finder((String) options.valueOf("pattern"))
              .find((String) options.valueOf("path"))
              .getURIs();
      final Integer threads = (Integer) options.valueOf("threads");

      logger.info("found {} input files", input.size());

      URI esURI = URI.create((String) options.valueOf("elasticsearch"));
      index = (String) options.valueOf("index");
      type = (String) options.valueOf("type");
      int maxbulkactions = (Integer) options.valueOf("maxbulkactions");
      int maxconcurrentbulkrequests = (Integer) options.valueOf("maxconcurrentbulkrequests");
      boolean mock = (Boolean) options.valueOf("mock");

      final IngestClient es = mock ? new MockIngestClient() : new IngestClient();

      es.maxBulkActions(maxbulkactions)
          .maxConcurrentBulkRequests(maxconcurrentbulkrequests)
          .newClient(esURI)
          .waitForCluster(ClusterHealthStatus.YELLOW, TimeValue.timeValueSeconds(30));

      logger.info("creating new index ...");
      es.setIndex(index).setType(type).newIndex();
      logger.info("... new index created");

      final ResourceSink sink = new ResourceSink(es);

      ImportService service =
          new ImportService()
              .threads(threads)
              .factory(
                  new ImporterFactory() {
                    @Override
                    public Importer newImporter() {
                      return new SpringerCitations(sink);
                    }
                  })
              .execute();

      logger.info(
          "finished, number of files = {}, resources indexed = {}", fileCounter, sink.getCounter());

      service.shutdown();
      logger.info("service shutdown");

      es.shutdown();
      logger.info("elasticsearch client shutdown");

    } catch (IOException | InterruptedException | ExecutionException e) {
      logger.error(e.getMessage(), e);
      exitcode = 1;
    }
    System.exit(exitcode);
  }