/** 加载量词词典 */
  private void loadQuantifierDict() {
    // 建立一个量词典实例
    _QuantifierDict = new DictSegment((char) 0);
    // 读取量词词典文件
    File file = new File(configuration.getDictRoot(), Dictionary.PATH_DIC_QUANTIFIER);
    InputStream is = null;
    try {
      is = new FileInputStream(file);
    } catch (FileNotFoundException e) {
      logger.error("ik-analyzer", e);
    }
    try {
      BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512);
      String theWord = null;
      do {
        theWord = br.readLine();
        if (theWord != null && !"".equals(theWord.trim())) {
          _QuantifierDict.fillSegment(theWord.trim().toCharArray());
        }
      } while (theWord != null);

    } catch (IOException ioe) {
      logger.error("Quantifier Dictionary loading exception.");

    } finally {
      try {
        if (is != null) {
          is.close();
          is = null;
        }
      } catch (IOException e) {
        logger.error("ik-analyzer", e);
      }
    }
  }
Exemple #2
0
  @Override
  public void run() {
    if (logger.isDebugEnabled()) {
      logger.debug("Create task manager thread.");
    }

    do {
      logger.info("TaskManager: current task index: " + Integer.toString(currentTaskIndex));
      try {
        String output = currentTask.Run();
        logger.info("Task {[]} output: {[]}", currentTask.id(), output);
        logger.info("Task [" + currentTask.id() + "] output: " + output);
      } catch (IOException ex) {
        logger.error("TaskManager: IOException");
      } catch (InterruptedException ex) {
        logger.error("TaskManager: Interrupted Exception");
      }

      currentTask = GetNextTask();
    } while (null != currentTask);

    DeleteMappingRequest req = new DeleteMappingRequest("_river");
    req.type(river.riverName().name());
    DeleteMappingResponse resp = client.admin().indices().deleteMapping(req).actionGet();
    logger.info("TaskManager: delete request: " + resp.toString());
  }
  /**
   * Queries the {@link #rdfEndpoint(String)} with each of the {@link #rdfQueries} and harvests the
   * results of the query.
   */
  private void harvestFromEndpoint() {

    Query query;
    QueryExecution qExec;

    for (String rdfQuery : rdfQueries) {
      if (closed) break;

      logger.info(
          "Harvesting with query: [{}] on index [{}] and type [{}]", rdfQuery, indexName, typeName);

      try {
        query = QueryFactory.create(rdfQuery);
      } catch (QueryParseException qpe) {
        logger.error("Could not parse [{}]. Please provide a relevant query. {}", rdfQuery, qpe);
        continue;
      }

      qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);

      try {
        harvest(qExec);
      } catch (Exception e) {
        logger.error("Exception [{}] occurred while harvesting", e.getLocalizedMessage());
      } finally {
        qExec.close();
      }
    }
  }
  private void loadPrepDict() {

    _PrepDict = new DictSegment((char) 0);
    File file = new File(configuration.getDictRoot(), Dictionary.PATH_DIC_PREP);
    InputStream is = null;
    try {
      is = new FileInputStream(file);
    } catch (FileNotFoundException e) {
      logger.error("ik-analyzer", e);
    }
    if (is == null) {
      throw new RuntimeException("Preposition Dictionary not found!!!");
    }
    try {

      BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512);
      String theWord;
      do {
        theWord = br.readLine();
        if (theWord != null && !"".equals(theWord.trim())) {

          _PrepDict.fillSegment(theWord.trim().toCharArray());
        }
      } while (theWord != null);
    } catch (IOException e) {
      logger.error("ik-analyzer", e);
    } finally {
      try {
        is.close();
        is = null;
      } catch (IOException e) {
        logger.error("ik-analyzer", e);
      }
    }
  }
 public static void removeAMB(String content) {
   try {
     File file =
         new File(AnsjElasticConfigurator.environment.configFile().toFile(), "ansj/ambiguity.dic");
     removeFile(content, file, true);
   } catch (FileNotFoundException e) {
     logger.error("file not found $ES_HOME/config/ansj/user/ext.dic", e);
     e.printStackTrace();
   } catch (IOException e) {
     logger.error("read exception", e);
     e.printStackTrace();
   }
 }
 @After
 public void stopNodes() {
   try {
     closeNodes();
   } catch (Exception e) {
     logger.error("can not close nodes", e);
   } finally {
     try {
       deleteFiles();
       logger.info("data files wiped");
     } catch (IOException e) {
       logger.error(e.getMessage(), e);
     }
   }
 }
  /**
   * Query SPARQL endpoint with a SELECT query
   *
   * @param qExec QueryExecution encapsulating the query
   * @return model retrieved by querying the endpoint
   */
  private Model getSelectModel(QueryExecution qExec) {
    Model model = ModelFactory.createDefaultModel();
    Graph graph = model.getGraph();
    ResultSet results = qExec.execSelect();

    while (results.hasNext()) {
      QuerySolution sol = results.next();
      String subject;
      String predicate;
      RDFNode object;

      try {
        subject = sol.getResource("s").toString();
        predicate = sol.getResource("p").toString();
        object = sol.get("o");
      } catch (NoSuchElementException e) {
        logger.error("SELECT query does not return a (?s ?p ?o) Triple");
        continue;
      }

      Node objNode;
      if (object.isLiteral()) {
        Literal obj = object.asLiteral();
        objNode = NodeFactory.createLiteral(obj.getString(), obj.getDatatype());
      } else {
        objNode = NodeFactory.createLiteral(object.toString());
      }

      graph.add(
          new Triple(NodeFactory.createURI(subject), NodeFactory.createURI(predicate), objNode));
    }

    return model;
  }
 @Override
 public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
   if (e.getCause() instanceof ClosedChannelException) {
     // do nothing
   } else if (e.getCause() instanceof UnresolvedAddressException) {
     logger.error(
         "Unable to resolve one of the server addresses: [{}]", e.getCause().getMessage());
   } else if (e.getCause() instanceof ConnectException) {
     logger.error("Unable to connect: [{}]", e.getCause().getMessage());
   } else if (e.getCause().getMessage() != null
       && e.getCause().getMessage().contains("Connection reset by peer")) {
     // still do nothing
   } else {
     super.exceptionCaught(ctx, e);
   }
 }
  @Test
  public void testRandomDocsNodeClient() throws Exception {
    final NodeClient es =
        new NodeClient()
            .maxActionsPerBulkRequest(1000)
            .flushInterval(TimeValue.timeValueSeconds(10))
            .newClient(client("1"))
            .newIndex("test");

    try {
      for (int i = 0; i < 12345; i++) {
        es.index("test", "test", null, "{ \"name\" : \"" + randomString(32) + "\"}");
      }
      es.flush();
    } catch (NoNodeAvailableException e) {
      logger.warn("skipping, no node available");
    } finally {
      es.shutdown();
      assertEquals(13, es.getState().getTotalIngest().count());
      if (es.hasThrowable()) {
        logger.error("error", es.getThrowable());
      }
      assertFalse(es.hasThrowable());
    }
  }
 @Before
 public void startNodes() {
   try {
     setClusterName();
     startNode("1");
     startNode("2"); // we need 2 nodes for knapsack
     findNodeAddress();
     try {
       ClusterHealthResponse healthResponse =
           client("1")
               .execute(
                   ClusterHealthAction.INSTANCE,
                   new ClusterHealthRequest()
                       .waitForYellowStatus()
                       .timeout(TimeValue.timeValueSeconds(30)))
               .actionGet();
       if (healthResponse != null && healthResponse.isTimedOut()) {
         throw new IOException(
             "cluster state is "
                 + healthResponse.getStatus().name()
                 + ", from here on, everything will fail!");
       }
     } catch (ElasticsearchTimeoutException e) {
       throw new IOException(
           "timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
     }
     logger.info("ready");
   } catch (Throwable t) {
     logger.error("startNodes failed", t);
   }
 }
Exemple #11
0
  @Override
  public void testFailure(Failure failure) throws Exception {
    // Ignore assumptions.
    if (failure.getException() instanceof AssumptionViolatedException) {
      return;
    }

    final Description d = failure.getDescription();
    final StringBuilder b = new StringBuilder();
    b.append("FAILURE  : ").append(d.getDisplayName()).append("\n");
    b.append("REPRODUCE WITH  : mvn test");
    ReproduceErrorMessageBuilder builder =
        reproduceErrorMessageBuilder(b).appendAllOpts(failure.getDescription());

    if (mustAppendClusterSeed(failure)) {
      appendClusterSeed(builder);
    }

    b.append("\n");
    b.append("Throwable:\n");
    if (failure.getException() != null) {
      traces().formatThrowable(b, failure.getException());
    }

    logger.error(b.toString());
  }
 @Override
 protected void doExecute(
     final KnapsackPullRequest request, ActionListener<KnapsackPullResponse> listener) {
   final KnapsackState state =
       new KnapsackState().setMode("pull").setNodeName(nodeService.nodeName());
   final KnapsackPullResponse response = new KnapsackPullResponse().setState(state);
   try {
     final BulkTransportClient transportClient =
         ClientBuilder.builder()
             .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
             .put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
             .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
             .put(clientSettings(client, request))
             .toBulkTransportClient();
     final BulkNodeClient nodeClient =
         ClientBuilder.builder()
             .put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, request.getMaxActionsPerBulkRequest())
             .put(ClientBuilder.MAX_CONCURRENT_REQUESTS, request.getMaxBulkConcurrency())
             .put(ClientBuilder.FLUSH_INTERVAL, TimeValue.timeValueSeconds(5))
             .toBulkNodeClient(client);
     state.setTimestamp(new DateTime());
     response.setRunning(true);
     knapsack.submit(
         new Thread() {
           public void run() {
             performPull(request, state, transportClient, nodeClient);
           }
         });
     listener.onResponse(response);
   } catch (Throwable e) {
     logger.error(e.getMessage(), e);
     listener.onFailure(e);
   }
 }
 private void handleException(final TransportResponseHandler handler, Throwable error) {
   if (!(error instanceof RemoteTransportException)) {
     error = new RemoteTransportException(error.getMessage(), error);
   }
   final RemoteTransportException rtx = (RemoteTransportException) error;
   if (handler.executor() == ThreadPool.Names.SAME) {
     try {
       handler.handleException(rtx);
     } catch (Throwable e) {
       logger.error("failed to handle exception response [{}]", e, handler);
     }
   } else {
     threadPool
         .executor(handler.executor())
         .execute(
             new Runnable() {
               @Override
               public void run() {
                 try {
                   handler.handleException(rtx);
                 } catch (Throwable e) {
                   logger.error("failed to handle exception response [{}]", e, handler);
                 }
               }
             });
   }
 }
 private boolean handleUnknownRevisionResponse(ChannelHandlerContext ctx) {
   logger.error(
       "Unknown revision response received on connection [{}]",
       ctx.getChannel().getRemoteAddress(),
       ctx.getChannel().getLocalAddress());
   return false;
 }
 @Override
 protected void doClose() throws ElasticsearchException {
   int size = tasks.size();
   if (size > 0) {
     for (Future<?> f : tasks) {
       if (!f.isDone()) {
         logger.info("aborting knapsack task {}", f);
         boolean b = f.cancel(true);
         if (!b) {
           logger.error("knapsack task {} could not be cancelled", f);
         }
       }
     }
     tasks.clear();
   }
   logger.info("knapsack shutdown...");
   executor.shutdown();
   try {
     this.executor.awaitTermination(5, TimeUnit.SECONDS);
   } catch (InterruptedException e) {
     throw new ElasticsearchException(e.getMessage());
   }
   if (!executor.isShutdown()) {
     logger.info("knapsack shutdown now");
     executor.shutdownNow();
     try {
       this.executor.awaitTermination(5, TimeUnit.SECONDS);
     } catch (InterruptedException e) {
       throw new ElasticsearchException(e.getMessage());
     }
   }
   logger.info("knapsack shutdown complete");
 }
 @Override
 public BaseIngestTransportClient newIndex(String index) {
   if (client == null) {
     logger.warn("no client for create index");
     return this;
   }
   if (index == null) {
     logger.warn("no index name given to create index");
     return this;
   }
   CreateIndexRequest request = new CreateIndexRequest(index);
   if (getSettings() != null) {
     request.settings(getSettings());
   }
   if (getMappings() != null) {
     for (Map.Entry<String, String> me : getMappings().entrySet()) {
       request.mapping(me.getKey(), me.getValue());
     }
   }
   logger.info(
       "creating index {} with settings = {}, mappings = {}",
       index,
       getSettings() != null ? getSettings().getAsMap() : null,
       getMappings());
   try {
     client.admin().indices().create(request).actionGet();
   } catch (Exception e) {
     logger.error(e.getMessage(), e);
   }
   return this;
 }
 private List<KnapsackState> get(String name) throws IOException {
   ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
   try {
     logger.debug("get knapsack states: {}", name);
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     GetResponse getResponse =
         client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
     if (!getResponse.isExists()) {
       return builder.build();
     }
     XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
     while (parser.nextToken() != START_ARRAY) {
       // forward
     }
     while (parser.nextToken() != END_ARRAY) {
       KnapsackState state = new KnapsackState();
       builder.add(state.fromXContent(parser));
     }
     return builder.build();
   } catch (Throwable t) {
     logger.error("get settings failed", t);
     return null;
   }
 }
Exemple #18
0
  public static DigestBlob resumeTransfer(
      BlobContainer blobContainer, String digest, UUID transferId, long currentPos) {
    DigestBlob digestBlob = new DigestBlob(blobContainer, digest, transferId);
    digestBlob.file = getTmpFilePath(blobContainer, digest, transferId);

    try {
      logger.trace("Resuming DigestBlob {}. CurrentPos {}", digest, currentPos);

      digestBlob.headFileChannel = new FileOutputStream(digestBlob.file, false).getChannel();
      digestBlob.headLength = currentPos;
      digestBlob.headSize = new AtomicLong();
      digestBlob.headCatchedUpLatch = new CountDownLatch(1);

      RandomAccessFile raf = new RandomAccessFile(digestBlob.file, "rw");
      raf.setLength(currentPos);
      raf.close();

      FileOutputStream outputStream = new FileOutputStream(digestBlob.file, true);
      digestBlob.fileChannel = outputStream.getChannel();
    } catch (IOException ex) {
      logger.error("error resuming transfer of {}, id: {}", ex, digest, transferId);
      return null;
    }

    return digestBlob;
  }
  private boolean handleRevision1Response(ChannelHandlerContext ctx, int payloadLength)
      throws Exception {
    int code = buffered.readInt();

    int descriptionLength = buffered.readInt();
    byte[] descBytes = new byte[descriptionLength];
    buffered.readBytes(descBytes, 0, descBytes.length);

    String description = new String(descBytes, StandardCharsets.UTF_8);

    logger.debug(
        "Decoded payload with length:[{}], code:[{}], descriptionLength:[{}], description:[{}] on connection [{}]",
        payloadLength,
        code,
        descriptionLength,
        description,
        ctx.getChannel().getLocalAddress());

    if (200 <= code && code <= 299) {
      logger.info(
          "Connected to Found Elasticsearch: [{}]: [{}] on connection [{}]",
          code,
          description,
          ctx.getChannel().getLocalAddress());
      return true;
    } else {
      logger.error(
          "Unable to connect to Found Elasticsearch: [{}]: [{}] on connection [{}]",
          code,
          description,
          ctx.getChannel().getLocalAddress());
      return false;
    }
  }
 @Override
 public void onFailure(Throwable e) {
   try {
     restChannel.sendResponse(new XContentThrowableRestResponse(restRequest, e));
   } catch (IOException e1) {
     logger.error("Failed to send failure response", e1);
   }
 }
  /** 加载用户配置的扩展词典到主词库表 */
  private void loadExtDict() {
    // 加载扩展词典配置
    List<String> extDictFiles = configuration.getExtDictionarys();
    if (extDictFiles != null) {
      InputStream is = null;
      for (String extDictName : extDictFiles) {
        // 读取扩展词典文件
        logger.info("[Dict Loading]" + extDictName);
        File file = new File(configuration.getDictRoot(), extDictName);
        try {
          is = new FileInputStream(file);
        } catch (FileNotFoundException e) {
          logger.error("ik-analyzer", e);
        }

        // 如果找不到扩展的字典,则忽略
        if (is == null) {
          continue;
        }
        try {
          BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"), 512);
          String theWord = null;
          do {
            theWord = br.readLine();
            if (theWord != null && !"".equals(theWord.trim())) {
              // 加载扩展词典数据到主内存词典中
              _MainDict.fillSegment(theWord.trim().toCharArray());
            }
          } while (theWord != null);

        } catch (IOException e) {
          logger.error("ik-analyzer", e);
        } finally {
          try {
            if (is != null) {
              is.close();
              is = null;
            }
          } catch (IOException e) {
            logger.error("ik-analyzer", e);
          }
        }
      }
    }
  }
 /**
  * 关闭数据库连接
  *
  * @param jedis redis.
  */
 public static void closeConnection(Jedis jedis) {
   if (null != jedis) {
     try {
       jedisPool.close();
     } catch (Exception e) {
       // e.printStackTrace();
       logger.error(e.getMessage(), e);
     }
   }
 }
  /** Harvests all the triplets from each URI in the @rdfUris list */
  private void harvestFromDumps() {
    for (String uri : rdfUris) {
      if (uri.isEmpty()) continue;

      logger.info("Harvesting uri [{}]", uri);

      Model model = ModelFactory.createDefaultModel();
      try {
        RDFDataMgr.read(model, uri.trim(), RDFLanguages.RDFXML);
        BulkRequestBuilder bulkRequest = client.prepareBulk();
        addModelToES(model, bulkRequest, true);
      } catch (RiotException re) {
        logger.error("Illegal xml character [{}]", re.getLocalizedMessage());
      } catch (Exception e) {
        logger.error(
            "Exception when harvesting url: {}. Details: {}", uri, e.getLocalizedMessage());
      }
    }
  }
 public static void appendAMB(String content) {
   try {
     File file =
         new File(AnsjElasticConfigurator.environment.configFile().toFile(), "ansj/ambiguity.dic");
     appendFile(content, file);
   } catch (IOException e) {
     logger.error("read exception", e);
     e.printStackTrace();
   }
 }
 /**
  * 获取数据库连接
  *
  * @return conn
  */
 public static Jedis getConnection() {
   Jedis jedis = null;
   try {
     jedis = jedisPool.getResource();
   } catch (Exception e) {
     e.printStackTrace();
     logger.error(e.getMessage(), e);
   }
   return jedis;
 }
 private void remove(final String name) {
   try {
     logger.debug("remove: {}", name);
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     client.prepareDelete(INDEX_NAME, MAPPING_NAME, name).setRefresh(true).execute().actionGet();
   } catch (Throwable t) {
     logger.error("remove failed", t);
   }
 }
  @Test
  public void testRenameAttribute() throws Throwable {
    logger.debug("Start testRenameAttribute");
    try {
      logger.debug("Create river {}", getRiver());
      String script = "ctx.document.score2 = ctx.document.score; delete ctx.document.score;";
      super.createRiver(
          "/test/elasticsearch/plugin/river/mongodb/script/test-mongodb-river-with-script.json",
          getRiver(),
          String.valueOf(getMongoPort1()),
          String.valueOf(getMongoPort2()),
          String.valueOf(getMongoPort3()),
          getDatabase(),
          getCollection(),
          script,
          getIndex(),
          getDatabase());

      String mongoDocument =
          copyToStringFromClasspath(
              "/test/elasticsearch/plugin/river/mongodb/script/test-simple-mongodb-document.json");
      DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
      WriteResult result = mongoCollection.insert(dbObject);
      Thread.sleep(wait);
      String id = dbObject.get("_id").toString();
      logger.info("WriteResult: {}", result.toString());
      refreshIndex();

      ActionFuture<IndicesExistsResponse> response =
          getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex()));
      assertThat(response.actionGet().isExists(), equalTo(true));

      SearchResponse sr =
          getNode()
              .client()
              .prepareSearch(getIndex())
              .setQuery(fieldQuery("_id", id))
              .execute()
              .actionGet();
      logger.debug("SearchResponse {}", sr.toString());
      long totalHits = sr.getHits().getTotalHits();
      logger.debug("TotalHits: {}", totalHits);
      assertThat(totalHits, equalTo(1l));

      assertThat(sr.getHits().getHits()[0].sourceAsMap().containsKey("score2"), equalTo(true));
      mongoCollection.remove(dbObject);
    } catch (Throwable t) {
      logger.error("testRenameAttribute failed.", t);
      t.printStackTrace();
      throw t;
    } finally {
      super.deleteRiver();
      super.deleteIndex();
    }
  }
  /**
   * Sets the {@link Harvester}'s {@link #rdfQueryType} parameter
   *
   * @param queryType - the type of any possible query
   * @return the same {@link Harvester} with the {@link #rdfQueryType} parameter set
   */
  public Harvester rdfQueryType(String queryType) {
    try {
      rdfQueryType = QueryType.valueOf(queryType.toUpperCase());
    } catch (IllegalArgumentException e) {
      logger.error("Bad query type: {}", queryType);

      /* River process can't continue */
      throw e;
    }
    return this;
  }
 public static Client getClient() {
   if (client == null) {
     try {
       client = startClient();
     } catch (IOException e) {
       logger.error("can not start the client", e);
     }
     assertThat(client, notNullValue());
   }
   return client;
 }
 private String getLastUpdate() {
   String res;
   SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
   try {
     GetResponse response =
         client.prepareGet(indexName, "stats", "1").setFields("last_update").execute().actionGet();
     if (!response.getFields().isEmpty()) {
       res = (String) response.getField("last_update").getValue();
     } else {
       res = sdf.format(new Date(0));
     }
   } catch (ElasticsearchIllegalStateException ex) {
     logger.error("Could not get last_update, use Date(0)", ex.getLocalizedMessage());
     res = sdf.format(new Date(0));
   } catch (NullPointerException ex) {
     logger.error("Could not get last_update, use Date(0)", ex.getLocalizedMessage());
     res = sdf.format(new Date(0));
   }
   return res;
 }