/**
   * Because we're running with a security manager (most likely), we need to scan for Lingo3G
   * license in ES configuration directories.
   */
  private Path scanForLingo3GLicense(Environment environment, Path pluginConfigPath) {
    List<Path> licenses = new ArrayList<>();

    for (Path candidate :
        new Path[] {
          pluginConfigPath.resolve("license.xml"),
          pluginConfigPath.resolve(".license.xml"),
          environment.configFile().resolve("license.xml"),
          environment.configFile().resolve(".license.xml")
        }) {
      logger.debug(
          "Lingo3G license location scan: {} {}.",
          candidate.toAbsolutePath().normalize(),
          Files.isRegularFile(candidate) ? "(found)" : "(not found)");
      if (Files.isRegularFile(candidate)) {
        licenses.add(candidate);
      }
    }

    if (licenses.size() > 1) {
      throw new ElasticsearchException(
          "There should be exactly one Lingo3G license on scan paths: {}", licenses);
    }

    if (licenses.size() == 1) {
      return licenses.iterator().next();
    } else {
      return null;
    }
  }
Exemple #2
0
  @Override
  public void run() {
    try {
      URL url = buildPingUrl();
      if (logger.isDebugEnabled()) {
        logger.debug("Sending UDC information to {}...", url);
      }
      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
      conn.setConnectTimeout((int) HTTP_TIMEOUT.millis());
      conn.setReadTimeout((int) HTTP_TIMEOUT.millis());

      if (conn.getResponseCode() >= 300) {
        throw new Exception(
            String.format("%s Responded with Code %d", url.getHost(), conn.getResponseCode()));
      }
      if (logger.isDebugEnabled()) {
        BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String line = reader.readLine();
        while (line != null) {
          logger.debug(line);
          line = reader.readLine();
        }
        reader.close();
      } else {
        conn.getInputStream().close();
      }
      successCounter.incrementAndGet();
    } catch (Exception e) {
      if (logger.isDebugEnabled()) {
        logger.debug("Error sending UDC information", e);
      }
      failCounter.incrementAndGet();
    }
  }
  /**
   * Start a river
   *
   * @param river
   */
  public void start(T river) {
    if (logger.isDebugEnabled()) logger.debug("start({})", river);
    // We only add the river if the river is started
    if (river == null || !river.isStart()) return;

    riverService.start(river, getHelper().toXContent(river));

    if (logger.isDebugEnabled()) logger.debug("/start({})", river);
  }
 protected String getJsonSettings(String jsonDefinition, Object... args) throws Exception {
   logger.debug("Get river setting");
   String setting = copyToStringFromClasspath(jsonDefinition);
   if (args != null) {
     setting = String.format(setting, args);
   }
   logger.debug("River setting: {}", setting);
   return setting;
 }
  /**
   * Stop a river
   *
   * @param river
   */
  public void stop(T river) {
    if (logger.isDebugEnabled()) logger.debug("stop({})", river);
    // We only add the river if the river is started
    if (river == null || !river.isStart()) return;

    riverService.stop(river);

    if (logger.isDebugEnabled()) logger.debug("/stop({})", river);
  }
  @Test
  public void testRenameAttribute() throws Throwable {
    logger.debug("Start testRenameAttribute");
    try {
      logger.debug("Create river {}", getRiver());
      String script = "ctx.document.score2 = ctx.document.score; delete ctx.document.score;";
      super.createRiver(
          "/test/elasticsearch/plugin/river/mongodb/script/test-mongodb-river-with-script.json",
          getRiver(),
          String.valueOf(getMongoPort1()),
          String.valueOf(getMongoPort2()),
          String.valueOf(getMongoPort3()),
          getDatabase(),
          getCollection(),
          script,
          getIndex(),
          getDatabase());

      String mongoDocument =
          copyToStringFromClasspath(
              "/test/elasticsearch/plugin/river/mongodb/script/test-simple-mongodb-document.json");
      DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
      WriteResult result = mongoCollection.insert(dbObject);
      Thread.sleep(wait);
      String id = dbObject.get("_id").toString();
      logger.info("WriteResult: {}", result.toString());
      refreshIndex();

      ActionFuture<IndicesExistsResponse> response =
          getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex()));
      assertThat(response.actionGet().isExists(), equalTo(true));

      SearchResponse sr =
          getNode()
              .client()
              .prepareSearch(getIndex())
              .setQuery(fieldQuery("_id", id))
              .execute()
              .actionGet();
      logger.debug("SearchResponse {}", sr.toString());
      long totalHits = sr.getHits().getTotalHits();
      logger.debug("TotalHits: {}", totalHits);
      assertThat(totalHits, equalTo(1l));

      assertThat(sr.getHits().getHits()[0].sourceAsMap().containsKey("score2"), equalTo(true));
      mongoCollection.remove(dbObject);
    } catch (Throwable t) {
      logger.error("testRenameAttribute failed.", t);
      t.printStackTrace();
      throw t;
    } finally {
      super.deleteRiver();
      super.deleteIndex();
    }
  }
  /**
   * Update (or add) a river
   *
   * @param river
   */
  public void update(T river) {
    if (logger.isDebugEnabled()) logger.debug("update({})", river);
    XContentBuilder xb = getHelper().toXContent(river);

    client
        .prepareIndex(
            SMDSearchProperties.ES_META_INDEX, SMDSearchProperties.ES_META_RIVERS, river.getId())
        .setSource(xb)
        .setRefresh(true)
        .execute()
        .actionGet();

    if (logger.isDebugEnabled()) logger.debug("/update({})", river);
  }
Exemple #8
0
  @Override
  public void run() {
    if (logger.isDebugEnabled()) {
      logger.debug("Create task manager thread.");
    }

    do {
      logger.info("TaskManager: current task index: " + Integer.toString(currentTaskIndex));
      try {
        String output = currentTask.Run();
        logger.info("Task {[]} output: {[]}", currentTask.id(), output);
        logger.info("Task [" + currentTask.id() + "] output: " + output);
      } catch (IOException ex) {
        logger.error("TaskManager: IOException");
      } catch (InterruptedException ex) {
        logger.error("TaskManager: Interrupted Exception");
      }

      currentTask = GetNextTask();
    } while (null != currentTask);

    DeleteMappingRequest req = new DeleteMappingRequest("_river");
    req.type(river.riverName().name());
    DeleteMappingResponse resp = client.admin().indices().deleteMapping(req).actionGet();
    logger.info("TaskManager: delete request: " + resp.toString());
  }
 @Override
 @SuppressWarnings("unchecked")
 public synchronized void write(ArchivePacket packet) throws IOException {
   if (!isOpen()) {
     throw new IOException("not open");
   }
   if (out == null) {
     throw new IOException("no output stream found");
   }
   if (packet == null || packet.payload() == null) {
     throw new IOException("no payload to write for entry");
   }
   byte[] buf = packet.payload().toString().getBytes();
   String name = ArchiveUtils.encodeArchiveEntryName(packet);
   ArchiveEntry entry = out.newArchiveEntry();
   entry.setName(name);
   entry.setLastModified(new Date());
   entry.setEntrySize(buf.length);
   out.putArchiveEntry(entry);
   out.write(buf);
   out.closeArchiveEntry();
   packetCounter++;
   if (watcher.getBytesToTransfer() != 0
       && watcher.getBytesTransferred() > watcher.getBytesToTransfer()) {
     logger.debug(
         "bytes watcher: transferred = {}, rate {}",
         watcher.getBytesTransferred(),
         watcher.getRecentByteRatePerSecond());
     switchToNextArchive();
     watcher.resetWatcher();
   }
 }
  /**
   * Detects if we're connecting to a Found Elasticsearch cluster (using pre-configured host
   * suffixes) and adds a SSL handler at the beginning of the pipeline if we're connecting to a
   * SSL-endpoint (using a list of pre-configured ports).
   */
  @Override
  public void connectRequested(final ChannelHandlerContext ctx, final ChannelStateEvent e)
      throws Exception {
    if (e.getValue() instanceof InetSocketAddress) {
      InetSocketAddress inetSocketAddress = (InetSocketAddress) e.getValue();

      for (String suffix : hostSuffixes) {
        isFoundCluster = isFoundCluster || inetSocketAddress.getHostString().endsWith(suffix);
      }

      if (isFoundCluster) {
        for (int sslPort : sslPorts) {
          if (inetSocketAddress.getPort() == sslPort) {
            logger.debug(
                "Enabling SSL on transport layer with unsafeAllowSelfSigned=[{}].",
                unsafeAllowSelfSigned);
            FoundSSLHandler handler =
                FoundSSLUtils.getSSLHandler(unsafeAllowSelfSigned, inetSocketAddress);
            ctx.getPipeline().addFirst("ssl", handler);
            break;
          }
        }
      } else {
        ctx.getPipeline().remove(this);
      }
    }
    super.connectRequested(ctx, e);
  }
  private boolean handleRevision1Response(ChannelHandlerContext ctx, int payloadLength)
      throws Exception {
    int code = buffered.readInt();

    int descriptionLength = buffered.readInt();
    byte[] descBytes = new byte[descriptionLength];
    buffered.readBytes(descBytes, 0, descBytes.length);

    String description = new String(descBytes, StandardCharsets.UTF_8);

    logger.debug(
        "Decoded payload with length:[{}], code:[{}], descriptionLength:[{}], description:[{}] on connection [{}]",
        payloadLength,
        code,
        descriptionLength,
        description,
        ctx.getChannel().getLocalAddress());

    if (200 <= code && code <= 299) {
      logger.info(
          "Connected to Found Elasticsearch: [{}]: [{}] on connection [{}]",
          code,
          description,
          ctx.getChannel().getLocalAddress());
      return true;
    } else {
      logger.error(
          "Unable to connect to Found Elasticsearch: [{}]: [{}] on connection [{}]",
          code,
          description,
          ctx.getChannel().getLocalAddress());
      return false;
    }
  }
  @Override
  public BytesRef writeToBytes() {
    long start = System.nanoTime();
    int size = set.size();

    BytesRef bytes = new BytesRef(new byte[HEADER_SIZE + (int) bytesUsed.get()]);

    // Encode encoding type
    Bytes.writeInt(bytes, this.getEncoding().ordinal());

    // Encode flag
    bytes.bytes[bytes.offset++] = (byte) (this.isPruned() ? 1 : 0);

    // Encode size of the set
    Bytes.writeInt(bytes, size);

    // Encode longs
    BytesRef reusable = new BytesRef();
    for (int i = 0; i < this.set.size(); i++) {
      this.set.get(i, reusable);
      Bytes.writeBytesRef(reusable, bytes);
    }

    logger.debug(
        "Serialized {} terms - took {} ms", this.size(), (System.nanoTime() - start) / 1000000);

    bytes.length = bytes.offset;
    bytes.offset = 0;
    return bytes;
  }
 private void add(String name, List<KnapsackState> values, KnapsackState targetValue)
     throws IOException {
   logger.debug("add: {} -> {}", name, values);
   put(
       name,
       generate(ImmutableList.<KnapsackState>builder().addAll(values).add(targetValue).build()));
 }
 private List<KnapsackState> get(String name) throws IOException {
   ImmutableList.Builder<KnapsackState> builder = ImmutableList.builder();
   try {
     logger.debug("get knapsack states: {}", name);
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     GetResponse getResponse =
         client.prepareGet(INDEX_NAME, MAPPING_NAME, name).execute().actionGet();
     if (!getResponse.isExists()) {
       return builder.build();
     }
     XContentParser parser = xContent(JSON).createParser(getResponse.getSourceAsBytes());
     while (parser.nextToken() != START_ARRAY) {
       // forward
     }
     while (parser.nextToken() != END_ARRAY) {
       KnapsackState state = new KnapsackState();
       builder.add(state.fromXContent(parser));
     }
     return builder.build();
   } catch (Throwable t) {
     logger.error("get settings failed", t);
     return null;
   }
 }
  public ACLRequest(RestRequest request, RestChannel channel) {
    this(
        request.uri(),
        getAddress(channel),
        request.header("X-Api-Key"),
        request.header("Auth"),
        request.content().length(),
        request.method(),
        getXForwardedForHeader(request));

    ESLogger logger = ESLoggerFactory.getLogger(ACLRequest.class.getName());
    logger.debug("Headers:\n");
    for (Map.Entry<String, String> header : request.headers()) {
      logger.debug(header.getKey() + "=" + header.getValue());
    }
  }
  @Override
  public void run() {
    try {
      socket = new DatagramSocket(port);

      while (!isClosed) {
        if (socket.isClosed()) return;

        byte[] buf = new byte[256];

        // receive request
        DatagramPacket packet = new DatagramPacket(buf, buf.length);
        socket.receive(packet);

        ByteArrayInputStream bis = new ByteArrayInputStream(buf, 0, packet.getLength());
        BufferedReader in = new BufferedReader(new InputStreamReader(bis));

        String msg;
        while ((msg = in.readLine()) != null) {
          logger.debug("Read from socket: " + msg);
          content.add(msg.trim());
        }
        in.close();
      }

    } catch (IOException e) {
      e.printStackTrace();
    }
  }
  /**
   * Remove river
   *
   * @param river
   */
  public void remove(T river) {
    if (logger.isDebugEnabled()) logger.debug("remove({})", river);

    // We stop the river if running
    if (riverService.checkState(river)) {
      riverService.stop(river);
    }

    // We remove the river in the database
    client
        .prepareDelete(
            SMDSearchProperties.ES_META_INDEX, SMDSearchProperties.ES_META_RIVERS, river.getId())
        .execute()
        .actionGet();

    if (logger.isDebugEnabled()) logger.debug("/remove({})", river);
  }
  @Test
  public void testIgnoreScript() throws Throwable {
    logger.debug("Start testIgnoreScript");
    try {
      logger.debug("Create river {}", getRiver());
      String script = "ctx.ignore = true;";
      super.createRiver(
          "/test/elasticsearch/plugin/river/mongodb/script/test-mongodb-river-with-script.json",
          getRiver(),
          String.valueOf(getMongoPort1()),
          String.valueOf(getMongoPort2()),
          String.valueOf(getMongoPort3()),
          getDatabase(),
          getCollection(),
          script,
          getIndex(),
          getDatabase());

      String mongoDocument =
          copyToStringFromClasspath(
              "/test/elasticsearch/plugin/river/mongodb/script/test-simple-mongodb-document.json");
      DBObject dbObject = (DBObject) JSON.parse(mongoDocument);
      WriteResult result = mongoCollection.insert(dbObject);
      Thread.sleep(wait);
      logger.info("WriteResult: {}", result.toString());
      refreshIndex();

      ActionFuture<IndicesExistsResponse> response =
          getNode().client().admin().indices().exists(new IndicesExistsRequest(getIndex()));
      assertThat(response.actionGet().isExists(), equalTo(true));
      CountResponse countResponse = getNode().client().count(countRequest(getIndex())).actionGet();
      logger.info("Document count: {}", countResponse.getCount());
      assertThat(countResponse.getCount(), equalTo(0l));

      mongoCollection.remove(dbObject);

    } catch (Throwable t) {
      logger.error("testIgnoreScript failed.", t);
      t.printStackTrace();
      throw t;
    } finally {
      super.deleteRiver();
      super.deleteIndex();
    }
  }
 @Override
 public BulkNodeClient flushIngest() {
   if (closed) {
     throw new ElasticsearchIllegalStateException("client is closed");
   }
   logger.debug("flushing bulk processor");
   BulkProcessorHelper.flush(bulkProcessor);
   return this;
 }
  protected long getCount(final String index, final String type) {
    logger.debug("getCount()");

    esSetup.client().admin().indices().refresh(new RefreshRequest()).actionGet();

    final CountResponse count =
        esSetup.client().count(new CountRequest(index).types(type)).actionGet();

    return count.getCount();
  }
 private void remove(final String name) {
   try {
     logger.debug("remove: {}", name);
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     client.prepareDelete(INDEX_NAME, MAPPING_NAME, name).setRefresh(true).execute().actionGet();
   } catch (Throwable t) {
     logger.error("remove failed", t);
   }
 }
  public void initCities() {
    if (log.isDebugEnabled()) {
      log.debug("Initializing all registry cities");
    }

    // File bulkFile = createCitiesBulkFile2();

    // Insert cities
    // bulkFromFile(bulkFile, INDEX_NAME, INDEX_TYPE);
  }
 private void shutdownMongoInstances() {
   logger.debug("*** shutdownMongoInstances ***");
   mongo.close();
   try {
     logger.debug("Start shutdown {}", mongod1);
     mongod1.stop();
   } catch (Throwable t) {
   }
   try {
     logger.debug("Start shutdown {}", mongod2);
     mongod2.stop();
   } catch (Throwable t) {
   }
   try {
     logger.debug("Start shutdown {}", mongod3);
     mongod3.stop();
   } catch (Throwable t) {
   }
 }
 private void remove(String name, List<KnapsackState> values, KnapsackState targetValue)
     throws IOException {
   logger.debug("remove: {} -> {}", name, values);
   ImmutableList.Builder<KnapsackState> updatedValues = ImmutableList.builder();
   for (KnapsackState value : values) {
     if (!value.equals(targetValue)) {
       updatedValues.add(value);
     }
   }
   put(name, generate(updatedValues.build()));
 }
 @Override
 public synchronized void shutdown() {
   try {
     if (bulkProcessor != null) {
       logger.debug("closing bulk processor...");
       bulkProcessor.close();
     }
     if (metric != null && metric.indices() != null && !metric.indices().isEmpty()) {
       logger.debug("stopping bulk mode for indices {}...", metric.indices());
       for (String index : ImmutableSet.copyOf(metric.indices())) {
         stopBulk(index);
       }
     }
     logger.debug("shutting down...");
     client.close();
     logger.debug("shutting down completed");
   } catch (Exception e) {
     logger.error(e.getMessage(), e);
   }
 }
 protected void deleteIndex(String name) {
   logger.info("Delete index [{}]", name);
   node.client().admin().indices().delete(deleteIndexRequest(name)).actionGet();
   logger.debug("Running Cluster Health");
   ClusterHealthResponse clusterHealth =
       node.client()
           .admin()
           .cluster()
           .health(clusterHealthRequest().waitForGreenStatus())
           .actionGet();
   logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
 }
 protected void deleteRiver(String name) {
   logger.info("Delete river [{}]", name);
   DeleteMappingRequest deleteMapping = new DeleteMappingRequest("_river").type(name);
   node.client().admin().indices().deleteMapping(deleteMapping).actionGet();
   logger.debug("Running Cluster Health");
   ClusterHealthResponse clusterHealth =
       node.client()
           .admin()
           .cluster()
           .health(clusterHealthRequest().waitForGreenStatus())
           .actionGet();
   logger.info("Done Cluster Health, status " + clusterHealth.getStatus());
 }
  /**
   * Get the river definition by its name
   *
   * @param name
   * @return
   */
  public T get(String name) {
    if (logger.isDebugEnabled()) logger.debug("get({})", name);
    T river = null;

    if (name != null) {
      GetRequestBuilder rb = new GetRequestBuilder(client, SMDSearchProperties.ES_META_INDEX);
      rb.setType(SMDSearchProperties.ES_META_RIVERS);
      rb.setId(name);

      try {
        GetResponse response = rb.execute().actionGet();
        if (response.isExists()) {
          river = getHelper().toRiver(buildInstance(), response.getSourceAsMap());
        }
      } catch (IndexMissingException e) {
        // Index does not exists, so RIVER does not exist...
      }
    }

    if (logger.isDebugEnabled()) logger.debug("/get({})={}", name, river);
    return river;
  }
  /**
   * Get all active rivers
   *
   * @return
   */
  public List<T> get() {
    if (logger.isDebugEnabled()) logger.debug("get()");
    List<T> rivers = new ArrayList<T>();

    SearchRequestBuilder srb = new SearchRequestBuilder(client);

    try {
      srb.setIndices(SMDSearchProperties.ES_META_INDEX);
      srb.setTypes(SMDSearchProperties.ES_META_RIVERS);

      SearchResponse response = srb.execute().actionGet();

      if (response.getHits().totalHits() > 0) {

        for (int i = 0; i < response.getHits().hits().length; i++) {
          T river = buildInstance();

          SearchHit hit = response.getHits().hits()[i];

          // We only manage rivers for type getHelper().type()
          river = getHelper().toRiver(river, hit.sourceAsMap());

          if (river.getType().equals(getHelper().type())) {
            // For each river, we check if the river is started or not
            river.setStart(riverService.checkState(river));
            rivers.add(river);
          }
        }
      }

    } catch (IndexMissingException e) {
      // That's a common use case. We started with an empty index
    }

    if (logger.isDebugEnabled()) logger.debug("/get()={}", rivers);
    return rivers;
  }
 private void put(final String name, final XContentBuilder builder) {
   try {
     logger.debug("put knapsack state: {} -> {}", name, builder.string());
     final Client client = injector.getInstance(Client.class);
     createIndexIfNotExist(client);
     client
         .prepareIndex(INDEX_NAME, MAPPING_NAME, name)
         .setSource(builder)
         .setRefresh(true)
         .execute()
         .actionGet();
   } catch (Throwable t) {
     logger.error("update settings failed", t);
   }
 }