void init(ESInputSplit esSplit, Configuration cfg) {
      size = esSplit.size;

      Settings settings = SettingsManager.loadFrom(cfg);
      query = settings.getTargetResource();
      // initialize REST client
      client = new BufferedRestClient(settings);
    }
  static void fixHive13InvalidComments(Settings settings, Properties tbl) {
    if (Booleans.parseBoolean(settings.getProperty("es.hive.disable.columns.comments.fix"))) {
      return;
    }

    settings.setProperty(HiveConstants.COLUMN_COMMENTS, "");
    tbl.remove(HiveConstants.COLUMN_COMMENTS);
  }
  public static void setFilters(Settings settings, String... filters) {
    // clear any filters inside the settings
    settings.setProperty(InternalConfigurationOptions.INTERNAL_ES_QUERY_FILTERS, "");

    if (ObjectUtils.isEmpty(filters)) {
      return;
    }

    settings.setProperty(
        InternalConfigurationOptions.INTERNAL_ES_QUERY_FILTERS, IOUtils.serializeToBase64(filters));
  }
  public RestClient(Settings settings) {
    network = new NetworkClient(settings, SettingsUtils.nodes(settings));

    scrollKeepAlive = TimeValue.timeValueMillis(settings.getScrollKeepAlive());
    indexReadMissingAsEmpty = settings.getIndexReadMissingAsEmpty();

    String retryPolicyName = settings.getBatchWriteRetryPolicy();

    if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_SIMPLE.equals(retryPolicyName)) {
      retryPolicyName = SimpleHttpRetryPolicy.class.getName();
    } else if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_NONE.equals(retryPolicyName)) {
      retryPolicyName = NoHttpRetryPolicy.class.getName();
    }

    retryPolicy = ObjectUtils.instantiate(retryPolicyName, settings);
  }
 public static List<String> discoveredOrDeclaredNodes(Settings settings) {
   // returned the discovered nodes or, if not defined, the set nodes
   String discoveredNodes =
       settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_DISCOVERED_NODES);
   return (StringUtils.hasText(discoveredNodes)
       ? StringUtils.tokenize(discoveredNodes)
       : declaredNodes(settings));
 }
  /**
   * Whether the settings indicate a ES 2.x (which introduces breaking changes) or 1.x.
   *
   * @param settings
   * @return
   */
  public static boolean isEs20(Settings settings) {
    String version = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_VERSION);
    // assume ES 1.0 by default
    if (!StringUtils.hasText(version)) {
      return true;
    }

    return version.startsWith("2.");
  }
 private Object wrapWithFormatter(String format, final FieldExtractor createFieldExtractor) {
   // instantiate field extractor
   final IndexFormatter iformatter =
       ObjectUtils.instantiate(settings.getMappingIndexFormatterClassName(), settings);
   iformatter.configure(format);
   return new FieldExtractor() {
     @Override
     public String field(Object target) {
       return iformatter.format(createFieldExtractor.field(target));
     }
   };
 }
  public JsonFieldExtractors(Settings settings) {
    final List<String> jsonPaths = new ArrayList<String>();

    id = init(settings.getMappingId(), jsonPaths);
    parent = init(settings.getMappingParent(), jsonPaths);
    routing = init(settings.getMappingRouting(), jsonPaths);
    ttl = init(settings.getMappingTtl(), jsonPaths);
    version = init(settings.getMappingVersion(), jsonPaths);
    timestamp = init(settings.getMappingTimestamp(), jsonPaths);

    // create index format
    indexFormat =
        new AbstractIndexFormat() {
          @Override
          protected Object createFieldExtractor(String fieldName) {
            return createJsonFieldExtractor(fieldName, jsonPaths);
          }
        };
    indexFormat.compile(new Resource(settings, false).toString());

    // if there's no pattern, simply remove it
    indexFormat = (indexFormat.hasPattern() ? indexFormat : null);

    paths = jsonPaths.toArray(new String[jsonPaths.size()]);
  }
  @Override
  public void setSettings(Settings settings) {
    this.settings = settings;

    String paramString = settings.getUpdateScriptParams();
    List<String> fields = StringUtils.tokenize(paramString);
    for (String string : fields) {
      List<String> param = StringUtils.tokenize(string, ":");
      Assert.isTrue(param.size() == 2, "Invalid param definition " + string);

      params.put(param.get(0), createFieldExtractor(param.get(1)));
    }
  }
  static Settings init(
      Settings settings, String nodes, int port, String resource, String query, boolean read) {
    if (StringUtils.hasText(nodes)) {
      settings.setHosts(nodes);
    }

    if (port > 0) {
      settings.setPort(port);
    }

    if (StringUtils.hasText(query)) {
      settings.setQuery(query);
    }

    if (StringUtils.hasText(resource)) {
      if (read) {
        settings.setResourceRead(resource);
      } else {
        settings.setResourceWrite(resource);
      }
    }

    return settings;
  }
    @Test
    public void exhaustConnections() throws InterruptedException {
      List<Thread> threads = new ArrayList<>();
      int workerNum = 0;
      for (String jobKey : JOB_KEYS) {
        final Settings workerSettings = SETTINGS.copy();
        if (POOLED) {
          SettingsUtils.setJobTransportPoolingKey(workerSettings, jobKey);
        }
        Thread worker = new Thread(new Exhauster(++workerNum, workerSettings));
        worker.start();
        threads.add(worker);
      }

      for (Thread thread : threads) {
        thread.join();
      }
    }
  static Collection<String> columnToAlias(Settings settings) {
    FieldAlias fa = alias(settings);
    List<String> columnNames =
        StringUtils.tokenize(settings.getProperty(HiveConstants.COLUMNS), ",");
    // eliminate virtual columns
    // we can't use virtual columns since some distro don't have this field...
    //        for (VirtualColumn vc : VirtualColumn.VIRTUAL_COLUMNS) {
    //            columnNames.remove(vc.getName());
    //        }

    for (String vc : HiveConstants.VIRTUAL_COLUMNS) {
      columnNames.remove(vc);
    }

    for (int i = 0; i < columnNames.size(); i++) {
      String original = columnNames.get(i);
      String alias = fa.toES(original);
      if (alias != null) {
        columnNames.set(i, alias);
      }
    }
    return columnNames;
  }
  static FieldAlias alias(Settings settings) {
    Map<String, String> aliasMap =
        SettingsUtils.aliases(settings.getProperty(HiveConstants.MAPPING_NAMES));

    // add default aliases for serialization (_colX -> mapping name)
    Map<String, String> columnMap = columnMap(settings);

    for (Entry<String, String> entry : columnMap.entrySet()) {
      String columnName = entry.getKey();
      String columnIndex = entry.getValue();

      if (!aliasMap.isEmpty()) {
        String alias = aliasMap.get(columnName);
        if (alias != null) {
          columnName = alias;
        }
      }

      aliasMap.put(columnIndex, columnName);
    }

    return new FieldAlias(aliasMap);
  }
  public NetworkClient(Settings settings, TransportFactory transportFactory) {
    this.settings = settings.copy();
    this.nodes = SettingsUtils.discoveredOrDeclaredNodes(settings);
    this.transportFactory = transportFactory;

    // shuffle the list of nodes so in case of failures, the fallback is spread
    Collections.shuffle(nodes);

    if (SettingsUtils.hasPinnedNode(settings)) {
      // move pinned node in front to be selected (only once)
      String pinnedNode = SettingsUtils.getPinnedNode(settings);

      if (log.isDebugEnabled()) {
        log.debug("Opening (pinned) network client to " + pinnedNode);
      }

      nodes.remove(pinnedNode);
      nodes.add(0, pinnedNode);
    }

    selectNextNode();

    Assert.notNull(currentTransport, "no node information provided");
  }
 public static String[] getFilters(Settings settings) {
   return IOUtils.deserializeFromBase64(
       settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_QUERY_FILTERS));
 }
 public static String getPinnedNode(Settings settings) {
   String node = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE);
   Assert.hasText(node, "Task has not been pinned to a node...");
   return node;
 }
 private Resource createResource(String target) {
   Settings s = new TestSettings();
   s.setProperty(ConfigurationOptions.ES_RESOURCE, target);
   return new Resource(s, true);
 }
 public static void pinNode(Settings settings, String node) {
   pinNode(settings, node, settings.getPort());
 }
 static FieldAlias alias(Settings settings) {
   return new FieldAlias(SettingsUtils.aliases(settings.getProperty(MAPPING_NAMES), false), false);
 }
 public static void pinNode(Settings settings, String node, int port) {
   if (StringUtils.hasText(node) && port > 0) {
     settings.setProperty(
         InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE, qualifyNode(node, port));
   }
 }
 @Override
 public void setSettings(Settings settings) {
   emptyAsNull = settings.getFieldReadEmptyAsNull();
 }
 public static boolean hasPinnedNode(Settings settings) {
   return StringUtils.hasText(
       settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE));
 }
 static Map<String, String> columnMap(Settings settings) {
   return columnMap(settings.getProperty(HiveConstants.COLUMNS));
 }
 public static void setDiscoveredNodes(Settings settings, Collection<String> nodes) {
   settings.setProperty(
       InternalConfigurationOptions.INTERNAL_ES_DISCOVERED_NODES,
       StringUtils.concatenate(nodes, StringUtils.DEFAULT_DELIMITER));
 }
 public static List<String> declaredNodes(Settings settings) {
   return qualifyNodes(settings.getNodes(), settings.getPort());
 }