private Object wrapWithFormatter(String format, final FieldExtractor createFieldExtractor) {
   // instantiate field extractor
   final IndexFormatter iformatter =
       ObjectUtils.instantiate(settings.getMappingIndexFormatterClassName(), settings);
   iformatter.configure(format);
   return new FieldExtractor() {
     @Override
     public String field(Object target) {
       return iformatter.format(createFieldExtractor.field(target));
     }
   };
 }
  @Override
  public void compile(String pattern) {
    this.pattern = pattern;
    // break it down into index/type
    String[] split = pattern.split("/");
    Assert.isTrue(!ObjectUtils.isEmpty(split), "invalid pattern given " + pattern);
    Assert.isTrue(split.length == 2, "invalid pattern given " + pattern);

    // check pattern
    hasPattern = pattern.contains("{") && pattern.contains("}");
    index = parse(split[0].trim());
    type = parse(split[1].trim());
  }
  public RestClient(Settings settings) {
    network = new NetworkClient(settings, SettingsUtils.nodes(settings));

    scrollKeepAlive = TimeValue.timeValueMillis(settings.getScrollKeepAlive());
    indexReadMissingAsEmpty = settings.getIndexReadMissingAsEmpty();

    String retryPolicyName = settings.getBatchWriteRetryPolicy();

    if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_SIMPLE.equals(retryPolicyName)) {
      retryPolicyName = SimpleHttpRetryPolicy.class.getName();
    } else if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_NONE.equals(retryPolicyName)) {
      retryPolicyName = NoHttpRetryPolicy.class.getName();
    }

    retryPolicy = ObjectUtils.instantiate(retryPolicyName, settings);
  }
  public void process(BytesArray storage) {
    // no extractors, no lookups
    if (ObjectUtils.isEmpty(paths)) {
      return;
    }

    results.clear();

    if (log.isTraceEnabled()) {
      log.trace(
          String.format(
              "About to look for paths [%s] in doc [%s]", Arrays.toString(paths), storage));
    }
    results.addAll(
        ParsingUtils.values(new JacksonJsonParser(storage.bytes(), 0, storage.length()), paths));
  }
public abstract class CascadingUtils {

  private static final String MAPPING_NAMES = "es.mapping.names";
  private static final boolean CASCADING_22_AVAILABLE =
      ObjectUtils.isClassPresent("cascading.tuple.type.CoercibleType", Tap.class.getClassLoader());

  static Settings addDefaultsToSettings(
      Properties flowProperties, Properties tapProperties, Log log) {
    Settings settings =
        HadoopSettingsManager.loadFrom(CascadingUtils.extractOriginalProperties(flowProperties))
            .merge(tapProperties);

    InitializationUtils.discoverNodesIfNeeded(settings, log);
    InitializationUtils.filterNonClientNodesIfNeeded(settings, log);
    InitializationUtils.discoverEsVersion(settings, log);

    InitializationUtils.setValueWriterIfNotSet(settings, CascadingValueWriter.class, log);
    InitializationUtils.setValueReaderIfNotSet(settings, JdkValueReader.class, log);
    InitializationUtils.setBytesConverterIfNeeded(
        settings, CascadingLocalBytesConverter.class, log);
    InitializationUtils.setFieldExtractorIfNotSet(settings, CascadingFieldExtractor.class, log);

    return settings;
  }

  static void addSerializationToken(Object config) {
    Configuration cfg = (Configuration) config;
    String tokens = cfg.get(TupleSerializationProps.SERIALIZATION_TOKENS);

    String lmw = LinkedMapWritable.class.getName();

    // if no tokens are defined, add one starting with 140
    if (tokens == null) {
      cfg.set(TupleSerializationProps.SERIALIZATION_TOKENS, "140=" + lmw);
      LogFactory.getLog(EsTap.class)
          .trace(String.format("Registered Cascading serialization token %s for %s", 140, lmw));
    } else {
      // token already registered
      if (tokens.contains(lmw)) {
        return;
      }

      // find token id
      Map<Integer, String> mapping = new LinkedHashMap<Integer, String>();
      tokens = tokens.replaceAll("\\s", ""); // allow for whitespace in token set

      for (String pair : tokens.split(",")) {
        String[] elements = pair.split("=");
        mapping.put(Integer.parseInt(elements[0]), elements[1]);
      }

      for (int id = 140; id < 255; id++) {
        if (!mapping.containsKey(Integer.valueOf(id))) {
          cfg.set(
              TupleSerializationProps.SERIALIZATION_TOKENS,
              Util.join(",", Util.removeNulls(tokens, id + "=" + lmw)));
          LogFactory.getLog(EsTap.class)
              .trace(String.format("Registered Cascading serialization token %s for %s", id, lmw));
          return;
        }
      }
    }
  }

  static FieldAlias alias(Settings settings) {
    return new FieldAlias(SettingsUtils.aliases(settings.getProperty(MAPPING_NAMES), false), false);
  }

  static List<String> asStrings(Fields fields) {
    if (fields == null || !fields.isDefined()) {
      // use auto-generated name
      return Collections.emptyList();
    }

    int size = fields.size();
    List<String> names = new ArrayList<String>(size);
    for (int fieldIndex = 0; fieldIndex < size; fieldIndex++) {
      names.add(fields.get(fieldIndex).toString());
    }

    return names;
  }

  static Collection<String> fieldToAlias(Settings settings, Fields fields) {
    FieldAlias fa = alias(settings);
    List<String> names = asStrings(fields);
    for (int i = 0; i < names.size(); i++) {
      String original = names.get(i);
      String alias = fa.toES(original);
      if (alias != null) {
        names.set(i, alias);
      }
    }
    return names;
  }

  static Properties extractOriginalProperties(Properties copy) {
    Field field = ReflectionUtils.findField(Properties.class, "defaults", Properties.class);
    ReflectionUtils.makeAccessible(field);
    return ReflectionUtils.getField(field, copy);
  }

  static Settings init(
      Settings settings, String nodes, int port, String resource, String query, boolean read) {
    if (StringUtils.hasText(nodes)) {
      settings.setHosts(nodes);
    }

    if (port > 0) {
      settings.setPort(port);
    }

    if (StringUtils.hasText(query)) {
      settings.setQuery(query);
    }

    if (StringUtils.hasText(resource)) {
      if (read) {
        settings.setResourceRead(resource);
      } else {
        settings.setResourceWrite(resource);
      }
    }

    return settings;
  }

  private abstract static class CoercibleOps {
    static void setObject(TupleEntry entry, Comparable<?> field, Object object) {
      if (object != null && entry.getFields().getType(field) instanceof CoercibleType) {
        entry.setObject(field, object.toString());
      } else {
        entry.setObject(field, object);
      }
    }

    static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
      TupleEntry entry = sinkCall.getOutgoingEntry();
      Fields fields = entry.getFields();
      Tuple tuple = entry.getTuple();

      if (fields.hasTypes()) {
        Type types[] = new Type[fields.size()];
        for (int index = 0; index < fields.size(); index++) {
          Type type = fields.getType(index);
          if (type instanceof CoercibleType<?>) {
            types[index] = String.class;
          } else {
            types[index] = type;
          }
        }

        tuple = entry.getCoercedTuple(types);
      }
      return tuple;
    }
  }

  private abstract static class LegacyOps {
    static void setObject(TupleEntry entry, Comparable<?> field, Object object) {
      entry.setObject(field, object);
    }

    static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
      return sinkCall.getOutgoingEntry().getTuple();
    }
  }

  static void setObject(TupleEntry entry, Comparable<?> field, Object object) {
    if (CASCADING_22_AVAILABLE) {
      CoercibleOps.setObject(entry, field, object);
    } else {
      LegacyOps.setObject(entry, field, object);
    }
  }

  static Tuple coerceToString(SinkCall<?, ?> sinkCall) {
    return (CASCADING_22_AVAILABLE
        ? CoercibleOps.coerceToString(sinkCall)
        : LegacyOps.coerceToString(sinkCall));
  }

  @SuppressWarnings("rawtypes")
  public static Tap hadoopTap(
      String host, int port, String path, String query, Fields fields, Properties props) {
    return new EsHadoopTap(host, port, path, query, fields, props);
  }
}
abstract class HiveUtils {

  // Date type available since Hive 0.12
  static final boolean DATE_WRITABLE_AVAILABLE =
      ObjectUtils.isClassPresent(
          HiveConstants.DATE_WRITABLE, TimestampWritable.class.getClassLoader());

  static StandardStructObjectInspector structObjectInspector(Properties tableProperties) {
    // extract column info - don't use Hive constants as they were renamed in 0.9 breaking
    // compatibility
    // the column names are saved as the given inspector to #serialize doesn't preserves them (maybe
    // because it's an external table)
    // use the class since StructType requires it ...
    List<String> columnNames =
        StringUtils.tokenize(tableProperties.getProperty(HiveConstants.COLUMNS), ",");
    List<TypeInfo> colTypes =
        TypeInfoUtils.getTypeInfosFromTypeString(
            tableProperties.getProperty(HiveConstants.COLUMNS_TYPES));

    // create a standard writable Object Inspector - used later on by serialization/deserialization
    List<ObjectInspector> inspectors = new ArrayList<ObjectInspector>();

    for (TypeInfo typeInfo : colTypes) {
      inspectors.add(TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(typeInfo));
    }

    return ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
  }

  static StructTypeInfo typeInfo(StructObjectInspector inspector) {
    return (StructTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(inspector);
  }

  static Collection<String> columnToAlias(Settings settings) {
    FieldAlias fa = alias(settings);
    List<String> columnNames =
        StringUtils.tokenize(settings.getProperty(HiveConstants.COLUMNS), ",");
    // eliminate virtual columns
    // we can't use virtual columns since some distro don't have this field...
    //        for (VirtualColumn vc : VirtualColumn.VIRTUAL_COLUMNS) {
    //            columnNames.remove(vc.getName());
    //        }

    for (String vc : HiveConstants.VIRTUAL_COLUMNS) {
      columnNames.remove(vc);
    }

    for (int i = 0; i < columnNames.size(); i++) {
      String original = columnNames.get(i);
      String alias = fa.toES(original);
      if (alias != null) {
        columnNames.set(i, alias);
      }
    }
    return columnNames;
  }

  static FieldAlias alias(Settings settings) {
    Map<String, String> aliasMap =
        SettingsUtils.aliases(settings.getProperty(HiveConstants.MAPPING_NAMES));

    // add default aliases for serialization (_colX -> mapping name)
    Map<String, String> columnMap = columnMap(settings);

    for (Entry<String, String> entry : columnMap.entrySet()) {
      String columnName = entry.getKey();
      String columnIndex = entry.getValue();

      if (!aliasMap.isEmpty()) {
        String alias = aliasMap.get(columnName);
        if (alias != null) {
          columnName = alias;
        }
      }

      aliasMap.put(columnIndex, columnName);
    }

    return new FieldAlias(aliasMap);
  }

  static Map<String, String> columnMap(Settings settings) {
    return columnMap(settings.getProperty(HiveConstants.COLUMNS));
  }

  // returns a map of {<column-name>:_colX}
  private static Map<String, String> columnMap(String columnString) {
    // add default aliases for serialization (mapping name -> _colX)
    List<String> columnNames = StringUtils.tokenize(columnString, ",");
    if (columnNames.isEmpty()) {
      return Collections.emptyMap();
    }

    Map<String, String> columns = new LinkedHashMap<String, String>();
    for (int i = 0; i < columnNames.size(); i++) {
      columns.put(columnNames.get(i), HiveConstants.UNNAMED_COLUMN_PREFIX + i);
    }
    return columns;
  }

  static void init(Settings settings, Log log) {
    InitializationUtils.checkIdForOperation(settings);
    InitializationUtils.setFieldExtractorIfNotSet(settings, HiveFieldExtractor.class, log);
    try {
      InitializationUtils.discoverEsVersion(settings, log);
    } catch (IOException ex) {
      throw new EsHadoopIllegalStateException("Cannot discover Elasticsearch version", ex);
    }
  }

  static void fixHive13InvalidComments(Settings settings, Properties tbl) {
    if (Booleans.parseBoolean(settings.getProperty("es.hive.disable.columns.comments.fix"))) {
      return;
    }

    settings.setProperty(HiveConstants.COLUMN_COMMENTS, "");
    tbl.remove(HiveConstants.COLUMN_COMMENTS);
  }
}