void init(ESInputSplit esSplit, Configuration cfg) {
size = esSplit.size;
Settings settings = SettingsManager.loadFrom(cfg);
query = settings.getTargetResource();
// initialize REST client
client = new BufferedRestClient(settings);
}
static void fixHive13InvalidComments(Settings settings, Properties tbl) {
if (Booleans.parseBoolean(settings.getProperty("es.hive.disable.columns.comments.fix"))) {
return;
}
settings.setProperty(HiveConstants.COLUMN_COMMENTS, "");
tbl.remove(HiveConstants.COLUMN_COMMENTS);
}
public static void setFilters(Settings settings, String... filters) {
// clear any filters inside the settings
settings.setProperty(InternalConfigurationOptions.INTERNAL_ES_QUERY_FILTERS, "");
if (ObjectUtils.isEmpty(filters)) {
return;
}
settings.setProperty(
InternalConfigurationOptions.INTERNAL_ES_QUERY_FILTERS, IOUtils.serializeToBase64(filters));
}
public RestClient(Settings settings) {
network = new NetworkClient(settings, SettingsUtils.nodes(settings));
scrollKeepAlive = TimeValue.timeValueMillis(settings.getScrollKeepAlive());
indexReadMissingAsEmpty = settings.getIndexReadMissingAsEmpty();
String retryPolicyName = settings.getBatchWriteRetryPolicy();
if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_SIMPLE.equals(retryPolicyName)) {
retryPolicyName = SimpleHttpRetryPolicy.class.getName();
} else if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_NONE.equals(retryPolicyName)) {
retryPolicyName = NoHttpRetryPolicy.class.getName();
}
retryPolicy = ObjectUtils.instantiate(retryPolicyName, settings);
}
public static List<String> discoveredOrDeclaredNodes(Settings settings) {
// returned the discovered nodes or, if not defined, the set nodes
String discoveredNodes =
settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_DISCOVERED_NODES);
return (StringUtils.hasText(discoveredNodes)
? StringUtils.tokenize(discoveredNodes)
: declaredNodes(settings));
}
/**
* Whether the settings indicate a ES 2.x (which introduces breaking changes) or 1.x.
*
* @param settings
* @return
*/
public static boolean isEs20(Settings settings) {
String version = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_VERSION);
// assume ES 1.0 by default
if (!StringUtils.hasText(version)) {
return true;
}
return version.startsWith("2.");
}
private Object wrapWithFormatter(String format, final FieldExtractor createFieldExtractor) {
// instantiate field extractor
final IndexFormatter iformatter =
ObjectUtils.instantiate(settings.getMappingIndexFormatterClassName(), settings);
iformatter.configure(format);
return new FieldExtractor() {
@Override
public String field(Object target) {
return iformatter.format(createFieldExtractor.field(target));
}
};
}
public JsonFieldExtractors(Settings settings) {
final List<String> jsonPaths = new ArrayList<String>();
id = init(settings.getMappingId(), jsonPaths);
parent = init(settings.getMappingParent(), jsonPaths);
routing = init(settings.getMappingRouting(), jsonPaths);
ttl = init(settings.getMappingTtl(), jsonPaths);
version = init(settings.getMappingVersion(), jsonPaths);
timestamp = init(settings.getMappingTimestamp(), jsonPaths);
// create index format
indexFormat =
new AbstractIndexFormat() {
@Override
protected Object createFieldExtractor(String fieldName) {
return createJsonFieldExtractor(fieldName, jsonPaths);
}
};
indexFormat.compile(new Resource(settings, false).toString());
// if there's no pattern, simply remove it
indexFormat = (indexFormat.hasPattern() ? indexFormat : null);
paths = jsonPaths.toArray(new String[jsonPaths.size()]);
}
@Override
public void setSettings(Settings settings) {
this.settings = settings;
String paramString = settings.getUpdateScriptParams();
List<String> fields = StringUtils.tokenize(paramString);
for (String string : fields) {
List<String> param = StringUtils.tokenize(string, ":");
Assert.isTrue(param.size() == 2, "Invalid param definition " + string);
params.put(param.get(0), createFieldExtractor(param.get(1)));
}
}
static Settings init(
Settings settings, String nodes, int port, String resource, String query, boolean read) {
if (StringUtils.hasText(nodes)) {
settings.setHosts(nodes);
}
if (port > 0) {
settings.setPort(port);
}
if (StringUtils.hasText(query)) {
settings.setQuery(query);
}
if (StringUtils.hasText(resource)) {
if (read) {
settings.setResourceRead(resource);
} else {
settings.setResourceWrite(resource);
}
}
return settings;
}
@Test
public void exhaustConnections() throws InterruptedException {
List<Thread> threads = new ArrayList<>();
int workerNum = 0;
for (String jobKey : JOB_KEYS) {
final Settings workerSettings = SETTINGS.copy();
if (POOLED) {
SettingsUtils.setJobTransportPoolingKey(workerSettings, jobKey);
}
Thread worker = new Thread(new Exhauster(++workerNum, workerSettings));
worker.start();
threads.add(worker);
}
for (Thread thread : threads) {
thread.join();
}
}
static Collection<String> columnToAlias(Settings settings) {
FieldAlias fa = alias(settings);
List<String> columnNames =
StringUtils.tokenize(settings.getProperty(HiveConstants.COLUMNS), ",");
// eliminate virtual columns
// we can't use virtual columns since some distro don't have this field...
// for (VirtualColumn vc : VirtualColumn.VIRTUAL_COLUMNS) {
// columnNames.remove(vc.getName());
// }
for (String vc : HiveConstants.VIRTUAL_COLUMNS) {
columnNames.remove(vc);
}
for (int i = 0; i < columnNames.size(); i++) {
String original = columnNames.get(i);
String alias = fa.toES(original);
if (alias != null) {
columnNames.set(i, alias);
}
}
return columnNames;
}
static FieldAlias alias(Settings settings) {
Map<String, String> aliasMap =
SettingsUtils.aliases(settings.getProperty(HiveConstants.MAPPING_NAMES));
// add default aliases for serialization (_colX -> mapping name)
Map<String, String> columnMap = columnMap(settings);
for (Entry<String, String> entry : columnMap.entrySet()) {
String columnName = entry.getKey();
String columnIndex = entry.getValue();
if (!aliasMap.isEmpty()) {
String alias = aliasMap.get(columnName);
if (alias != null) {
columnName = alias;
}
}
aliasMap.put(columnIndex, columnName);
}
return new FieldAlias(aliasMap);
}
public NetworkClient(Settings settings, TransportFactory transportFactory) {
this.settings = settings.copy();
this.nodes = SettingsUtils.discoveredOrDeclaredNodes(settings);
this.transportFactory = transportFactory;
// shuffle the list of nodes so in case of failures, the fallback is spread
Collections.shuffle(nodes);
if (SettingsUtils.hasPinnedNode(settings)) {
// move pinned node in front to be selected (only once)
String pinnedNode = SettingsUtils.getPinnedNode(settings);
if (log.isDebugEnabled()) {
log.debug("Opening (pinned) network client to " + pinnedNode);
}
nodes.remove(pinnedNode);
nodes.add(0, pinnedNode);
}
selectNextNode();
Assert.notNull(currentTransport, "no node information provided");
}
public static String[] getFilters(Settings settings) {
return IOUtils.deserializeFromBase64(
settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_QUERY_FILTERS));
}
public static String getPinnedNode(Settings settings) {
String node = settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE);
Assert.hasText(node, "Task has not been pinned to a node...");
return node;
}
private Resource createResource(String target) {
Settings s = new TestSettings();
s.setProperty(ConfigurationOptions.ES_RESOURCE, target);
return new Resource(s, true);
}
public static void pinNode(Settings settings, String node) {
pinNode(settings, node, settings.getPort());
}
static FieldAlias alias(Settings settings) {
return new FieldAlias(SettingsUtils.aliases(settings.getProperty(MAPPING_NAMES), false), false);
}
public static void pinNode(Settings settings, String node, int port) {
if (StringUtils.hasText(node) && port > 0) {
settings.setProperty(
InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE, qualifyNode(node, port));
}
}
@Override
public void setSettings(Settings settings) {
emptyAsNull = settings.getFieldReadEmptyAsNull();
}
public static boolean hasPinnedNode(Settings settings) {
return StringUtils.hasText(
settings.getProperty(InternalConfigurationOptions.INTERNAL_ES_PINNED_NODE));
}
static Map<String, String> columnMap(Settings settings) {
return columnMap(settings.getProperty(HiveConstants.COLUMNS));
}
public static void setDiscoveredNodes(Settings settings, Collection<String> nodes) {
settings.setProperty(
InternalConfigurationOptions.INTERNAL_ES_DISCOVERED_NODES,
StringUtils.concatenate(nodes, StringUtils.DEFAULT_DELIMITER));
}
public static List<String> declaredNodes(Settings settings) {
return qualifyNodes(settings.getNodes(), settings.getPort());
}