private void updateTemplate(
DocIndexMetaData md,
TransportPutIndexTemplateAction transportPutIndexTemplateAction,
Settings updateSettings) {
String templateName = PartitionName.templateName(ident.schema(), ident.name());
PutIndexTemplateRequest request =
new PutIndexTemplateRequest(templateName)
.mapping(Constants.DEFAULT_MAPPING_TYPE, md.defaultMappingMap)
.create(false)
.settings(updateSettings)
.template(templateName + "*");
for (String alias : md.aliases()) {
request = request.alias(new Alias(alias));
}
transportPutIndexTemplateAction.execute(request);
}
@Override
protected void bindSchemas() {
super.bindSchemas();
SchemaInfo schemaInfo = mock(SchemaInfo.class);
when(schemaInfo.getTableInfo(USER_TABLE_IDENT.name())).thenReturn(USER_TABLE_INFO);
when(schemaInfo.getTableInfo(TEST_ALIAS_TABLE_IDENT.name()))
.thenReturn(TEST_ALIAS_TABLE_INFO);
when(schemaInfo.getTableInfo(USER_TABLE_IDENT_CLUSTERED_BY_ONLY.name()))
.thenReturn(USER_TABLE_INFO_CLUSTERED_BY_ONLY);
when(schemaInfo.getTableInfo(TEST_PARTITIONED_TABLE_IDENT.name()))
.thenReturn(TEST_PARTITIONED_TABLE_INFO);
when(schemaInfo.getTableInfo(DEEPLY_NESTED_TABLE_IDENT.name()))
.thenReturn(DEEPLY_NESTED_TABLE_INFO);
when(schemaInfo.getTableInfo(NESTED_CLUSTERED_BY_TABLE_IDENT.name()))
.thenReturn(NESTED_CLUSTERED_BY_TABLE_INFO);
schemaBinder.addBinding(Schemas.DEFAULT_SCHEMA_NAME).toInstance(schemaInfo);
}
@Test
public void testStreaming() throws Exception {
TableIdent t1 = new TableIdent(null, "t1");
TreeMap<String, Integer> bases = new TreeMap<String, Integer>();
bases.put(t1.name(), 0);
bases.put("i2", 1);
Multimap<TableIdent, String> tableIndices = HashMultimap.create();
tableIndices.put(t1, t1.name());
tableIndices.put(new TableIdent(null, "i2"), "i2_s1");
tableIndices.put(new TableIdent(null, "i2"), "i2_s2");
ReferenceIdent nameIdent = new ReferenceIdent(t1, "name");
Reference name =
new Reference(new ReferenceInfo(nameIdent, RowGranularity.DOC, DataTypes.STRING));
FetchPhase orig =
new FetchPhase(
1,
ImmutableSet.<String>of("node1", "node2"),
bases,
tableIndices,
ImmutableList.of(name));
BytesStreamOutput out = new BytesStreamOutput();
ExecutionPhases.toStream(out, orig);
BytesStreamInput in = new BytesStreamInput(out.bytes());
FetchPhase streamed = (FetchPhase) ExecutionPhases.fromStream(in);
assertThat(orig.executionPhaseId(), is(streamed.executionPhaseId()));
assertThat(orig.executionNodes(), is(streamed.executionNodes()));
assertThat(orig.fetchRefs(), is(streamed.fetchRefs()));
assertThat(orig.bases(), is(streamed.bases()));
assertThat(orig.tableIndices(), is(streamed.tableIndices()));
}
public DocTableInfo build() {
DocIndexMetaData md = docIndexMetaData();
List<PartitionName> partitions = new ArrayList<>();
if (md.partitionedBy().size() > 0) {
for (String index : concreteIndices) {
if (PartitionName.isPartition(index)) {
try {
PartitionName partitionName = PartitionName.fromIndexOrTemplate(index);
assert partitionName.tableIdent().equals(ident);
partitions.add(partitionName);
} catch (IllegalArgumentException e) {
// ignore
logger.warn(
String.format(
Locale.ENGLISH,
"Cannot build partition %s of index %s",
index,
ident.indexName()));
}
}
}
}
return new DocTableInfo(
ident,
md.columns(),
md.partitionedByColumns(),
md.generatedColumnReferences(),
md.indices(),
md.references(),
md.analyzers(),
md.primaryKey(),
md.routingCol(),
md.isAlias(),
md.hasAutoGeneratedPrimaryKey(),
concreteIndices,
clusterService,
md.numberOfShards(),
md.numberOfReplicas(),
md.tableParameters(),
md.partitionedBy(),
partitions,
md.columnPolicy(),
executorService);
}
public DocIndexMetaData(IndexMetaData metaData, TableIdent ident) throws IOException {
this.ident = ident;
this.metaData = metaData;
this.isAlias = !metaData.getIndex().equals(ident.esName());
this.numberOfShards = metaData.numberOfShards();
final Settings settings = metaData.getSettings();
this.numberOfReplicas = NumberOfReplicas.fromSettings(settings);
this.aliases = ImmutableSet.copyOf(metaData.aliases().keys().toArray(String.class));
this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE);
if (defaultMappingMetaData == null) {
this.defaultMappingMap = new HashMap<>();
} else {
this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap();
}
this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData);
prepareCrateMeta();
}
@Override
protected ShardResponse processRequestItems(
ShardId shardId, ShardUpsertRequest request, AtomicBoolean killed) {
ShardResponse shardResponse = new ShardResponse();
DocTableInfo tableInfo = schemas.getWritableTable(TableIdent.fromIndexName(request.index()));
for (int i = 0; i < request.itemIndices().size(); i++) {
int location = request.itemIndices().get(i);
ShardUpsertRequest.Item item = request.items().get(i);
if (killed.get()) {
throw new CancellationException();
}
try {
indexItem(
tableInfo,
request,
item,
shardId,
item.insertValues() != null, // try insert first
0);
shardResponse.add(location);
} catch (Throwable t) {
if (!TransportActions.isShardNotAvailableException(t) && !request.continueOnError()) {
throw t;
} else {
logger.debug(
"{} failed to execute upsert for [{}]/[{}]",
t,
request.shardId(),
request.type(),
item.id());
shardResponse.add(
location,
new ShardResponse.Failure(
item.id(),
ExceptionsHelper.detailedMessage(t),
(t instanceof VersionConflictEngineException)));
}
}
}
return shardResponse;
}
private DocIndexMetaData docIndexMetaData() {
DocIndexMetaData docIndexMetaData;
String templateName = PartitionName.templateName(ident.schema(), ident.name());
boolean createdFromTemplate = false;
if (metaData.getTemplates().containsKey(templateName)) {
docIndexMetaData = buildDocIndexMetaDataFromTemplate(ident.indexName(), templateName);
createdFromTemplate = true;
concreteIndices =
metaData.concreteIndices(IndicesOptions.lenientExpandOpen(), ident.indexName());
} else {
try {
concreteIndices =
metaData.concreteIndices(IndicesOptions.strictExpandOpen(), ident.indexName());
if (concreteIndices.length == 0) {
// no matching index found
throw new TableUnknownException(ident);
}
docIndexMetaData = buildDocIndexMetaData(concreteIndices[0]);
} catch (IndexMissingException ex) {
throw new TableUnknownException(ident.fqn(), ex);
}
}
if ((!createdFromTemplate && concreteIndices.length == 1) || !checkAliasSchema) {
return docIndexMetaData;
}
for (int i = 0; i < concreteIndices.length; i++) {
try {
docIndexMetaData =
docIndexMetaData.merge(
buildDocIndexMetaData(concreteIndices[i]),
transportPutIndexTemplateAction,
createdFromTemplate);
} catch (IOException e) {
throw new UnhandledServerException("Unable to merge/build new DocIndexMetaData", e);
}
}
return docIndexMetaData;
}