Example #1
0
 private void updateTemplate(
     DocIndexMetaData md,
     TransportPutIndexTemplateAction transportPutIndexTemplateAction,
     Settings updateSettings) {
   String templateName = PartitionName.templateName(ident.schema(), ident.name());
   PutIndexTemplateRequest request =
       new PutIndexTemplateRequest(templateName)
           .mapping(Constants.DEFAULT_MAPPING_TYPE, md.defaultMappingMap)
           .create(false)
           .settings(updateSettings)
           .template(templateName + "*");
   for (String alias : md.aliases()) {
     request = request.alias(new Alias(alias));
   }
   transportPutIndexTemplateAction.execute(request);
 }
 @Override
 protected void bindSchemas() {
   super.bindSchemas();
   SchemaInfo schemaInfo = mock(SchemaInfo.class);
   when(schemaInfo.getTableInfo(USER_TABLE_IDENT.name())).thenReturn(USER_TABLE_INFO);
   when(schemaInfo.getTableInfo(TEST_ALIAS_TABLE_IDENT.name()))
       .thenReturn(TEST_ALIAS_TABLE_INFO);
   when(schemaInfo.getTableInfo(USER_TABLE_IDENT_CLUSTERED_BY_ONLY.name()))
       .thenReturn(USER_TABLE_INFO_CLUSTERED_BY_ONLY);
   when(schemaInfo.getTableInfo(TEST_PARTITIONED_TABLE_IDENT.name()))
       .thenReturn(TEST_PARTITIONED_TABLE_INFO);
   when(schemaInfo.getTableInfo(DEEPLY_NESTED_TABLE_IDENT.name()))
       .thenReturn(DEEPLY_NESTED_TABLE_INFO);
   when(schemaInfo.getTableInfo(NESTED_CLUSTERED_BY_TABLE_IDENT.name()))
       .thenReturn(NESTED_CLUSTERED_BY_TABLE_INFO);
   schemaBinder.addBinding(Schemas.DEFAULT_SCHEMA_NAME).toInstance(schemaInfo);
 }
Example #3
0
  @Test
  public void testStreaming() throws Exception {

    TableIdent t1 = new TableIdent(null, "t1");

    TreeMap<String, Integer> bases = new TreeMap<String, Integer>();
    bases.put(t1.name(), 0);
    bases.put("i2", 1);

    Multimap<TableIdent, String> tableIndices = HashMultimap.create();
    tableIndices.put(t1, t1.name());
    tableIndices.put(new TableIdent(null, "i2"), "i2_s1");
    tableIndices.put(new TableIdent(null, "i2"), "i2_s2");

    ReferenceIdent nameIdent = new ReferenceIdent(t1, "name");
    Reference name =
        new Reference(new ReferenceInfo(nameIdent, RowGranularity.DOC, DataTypes.STRING));

    FetchPhase orig =
        new FetchPhase(
            1,
            ImmutableSet.<String>of("node1", "node2"),
            bases,
            tableIndices,
            ImmutableList.of(name));

    BytesStreamOutput out = new BytesStreamOutput();
    ExecutionPhases.toStream(out, orig);

    BytesStreamInput in = new BytesStreamInput(out.bytes());
    FetchPhase streamed = (FetchPhase) ExecutionPhases.fromStream(in);

    assertThat(orig.executionPhaseId(), is(streamed.executionPhaseId()));
    assertThat(orig.executionNodes(), is(streamed.executionNodes()));
    assertThat(orig.fetchRefs(), is(streamed.fetchRefs()));
    assertThat(orig.bases(), is(streamed.bases()));
    assertThat(orig.tableIndices(), is(streamed.tableIndices()));
  }
Example #4
0
  public DocTableInfo build() {
    DocIndexMetaData md = docIndexMetaData();

    List<PartitionName> partitions = new ArrayList<>();
    if (md.partitionedBy().size() > 0) {
      for (String index : concreteIndices) {
        if (PartitionName.isPartition(index)) {
          try {
            PartitionName partitionName = PartitionName.fromIndexOrTemplate(index);
            assert partitionName.tableIdent().equals(ident);
            partitions.add(partitionName);
          } catch (IllegalArgumentException e) {
            // ignore
            logger.warn(
                String.format(
                    Locale.ENGLISH,
                    "Cannot build partition %s of index %s",
                    index,
                    ident.indexName()));
          }
        }
      }
    }
    return new DocTableInfo(
        ident,
        md.columns(),
        md.partitionedByColumns(),
        md.generatedColumnReferences(),
        md.indices(),
        md.references(),
        md.analyzers(),
        md.primaryKey(),
        md.routingCol(),
        md.isAlias(),
        md.hasAutoGeneratedPrimaryKey(),
        concreteIndices,
        clusterService,
        md.numberOfShards(),
        md.numberOfReplicas(),
        md.tableParameters(),
        md.partitionedBy(),
        partitions,
        md.columnPolicy(),
        executorService);
  }
Example #5
0
  public DocIndexMetaData(IndexMetaData metaData, TableIdent ident) throws IOException {
    this.ident = ident;
    this.metaData = metaData;
    this.isAlias = !metaData.getIndex().equals(ident.esName());
    this.numberOfShards = metaData.numberOfShards();
    final Settings settings = metaData.getSettings();
    this.numberOfReplicas = NumberOfReplicas.fromSettings(settings);
    this.aliases = ImmutableSet.copyOf(metaData.aliases().keys().toArray(String.class));
    this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE);
    if (defaultMappingMetaData == null) {
      this.defaultMappingMap = new HashMap<>();
    } else {
      this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap();
    }
    this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData);

    prepareCrateMeta();
  }
  @Override
  protected ShardResponse processRequestItems(
      ShardId shardId, ShardUpsertRequest request, AtomicBoolean killed) {
    ShardResponse shardResponse = new ShardResponse();
    DocTableInfo tableInfo = schemas.getWritableTable(TableIdent.fromIndexName(request.index()));
    for (int i = 0; i < request.itemIndices().size(); i++) {
      int location = request.itemIndices().get(i);
      ShardUpsertRequest.Item item = request.items().get(i);
      if (killed.get()) {
        throw new CancellationException();
      }
      try {
        indexItem(
            tableInfo,
            request,
            item,
            shardId,
            item.insertValues() != null, // try insert first
            0);
        shardResponse.add(location);
      } catch (Throwable t) {
        if (!TransportActions.isShardNotAvailableException(t) && !request.continueOnError()) {
          throw t;
        } else {
          logger.debug(
              "{} failed to execute upsert for [{}]/[{}]",
              t,
              request.shardId(),
              request.type(),
              item.id());
          shardResponse.add(
              location,
              new ShardResponse.Failure(
                  item.id(),
                  ExceptionsHelper.detailedMessage(t),
                  (t instanceof VersionConflictEngineException)));
        }
      }
    }

    return shardResponse;
  }
Example #7
0
  private DocIndexMetaData docIndexMetaData() {
    DocIndexMetaData docIndexMetaData;
    String templateName = PartitionName.templateName(ident.schema(), ident.name());
    boolean createdFromTemplate = false;
    if (metaData.getTemplates().containsKey(templateName)) {
      docIndexMetaData = buildDocIndexMetaDataFromTemplate(ident.indexName(), templateName);
      createdFromTemplate = true;
      concreteIndices =
          metaData.concreteIndices(IndicesOptions.lenientExpandOpen(), ident.indexName());
    } else {
      try {
        concreteIndices =
            metaData.concreteIndices(IndicesOptions.strictExpandOpen(), ident.indexName());
        if (concreteIndices.length == 0) {
          // no matching index found
          throw new TableUnknownException(ident);
        }
        docIndexMetaData = buildDocIndexMetaData(concreteIndices[0]);
      } catch (IndexMissingException ex) {
        throw new TableUnknownException(ident.fqn(), ex);
      }
    }

    if ((!createdFromTemplate && concreteIndices.length == 1) || !checkAliasSchema) {
      return docIndexMetaData;
    }
    for (int i = 0; i < concreteIndices.length; i++) {
      try {
        docIndexMetaData =
            docIndexMetaData.merge(
                buildDocIndexMetaData(concreteIndices[i]),
                transportPutIndexTemplateAction,
                createdFromTemplate);
      } catch (IOException e) {
        throw new UnhandledServerException("Unable to merge/build new DocIndexMetaData", e);
      }
    }
    return docIndexMetaData;
  }