Example #1
0
  @Test
  public void testClearDerefMap() throws Exception {
    DerefMap derefMap =
        DerefMapHbaseImpl.create(
            INDEX_NAME, lilyProxy.getHBaseProxy().getConf(), null, repository.getIdGenerator());

    Record linkedRecord =
        repository
            .recordBuilder()
            .id("deref-test-linkedrecord")
            .recordType(rt1.getName())
            .field(ft1.getName(), "deref test linkedrecord")
            .create();

    Record record =
        repository
            .recordBuilder()
            .id("deref-test-main")
            .recordType(rt1.getName())
            .field(ft1.getName(), "deref test main")
            .field(ft2.getName(), new Link(linkedRecord.getId()))
            .create();

    SchemaId vtag = typeManager.getFieldTypeByName(VersionTag.LAST).getId();
    DependantRecordIdsIterator it = null;

    try {
      it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
      assertTrue(!it.hasNext());
    } finally {
      it.close();
    }

    setBatchIndexConf(
        getResourceAsByteArray("batchIndexConf-testClearDerefmap-false.json"), null, false);

    buildAndCommit();

    QueryResponse response = solrServer.query(new SolrQuery("field1:deref\\ test\\ main"));
    assertEquals(1, response.getResults().size());

    try {
      it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
      assertTrue(it.hasNext());
    } finally {
      it.close();
    }

    setBatchIndexConf(
        null, getResourceAsByteArray("batchIndexConf-testClearDerefmap-true.json"), true);
    waitForIndexAndCommit(BUILD_TIMEOUT);

    try {
      it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
      assertTrue(!it.hasNext());
    } finally {
      it.close();
    }
  }
Example #2
0
  private void addIndexUpdater(IndexDefinition index) {
    IndexUpdaterHandle handle = null;
    try {
      IndexerConf indexerConf =
          IndexerConfBuilder.build(new ByteArrayInputStream(index.getConfiguration()), repository);

      ShardSelector shardSelector;
      if (index.getShardingConfiguration() == null) {
        shardSelector = DefaultShardSelectorBuilder.createDefaultSelector(index.getSolrShards());
      } else {
        shardSelector = JsonShardSelectorBuilder.build(index.getShardingConfiguration());
      }

      checkShardUsage(index.getName(), index.getSolrShards().keySet(), shardSelector.getShards());

      SolrShardManager solrShardMgr =
          new SolrShardManagerImpl(
              index.getName(),
              index.getSolrShards(),
              shardSelector,
              httpClient,
              solrClientConfig,
              true);
      IndexLocker indexLocker = new IndexLocker(zk, settings.getEnableLocking());
      IndexerMetrics indexerMetrics = new IndexerMetrics(index.getName());

      // create a deref map in case the indexer configuration contains deref fields
      DerefMap derefMap =
          indexerConf.containsDerefExpressions()
              ? DerefMapHbaseImpl.create(index.getName(), hbaseConf, repository.getIdGenerator())
              : null;

      // create and register the indexer
      Indexer indexer =
          new Indexer(
              index.getName(),
              indexerConf,
              repository,
              solrShardMgr,
              indexLocker,
              indexerMetrics,
              derefMap);
      indexerRegistry.register(indexer);

      IndexUpdaterMetrics updaterMetrics = new IndexUpdaterMetrics(index.getName());
      IndexUpdater indexUpdater =
          new IndexUpdater(
              indexer,
              repository,
              indexLocker,
              rowLog,
              updaterMetrics,
              derefMap,
              index.getQueueSubscriptionId());

      List<RemoteListenerHandler> listenerHandlers = new ArrayList<RemoteListenerHandler>();

      for (int i = 0; i < settings.getListenersPerIndex(); i++) {
        RemoteListenerHandler handler =
            new RemoteListenerHandler(
                rowLog, index.getQueueSubscriptionId(), indexUpdater, rowLogConfMgr, hostName);
        listenerHandlers.add(handler);
      }

      handle =
          new IndexUpdaterHandle(
              index, listenerHandlers, solrShardMgr, indexerMetrics, updaterMetrics);
      handle.start();

      indexUpdaters.put(index.getName(), handle);

      log.info("Started index updater for index " + index.getName());
    } catch (Throwable t) {
      if (t instanceof InterruptedException) {
        Thread.currentThread().interrupt();
      }

      log.error("Problem starting index updater for index " + index.getName(), t);

      if (handle != null) {
        // stop any listeners that might have been started
        try {
          handle.stop();
        } catch (Throwable t2) {
          if (t instanceof InterruptedException) {
            Thread.currentThread().interrupt();
          }
          log.error(
              "Problem stopping listeners for failed-to-start index updater for index '"
                  + index.getName()
                  + "'",
              t2);
        }
      }
    }
  }