예제 #1
0
  public boolean sync(
      ZkController zkController,
      SolrCore core,
      ZkNodeProps leaderProps,
      boolean peerSyncOnlyWithActive) {
    if (SKIP_AUTO_RECOVERY) {
      return true;
    }
    boolean success;
    SolrQueryRequest req = new LocalSolrQueryRequest(core, new ModifiableSolrParams());
    SolrQueryResponse rsp = new SolrQueryResponse();
    SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
    try {
      if (isClosed) {
        log.warn("Closed, skipping sync up.");
        return false;
      }
      log.info("Sync replicas to " + ZkCoreNodeProps.getCoreUrl(leaderProps));

      if (core.getUpdateHandler().getUpdateLog() == null) {
        log.error("No UpdateLog found - cannot sync");
        return false;
      }

      success = syncReplicas(zkController, core, leaderProps, peerSyncOnlyWithActive);
    } finally {
      SolrRequestInfo.clearRequestInfo();
    }
    return success;
  }
예제 #2
0
  public static SolrInputDocument getInputDocument(SolrCore core, BytesRef idBytes)
      throws IOException {
    SolrInputDocument sid = null;
    RefCounted<SolrIndexSearcher> searcherHolder = null;
    try {
      SolrIndexSearcher searcher = null;
      UpdateLog ulog = core.getUpdateHandler().getUpdateLog();

      if (ulog != null) {
        Object o = ulog.lookup(idBytes);
        if (o != null) {
          // should currently be a List<Oper,Ver,Doc/Id>
          List entry = (List) o;
          assert entry.size() >= 3;
          int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
          switch (oper) {
            case UpdateLog.ADD:
              sid = (SolrInputDocument) entry.get(entry.size() - 1);
              break;
            case UpdateLog.DELETE:
              return null;
            default:
              throw new SolrException(
                  SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
          }
        }
      }

      if (sid == null) {
        // didn't find it in the update log, so it should be in the newest searcher opened
        if (searcher == null) {
          searcherHolder = core.getRealtimeSearcher();
          searcher = searcherHolder.get();
        }

        // SolrCore.verbose("RealTimeGet using searcher ", searcher);
        SchemaField idField = core.getLatestSchema().getUniqueKeyField();

        int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
        if (docid < 0) return null;
        StoredDocument luceneDocument = searcher.doc(docid);
        sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
      }
    } finally {
      if (searcherHolder != null) {
        searcherHolder.decref();
      }
    }

    return sid;
  }
예제 #3
0
 @Override
 public void deleteAll() {
   DeleteUpdateCommand cmd = new DeleteUpdateCommand(makeSolrQueryRequest());
   /*
   cmd.fromCommitted = true;
   cmd.fromPending = true;
   */
   cmd.query = "*:*";
   try {
     if (isCloud()) {
       deleteCloud(cmd);
     } else {
       core.getUpdateHandler().deleteByQuery(cmd);
     }
   } catch (IOException e) {
     throw new LuxException(e);
   }
 }
예제 #4
0
  public void testMaxDocs() throws Exception {
    SolrCore core = h.getCore();

    NewSearcherListener trigger = new NewSearcherListener();

    DirectUpdateHandler2 updateHandler = (DirectUpdateHandler2) core.getUpdateHandler();
    CommitTracker tracker = updateHandler.softCommitTracker;
    tracker.setTimeUpperBound(-1);
    tracker.setDocsUpperBound(14);
    core.registerNewSearcherListener(trigger);

    UpdateRequestHandler handler = new UpdateRequestHandler();
    handler.init(null);

    MapSolrParams params = new MapSolrParams(new HashMap<String, String>());

    // Add documents
    SolrQueryResponse rsp = new SolrQueryResponse();
    SolrQueryRequestBase req = new SolrQueryRequestBase(core, params) {};
    for (int i = 0; i < 14; i++) {
      req.setContentStreams(
          toContentStreams(adoc("id", Integer.toString(i), "subject", "info"), null));
      handler.handleRequest(req, rsp);
    }
    // It should not be there right away
    assertQ("shouldn't find any", req("id:1"), "//result[@numFound=0]");
    assertEquals(0, tracker.getCommitCount());

    req.setContentStreams(toContentStreams(adoc("id", "14", "subject", "info"), null));
    handler.handleRequest(req, rsp);

    assertTrue(trigger.waitForNewSearcher(15000));

    req.setContentStreams(toContentStreams(adoc("id", "15", "subject", "info"), null));
    handler.handleRequest(req, rsp);

    // Now make sure we can find it
    assertQ("should find one", req("id:14"), "//result[@numFound=1]");
    assertEquals(1, tracker.getCommitCount());
    // But not the one added afterward
    assertQ("should not find one", req("id:15"), "//result[@numFound=0]");
    assertEquals(1, tracker.getCommitCount());
  }
예제 #5
0
 private void writeLocal(SolrInputDocument solrDoc, NodeInfo node, String uri) {
   XmlIndexer indexer = null;
   try {
     indexer = xqueryComponent.getSolrIndexConfig().checkoutXmlIndexer();
     try {
       indexer.index(node, uri);
     } catch (XMLStreamException e) {
       throw new LuxException(e);
     }
     UpdateDocCommand cmd = new UpdateDocCommand(core, indexer.createLuceneDocument(), uri);
     cmd.solrDoc = solrDoc;
     core.getUpdateHandler().addDoc(cmd);
   } catch (IOException e) {
     throw new LuxException(e);
   } finally {
     if (indexer != null) {
       xqueryComponent.getSolrIndexConfig().returnXmlIndexer(indexer);
     }
   }
 }
예제 #6
0
  private boolean syncWithReplicas(
      ZkController zkController,
      SolrCore core,
      ZkNodeProps props,
      String collection,
      String shardId,
      boolean peerSyncOnlyWithActive) {
    List<ZkCoreNodeProps> nodes =
        zkController
            .getZkStateReader()
            .getReplicaProps(
                collection,
                shardId,
                core.getCoreDescriptor().getCloudDescriptor().getCoreNodeName());

    if (nodes == null) {
      // I have no replicas
      return true;
    }

    List<String> syncWith = new ArrayList<>();
    for (ZkCoreNodeProps node : nodes) {
      syncWith.add(node.getCoreUrl());
    }

    // if we can't reach a replica for sync, we still consider the overall sync a success
    // TODO: as an assurance, we should still try and tell the sync nodes that we couldn't reach
    // to recover once more?
    PeerSync peerSync =
        new PeerSync(
            core,
            syncWith,
            core.getUpdateHandler().getUpdateLog().getNumRecordsToKeep(),
            true,
            true,
            peerSyncOnlyWithActive);
    return peerSync.sync();
  }
예제 #7
0
  @Override
  public void write(NodeInfo node, String uri) {
    UpdateHandler updateHandler = core.getUpdateHandler();

    // Create a version of the document for saving to the transaction log,
    // or for cloud update via HTTP
    SolrInputDocument solrDoc = new SolrInputDocument();
    solrDoc.addField(uriFieldName, uri);
    if (isCloud()) {
      // TODO: write as binary, but we need to enable the binary update request writer for this
      // TinyBinary tinybin = new TinyBinary(((TinyNodeImpl)node).getTree());
      // solrDoc.addField(xmlFieldName, tinybin.getByteBuffer().array());
      Serializer serializer = xqueryComponent.solrIndexConfig.checkoutSerializer();
      try {
        String xmlString = serializer.serializeNodeToString(new XdmNode(node));
        solrDoc.addField(xmlFieldName, xmlString);
      } catch (SaxonApiException e) {
        throw new LuxException(e);
      } finally {
        xqueryComponent.solrIndexConfig.returnSerializer(serializer);
      }
      // TODO -- if we can determine this doc only gets added locally??
      // solrDoc.addField(xmlFieldName, node);
    } else if (updateHandler.getUpdateLog() != null) {
      if (node instanceof TinyNodeImpl) {
        TinyBinary tinybin = new TinyBinary(((TinyNodeImpl) node).getTree());
        solrDoc.addField(xmlFieldName, tinybin.getByteBuffer());
      } else {
        String xml = node.toString();
        solrDoc.addField(xmlFieldName, xml);
      }
    }
    if (isCloud()) {
      writeToCloud(solrDoc, uri);
    } else {
      writeLocal(solrDoc, node, uri);
    }
  }
예제 #8
0
  private boolean doRestore() throws Exception {

    Path backupPath = Paths.get(backupLocation).resolve(backupName);
    SimpleDateFormat dateFormat = new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT);
    String restoreIndexName = "restore." + dateFormat.format(new Date());
    String restoreIndexPath = core.getDataDir() + restoreIndexName;

    Directory restoreIndexDir = null;
    Directory indexDir = null;
    try (Directory backupDir = FSDirectory.open(backupPath)) {

      final Version version =
          IndexFetcher.checkOldestVersion(SegmentInfos.readLatestCommit(backupDir));

      restoreIndexDir =
          core.getDirectoryFactory()
              .get(
                  restoreIndexPath,
                  DirectoryFactory.DirContext.DEFAULT,
                  core.getSolrConfig().indexConfig.lockType);

      // Prefer local copy.
      indexDir =
          core.getDirectoryFactory()
              .get(
                  core.getIndexDir(),
                  DirectoryFactory.DirContext.DEFAULT,
                  core.getSolrConfig().indexConfig.lockType);

      // Move all files from backupDir to restoreIndexDir
      for (String filename : backupDir.listAll()) {
        checkInterrupted();
        log.info("Copying file {} to restore directory ", filename);
        try (IndexInput indexInput = backupDir.openInput(filename, IOContext.READONCE)) {
          Long checksum = null;
          try {
            checksum = CodecUtil.retrieveChecksum(indexInput);
          } catch (Exception e) {
            log.warn("Could not read checksum from index file: " + filename, e);
          }
          long length = indexInput.length();
          IndexFetcher.CompareResult compareResult =
              IndexFetcher.compareFile(indexDir, version, filename, length, checksum);
          if (!compareResult.equal
              || (!compareResult.checkSummed
                  && (filename.endsWith(".si")
                      || filename.endsWith(".liv")
                      || filename.startsWith("segments_")))) {
            restoreIndexDir.copyFrom(backupDir, filename, filename, IOContext.READONCE);
          } else {
            // prefer local copy
            restoreIndexDir.copyFrom(indexDir, filename, filename, IOContext.READONCE);
          }
        } catch (Exception e) {
          throw new SolrException(
              SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
        }
      }
      log.debug("Switching directories");
      IndexFetcher.modifyIndexProps(core, restoreIndexName);

      boolean success;
      try {
        core.getUpdateHandler().newIndexWriter(false);
        openNewSearcher();
        success = true;
        log.info("Successfully restored to the backup index");
      } catch (Exception e) {
        // Rollback to the old index directory. Delete the restore index directory and mark the
        // restore as failed.
        log.warn("Could not switch to restored index. Rolling back to the current index");
        Directory dir = null;
        try {
          dir =
              core.getDirectoryFactory()
                  .get(
                      core.getDataDir(),
                      DirectoryFactory.DirContext.META_DATA,
                      core.getSolrConfig().indexConfig.lockType);
          dir.deleteFile(IndexFetcher.INDEX_PROPERTIES);
        } finally {
          if (dir != null) {
            core.getDirectoryFactory().release(dir);
          }
        }

        core.getDirectoryFactory().doneWithDirectory(restoreIndexDir);
        core.getDirectoryFactory().remove(restoreIndexDir);
        core.getUpdateHandler().newIndexWriter(false);
        openNewSearcher();
        throw new SolrException(
            SolrException.ErrorCode.UNKNOWN, "Exception while restoring the backup index", e);
      }
      if (success) {
        core.getDirectoryFactory().doneWithDirectory(indexDir);
        core.getDirectoryFactory().remove(indexDir);
      }

      return true;
    } finally {
      if (restoreIndexDir != null) {
        core.getDirectoryFactory().release(restoreIndexDir);
      }
      if (indexDir != null) {
        core.getDirectoryFactory().release(indexDir);
      }
    }
  }
예제 #9
0
  public void testCommitWithin() throws Exception {
    SolrCore core = h.getCore();

    NewSearcherListener trigger = new NewSearcherListener();
    core.registerNewSearcherListener(trigger);
    DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
    CommitTracker tracker = updater.softCommitTracker;
    tracker.setTimeUpperBound(0);
    tracker.setDocsUpperBound(-1);

    UpdateRequestHandler handler = new UpdateRequestHandler();
    handler.init(null);

    MapSolrParams params = new MapSolrParams(new HashMap<String, String>());

    // Add a single document with commitWithin == 4 second
    SolrQueryResponse rsp = new SolrQueryResponse();
    SolrQueryRequestBase req = new SolrQueryRequestBase(core, params) {};
    req.setContentStreams(
        toContentStreams(
            adoc(4000, "id", "529", "field_t", "what's inside?", "subject", "info"), null));
    trigger.reset();
    handler.handleRequest(req, rsp);

    // Check it isn't in the index
    assertQ("shouldn't find any", req("id:529"), "//result[@numFound=0]");

    // Wait longer than the commitWithin time
    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));

    // Add one document without commitWithin
    req.setContentStreams(
        toContentStreams(adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null));
    trigger.reset();
    handler.handleRequest(req, rsp);

    // Check it isn't in the index
    assertQ("shouldn't find any", req("id:530"), "//result[@numFound=0]");

    // Delete one document with commitWithin
    req.setContentStreams(toContentStreams(delI("529", "commitWithin", "1000"), null));
    trigger.reset();
    handler.handleRequest(req, rsp);

    // Now make sure we can find it
    assertQ("should find one", req("id:529"), "//result[@numFound=1]");

    // Wait for the commit to happen
    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));

    // Now we shouldn't find it
    assertQ("should find none", req("id:529"), "//result[@numFound=0]");
    // ... but we should find the new one
    assertQ("should find one", req("id:530"), "//result[@numFound=1]");

    trigger.reset();

    // now make the call 10 times really fast and make sure it
    // only commits once
    req.setContentStreams(toContentStreams(adoc(2000, "id", "500"), null));
    for (int i = 0; i < 10; i++) {
      handler.handleRequest(req, rsp);
    }
    assertQ("should not be there yet", req("id:500"), "//result[@numFound=0]");

    // the same for the delete
    req.setContentStreams(toContentStreams(delI("530", "commitWithin", "1000"), null));
    for (int i = 0; i < 10; i++) {
      handler.handleRequest(req, rsp);
    }
    assertQ("should be there", req("id:530"), "//result[@numFound=1]");

    assertTrue("commitWithin failed to commit", trigger.waitForNewSearcher(30000));
    assertQ("should be there", req("id:500"), "//result[@numFound=1]");
    assertQ("should not be there", req("id:530"), "//result[@numFound=0]");

    assertEquals(3, tracker.getCommitCount());
  }
예제 #10
0
  public void testMaxTime() throws Exception {
    SolrCore core = h.getCore();

    NewSearcherListener trigger = new NewSearcherListener();
    core.registerNewSearcherListener(trigger);
    DirectUpdateHandler2 updater = (DirectUpdateHandler2) core.getUpdateHandler();
    CommitTracker tracker = updater.softCommitTracker;
    // too low of a number can cause a slow host to commit before the test code checks that it
    // isn't there... causing a failure at "shouldn't find any"
    tracker.setTimeUpperBound(1000);
    tracker.setDocsUpperBound(-1);
    // updater.commitCallbacks.add(trigger);

    UpdateRequestHandler handler = new UpdateRequestHandler();
    handler.init(null);

    MapSolrParams params = new MapSolrParams(new HashMap<String, String>());

    // Add a single document
    SolrQueryResponse rsp = new SolrQueryResponse();
    SolrQueryRequestBase req = new SolrQueryRequestBase(core, params) {};
    req.setContentStreams(
        toContentStreams(adoc("id", "529", "field_t", "what's inside?", "subject", "info"), null));
    trigger.reset();
    handler.handleRequest(req, rsp);

    // Check it it is in the index
    assertQ("shouldn't find any", req("id:529"), "//result[@numFound=0]");

    // Wait longer than the autocommit time
    assertTrue(trigger.waitForNewSearcher(45000));
    trigger.reset();
    req.setContentStreams(
        toContentStreams(adoc("id", "530", "field_t", "what's inside?", "subject", "info"), null));
    handler.handleRequest(req, rsp);

    // Now make sure we can find it
    assertQ("should find one", req("id:529"), "//result[@numFound=1]");
    // But not this one
    assertQ("should find none", req("id:530"), "//result[@numFound=0]");

    // Delete the document
    assertU(delI("529"));
    assertQ("deleted, but should still be there", req("id:529"), "//result[@numFound=1]");
    // Wait longer than the autocommit time
    assertTrue(trigger.waitForNewSearcher(30000));
    trigger.reset();
    req.setContentStreams(
        toContentStreams(adoc("id", "550", "field_t", "what's inside?", "subject", "info"), null));
    handler.handleRequest(req, rsp);
    assertEquals(2, tracker.getCommitCount());
    assertQ("deleted and time has passed", req("id:529"), "//result[@numFound=0]");

    // now make the call 10 times really fast and make sure it
    // only commits once
    req.setContentStreams(toContentStreams(adoc("id", "500"), null));
    for (int i = 0; i < 10; i++) {
      handler.handleRequest(req, rsp);
    }
    assertQ("should not be there yet", req("id:500"), "//result[@numFound=0]");

    // Wait longer than the autocommit time
    assertTrue(trigger.waitForNewSearcher(45000));
    trigger.reset();

    req.setContentStreams(
        toContentStreams(adoc("id", "531", "field_t", "what's inside?", "subject", "info"), null));
    handler.handleRequest(req, rsp);
    assertEquals(3, tracker.getCommitCount());

    assertQ("now it should", req("id:500"), "//result[@numFound=1]");
  }
예제 #11
0
  @Override
  public void process(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();

    if (!params.getBool(COMPONENT_NAME, true)) {
      return;
    }

    String val = params.get("getVersions");
    if (val != null) {
      processGetVersions(rb);
      return;
    }

    val = params.get("getUpdates");
    if (val != null) {
      processGetUpdates(rb);
      return;
    }

    String id[] = params.getParams("id");
    String ids[] = params.getParams("ids");

    if (id == null && ids == null) {
      return;
    }

    String[] allIds = id == null ? new String[0] : id;

    if (ids != null) {
      List<String> lst = new ArrayList<String>();
      for (String s : allIds) {
        lst.add(s);
      }
      for (String idList : ids) {
        lst.addAll(StrUtils.splitSmart(idList, ",", true));
      }
      allIds = lst.toArray(new String[lst.size()]);
    }

    SolrCore core = req.getCore();
    SchemaField idField = core.getLatestSchema().getUniqueKeyField();
    FieldType fieldType = idField.getType();

    SolrDocumentList docList = new SolrDocumentList();
    UpdateLog ulog = core.getUpdateHandler().getUpdateLog();

    RefCounted<SolrIndexSearcher> searcherHolder = null;

    DocTransformer transformer = rsp.getReturnFields().getTransformer();
    if (transformer != null) {
      TransformContext context = new TransformContext();
      context.req = req;
      transformer.setContext(context);
    }
    try {
      SolrIndexSearcher searcher = null;

      BytesRef idBytes = new BytesRef();
      for (String idStr : allIds) {
        fieldType.readableToIndexed(idStr, idBytes);
        if (ulog != null) {
          Object o = ulog.lookup(idBytes);
          if (o != null) {
            // should currently be a List<Oper,Ver,Doc/Id>
            List entry = (List) o;
            assert entry.size() >= 3;
            int oper = (Integer) entry.get(0) & UpdateLog.OPERATION_MASK;
            switch (oper) {
              case UpdateLog.ADD:
                SolrDocument doc =
                    toSolrDoc(
                        (SolrInputDocument) entry.get(entry.size() - 1), core.getLatestSchema());
                if (transformer != null) {
                  transformer.transform(doc, -1); // unknown docID
                }
                docList.add(doc);
                break;
              case UpdateLog.DELETE:
                break;
              default:
                throw new SolrException(
                    SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
            }
            continue;
          }
        }

        // didn't find it in the update log, so it should be in the newest searcher opened
        if (searcher == null) {
          searcherHolder = core.getRealtimeSearcher();
          searcher = searcherHolder.get();
        }

        // SolrCore.verbose("RealTimeGet using searcher ", searcher);

        int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
        if (docid < 0) continue;
        StoredDocument luceneDocument = searcher.doc(docid);
        SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
        if (transformer != null) {
          transformer.transform(doc, docid);
        }
        docList.add(doc);
      }

    } finally {
      if (searcherHolder != null) {
        searcherHolder.decref();
      }
    }

    // if the client specified a single id=foo, then use "doc":{
    // otherwise use a standard doclist

    if (ids == null && allIds.length <= 1) {
      // if the doc was not found, then use a value of null.
      rsp.add("doc", docList.size() > 0 ? docList.get(0) : null);
    } else {
      docList.setNumFound(docList.size());
      rsp.add("response", docList);
    }
  }