Example #1
0
  // make sure that log isn't needlessly replayed after a clean close
  @Test
  public void testCleanShutdown() throws Exception {
    DirectUpdateHandler2.commitOnClose = true;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);

    UpdateLog.testing_logReplayHook =
        new Runnable() {
          @Override
          public void run() {
            try {
              assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
          }
        };

    UpdateLog.testing_logReplayFinishHook =
        new Runnable() {
          @Override
          public void run() {
            logReplayFinish.release();
          }
        };

    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();

    try {
      clearIndex();
      assertU(commit());

      assertU(adoc("id", "E1", "val_i", "1"));
      assertU(adoc("id", "E2", "val_i", "1"));

      // set to a high enough number so this test won't hang on a bug
      logReplay.release(10);

      h.close();
      createCore();

      // make sure the docs got committed
      assertJQ(req("q", "*:*"), "/response/numFound==2");

      // make sure no replay happened
      assertEquals(10, logReplay.availablePermits());

    } finally {
      DirectUpdateHandler2.commitOnClose = true;
      UpdateLog.testing_logReplayHook = null;
      UpdateLog.testing_logReplayFinishHook = null;

      req().close();
    }
  }
Example #2
0
  @Override
  public void write(NodeInfo node, String uri) {
    UpdateHandler updateHandler = core.getUpdateHandler();

    // Create a version of the document for saving to the transaction log,
    // or for cloud update via HTTP
    SolrInputDocument solrDoc = new SolrInputDocument();
    solrDoc.addField(uriFieldName, uri);
    if (isCloud()) {
      // TODO: write as binary, but we need to enable the binary update request writer for this
      // TinyBinary tinybin = new TinyBinary(((TinyNodeImpl)node).getTree());
      // solrDoc.addField(xmlFieldName, tinybin.getByteBuffer().array());
      Serializer serializer = xqueryComponent.solrIndexConfig.checkoutSerializer();
      try {
        String xmlString = serializer.serializeNodeToString(new XdmNode(node));
        solrDoc.addField(xmlFieldName, xmlString);
      } catch (SaxonApiException e) {
        throw new LuxException(e);
      } finally {
        xqueryComponent.solrIndexConfig.returnSerializer(serializer);
      }
      // TODO -- if we can determine this doc only gets added locally??
      // solrDoc.addField(xmlFieldName, node);
    } else if (updateHandler.getUpdateLog() != null) {
      if (node instanceof TinyNodeImpl) {
        TinyBinary tinybin = new TinyBinary(((TinyNodeImpl) node).getTree());
        solrDoc.addField(xmlFieldName, tinybin.getByteBuffer());
      } else {
        String xml = node.toString();
        solrDoc.addField(xmlFieldName, xml);
      }
    }
    if (isCloud()) {
      writeToCloud(solrDoc, uri);
    } else {
      writeLocal(solrDoc, node, uri);
    }
  }
  public DistributedUpdateProcessor(
      SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
    super(next);
    this.rsp = rsp;
    this.next = next;
    this.idField = req.getSchema().getUniqueKeyField();
    // version init

    this.updateHandler = req.getCore().getUpdateHandler();
    this.ulog = updateHandler.getUpdateLog();
    this.vinfo = ulog == null ? null : ulog.getVersionInfo();
    versionsStored = this.vinfo != null && this.vinfo.getVersionField() != null;
    returnVersions = req.getParams().getBool(UpdateParams.VERSIONS, false);

    // TODO: better way to get the response, or pass back info to it?
    SolrRequestInfo reqInfo = returnVersions ? SolrRequestInfo.getRequestInfo() : null;

    this.req = req;

    CoreDescriptor coreDesc = req.getCore().getCoreDescriptor();

    this.zkEnabled = coreDesc.getCoreContainer().isZooKeeperAware();
    zkController = req.getCore().getCoreDescriptor().getCoreContainer().getZkController();
    if (zkEnabled) {
      numNodes = zkController.getZkStateReader().getClusterState().getLiveNodes().size();
      cmdDistrib =
          new SolrCmdDistributor(
              numNodes, coreDesc.getCoreContainer().getZkController().getCmdDistribExecutor());
    }
    // this.rsp = reqInfo != null ? reqInfo.getRsp() : null;

    cloudDesc = coreDesc.getCloudDescriptor();

    if (cloudDesc != null) {
      collection = cloudDesc.getCollectionName();
    }
  }
Example #4
0
  @Test
  public void testBufferingFlags() throws Exception {

    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplayFinish = new Semaphore(0);

    UpdateLog.testing_logReplayFinishHook =
        new Runnable() {
          @Override
          public void run() {
            logReplayFinish.release();
          }
        };

    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();

    try {
      clearIndex();
      assertU(commit());

      assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
      ulog.bufferUpdates();

      // simulate updates from a leader
      updateJ(
          jsonAdd(sdoc("id", "Q1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "Q2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "Q3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());

      req.close();
      h.close();
      createCore();

      req = req();
      uhandler = req.getCore().getUpdateHandler();
      ulog = uhandler.getUpdateLog();

      logReplayFinish.acquire(); // wait for replay to finish

      assertTrue(
          (ulog.getStartingOperation() & UpdateLog.FLAG_GAP)
              != 0); // since we died while buffering, we should see this last

      //
      // Try again to ensure that the previous log replay didn't wipe out our flags
      //

      req.close();
      h.close();
      createCore();

      req = req();
      uhandler = req.getCore().getUpdateHandler();
      ulog = uhandler.getUpdateLog();

      assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) != 0);

      // now do some normal non-buffered adds
      updateJ(
          jsonAdd(sdoc("id", "Q4", "_version_", "114")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "Q5", "_version_", "115")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "Q6", "_version_", "116")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      assertU(commit());

      req.close();
      h.close();
      createCore();

      req = req();
      uhandler = req.getCore().getUpdateHandler();
      ulog = uhandler.getUpdateLog();

      assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0);

      ulog.bufferUpdates();
      // simulate receiving no updates
      ulog.applyBufferedUpdates();
      updateJ(
          jsonAdd(sdoc("id", "Q7", "_version_", "117")),
          params(
              DISTRIB_UPDATE_PARAM,
              FROM_LEADER)); // do another add to make sure flags are back to normal

      req.close();
      h.close();
      createCore();

      req = req();
      uhandler = req.getCore().getUpdateHandler();
      ulog = uhandler.getUpdateLog();

      assertTrue((ulog.getStartingOperation() & UpdateLog.FLAG_GAP) == 0); // check flags on Q7

      logReplayFinish.acquire();
      assertEquals(
          UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
    } finally {
      DirectUpdateHandler2.commitOnClose = true;
      UpdateLog.testing_logReplayHook = null;
      UpdateLog.testing_logReplayFinishHook = null;

      req().close();
    }
  }
Example #5
0
  @Test
  @Ignore("HDFS-3107: no truncate support yet")
  public void testDropBuffered() throws Exception {

    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);

    UpdateLog.testing_logReplayHook =
        new Runnable() {
          @Override
          public void run() {
            try {
              assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
          }
        };

    UpdateLog.testing_logReplayFinishHook =
        new Runnable() {
          @Override
          public void run() {
            logReplayFinish.release();
          }
        };

    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();

    try {
      clearIndex();
      assertU(commit());

      assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
      ulog.bufferUpdates();
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
      Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
      assertTrue(rinfoFuture == null);
      assertEquals(UpdateLog.State.ACTIVE, ulog.getState());

      ulog.bufferUpdates();
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());

      // simulate updates from a leader
      updateJ(
          jsonAdd(sdoc("id", "C1", "_version_", "101")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C2", "_version_", "102")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C3", "_version_", "103")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      assertTrue(ulog.dropBufferedUpdates());
      ulog.bufferUpdates();
      updateJ(
          jsonAdd(sdoc("id", "C4", "_version_", "104")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C5", "_version_", "105")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      logReplay.release(1000);
      rinfoFuture = ulog.applyBufferedUpdates();
      UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
      assertEquals(2, rinfo.adds);

      assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[105,104]}");

      // this time add some docs first before buffering starts (so tlog won't be at pos 0)
      updateJ(
          jsonAdd(sdoc("id", "C100", "_version_", "200")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C101", "_version_", "201")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      ulog.bufferUpdates();
      updateJ(
          jsonAdd(sdoc("id", "C103", "_version_", "203")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C104", "_version_", "204")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      assertTrue(ulog.dropBufferedUpdates());
      ulog.bufferUpdates();
      updateJ(
          jsonAdd(sdoc("id", "C105", "_version_", "205")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C106", "_version_", "206")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      rinfoFuture = ulog.applyBufferedUpdates();
      rinfo = rinfoFuture.get();
      assertEquals(2, rinfo.adds);

      assertJQ(
          req("q", "*:*", "sort", "_version_ asc", "fl", "id,_version_"),
          "/response/docs==["
              + "{'id':'C4','_version_':104}"
              + ",{'id':'C5','_version_':105}"
              + ",{'id':'C100','_version_':200}"
              + ",{'id':'C101','_version_':201}"
              + ",{'id':'C105','_version_':205}"
              + ",{'id':'C106','_version_':206}"
              + "]");

      assertJQ(req("qt", "/get", "getVersions", "6"), "=={'versions':[206,205,201,200,105,104]}");

      ulog.bufferUpdates();
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
      updateJ(
          jsonAdd(sdoc("id", "C301", "_version_", "998")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C302", "_version_", "999")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      assertTrue(ulog.dropBufferedUpdates());

      // make sure we can overwrite with a lower version
      // TODO: is this functionality needed?
      updateJ(
          jsonAdd(sdoc("id", "C301", "_version_", "301")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "C302", "_version_", "302")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      assertU(commit());

      assertJQ(req("qt", "/get", "getVersions", "2"), "=={'versions':[302,301]}");

      assertJQ(
          req("q", "*:*", "sort", "_version_ desc", "fl", "id,_version_", "rows", "2"),
          "/response/docs==["
              + "{'id':'C302','_version_':302}"
              + ",{'id':'C301','_version_':301}"
              + "]");

      updateJ(
          jsonAdd(sdoc("id", "C2", "_version_", "302")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      assertEquals(
          UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
    } finally {
      DirectUpdateHandler2.commitOnClose = true;
      UpdateLog.testing_logReplayHook = null;
      UpdateLog.testing_logReplayFinishHook = null;

      req().close();
    }
  }
Example #6
0
  @Test
  public void testBuffering() throws Exception {

    DirectUpdateHandler2.commitOnClose = false;
    final Semaphore logReplay = new Semaphore(0);
    final Semaphore logReplayFinish = new Semaphore(0);

    UpdateLog.testing_logReplayHook =
        new Runnable() {
          @Override
          public void run() {
            try {
              assertTrue(logReplay.tryAcquire(timeout, TimeUnit.SECONDS));
            } catch (Exception e) {
              throw new RuntimeException(e);
            }
          }
        };

    UpdateLog.testing_logReplayFinishHook =
        new Runnable() {
          @Override
          public void run() {
            logReplayFinish.release();
          }
        };

    SolrQueryRequest req = req();
    UpdateHandler uhandler = req.getCore().getUpdateHandler();
    UpdateLog ulog = uhandler.getUpdateLog();

    try {
      clearIndex();
      assertU(commit());

      assertEquals(UpdateLog.State.ACTIVE, ulog.getState());
      ulog.bufferUpdates();
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());
      Future<UpdateLog.RecoveryInfo> rinfoFuture = ulog.applyBufferedUpdates();
      assertTrue(rinfoFuture == null);
      assertEquals(UpdateLog.State.ACTIVE, ulog.getState());

      ulog.bufferUpdates();
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());

      // simulate updates from a leader
      updateJ(
          jsonAdd(sdoc("id", "B1", "_version_", "1010")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "B11", "_version_", "1015")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonDelQ("id:B1 id:B11 id:B2 id:B3"),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-1017"));
      updateJ(
          jsonAdd(sdoc("id", "B2", "_version_", "1020")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "B3", "_version_", "1030")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      deleteAndGetVersion("B1", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2010"));

      assertJQ(
          req("qt", "/get", "getVersions", "6"),
          "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");

      assertU(commit());

      assertJQ(
          req("qt", "/get", "getVersions", "6"),
          "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");

      // updates should be buffered, so we should not see any results yet.
      assertJQ(req("q", "*:*"), "/response/numFound==0");

      // real-time get should also not show anything (this could change in the future,
      // but it's currently used for validating version numbers too, so it would
      // be bad for updates to be visible if we're just buffering.
      assertJQ(req("qt", "/get", "id", "B3"), "=={'doc':null}");

      rinfoFuture = ulog.applyBufferedUpdates();
      assertTrue(rinfoFuture != null);

      assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());

      logReplay.release(1000);

      UpdateLog.RecoveryInfo rinfo = rinfoFuture.get();
      assertEquals(UpdateLog.State.ACTIVE, ulog.getState());

      assertJQ(
          req("qt", "/get", "getVersions", "6"),
          "=={'versions':[-2010,1030,1020,-1017,1015,1010]}");

      assertJQ(req("q", "*:*"), "/response/numFound==2");

      // move back to recovering
      ulog.bufferUpdates();
      assertEquals(UpdateLog.State.BUFFERING, ulog.getState());

      Long ver = getVer(req("qt", "/get", "id", "B3"));
      assertEquals(1030L, ver.longValue());

      // add a reordered doc that shouldn't overwrite one in the index
      updateJ(
          jsonAdd(sdoc("id", "B3", "_version_", "3")), params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      // reorder two buffered updates
      updateJ(
          jsonAdd(sdoc("id", "B4", "_version_", "1040")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      deleteAndGetVersion(
          "B4",
          params(
              DISTRIB_UPDATE_PARAM,
              FROM_LEADER,
              "_version_",
              "-940")); // this update should not take affect
      updateJ(
          jsonAdd(sdoc("id", "B6", "_version_", "1060")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "B5", "_version_", "1050")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));
      updateJ(
          jsonAdd(sdoc("id", "B8", "_version_", "1080")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      // test that delete by query is at least buffered along with everything else so it will delete
      // the
      // currently buffered id:8 (even if it doesn't currently support versioning)
      updateJ(
          "{\"delete\": { \"query\":\"id:B2 OR id:B8\" }}",
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-3000"));

      assertJQ(
          req("qt", "/get", "getVersions", "13"),
          "=={'versions':[-3000,1080,1050,1060,-940,1040,3,-2010,1030,1020,-1017,1015,1010]}" // the
          // "3"
          // appears because versions aren't checked while buffering
          );

      logReplay.drainPermits();
      rinfoFuture = ulog.applyBufferedUpdates();
      assertTrue(rinfoFuture != null);
      assertEquals(UpdateLog.State.APPLYING_BUFFERED, ulog.getState());

      // apply a single update
      logReplay.release(1);

      // now add another update
      updateJ(
          jsonAdd(sdoc("id", "B7", "_version_", "1070")),
          params(DISTRIB_UPDATE_PARAM, FROM_LEADER));

      // a reordered update that should be dropped
      deleteAndGetVersion("B5", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-950"));

      deleteAndGetVersion("B6", params(DISTRIB_UPDATE_PARAM, FROM_LEADER, "_version_", "-2060"));

      logReplay.release(1000);
      UpdateLog.RecoveryInfo recInfo = rinfoFuture.get();

      assertJQ(
          req("q", "*:*", "sort", "id asc", "fl", "id,_version_"),
          "/response/docs==["
              + "{'id':'B3','_version_':1030}"
              + ",{'id':'B4','_version_':1040}"
              + ",{'id':'B5','_version_':1050}"
              + ",{'id':'B7','_version_':1070}"
              + "]");

      assertEquals(1, recInfo.deleteByQuery);

      assertEquals(
          UpdateLog.State.ACTIVE, ulog.getState()); // leave each test method in a good state
    } finally {
      DirectUpdateHandler2.commitOnClose = true;
      UpdateLog.testing_logReplayHook = null;
      UpdateLog.testing_logReplayFinishHook = null;

      req().close();
    }
  }
 private void addLazy(SolrCore core, String... fieldValues) throws IOException {
   UpdateHandler updater = core.getUpdateHandler();
   AddUpdateCommand cmd = new AddUpdateCommand(makeReq(core));
   cmd.solrDoc = sdoc((Object[]) fieldValues);
   updater.addDoc(cmd);
 }