Esempio n. 1
0
 private void recoverFromTranslog(TranslogRecoveryPerformer handler) throws IOException {
   Translog.TranslogGeneration translogGeneration = translog.getGeneration();
   final int opsRecovered;
   try {
     Translog.Snapshot snapshot = translog.newSnapshot();
     opsRecovered = handler.recoveryFromSnapshot(this, snapshot);
   } catch (Throwable e) {
     throw new EngineException(shardId, "failed to recover from translog", e);
   }
   // flush if we recovered something or if we have references to older translogs
   // note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0
   // length.
   assert allowCommits.get() == false : "commits are allowed but shouldn't";
   allowCommits.set(true); // we are good - now we can commit
   if (opsRecovered > 0) {
     logger.trace(
         "flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]",
         opsRecovered,
         translogGeneration == null ? null : translogGeneration.translogFileGeneration,
         translog.currentFileGeneration());
     flush(true, true);
   } else if (translog.isCurrent(translogGeneration) == false) {
     commitIndexWriter(
         indexWriter,
         translog,
         lastCommittedSegmentInfos.getUserData().get(Engine.SYNC_COMMIT_ID));
   }
 }
Esempio n. 2
0
  final boolean tryRenewSyncCommit() {
    boolean renewed = false;
    try (ReleasableLock lock = writeLock.acquire()) {
      ensureOpen();
      ensureCanFlush();
      String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID);
      if (syncId != null
          && translog.totalOperations() == 0
          && indexWriter.hasUncommittedChanges()) {
        logger.trace("start renewing sync commit [{}]", syncId);
        commitIndexWriter(indexWriter, translog, syncId);
        logger.debug("successfully sync committed. sync id [{}].", syncId);
        lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
        renewed = true;
      }
    } catch (IOException ex) {
      maybeFailEngine("renew sync commit", ex);
      throw new EngineException(shardId, "failed to renew sync commit", ex);
    }
    if (renewed) { // refresh outside of the write lock
      refresh("renew sync commit");
    }

    return renewed;
  }
Esempio n. 3
0
 ReaderCommit(SegmentInfos infos, Directory dir) throws IOException {
   segmentsFileName = infos.getCurrentSegmentFileName();
   this.dir = dir;
   userData = infos.getUserData();
   files = Collections.unmodifiableCollection(infos.files(dir, true));
   version = infos.getVersion();
   generation = infos.getGeneration();
   segmentCount = infos.size();
 }
Esempio n. 4
0
 public CommitStats(SegmentInfos segmentInfos) {
   // clone the map to protect against concurrent changes
   userData =
       MapBuilder.<String, String>newMapBuilder()
           .putAll(segmentInfos.getUserData())
           .immutableMap();
   // lucene calls the current generation, last generation.
   generation = segmentInfos.getLastGeneration();
   if (segmentInfos.getId() != null) { // id is only written starting with Lucene 5.0
     id = Base64.encodeBytes(segmentInfos.getId());
   }
   numDocs = Lucene.getNumDocs(segmentInfos);
 }
Esempio n. 5
0
  /*
   * Test "by time expiration" deletion policy:
   */
  public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {

    final double SECONDS = 2.0;

    Directory dir = newDirectory();
    if (dir instanceof MockDirectoryWrapper) {
      // test manually deletes files
      ((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
    }
    IndexWriterConfig conf =
        newIndexWriterConfig(new MockAnalyzer(random()))
            .setIndexDeletionPolicy(new ExpirationTimeDeletionPolicy(dir, SECONDS));
    MergePolicy mp = conf.getMergePolicy();
    mp.setNoCFSRatio(1.0);
    IndexWriter writer = new IndexWriter(dir, conf);
    ExpirationTimeDeletionPolicy policy =
        (ExpirationTimeDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
    Map<String, String> commitData = new HashMap<>();
    commitData.put("commitTime", String.valueOf(System.currentTimeMillis()));
    writer.setCommitData(commitData);
    writer.commit();
    writer.close();

    long lastDeleteTime = 0;
    final int targetNumDelete = TestUtil.nextInt(random(), 1, 5);
    while (policy.numDelete < targetNumDelete) {
      // Record last time when writer performed deletes of
      // past commits
      lastDeleteTime = System.currentTimeMillis();
      conf =
          newIndexWriterConfig(new MockAnalyzer(random()))
              .setOpenMode(OpenMode.APPEND)
              .setIndexDeletionPolicy(policy);
      mp = conf.getMergePolicy();
      mp.setNoCFSRatio(1.0);
      writer = new IndexWriter(dir, conf);
      policy = (ExpirationTimeDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
      for (int j = 0; j < 17; j++) {
        addDoc(writer);
      }
      commitData = new HashMap<>();
      commitData.put("commitTime", String.valueOf(System.currentTimeMillis()));
      writer.setCommitData(commitData);
      writer.commit();
      writer.close();

      Thread.sleep((int) (1000.0 * (SECONDS / 5.0)));
    }

    // Then simplistic check: just verify that the
    // segments_N's that still exist are in fact within SECONDS
    // seconds of the last one's mod time, and, that I can
    // open a reader on each:
    long gen = SegmentInfos.getLastCommitGeneration(dir);

    String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);
    boolean oneSecondResolution = true;

    while (gen > 0) {
      try {
        IndexReader reader = DirectoryReader.open(dir);
        reader.close();
        fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen);

        // if we are on a filesystem that seems to have only
        // 1 second resolution, allow +1 second in commit
        // age tolerance:
        SegmentInfos sis = SegmentInfos.readCommit(dir, fileName);
        long modTime = Long.parseLong(sis.getUserData().get("commitTime"));
        oneSecondResolution &= (modTime % 1000) == 0;
        final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0 : 0.0)) * 1000);

        assertTrue(
            "commit point was older than "
                + SECONDS
                + " seconds ("
                + (lastDeleteTime - modTime)
                + " msec) but did not get deleted ",
            lastDeleteTime - modTime <= leeway);
      } catch (IOException e) {
        // OK
        break;
      }

      dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
      gen--;
    }

    dir.close();
  }
Esempio n. 6
0
 @Override
 public Map<String, String> getCommitUserData() {
   ensureOpen();
   return segmentInfos.getUserData();
 }
Esempio n. 7
0
 /**
  * Returns userData from latest segments file
  *
  * @throws CorruptIndexException if the index is corrupt
  * @throws IOException if there is a low-level IO error
  */
 public static Map<String, String> readCurrentUserData(Directory directory)
     throws CorruptIndexException, IOException {
   SegmentInfos sis = new SegmentInfos();
   sis.read(directory);
   return sis.getUserData();
 }
 private long getEpoch(Directory taxoDir) throws IOException {
   SegmentInfos infos = new SegmentInfos();
   infos.read(taxoDir);
   return Long.parseLong(infos.getUserData().get(DirectoryTaxonomyWriter.INDEX_EPOCH));
 }