Example #1
0
 public void assertDeleteContent(Store store, DirectoryService service) throws IOException {
   store.deleteContent();
   assertThat(
       Arrays.toString(store.directory().listAll()),
       store.directory().listAll().length,
       equalTo(0));
   assertThat(store.stats().sizeInBytes(), equalTo(0l));
   for (Directory dir : service.build()) {
     assertThat(dir.listAll().length, equalTo(0));
   }
 }
    public void seek(TermEnum terms) throws IOException {
      original.seek(terms);

      docFreq = terms.docFreq();
      pointer = -1;

      if (docFreq > postingMaps.length) { // grow postingsMap
        PostingMap[] newMap = new PostingMap[docFreq];
        System.arraycopy(postingMaps, 0, newMap, 0, postingMaps.length);
        for (int i = postingMaps.length; i < docFreq; i++) {
          newMap[i] = new PostingMap();
        }
        postingMaps = newMap;
      }

      out.reset();

      int i = 0;
      while (original.next()) {
        PostingMap map = postingMaps[i++];
        map.newDoc = oldToNew[original.doc()]; // remap the newDoc id
        map.offset = out.getFilePointer(); // save pointer to buffer

        final int tf = original.freq(); // buffer tf & positions
        out.writeVInt(tf);
        int prevPosition = 0;
        for (int j = tf; j > 0; j--) { // delta encode positions
          int p = original.nextPosition();
          out.writeVInt(p - prevPosition);
          prevPosition = p;
        }
      }
      out.flush();
      docFreq = i; // allow for deletions

      Arrays.sort(postingMaps, 0, docFreq); // resort by mapped doc ids
      // HeapSorter.sort(postingMaps,docFreq); // TODO MC - due to the lack of space

      // NOTE: this might be substantially faster if RAMInputStream were public
      // and supported a reset() operation.
      in = tempDir.openInput(TEMP_FILE);
    }
 public static void checkIndex(ESLogger logger, Store store, ShardId shardId) {
   if (store.tryIncRef()) {
     logger.info("start check index");
     try {
       Directory dir = store.directory();
       if (!Lucene.indexExists(dir)) {
         return;
       }
       if (IndexWriter.isLocked(dir)) {
         ESTestCase.checkIndexFailed = true;
         throw new IllegalStateException("IndexWriter is still open on shard " + shardId);
       }
       try (CheckIndex checkIndex = new CheckIndex(dir)) {
         BytesStreamOutput os = new BytesStreamOutput();
         PrintStream out = new PrintStream(os, false, StandardCharsets.UTF_8.name());
         checkIndex.setInfoStream(out);
         out.flush();
         CheckIndex.Status status = checkIndex.checkIndex();
         if (!status.clean) {
           ESTestCase.checkIndexFailed = true;
           logger.warn(
               "check index [failure] index files={}\n{}",
               Arrays.toString(dir.listAll()),
               new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
           throw new IOException("index check failure");
         } else {
           if (logger.isDebugEnabled()) {
             logger.debug(
                 "check index [success]\n{}",
                 new String(os.bytes().toBytes(), StandardCharsets.UTF_8));
           }
         }
       }
     } catch (Exception e) {
       logger.warn("failed to check index", e);
     } finally {
       logger.info("end check index");
       store.decRef();
     }
   }
 }
  @Inject
  @SuppressForbidden(reason = "System.out.*")
  public NodeEnvironment(Settings settings, Environment environment) throws IOException {
    super(settings);

    this.addNodeId = settings.getAsBoolean(ADD_NODE_ID_TO_CUSTOM_PATH, true);
    this.customPathsEnabled = settings.getAsBoolean(SETTING_CUSTOM_DATA_PATH_ENABLED, false);

    if (!DiscoveryNode.nodeRequiresLocalStorage(settings)) {
      nodePaths = null;
      locks = null;
      localNodeId = -1;
      return;
    }

    final NodePath[] nodePaths = new NodePath[environment.dataWithClusterFiles().length];
    final Lock[] locks = new Lock[nodePaths.length];

    int localNodeId = -1;
    IOException lastException = null;
    int maxLocalStorageNodes = settings.getAsInt("node.max_local_storage_nodes", 50);
    for (int possibleLockId = 0; possibleLockId < maxLocalStorageNodes; possibleLockId++) {
      for (int dirIndex = 0; dirIndex < environment.dataWithClusterFiles().length; dirIndex++) {
        Path dir =
            environment
                .dataWithClusterFiles()[dirIndex]
                .resolve(NODES_FOLDER)
                .resolve(Integer.toString(possibleLockId));
        Files.createDirectories(dir);

        try (Directory luceneDir = FSDirectory.open(dir, NativeFSLockFactory.INSTANCE)) {
          logger.trace("obtaining node lock on {} ...", dir.toAbsolutePath());
          try {
            locks[dirIndex] = Lucene.acquireLock(luceneDir, NODE_LOCK_FILENAME, 0);
            nodePaths[dirIndex] = new NodePath(dir, environment);
            localNodeId = possibleLockId;
          } catch (LockObtainFailedException ex) {
            logger.trace("failed to obtain node lock on {}", dir.toAbsolutePath());
            // release all the ones that were obtained up until now
            releaseAndNullLocks(locks);
            break;
          }

        } catch (IOException e) {
          logger.trace("failed to obtain node lock on {}", e, dir.toAbsolutePath());
          lastException = new IOException("failed to obtain lock on " + dir.toAbsolutePath(), e);
          // release all the ones that were obtained up until now
          releaseAndNullLocks(locks);
          break;
        }
      }
      if (locks[0] != null) {
        // we found a lock, break
        break;
      }
    }

    if (locks[0] == null) {
      throw new IllegalStateException(
          "Failed to obtain node lock, is the following location writable?: "
              + Arrays.toString(environment.dataWithClusterFiles()),
          lastException);
    }

    this.localNodeId = localNodeId;
    this.locks = locks;
    this.nodePaths = nodePaths;

    if (logger.isDebugEnabled()) {
      logger.debug("using node location [{}], local_node_id [{}]", nodePaths, localNodeId);
    }

    maybeLogPathDetails();

    if (settings.getAsBoolean(SETTING_ENABLE_LUCENE_SEGMENT_INFOS_TRACE, false)) {
      SegmentInfos.setInfoStream(System.out);
    }
  }
  // private static int[] oldToNew(IndexReader reader, Searcher searcher) throws IOException {
  private static DocScore[] newToOld(IndexReader reader, Searcher searcher) throws IOException {
    int readerMax = reader.maxDoc();
    DocScore[] newToOld = new DocScore[readerMax];

    // use site, an indexed, un-tokenized field to get boost
    // byte[] boosts = reader.norms("site"); TODO MC
    /* TODO MC */
    Document docMeta;
    Pattern includes = Pattern.compile("\\|");
    String value = NutchConfiguration.create().get(INCLUDE_EXTENSIONS_KEY, "");
    String includeExtensions[] = includes.split(value);
    Hashtable<String, Boolean> validExtensions = new Hashtable<String, Boolean>();
    for (int i = 0; i < includeExtensions.length; i++) {
      validExtensions.put(includeExtensions[i], true);
      System.out.println("extension boosted " + includeExtensions[i]);
    }
    /* TODO MC */

    for (int oldDoc = 0; oldDoc < readerMax; oldDoc++) {
      float score;
      if (reader.isDeleted(oldDoc)) {
        // score = 0.0f;
        score = -1f; // TODO MC
      } else {
        // score = Similarity.decodeNorm(boosts[oldDoc]); TODO MC
        /* TODO MC */
        docMeta = searcher.doc(oldDoc);
        if (validExtensions.get(docMeta.get("subType"))
            == null) { // searched extensions will have higher scores
          score = -0.5f;
        } else {
          score = Integer.parseInt(docMeta.get("inlinks"));
          /*
          if (score==0) {
          	score=0.001f; // TODO MC - to not erase
          }
          */
        }
        /* TODO MC */
        // System.out.println("Score for old document "+oldDoc+" is "+score+" and type
        // "+docMeta.get("subType")); // TODO MC debug remove
      }
      DocScore docScore = new DocScore();
      docScore.doc = oldDoc;
      docScore.score = score;
      newToOld[oldDoc] = docScore;
    }

    System.out.println("Sorting " + newToOld.length + " documents.");
    Arrays.sort(newToOld);
    // HeapSorter.sort(newToOld); // TODO MC - due to the lack of space

    /* TODO MC
    int[] oldToNew = new int[readerMax];
    for (int newDoc = 0; newDoc < readerMax; newDoc++) {
      DocScore docScore = newToOld[newDoc];
      //oldToNew[docScore.oldDoc] = docScore.score > 0.0f ? newDoc : -1; // TODO MC
      oldToNew[docScore.oldDoc] = newDoc; // TODO MC
    }
    */

    /* TODO MC *
    for (int newDoc = 0; newDoc < readerMax; newDoc++) {
    	DocScore docScore = newToOld[newDoc];
    	System.out.println("Score for new document "+newDoc+" is "+docScore.score); // TODO MC debug remove
    }
    * TODO MC */

    // return oldToNew; TODO MC
    return newToOld; // TODO MC
  }