コード例 #1
0
  public LexiconPostingChecker(String id, File dir, int revision, int findDocNo)
      throws IOException {
    this.id = id;
    this.dir = dir;
    this.rev = revision;
    this.findDocNo = findDocNo;

    System.out.println("Check dir = " + dir.getAbsolutePath());

    indexInput =
        new BufferedFileInput(
            IndexFileNames.getRevisionDir(dir, revision),
            IndexFileNames.getSearchLexiconFileName(id));
  }
コード例 #2
0
ファイル: BitSetTest.java プロジェクト: Zunonia/fastcatsearch
 public void test2() throws IOException {
   File segmentDir = new File("/Users/swsong/search/fastcat_basic/collection/news/data/1");
   int revision = 45;
   System.out.println(segmentDir);
   BitSet deleteSet =
       new BitSet(
           IndexFileNames.getRevisionDir(segmentDir, revision), IndexFileNames.docDeleteSet);
   for (int i = 0; i < 100; i++) {
     System.out.println(i);
     if (deleteSet.isSet(i)) {
       System.out.println(">> " + i);
     }
   }
 }
コード例 #3
0
  public void list(PrintStream output) throws IOException {
    int indexSize = indexInput.readInt();
    output.println("Memory indexsize = " + indexSize);

    StringBuilder findWord = new StringBuilder();

    for (int k = 0; k < indexSize; k++) {
      String string = new String(indexInput.readUString());
      long inputOffset = indexInput.readLong();

      // output.println("word="+string+" ,"+ inputOffset);
      BufferedFileInput postingInput =
          new BufferedFileInput(
              IndexFileNames.getRevisionDir(dir, rev), IndexFileNames.getSearchPostingFileName(id));

      BufferedFileInput clone = postingInput.clone();

      IndexFieldOption indexFieldOption = new IndexFieldOption(clone.readInt());

      // output.println("offset:" + inputOffset);

      clone.seek(inputOffset);

      int len = 0, postingCount = 0, lastDocNo = 0;

      // List<Integer> postingList = new ArrayList<Integer>();

      try {
        len = clone.readVInt();
        postingCount = clone.readInt();
        lastDocNo = clone.readInt();

        int postingRemain = postingCount;

        int prevId = -1;

        boolean isStorePosition = indexFieldOption.isStorePosition();

        for (int i = 0; postingRemain > 0; i++) {
          int docId = -1;
          int readed = clone.readVInt();
          if (prevId >= 0) {
            docId = readed + prevId + 1;
          } else {
            docId = readed;
          }

          if (findDocNo == docId) {
            findWord.append("\"").append(string).append("\", ");
          }

          int tf = clone.readVInt();
          if (tf > 0 && isStorePosition) {

            int prevPosition = -1;
            for (int j = 0; j < tf; j++) {
              int position = 0;
              if (prevPosition >= 0) {
                position = clone.readVInt() + prevPosition + 1;
              } else {
                position = clone.readVInt();
              }
              prevPosition = position;
            }
          }
          // postingList.add(docId);

          postingRemain--;
          prevId = docId;
        }

      } catch (IOException ex) {
        ex.printStackTrace();
      }

      postingInput.close();

      // output.println("poosting len : " + len + " postingCount : "
      //		+ postingCount + " / lastDocNo : " + lastDocNo);
      // output.println("postingList:"+postingList);
      if (findDocNo != -1) {
        // output.println("findDocNo:"+findDocNo+"
        // "+(postingList.contains(findDocNo)?"CONTAINS":"NOT"));
      }
    }

    output.println("docNo [" + findDocNo + "] has word " + findWord);
  }
コード例 #4
0
  @Override
  protected boolean done(RevisionInfo revisionInfo, IndexStatus indexStatus)
      throws IRException, IndexingStopException {

    int insertCount = revisionInfo.getInsertCount();
    int deleteCount = revisionInfo.getDeleteCount();
    FilePaths indexFilePaths = collectionContext.indexFilePaths();
    try {
      if ((insertCount > 0 || deleteCount > 0) && !stopRequested) {
        if (insertCount > 0) {
          revisionInfo.setRefWithRevision();
        } else {
          // 추가문서가 없고 삭제문서만 존재할 경우
          logger.debug("추가문서없이 삭제문서만 존재합니다.!!");
          if (workingSegmentInfo != null && !workingSegmentInfo.equals(workingSegmentInfo)) {
            // 기존색인문서수가 limit을 넘으면서 삭제문서만 색인될 경우 세그먼트가 바뀌는 현상이 나타날수 있다.
            // 색인후 문서가 0건이고 delete문서가 존재하면 이전 세그먼트의 다음 리비전으로 변경해주는 작업필요.
            // 세그먼트가 다르면, 즉 증가했으면 다시 원래의 세그먼트로 돌리고, rev를 증가시킨다.
            File segmentDir = indexFilePaths.file(workingSegmentInfo.getId());
            // FileUtils.deleteDirectory(segmentDir);
            FileUtils.forceDelete(segmentDir);

            logger.debug("# 추가문서가 없으므로, segment를 삭제합니다. {}", segmentDir.getAbsolutePath());
            workingSegmentInfo = workingSegmentInfo.copy();
            int revision = workingSegmentInfo.getRevision();
            workingSegmentInfo.getRevisionInfo().setInsertCount(0);
            workingSegmentInfo.getRevisionInfo().setUpdateCount(0);
            workingSegmentInfo.getRevisionInfo().setDeleteCount(deleteCount);

            // 이전 리비전의 delete.set.#을 현 리비전으로 복사해온다.
            // 원래 primarykeyindexeswriter에서 append일 경우 복사를 하나, 여기서는 추가문서가 0이므로
            String segmentId = workingSegmentInfo.getId();
            segmentDir = indexFilePaths.file(workingSegmentInfo.getId());
            File revisionDir = IndexFileNames.getRevisionDir(segmentDir, revision);
            File prevRevisionDir = IndexFileNames.getRevisionDir(segmentDir, revision - 1);
            String deleteFileName =
                IndexFileNames.getSuffixFileName(IndexFileNames.docDeleteSet, segmentId);
            FileUtils.copyFile(
                new File(prevRevisionDir, deleteFileName), new File(revisionDir, deleteFileName));
          }
          /*
           * else 세그먼트가 증가하지 않고 리비전이 증가한 경우.
           */
        }

        File segmentDir = indexFilePaths.file(workingSegmentInfo.getId());
        collectionHandler.updateCollection(
            collectionContext, workingSegmentInfo, segmentDir, deleteIdSet);

        // status.xml 업데이트
        collectionContext.updateCollectionStatus(
            IndexingType.ADD, revisionInfo, startTime, System.currentTimeMillis());
        collectionContext.indexStatus().setAddIndexStatus(indexStatus);

        return true;
      } else {
        // 추가,삭제 문서 모두 없을때.
        // 리비전 디렉토리 삭제.
        File segmentDir = indexFilePaths.file(workingSegmentInfo.getId());
        File revisionDir = IndexFileNames.getRevisionDir(segmentDir, revisionInfo.getId());
        if (workingSegmentInfo != null && !workingSegmentInfo.equals(workingSegmentInfo)) {
          // 세그먼트 증가시 segment디렉토리 삭제.
          FileUtils.deleteDirectory(segmentDir);
          logger.info("delete segment dir ={}", segmentDir.getAbsolutePath());
        } else {
          // 리비전 증가시 revision디렉토리 삭제.
          FileUtils.deleteDirectory(revisionDir);
          logger.info("delete revision dir ={}", revisionDir.getAbsolutePath());
        }

        if (!stopRequested) {
          logger.info(
              "[{}] Indexing Canceled due to no documents.", collectionContext.collectionId());
          throw new IndexingStopException(
              collectionContext.collectionId() + " Indexing Canceled due to no documents.");
        } else {
          logger.info(
              "[{}] Indexing Canceled due to Stop Requested!", collectionContext.collectionId());
          throw new IndexingStopException(
              collectionContext.collectionId() + " Indexing Canceled due to Stop Requested");
        }
      }

    } catch (IOException e) {
      throw new IRException(e);
    }
  }