예제 #1
0
  public static void addField(
      String indexDir, String newFieldName, FlamdexReader docReader, final String[] values)
      throws IOException {
    final int[] indices = new int[docReader.getNumDocs()];
    for (int i = 0; i < indices.length; i++) {
      indices[i] = i;
    }
    log.debug("sorting");
    Quicksortables.sort(
        new Quicksortable() {
          @Override
          public void swap(int i, int j) {
            Quicksortables.swap(indices, i, j);
          }

          @Override
          public int compare(int i, int j) {
            // Sorting logic: Primarily by value (String), secondarily by document ID (indices[i])
            final String left = values[indices[i]];
            final String right = values[indices[j]];
            if (left.compareTo(right) < 0) {
              return -1;
            } else if (left.compareTo(right) > 0) {
              return 1;
            } else { // left == right
              if (indices[i] < indices[j]) {
                return -1;
              } else if (indices[i] > indices[j]) {
                return 1;
              } else {
                return 0; // Both value & doc ID match
              }
            }
          }
        },
        values.length);

    log.debug("writing field " + newFieldName);
    final SimpleFlamdexWriter w = new SimpleFlamdexWriter(indexDir, docReader.getNumDocs(), false);
    final StringFieldWriter sfw = w.getStringFieldWriter(newFieldName, true);
    final IntArrayList docList = new IntArrayList();
    docList.add(indices[0]);
    for (int i = 1; i < indices.length; ++i) {
      final String prev = values[indices[i - 1]];
      final String cur = values[indices[i]];
      if (cur.compareTo(prev) != 0) {
        sfw.nextTerm(prev);
        for (int j = 0; j < docList.size(); ++j) {
          sfw.nextDoc(docList.getInt(j));
        }
        docList.clear();
      }
      docList.add(indices[i]);
    }
    if (docList.size() > 0) {
      sfw.nextTerm(values[indices[indices.length - 1]]);
      for (int j = 0; j < docList.size(); ++j) {
        sfw.nextDoc(docList.getInt(j));
      }
    }

    sfw.close();
    w.close();
  }
예제 #2
0
  public static void addField(String dir, String fieldName, FlamdexReader r, final long[] cache)
      throws IOException {
    final File tempFile =
        new File(dir, "temp-" + fieldName + "-" + UUID.randomUUID() + ".intarray.bin");
    try {
      final MMapBuffer buffer =
          new MMapBuffer(
              tempFile,
              0,
              4 * cache.length,
              FileChannel.MapMode.READ_WRITE,
              ByteOrder.nativeOrder());
      try {
        final IntArray indices = buffer.memory().intArray(0, cache.length);
        for (int i = 0; i < cache.length; ++i) {
          indices.set(i, i);
        }
        log.debug("sorting");
        Quicksortables.sort(
            new Quicksortable() {
              @Override
              public void swap(int i, int j) {
                final int t = indices.get(i);
                indices.set(i, indices.get(j));
                indices.set(j, t);
              }

              @Override
              public int compare(int i, int j) {
                final long ii = cache[indices.get(i)];
                final long ij = cache[indices.get(j)];
                return ii < ij
                    ? -1
                    : ii > ij
                        ? 1
                        : indices.get(i) < indices.get(j)
                            ? -1
                            : indices.get(i) > indices.get(j) ? 1 : 0;
              }
            },
            cache.length);

        log.debug("writing field " + fieldName);
        final SimpleFlamdexWriter w = new SimpleFlamdexWriter(dir, r.getNumDocs(), false);
        final IntFieldWriter ifw = w.getIntFieldWriter(fieldName, true);
        long prev = 0;
        boolean prevInitialized = false;
        for (int i = 0; i < cache.length; ++i) {
          final long cur = cache[indices.get(i)];
          if (!prevInitialized || cur != prev) {
            ifw.nextTerm(cur);
            prev = cur;
            prevInitialized = true;
          }
          ifw.nextDoc(indices.get(i));
        }

        ifw.close();
        w.close();
      } finally {
        try {
          buffer.close();
        } catch (IOException e) {
          log.error("error closing MMapBuffer", e);
        }
      }
    } finally {
      if (!tempFile.delete()) {
        log.warn("unable to delete temp file " + tempFile);
      }
    }
  }