예제 #1
0
  /**
   * Write a user or item index to the file.
   *
   * @param map The index to write.
   */
  private void writeIndex(Long2ObjectMap<IntList> map) throws IOException {
    LongSortedSet keys = LongUtils.packedSet(map.keySet());
    BinaryIndexTableWriter tableWriter =
        BinaryIndexTableWriter.create(format, channel, keys.size());

    SortComparator indexComparator = new SortComparator();

    LongIterator iter = keys.iterator();
    while (iter.hasNext()) {
      final long key = iter.nextLong();
      int[] indexes = map.get(key).toIntArray();
      if (needsSorting) {
        IntArrays.quickSort(indexes, indexComparator);
      }

      if (translationMap != null) {
        for (int i = 0; i < indexes.length; i++) {
          indexes[i] = translationMap[indexes[i]];
        }
      }

      logger.debug("writing {} indices for id {}", key, indexes.length);
      tableWriter.writeEntry(key, indexes);
    }
  }
예제 #2
0
  public static Block intersect(Type type, Block leftArray, Block rightArray) {
    int leftPositionCount = leftArray.getPositionCount();
    int rightPositionCount = rightArray.getPositionCount();

    if (leftPositionCount == 0) {
      return leftArray;
    }
    if (rightPositionCount == 0) {
      return rightArray;
    }

    int[] leftPositions = new int[leftPositionCount];
    int[] rightPositions = new int[rightPositionCount];

    for (int i = 0; i < leftPositionCount; i++) {
      leftPositions[i] = i;
    }
    for (int i = 0; i < rightPositionCount; i++) {
      rightPositions[i] = i;
    }
    IntArrays.quickSort(leftPositions, IntBlockCompare(type, leftArray));
    IntArrays.quickSort(rightPositions, IntBlockCompare(type, rightArray));

    int entrySize;
    if (leftPositionCount < rightPositionCount) {
      entrySize = (int) Math.ceil(leftArray.getSizeInBytes() / (double) leftPositionCount);
    } else {
      entrySize = (int) Math.ceil(rightArray.getSizeInBytes() / (double) rightPositionCount);
    }
    BlockBuilder resultBlockBuilder =
        type.createBlockBuilder(
            new BlockBuilderStatus(),
            Math.min(leftArray.getPositionCount(), rightArray.getPositionCount()),
            entrySize);

    int leftCurrentPosition = 0;
    int rightCurrentPosition = 0;
    int leftBasePosition;
    int rightBasePosition;

    while (leftCurrentPosition < leftPositionCount && rightCurrentPosition < rightPositionCount) {
      leftBasePosition = leftCurrentPosition;
      rightBasePosition = rightCurrentPosition;
      int compareValue =
          type.compareTo(
              leftArray,
              leftPositions[leftCurrentPosition],
              rightArray,
              rightPositions[rightCurrentPosition]);
      if (compareValue > 0) {
        rightCurrentPosition++;
      } else if (compareValue < 0) {
        leftCurrentPosition++;
      } else {
        type.appendTo(leftArray, leftPositions[leftCurrentPosition], resultBlockBuilder);
        leftCurrentPosition++;
        rightCurrentPosition++;
        while (leftCurrentPosition < leftPositionCount
            && type.equalTo(
                leftArray,
                leftPositions[leftBasePosition],
                leftArray,
                leftPositions[leftCurrentPosition])) {
          leftCurrentPosition++;
        }
        while (rightCurrentPosition < rightPositionCount
            && type.equalTo(
                rightArray,
                rightPositions[rightBasePosition],
                rightArray,
                rightPositions[rightCurrentPosition])) {
          rightCurrentPosition++;
        }
      }
    }

    return resultBlockBuilder.build();
  }