public TermVectorsWriter(Directory directory, String segment, FieldInfos fieldInfos)
      throws IOException {
    // Open files for TermVector storage
    tvx = directory.createOutput(segment + TVX_EXTENSION);
    tvx.writeInt(FORMAT_VERSION);
    tvd = directory.createOutput(segment + TVD_EXTENSION);
    tvd.writeInt(FORMAT_VERSION);
    tvf = directory.createOutput(segment + TVF_EXTENSION);
    tvf.writeInt(FORMAT_VERSION);

    this.fieldInfos = fieldInfos;
    fields = new Vector(fieldInfos.size());
    terms = new Vector();
  }
 private void testOn(Directory dir, int writeSize, int readSize, Cache cache) throws IOException {
   if (cache != null)
     cache
         .clear(); // needed to make sure no chunks are left over in case of Infinispan
                   // implementation
   final String filename = "chunkTest";
   IndexOutput indexOutput = dir.createOutput(filename);
   byte[] toWrite = fillBytes(writeSize);
   indexOutput.writeBytes(toWrite, writeSize);
   indexOutput.close();
   if (cache != null) {
     AssertJUnit.assertEquals(
         writeSize,
         DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME, filename), cache));
   }
   AssertJUnit.assertEquals(writeSize, indexOutput.length());
   byte[] results = new byte[readSize];
   IndexInput openInput = dir.openInput(filename);
   try {
     openInput.readBytes(results, 0, readSize);
     for (int i = 0; i < writeSize && i < readSize; i++) {
       AssertJUnit.assertEquals(results[i], toWrite[i]);
     }
     if (readSize > writeSize)
       AssertJUnit.fail("should have thrown an IOException for reading past EOF");
   } catch (IOException ioe) {
     if (readSize <= writeSize)
       AssertJUnit.fail("should not have thrown an IOException" + ioe.getMessage());
   }
 }
Esempio n. 3
0
 public IndexOutput createOutput(String name, IOContext context, boolean raw)
     throws IOException {
   Directory directory;
   if (isChecksum(name)) {
     directory = distributor.primary();
   } else {
     directory = distributor.any();
   }
   IndexOutput out = directory.createOutput(name, context);
   synchronized (mutex) {
     StoreFileMetaData metaData = new StoreFileMetaData(name, -1, null, directory);
     filesMetadata = MapBuilder.newMapBuilder(filesMetadata).put(name, metaData).immutableMap();
     files = filesMetadata.keySet().toArray(new String[filesMetadata.size()]);
     boolean computeChecksum = !raw;
     if (computeChecksum) {
       // don't compute checksum for segment based files
       if ("segments.gen".equals(name) || name.startsWith("segments")) {
         computeChecksum = false;
       }
     }
     if (computeChecksum) {
       out = new BufferedChecksumIndexOutput(out, new Adler32());
     }
     return new StoreIndexOutput(metaData, out, name);
   }
 }
  // Make sure we don't somehow use more than 1 descriptor
  // when reading a CFS with many subs:
  public void testManySubFiles() throws IOException {

    final Directory d = newFSDirectory(_TestUtil.getTempDir("CFSManySubFiles"));
    final int FILE_COUNT = 10000;

    for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
      IndexOutput out = d.createOutput("file." + fileIdx);
      out.writeByte((byte) fileIdx);
      out.close();
    }

    final CompoundFileWriter cfw = new CompoundFileWriter(d, "c.cfs");
    for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
      cfw.addFile("file." + fileIdx);
    }
    cfw.close();

    final IndexInput[] ins = new IndexInput[FILE_COUNT];
    final CompoundFileReader cfr = new CompoundFileReader(d, "c.cfs");
    for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
      ins[fileIdx] = cfr.openInput("file." + fileIdx);
    }

    for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
      assertEquals((byte) fileIdx, ins[fileIdx].readByte());
    }

    for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
      ins[fileIdx].close();
    }
    cfr.close();
    d.close();
  }
 public void multipleFlushTest() throws IOException {
   final String filename = "longFile.writtenInMultipleFlushes";
   final int bufferSize = 300;
   Cache cache = cacheManager.getCache();
   cache.clear();
   Directory dir =
       DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
           .chunkSize(13)
           .create();
   byte[] manyBytes = fillBytes(bufferSize);
   IndexOutput indexOutput = dir.createOutput(filename);
   for (int i = 0; i < 10; i++) {
     indexOutput.writeBytes(manyBytes, bufferSize);
     indexOutput.flush();
   }
   indexOutput.close();
   IndexInput input = dir.openInput(filename);
   final int finalSize = (10 * bufferSize);
   AssertJUnit.assertEquals(finalSize, input.length());
   final byte[] resultingBuffer = new byte[finalSize];
   input.readBytes(resultingBuffer, 0, finalSize);
   int index = 0;
   for (int i = 0; i < 10; i++) {
     for (int j = 0; j < bufferSize; j++)
       AssertJUnit.assertEquals(resultingBuffer[index++], manyBytes[j]);
   }
 }
  private void demo_FSIndexInputBug(Directory fsdir, String file) throws IOException {
    // Setup the test file - we need more than 1024 bytes
    IndexOutput os = fsdir.createOutput(file);
    for (int i = 0; i < 2000; i++) {
      os.writeByte((byte) i);
    }
    os.close();

    IndexInput in = fsdir.openInput(file);

    // This read primes the buffer in IndexInput
    in.readByte();

    // Close the file
    in.close();

    // ERROR: this call should fail, but succeeds because the buffer
    // is still filled
    in.readByte();

    // ERROR: this call should fail, but succeeds for some reason as well
    in.seek(1099);

    try {
      // OK: this call correctly fails. We are now past the 1024 internal
      // buffer, so an actual IO is attempted, which fails
      in.readByte();
      fail("expected readByte() to throw exception");
    } catch (IOException e) {
      // expected exception
    }
  }
  public void testWriteChunks() throws Exception {
    final int BUFFER_SIZE = 64;

    Cache cache = cacheManager.getCache();
    Directory dir =
        DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
            .chunkSize(BUFFER_SIZE)
            .create();

    IndexOutput io = dir.createOutput("MyNewFile.txt");

    io.writeByte((byte) 66);
    io.writeByte((byte) 69);

    io.flush();
    io.close();

    assert dir.fileExists("MyNewFile.txt");
    assert null != cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));

    // test contents by reading:
    byte[] buf = new byte[9];
    IndexInput ii = dir.openInput("MyNewFile.txt");
    ii.readBytes(buf, 0, (int) ii.length());
    ii.close();

    assert new String(new byte[] {66, 69}).equals(new String(buf).trim());

    String testText =
        "This is some rubbish again that will span more than one chunk - one hopes.  Who knows, maybe even three or four chunks.";
    io = dir.createOutput("MyNewFile.txt");
    io.seek(0);
    io.writeBytes(testText.getBytes(), 0, testText.length());
    io.close();
    // now compare.
    byte[] chunk1 =
        (byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 0, BUFFER_SIZE));
    byte[] chunk2 =
        (byte[]) cache.get(new ChunkCacheKey(INDEXNAME, "MyNewFile.txt", 1, BUFFER_SIZE));
    assert null != chunk1;
    assert null != chunk2;

    assert testText.equals(new String(chunk1) + new String(chunk2).trim());

    dir.close();
    DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
  }
 /** Creates a file of the specified size with random data. */
 private void createRandomFile(Directory dir, String name, int size) throws IOException {
   IndexOutput os = dir.createOutput(name);
   for (int i = 0; i < size; i++) {
     byte b = (byte) (Math.random() * 256);
     os.writeByte(b);
   }
   os.close();
 }
 /**
  * Creates a file of the specified size with sequential data. The first byte is written as the
  * start byte provided. All subsequent bytes are computed as start + offset where offset is the
  * number of the byte.
  */
 private void createSequenceFile(Directory dir, String name, byte start, int size)
     throws IOException {
   IndexOutput os = dir.createOutput(name);
   for (int i = 0; i < size; i++) {
     os.writeByte(start);
     start++;
   }
   os.close();
 }
 /**
  * It creates a file with fixed size using a RepeatableLongByteSequence object to generate a
  * repeatable content
  *
  * @param dir The Directory containing the file to create
  * @param fileName The file name to create
  * @param contentFileSize The size content file to create
  * @throws IOException
  */
 private void createFileWithRepeatableContent(
     Directory dir, String fileName, final int contentFileSize) throws IOException {
   IndexOutput indexOutput = dir.createOutput(fileName);
   RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
   for (int i = 0; i < contentFileSize; i++) {
     indexOutput.writeByte(bytesGenerator.nextByte());
   }
   indexOutput.close();
 }
Esempio n. 11
0
  final void finishCommit(Directory dir) throws IOException {
    if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called");
    boolean success = false;
    try {
      pendingSegnOutput.finishCommit();
      pendingSegnOutput.close();
      pendingSegnOutput = null;
      success = true;
    } finally {
      if (!success) rollbackCommit(dir);
    }

    // NOTE: if we crash here, we have left a segments_N
    // file in the directory in a possibly corrupt state (if
    // some bytes made it to stable storage and others
    // didn't).  But, the segments_N file includes checksum
    // at the end, which should catch this case.  So when a
    // reader tries to read it, it will throw a
    // CorruptIndexException, which should cause the retry
    // logic in SegmentInfos to kick in and load the last
    // good (previous) segments_N-1 file.

    final String fileName =
        IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", generation);
    success = false;
    try {
      dir.sync(Collections.singleton(fileName));
      success = true;
    } finally {
      if (!success) {
        try {
          dir.deleteFile(fileName);
        } catch (Throwable t) {
          // Suppress so we keep throwing the original exception
        }
      }
    }

    lastGeneration = generation;

    try {
      IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
      try {
        genOutput.writeInt(FORMAT_LOCKLESS);
        genOutput.writeLong(generation);
        genOutput.writeLong(generation);
      } finally {
        genOutput.close();
      }
    } catch (ThreadInterruptedException t) {
      throw t;
    } catch (Throwable t) {
      // It's OK if we fail to write this file since it's
      // used only as one of the retry fallbacks.
    }
  }
  public void testEncodeDecode() throws IOException {
    final int iterations = RandomInts.randomIntBetween(random(), 1, 1000);
    final float acceptableOverheadRatio = random().nextFloat();
    final int[] values = new int[(iterations - 1) * BLOCK_SIZE + ForUtil.MAX_DATA_SIZE];
    for (int i = 0; i < iterations; ++i) {
      final int bpv = random().nextInt(32);
      if (bpv == 0) {
        final int value = RandomInts.randomIntBetween(random(), 0, Integer.MAX_VALUE);
        for (int j = 0; j < BLOCK_SIZE; ++j) {
          values[i * BLOCK_SIZE + j] = value;
        }
      } else {
        for (int j = 0; j < BLOCK_SIZE; ++j) {
          values[i * BLOCK_SIZE + j] =
              RandomInts.randomIntBetween(random(), 0, (int) PackedInts.maxValue(bpv));
        }
      }
    }

    final Directory d = new RAMDirectory();
    final long endPointer;

    {
      // encode
      IndexOutput out = d.createOutput("test.bin", IOContext.DEFAULT);
      final ForUtil forUtil = new ForUtil(acceptableOverheadRatio, out);

      for (int i = 0; i < iterations; ++i) {
        forUtil.writeBlock(
            Arrays.copyOfRange(values, i * BLOCK_SIZE, values.length),
            new byte[MAX_ENCODED_SIZE],
            out);
      }
      endPointer = out.getFilePointer();
      out.close();
    }

    {
      // decode
      IndexInput in = d.openInput("test.bin", IOContext.READONCE);
      final ForUtil forUtil = new ForUtil(in);
      for (int i = 0; i < iterations; ++i) {
        if (random().nextBoolean()) {
          forUtil.skipBlock(in);
          continue;
        }
        final int[] restored = new int[MAX_DATA_SIZE];
        forUtil.readBlock(in, new byte[MAX_ENCODED_SIZE], restored);
        assertArrayEquals(
            Arrays.copyOfRange(values, i * BLOCK_SIZE, (i + 1) * BLOCK_SIZE),
            Arrays.copyOf(restored, BLOCK_SIZE));
      }
      assertEquals(endPointer, in.getFilePointer());
      in.close();
    }
  }
  @Override
  public void write(Directory dir, SegmentInfo si, IOContext context) throws IOException {
    String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);

    int numFiles = si.files().size();
    String names[] = si.files().toArray(new String[numFiles]);
    Arrays.sort(names);
    long startOffsets[] = new long[numFiles];
    long endOffsets[] = new long[numFiles];

    BytesRefBuilder scratch = new BytesRefBuilder();

    try (IndexOutput out = dir.createOutput(dataFile, context)) {
      for (int i = 0; i < names.length; i++) {
        // write header for file
        SimpleTextUtil.write(out, HEADER);
        SimpleTextUtil.write(out, names[i], scratch);
        SimpleTextUtil.writeNewline(out);

        // write bytes for file
        startOffsets[i] = out.getFilePointer();
        try (IndexInput in = dir.openInput(names[i], IOContext.READONCE)) {
          out.copyBytes(in, in.length());
        }
        endOffsets[i] = out.getFilePointer();
      }

      long tocPos = out.getFilePointer();

      // write CFS table
      SimpleTextUtil.write(out, TABLE);
      SimpleTextUtil.write(out, Integer.toString(numFiles), scratch);
      SimpleTextUtil.writeNewline(out);

      for (int i = 0; i < names.length; i++) {
        SimpleTextUtil.write(out, TABLENAME);
        SimpleTextUtil.write(out, names[i], scratch);
        SimpleTextUtil.writeNewline(out);

        SimpleTextUtil.write(out, TABLESTART);
        SimpleTextUtil.write(out, Long.toString(startOffsets[i]), scratch);
        SimpleTextUtil.writeNewline(out);

        SimpleTextUtil.write(out, TABLEEND);
        SimpleTextUtil.write(out, Long.toString(endOffsets[i]), scratch);
        SimpleTextUtil.writeNewline(out);
      }

      DecimalFormat df =
          new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT));
      SimpleTextUtil.write(out, TABLEPOS);
      SimpleTextUtil.write(out, df.format(tocPos), scratch);
      SimpleTextUtil.writeNewline(out);
    }
  }
Esempio n. 14
0
 @Test
 public void testVerifyingIndexOutput() throws IOException {
   Directory dir = newDirectory();
   IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
   int iters = scaledRandomIntBetween(10, 100);
   for (int i = 0; i < iters; i++) {
     BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
     output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
   }
   CodecUtil.writeFooter(output);
   output.close();
   IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
   String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
   indexInput.seek(0);
   BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
   long length = indexInput.length();
   IndexOutput verifyingOutput =
       new Store.LuceneVerifyingIndexOutput(
           new StoreFileMetaData("foo1.bar", length, checksum),
           dir.createOutput("foo1.bar", IOContext.DEFAULT));
   while (length > 0) {
     if (random().nextInt(10) == 0) {
       verifyingOutput.writeByte(indexInput.readByte());
       length--;
     } else {
       int min = (int) Math.min(length, ref.bytes.length);
       indexInput.readBytes(ref.bytes, ref.offset, min);
       verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
       length -= min;
     }
   }
   Store.verify(verifyingOutput);
   verifyingOutput.writeByte((byte) 0x0);
   try {
     Store.verify(verifyingOutput);
     fail("should be a corrupted index");
   } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
     // ok
   }
   IOUtils.close(indexInput, verifyingOutput, dir);
 }
  /**
   * Sets some otherwise hard-to-test properties: random segment names, ID values, document count,
   * etc and round-trips
   */
  public void testRandom() throws Exception {
    Codec codec = getCodec();
    Version[] versions = getVersions();
    for (int i = 0; i < 10; i++) {
      Directory dir = newDirectory();
      Version version = versions[random().nextInt(versions.length)];
      String name =
          "_" + Integer.toString(random().nextInt(Integer.MAX_VALUE), Character.MAX_RADIX);
      int docCount = TestUtil.nextInt(random(), 1, IndexWriter.MAX_DOCS);
      boolean isCompoundFile = random().nextBoolean();
      Set<String> files = new HashSet<>();
      int numFiles = random().nextInt(10);
      for (int j = 0; j < numFiles; j++) {
        String file = IndexFileNames.segmentFileName(name, "", Integer.toString(j));
        files.add(file);
        dir.createOutput(file, IOContext.DEFAULT).close();
      }
      Map<String, String> diagnostics = new HashMap<>();
      int numDiags = random().nextInt(10);
      for (int j = 0; j < numDiags; j++) {
        diagnostics.put(
            TestUtil.randomUnicodeString(random()), TestUtil.randomUnicodeString(random()));
      }
      byte id[] = new byte[StringHelper.ID_LENGTH];
      random().nextBytes(id);

      Map<String, String> attributes = new HashMap<>();
      int numAttributes = random().nextInt(10);
      for (int j = 0; j < numAttributes; j++) {
        attributes.put(
            TestUtil.randomUnicodeString(random()), TestUtil.randomUnicodeString(random()));
      }

      SegmentInfo info =
          new SegmentInfo(
              dir,
              version,
              name,
              docCount,
              isCompoundFile,
              codec,
              diagnostics,
              id,
              attributes,
              null);
      info.setFiles(files);
      codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
      SegmentInfo info2 = codec.segmentInfoFormat().read(dir, name, id, IOContext.DEFAULT);
      assertEquals(info, info2);

      dir.close();
    }
  }
 /**
  * creates compressed output.
  *
  * @param pCompressed compressed directory to inform when file is closed
  * @param pTempDir temporary directory that can be the asame as compressed one
  * @param pOut actual IndexOutput from compresed directory (nested storage)
  * @param pName name of output
  * @param pChunkSize chunk size
  * @param pDeflateMethod deflate method
  * @param deflateCount number of times to deflate data
  * @throws IOException
  */
 public TransformedIndexOutput(
     TransformedDirectory pCompressed,
     Directory pTempDir,
     IndexOutput pOut,
     String pName,
     int pChunkSize,
     StoreDataTransformer pTransformer)
     throws IOException {
   super(pName, pOut, pTransformer, pCompressed);
   this.tempDirectory = pTempDir;
   this.chunkSize = pChunkSize;
   tmpName = pName + ".plain." + pCompressed.nextSequence() + ".tmp";
   tempOut = pTempDir.createOutput(tmpName);
 }
  @Override
  public void write(
      Directory directory,
      SegmentInfo segmentInfo,
      String segmentSuffix,
      FieldInfos infos,
      IOContext context)
      throws IOException {
    final String fileName =
        IndexFileNames.segmentFileName(
            segmentInfo.name, segmentSuffix, Lucene46FieldInfosFormat.EXTENSION);
    try (IndexOutput output = directory.createOutput(fileName, context)) {
      CodecUtil.writeHeader(
          output, Lucene46FieldInfosFormat.CODEC_NAME, Lucene46FieldInfosFormat.FORMAT_CURRENT);
      output.writeVInt(infos.size());
      for (FieldInfo fi : infos) {
        IndexOptions indexOptions = fi.getIndexOptions();
        byte bits = 0x0;
        if (fi.hasVectors()) bits |= Lucene46FieldInfosFormat.STORE_TERMVECTOR;
        if (fi.omitsNorms()) bits |= Lucene46FieldInfosFormat.OMIT_NORMS;
        if (fi.hasPayloads()) bits |= Lucene46FieldInfosFormat.STORE_PAYLOADS;
        if (fi.getIndexOptions() != IndexOptions.NONE) {
          bits |= Lucene46FieldInfosFormat.IS_INDEXED;
          assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0
              || !fi.hasPayloads();
          if (indexOptions == IndexOptions.DOCS) {
            bits |= Lucene46FieldInfosFormat.OMIT_TERM_FREQ_AND_POSITIONS;
          } else if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
            bits |= Lucene46FieldInfosFormat.STORE_OFFSETS_IN_POSTINGS;
          } else if (indexOptions == IndexOptions.DOCS_AND_FREQS) {
            bits |= Lucene46FieldInfosFormat.OMIT_POSITIONS;
          }
        }
        output.writeString(fi.name);
        output.writeVInt(fi.number);
        output.writeByte(bits);

        // pack the DV types in one byte
        final byte dv = docValuesByte(fi.getDocValuesType());
        final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : DocValuesType.NONE);
        assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
        byte val = (byte) (0xff & ((nrm << 4) | dv));
        output.writeByte(val);
        output.writeLong(fi.getDocValuesGen());
        output.writeStringStringMap(fi.attributes());
      }
      CodecUtil.writeFooter(output);
    }
  }
  /**
   * This test that writes larger than the size of the buffer output will correctly increment the
   * file pointer.
   */
  public void testLargeWrites() throws IOException {
    IndexOutput os = dir.createOutput("testBufferStart.txt");

    byte[] largeBuf = new byte[2048];
    for (int i = 0; i < largeBuf.length; i++) {
      largeBuf[i] = (byte) (Math.random() * 256);
    }

    long currentPos = os.getFilePointer();
    os.writeBytes(largeBuf, largeBuf.length);

    try {
      assertEquals(currentPos + largeBuf.length, os.getFilePointer());
    } finally {
      os.close();
    }
  }
Esempio n. 19
0
  private final void write(Directory directory) throws IOException {

    String segmentFileName = getNextSegmentFileName();

    // Always advance the generation on write:
    if (generation == -1) {
      generation = 1;
    } else {
      generation++;
    }

    ChecksumIndexOutput segnOutput =
        new ChecksumIndexOutput(directory.createOutput(segmentFileName));

    boolean success = false;

    try {
      segnOutput.writeInt(CURRENT_FORMAT); // write FORMAT
      segnOutput.writeLong(version);
      segnOutput.writeInt(counter); // write counter
      segnOutput.writeInt(size()); // write infos
      for (SegmentInfo si : this) {
        si.write(segnOutput);
      }
      segnOutput.writeStringStringMap(userData);
      segnOutput.prepareCommit();
      pendingSegnOutput = segnOutput;
      success = true;
    } finally {
      if (!success) {
        // We hit an exception above; try to close the file
        // but suppress any exception:
        IOUtils.closeSafely(true, segnOutput);
        try {
          // Try not to leave a truncated segments_N file in
          // the index:
          directory.deleteFile(segmentFileName);
        } catch (Throwable t) {
          // Suppress so we keep throwing the original exception
        }
      }
    }
  }
Esempio n. 20
0
 @Test
 public void testVerifyingIndexOutputWithBogusInput() throws IOException {
   Directory dir = newDirectory();
   int length = scaledRandomIntBetween(10, 1024);
   IndexOutput verifyingOutput =
       new Store.LuceneVerifyingIndexOutput(
           new StoreFileMetaData("foo1.bar", length, ""),
           dir.createOutput("foo1.bar", IOContext.DEFAULT));
   try {
     while (length > 0) {
       verifyingOutput.writeByte((byte) random().nextInt());
       length--;
     }
     fail("should be a corrupted index");
   } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
     // ok
   }
   IOUtils.close(verifyingOutput, dir);
 }
Esempio n. 21
0
 private void corruptFile(Directory dir, String fileIn, String fileOut) throws IOException {
   IndexInput input = dir.openInput(fileIn, IOContext.READONCE);
   IndexOutput output = dir.createOutput(fileOut, IOContext.DEFAULT);
   long len = input.length();
   byte[] b = new byte[1024];
   long broken = randomInt((int) len);
   long pos = 0;
   while (pos < len) {
     int min = (int) Math.min(input.length() - pos, b.length);
     input.readBytes(b, 0, min);
     if (broken >= pos && broken < pos + min) {
       // Flip one byte
       int flipPos = (int) (broken - pos);
       b[flipPos] = (byte) (b[flipPos] ^ 42);
     }
     output.writeBytes(b, min);
     pos += min;
   }
   IOUtils.close(input, output);
 }
  @Test
  public void testWriteChunksDefaultChunks() throws Exception {
    Cache cache = cacheManager.getCache();
    Directory dir = DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME).create();

    final String testText = "This is some rubbish";
    final byte[] testTextAsBytes = testText.getBytes();

    IndexOutput io = dir.createOutput("MyNewFile.txt");

    io.writeByte((byte) 1);
    io.writeByte((byte) 2);
    io.writeByte((byte) 3);
    io.writeBytes(testTextAsBytes, testTextAsBytes.length);
    io.close();
    DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);

    FileCacheKey fileCacheKey = new FileCacheKey(INDEXNAME, "MyNewFile.txt");
    assert null != cache.get(fileCacheKey);
    FileMetadata metadata = (FileMetadata) cache.get(fileCacheKey);
    AssertJUnit.assertEquals(testTextAsBytes.length + 3, metadata.getSize());
    assert null
        != cache.get(
            new ChunkCacheKey(
                INDEXNAME, "MyNewFile.txt", 0, DirectoryBuilderImpl.DEFAULT_BUFFER_SIZE));

    // test contents by reading:
    IndexInput ii = dir.openInput("MyNewFile.txt");
    assert ii.readByte() == 1;
    assert ii.readByte() == 2;
    assert ii.readByte() == 3;
    byte[] buf = new byte[testTextAsBytes.length];

    ii.readBytes(buf, 0, testTextAsBytes.length);
    ii.close();

    assert testText.equals(new String(buf).trim());

    dir.close();
    DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
  }
Esempio n. 23
0
    public IndexOutput createOutput(String name, IOContext context, boolean raw)
        throws IOException {
      ensureOpen();
      Directory directory;
      // we want to write the segments gen file to the same directory *all* the time
      // to make sure we don't create multiple copies of it
      if (isChecksum(name) || IndexFileNames.SEGMENTS_GEN.equals(name)) {
        directory = distributor.primary();
      } else {
        directory = distributor.any();
      }
      IndexOutput out = directory.createOutput(name, context);
      boolean success = false;
      try {
        synchronized (mutex) {
          StoreFileMetaData metaData = new StoreFileMetaData(name, -1, null, directory);
          filesMetadata = ImmutableOpenMap.builder(filesMetadata).fPut(name, metaData).build();
          files = filesMetadata.keys().toArray(String.class);
          boolean computeChecksum = !raw;
          if (computeChecksum) {
            // don't compute checksum for segment based files
            if (IndexFileNames.SEGMENTS_GEN.equals(name)
                || name.startsWith(IndexFileNames.SEGMENTS)) {
              computeChecksum = false;
            }
          }
          if (computeChecksum) {
            out = new BufferedChecksumIndexOutput(out, new Adler32());
          }

          final StoreIndexOutput storeIndexOutput = new StoreIndexOutput(metaData, out, name);
          success = true;
          return storeIndexOutput;
        }
      } finally {
        if (!success) {
          IOUtils.closeWhileHandlingException(out);
        }
      }
    }
 @Override
 public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context)
     throws IOException {
   IndexOutput out = dir.createOutput(fileName, context);
   boolean success = false;
   try {
     FixedIntBlockIndexOutput ret =
         new FixedIntBlockIndexOutput(out, blockSize) {
           @Override
           protected void flushBlock() throws IOException {
             for (int i = 0; i < buffer.length; i++) {
               out.writeVInt(buffer[i]);
             }
           }
         };
     success = true;
     return ret;
   } finally {
     if (!success) {
       IOUtils.closeWhileHandlingException(out);
     }
   }
 }
Esempio n. 25
0
  @Test
  public void testVerifyingIndexInput() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
      BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
      output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    CodecUtil.writeFooter(output);
    output.close();

    // Check file
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    long checksum = CodecUtil.retrieveChecksum(indexInput);
    indexInput.seek(0);
    IndexInput verifyingIndexInput =
        new Store.VerifyingIndexInput(dir.openInput("foo.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    Store.verify(verifyingIndexInput);
    assertThat(checksum, equalTo(((ChecksumIndexInput) verifyingIndexInput).getChecksum()));
    IOUtils.close(indexInput, verifyingIndexInput);

    // Corrupt file and check again
    corruptFile(dir, "foo.bar", "foo1.bar");
    verifyingIndexInput =
        new Store.VerifyingIndexInput(dir.openInput("foo1.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    try {
      Store.verify(verifyingIndexInput);
      fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
      // ok
    }
    IOUtils.close(verifyingIndexInput);
    IOUtils.close(dir);
  }
  // TODO ADD more unittests
  private void writeData(
      Directory dir, Completion090PostingsFormat.CompletionLookupProvider provider)
      throws IOException {
    IndexOutput output = dir.createOutput("foo.txt", IOContext.DEFAULT);
    FieldsConsumer consumer = provider.consumer(output);
    final List<TermPosAndPayload> terms = new ArrayList<>();
    terms.add(
        new TermPosAndPayload(
            "foofightersgenerator",
            256 - 2,
            provider.buildPayload(
                new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10"))));
    terms.add(
        new TermPosAndPayload(
            "generator",
            256 - 1,
            provider.buildPayload(
                new BytesRef("Generator - Foo Fighters"), 9, new BytesRef("id:10"))));
    Fields fields =
        new Fields() {
          @Override
          public Iterator<String> iterator() {
            return Arrays.asList("foo").iterator();
          }

          @Override
          public Terms terms(String field) throws IOException {
            if (field.equals("foo")) {
              return new Terms() {
                @Override
                public TermsEnum iterator(TermsEnum reuse) throws IOException {
                  final Iterator<TermPosAndPayload> iterator = terms.iterator();
                  return new TermsEnum() {
                    private TermPosAndPayload current = null;

                    @Override
                    public SeekStatus seekCeil(BytesRef text) throws IOException {
                      throw new UnsupportedOperationException();
                    }

                    @Override
                    public void seekExact(long ord) throws IOException {
                      throw new UnsupportedOperationException();
                    }

                    @Override
                    public BytesRef term() throws IOException {
                      return current == null ? null : current.term;
                    }

                    @Override
                    public long ord() throws IOException {
                      throw new UnsupportedOperationException();
                    }

                    @Override
                    public int docFreq() throws IOException {
                      return current == null ? 0 : 1;
                    }

                    @Override
                    public long totalTermFreq() throws IOException {
                      throw new UnsupportedOperationException();
                    }

                    @Override
                    public PostingsEnum postings(Bits liveDocs, PostingsEnum reuse, int flags)
                        throws IOException {
                      final TermPosAndPayload data = current;
                      return new PostingsEnum() {
                        boolean done = false;

                        @Override
                        public int nextPosition() throws IOException {
                          return current.pos;
                        }

                        @Override
                        public int startOffset() throws IOException {
                          return 0;
                        }

                        @Override
                        public int endOffset() throws IOException {
                          return 0;
                        }

                        @Override
                        public BytesRef getPayload() throws IOException {
                          return current.payload;
                        }

                        @Override
                        public int freq() throws IOException {
                          return 1;
                        }

                        @Override
                        public int docID() {
                          if (done) {
                            return NO_MORE_DOCS;
                          }
                          return 0;
                        }

                        @Override
                        public int nextDoc() throws IOException {
                          if (done) {
                            return NO_MORE_DOCS;
                          }
                          done = true;
                          return 0;
                        }

                        @Override
                        public int advance(int target) throws IOException {
                          if (done) {
                            return NO_MORE_DOCS;
                          }
                          done = true;
                          return 0;
                        }

                        @Override
                        public long cost() {
                          return 0;
                        }
                      };
                    }

                    @Override
                    public BytesRef next() throws IOException {
                      if (iterator.hasNext()) {
                        current = iterator.next();
                        return current.term;
                      }
                      current = null;
                      return null;
                    }
                  };
                }

                @Override
                public long size() throws IOException {
                  throw new UnsupportedOperationException();
                }

                @Override
                public long getSumTotalTermFreq() throws IOException {
                  throw new UnsupportedOperationException();
                }

                @Override
                public long getSumDocFreq() throws IOException {
                  throw new UnsupportedOperationException();
                }

                @Override
                public int getDocCount() throws IOException {
                  throw new UnsupportedOperationException();
                }

                @Override
                public boolean hasFreqs() {
                  throw new UnsupportedOperationException();
                }

                @Override
                public boolean hasOffsets() {
                  throw new UnsupportedOperationException();
                }

                @Override
                public boolean hasPositions() {
                  throw new UnsupportedOperationException();
                }

                @Override
                public boolean hasPayloads() {
                  throw new UnsupportedOperationException();
                }
              };
            }
            return null;
          }

          @Override
          public int size() {
            return 0;
          }
        };
    consumer.write(fields);
    consumer.close();
    output.close();
  }
Esempio n. 27
0
  private void write(Directory directory) throws IOException {

    long nextGeneration = getNextPendingGeneration();
    String segmentFileName =
        IndexFileNames.fileNameFromGeneration(IndexFileNames.PENDING_SEGMENTS, "", nextGeneration);

    // Always advance the generation on write:
    generation = nextGeneration;

    IndexOutput segnOutput = null;
    boolean success = false;

    try {
      segnOutput = directory.createOutput(segmentFileName, IOContext.DEFAULT);
      CodecUtil.writeIndexHeader(
          segnOutput,
          "segments",
          VERSION_CURRENT,
          StringHelper.randomId(),
          Long.toString(nextGeneration, Character.MAX_RADIX));
      segnOutput.writeVInt(Version.LATEST.major);
      segnOutput.writeVInt(Version.LATEST.minor);
      segnOutput.writeVInt(Version.LATEST.bugfix);

      segnOutput.writeLong(version);
      segnOutput.writeInt(counter); // write counter
      segnOutput.writeInt(size());

      if (size() > 0) {

        Version minSegmentVersion = null;

        // We do a separate loop up front so we can write the minSegmentVersion before
        // any SegmentInfo; this makes it cleaner to throw IndexFormatTooOldExc at read time:
        for (SegmentCommitInfo siPerCommit : this) {
          Version segmentVersion = siPerCommit.info.getVersion();
          if (minSegmentVersion == null || segmentVersion.onOrAfter(minSegmentVersion) == false) {
            minSegmentVersion = segmentVersion;
          }
        }

        segnOutput.writeVInt(minSegmentVersion.major);
        segnOutput.writeVInt(minSegmentVersion.minor);
        segnOutput.writeVInt(minSegmentVersion.bugfix);
      }

      // write infos
      for (SegmentCommitInfo siPerCommit : this) {
        SegmentInfo si = siPerCommit.info;
        segnOutput.writeString(si.name);
        byte segmentID[] = si.getId();
        // TODO: remove this in lucene 6, we don't need to include 4.x segments in commits anymore
        if (segmentID == null) {
          segnOutput.writeByte((byte) 0);
        } else {
          if (segmentID.length != StringHelper.ID_LENGTH) {
            throw new IllegalStateException(
                "cannot write segment: invalid id segment="
                    + si.name
                    + "id="
                    + StringHelper.idToString(segmentID));
          }
          segnOutput.writeByte((byte) 1);
          segnOutput.writeBytes(segmentID, segmentID.length);
        }
        segnOutput.writeString(si.getCodec().getName());
        segnOutput.writeLong(siPerCommit.getDelGen());
        int delCount = siPerCommit.getDelCount();
        if (delCount < 0 || delCount > si.maxDoc()) {
          throw new IllegalStateException(
              "cannot write segment: invalid maxDoc segment="
                  + si.name
                  + " maxDoc="
                  + si.maxDoc()
                  + " delCount="
                  + delCount);
        }
        segnOutput.writeInt(delCount);
        segnOutput.writeLong(siPerCommit.getFieldInfosGen());
        segnOutput.writeLong(siPerCommit.getDocValuesGen());
        segnOutput.writeSetOfStrings(siPerCommit.getFieldInfosFiles());
        final Map<Integer, Set<String>> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles();
        segnOutput.writeInt(dvUpdatesFiles.size());
        for (Entry<Integer, Set<String>> e : dvUpdatesFiles.entrySet()) {
          segnOutput.writeInt(e.getKey());
          segnOutput.writeSetOfStrings(e.getValue());
        }
      }
      segnOutput.writeMapOfStrings(userData);
      CodecUtil.writeFooter(segnOutput);
      segnOutput.close();
      directory.sync(Collections.singleton(segmentFileName));
      success = true;
    } finally {
      if (success) {
        pendingCommit = true;
      } else {
        // We hit an exception above; try to close the file
        // but suppress any exception:
        IOUtils.closeWhileHandlingException(segnOutput);
        // Try not to leave a truncated segments_N file in
        // the index:
        IOUtils.deleteFilesIgnoringExceptions(directory, segmentFileName);
      }
    }
  }
  @Test
  public void testWriteUsingSeekMethod() throws IOException {
    final int BUFFER_SIZE = 64;

    Cache cache = cacheManager.getCache();
    Directory dir =
        DirectoryBuilder.newDirectoryInstance(cache, cache, cache, INDEXNAME)
            .chunkSize(BUFFER_SIZE)
            .create();

    String fileName = "SomeText.txt";
    IndexOutput io = dir.createOutput(fileName);
    RepeatableLongByteSequence bytesGenerator = new RepeatableLongByteSequence();
    // It writes repeatable text
    final int REPEATABLE_BUFFER_SIZE = 1501;
    for (int i = 0; i < REPEATABLE_BUFFER_SIZE; i++) {
      io.writeByte(bytesGenerator.nextByte());
    }
    io.flush();
    assert io.length() == REPEATABLE_BUFFER_SIZE;

    // Text to write on file with repeatable text
    final String someText = "This is some text";
    final byte[] someTextAsBytes = someText.getBytes();
    // 4 points in random order where writing someText: at begin of file, at end of file, within a
    // single chunk,
    // between 2 chunks
    final int[] pointers = {0, 635, REPEATABLE_BUFFER_SIZE, 135};
    for (int i = 0; i < pointers.length; i++) {
      io.seek(pointers[i]);
      io.writeBytes(someTextAsBytes, someTextAsBytes.length);
    }

    io.close();
    bytesGenerator.reset();
    final long finalSize = REPEATABLE_BUFFER_SIZE + someTextAsBytes.length;
    assert io.length() == finalSize;
    assert io.length()
        == DirectoryIntegrityCheck.deepCountFileSize(new FileCacheKey(INDEXNAME, fileName), cache);

    int indexPointer = 0;
    Arrays.sort(pointers);
    byte[] buffer = null;
    int chunkIndex = -1;
    // now testing the stream is equal to the produced repeatable but including the edits at pointed
    // positions
    for (int i = 0; i < REPEATABLE_BUFFER_SIZE + someTextAsBytes.length; i++) {
      if (i % BUFFER_SIZE == 0) {
        buffer =
            (byte[]) cache.get(new ChunkCacheKey(INDEXNAME, fileName, ++chunkIndex, BUFFER_SIZE));
      }

      byte predictableByte = bytesGenerator.nextByte();
      if (i < pointers[indexPointer]) {
        // Assert predictable text
        AssertJUnit.assertEquals(predictableByte, buffer[i % BUFFER_SIZE]);
      } else if (pointers[indexPointer] <= i
          && i < pointers[indexPointer] + someTextAsBytes.length) {
        // Assert someText
        AssertJUnit.assertEquals(
            someTextAsBytes[i - pointers[indexPointer]], buffer[i % BUFFER_SIZE]);
      }

      if (i == pointers[indexPointer] + someTextAsBytes.length) {
        // Change pointer
        indexPointer++;
      }
    }

    dir.close();
    DirectoryIntegrityCheck.verifyDirectoryStructure(cache, INDEXNAME);
  }
Esempio n. 29
0
  public void testCheckIntegrity() throws IOException {
    Directory dir = newDirectory();
    long luceneFileLength = 0;

    try (IndexOutput output = dir.createOutput("lucene_checksum.bin", IOContext.DEFAULT)) {
      int iters = scaledRandomIntBetween(10, 100);
      for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        luceneFileLength += bytesRef.length;
      }
      CodecUtil.writeFooter(output);
      luceneFileLength += CodecUtil.footerLength();
    }

    final Adler32 adler32 = new Adler32();
    long legacyFileLength = 0;
    try (IndexOutput output = dir.createOutput("legacy.bin", IOContext.DEFAULT)) {
      int iters = scaledRandomIntBetween(10, 100);
      for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        adler32.update(bytesRef.bytes, bytesRef.offset, bytesRef.length);
        legacyFileLength += bytesRef.length;
      }
    }
    final long luceneChecksum;
    final long adler32LegacyChecksum = adler32.getValue();
    try (IndexInput indexInput = dir.openInput("lucene_checksum.bin", IOContext.DEFAULT)) {
      assertEquals(luceneFileLength, indexInput.length());
      luceneChecksum = CodecUtil.retrieveChecksum(indexInput);
    }

    { // positive check
      StoreFileMetaData lucene =
          new StoreFileMetaData(
              "lucene_checksum.bin",
              luceneFileLength,
              Store.digestToString(luceneChecksum),
              Version.LUCENE_4_8_0);
      StoreFileMetaData legacy =
          new StoreFileMetaData(
              "legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
      assertTrue(legacy.hasLegacyChecksum());
      assertFalse(lucene.hasLegacyChecksum());
      assertTrue(Store.checkIntegrityNoException(lucene, dir));
      assertTrue(Store.checkIntegrityNoException(legacy, dir));
    }

    { // negative check - wrong checksum
      StoreFileMetaData lucene =
          new StoreFileMetaData(
              "lucene_checksum.bin",
              luceneFileLength,
              Store.digestToString(luceneChecksum + 1),
              Version.LUCENE_4_8_0);
      StoreFileMetaData legacy =
          new StoreFileMetaData(
              "legacy.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum + 1));
      assertTrue(legacy.hasLegacyChecksum());
      assertFalse(lucene.hasLegacyChecksum());
      assertFalse(Store.checkIntegrityNoException(lucene, dir));
      assertFalse(Store.checkIntegrityNoException(legacy, dir));
    }

    { // negative check - wrong length
      StoreFileMetaData lucene =
          new StoreFileMetaData(
              "lucene_checksum.bin",
              luceneFileLength + 1,
              Store.digestToString(luceneChecksum),
              Version.LUCENE_4_8_0);
      StoreFileMetaData legacy =
          new StoreFileMetaData(
              "legacy.bin", legacyFileLength + 1, Store.digestToString(adler32LegacyChecksum));
      assertTrue(legacy.hasLegacyChecksum());
      assertFalse(lucene.hasLegacyChecksum());
      assertFalse(Store.checkIntegrityNoException(lucene, dir));
      assertFalse(Store.checkIntegrityNoException(legacy, dir));
    }

    { // negative check - wrong file
      StoreFileMetaData lucene =
          new StoreFileMetaData(
              "legacy.bin",
              luceneFileLength,
              Store.digestToString(luceneChecksum),
              Version.LUCENE_4_8_0);
      StoreFileMetaData legacy =
          new StoreFileMetaData(
              "lucene_checksum.bin", legacyFileLength, Store.digestToString(adler32LegacyChecksum));
      assertTrue(legacy.hasLegacyChecksum());
      assertFalse(lucene.hasLegacyChecksum());
      assertFalse(Store.checkIntegrityNoException(lucene, dir));
      assertFalse(Store.checkIntegrityNoException(legacy, dir));
    }
    dir.close();
  }
Esempio n. 30
0
 @Override
 public IndexOutput createOutput(String name, IOContext context) throws IOException {
   return fsDir.createOutput(name, context);
 }