@Override
 public void setUp() throws Exception {
   super.setUp();
   // set the default codec, so adding test cases to this isn't fragile
   savedCodec = Codec.getDefault();
   Codec.setDefault(getCodec());
 }
Beispiel #2
0
  private SegmentCommitInfo merge(
      Directory dir,
      SegmentCommitInfo si1,
      SegmentCommitInfo si2,
      String merged,
      boolean useCompoundFile)
      throws Exception {
    IOContext context = newIOContext(random());
    SegmentReader r1 = new SegmentReader(si1, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, context);
    SegmentReader r2 = new SegmentReader(si2, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, context);

    final Codec codec = Codec.getDefault();
    TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
    final SegmentInfo si =
        new SegmentInfo(si1.info.dir, Version.LATEST, merged, -1, false, codec, null);

    SegmentMerger merger =
        new SegmentMerger(
            Arrays.<AtomicReader>asList(r1, r2),
            si,
            InfoStream.getDefault(),
            trackingDir,
            IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL,
            MergeState.CheckAbort.NONE,
            new FieldInfos.FieldNumbers(),
            context,
            true);

    MergeState mergeState = merger.merge();
    r1.close();
    r2.close();
    final SegmentInfo info =
        new SegmentInfo(
            si1.info.dir,
            Version.LATEST,
            merged,
            si1.info.getDocCount() + si2.info.getDocCount(),
            false,
            codec,
            null);
    info.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));

    if (useCompoundFile) {
      Collection<String> filesToDelete =
          IndexWriter.createCompoundFile(
              InfoStream.getDefault(),
              dir,
              MergeState.CheckAbort.NONE,
              info,
              newIOContext(random()));
      info.setUseCompoundFile(true);
      for (final String fileToDelete : filesToDelete) {
        si1.info.dir.deleteFile(fileToDelete);
      }
    }

    return new SegmentCommitInfo(info, 0, -1L, -1L, -1L);
  }
 @BeforeClass
 public static void beforeClass() throws Exception {
   savedCodec = Codec.getDefault();
   // currently only these codecs that support random access ordinals
   int victim = random().nextInt(2);
   switch (victim) {
     case 0:
       Codec.setDefault(TestUtil.alwaysDocValuesFormat(new DirectDocValuesFormat()));
     default:
       Codec.setDefault(TestUtil.alwaysDocValuesFormat(new Lucene410DocValuesFormat()));
   }
 }
@Listeners({ReproduceInfoPrinter.class})
@ThreadLeakFilters(
    defaultFilters = true,
    filters = {ElasticSearchThreadFilter.class})
@ThreadLeakScope(Scope.NONE)
@TimeoutSuite(millis = TimeUnits.HOUR)
@SuppressCodecs("Lucene3x")
public abstract class ElasticSearchLuceneTestCase extends LuceneTestCase {

  private static final Codec DEFAULT_CODEC = Codec.getDefault();

  /** Forcefully reset the default codec */
  public static void forceDefaultCodec() {
    Codec.setDefault(DEFAULT_CODEC);
  }
}
  public Lookup buildAnalyzingLookup(
      final CompletionFieldMapper mapper, String[] terms, String[] surfaces, long[] weights)
      throws IOException {
    RAMDirectory dir = new RAMDirectory();
    FilterCodec filterCodec =
        new FilterCodec("filtered", Codec.getDefault()) {
          @Override
          public PostingsFormat postingsFormat() {
            final PostingsFormat in = super.postingsFormat();
            return mapper.postingsFormat(in);
          }
        };
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(mapper.indexAnalyzer());

    indexWriterConfig.setCodec(filterCodec);
    IndexWriter writer = new IndexWriter(dir, indexWriterConfig);
    for (int i = 0; i < weights.length; i++) {
      Document doc = new Document();
      BytesRef payload =
          mapper.buildPayload(
              new BytesRef(surfaces[i]), weights[i], new BytesRef(Long.toString(weights[i])));
      doc.add(mapper.getCompletionField(ContextMapping.EMPTY_CONTEXT, terms[i], payload));
      if (randomBoolean()) {
        writer.commit();
      }
      writer.addDocument(doc);
    }
    writer.commit();
    writer.forceMerge(1, true);
    writer.commit();
    DirectoryReader reader = DirectoryReader.open(writer, true);
    assertThat(reader.leaves().size(), equalTo(1));
    assertThat(reader.leaves().get(0).reader().numDocs(), equalTo(weights.length));
    LeafReaderContext atomicReaderContext = reader.leaves().get(0);
    Terms luceneTerms = atomicReaderContext.reader().terms(mapper.name());
    Lookup lookup =
        ((Completion090PostingsFormat.CompletionTerms) luceneTerms)
            .getLookup(mapper, new CompletionSuggestionContext(null));
    reader.close();
    writer.close();
    dir.close();
    return lookup;
  }
 public void testAcceptDocValuesFormat() throws IOException {
   String mapping =
       XContentFactory.jsonBuilder()
           .startObject()
           .startObject("type")
           .startObject("properties")
           .startObject("field")
           .field("type", "string")
           .field("doc_values_format", Codec.getDefault().docValuesFormat().getName())
           .endObject()
           .endObject()
           .endObject()
           .endObject()
           .string();
   int i = 0;
   for (Version v : VersionUtils.allVersions()) {
     if (v.onOrAfter(Version.V_2_0_0) == false) {
       // no need to test, we don't support upgrading from these versions
       continue;
     }
     IndexService indexService =
         createIndex(
             "test-" + i++,
             Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, v).build());
     DocumentMapperParser parser = indexService.mapperService().documentMapperParser();
     try {
       parser.parse("type", new CompressedXContent(mapping));
       if (v.onOrAfter(Version.V_2_0_0_beta1)) {
         fail("Elasticsearch 2.0 should not support custom postings formats");
       }
     } catch (MapperParsingException e) {
       if (v.before(Version.V_2_0_0_beta1)) {
         // Elasticsearch 1.x should ignore custom postings formats
         throw e;
       }
       Assert.assertThat(
           e.getMessage(), containsString("unsupported parameters:  [doc_values_format"));
     }
   }
 }
  public void testWriteReadMerge() throws IOException {
    // get another codec, other than the default: so we are merging segments across different codecs
    final Codec otherCodec;
    if ("SimpleText".equals(Codec.getDefault().getName())) {
      otherCodec = new Lucene46Codec();
    } else {
      otherCodec = new SimpleTextCodec();
    }
    Directory dir = newDirectory();
    IndexWriterConfig iwConf =
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf.clone());

    final int docCount = atLeast(200);
    final byte[][][] data = new byte[docCount][][];
    for (int i = 0; i < docCount; ++i) {
      final int fieldCount =
          rarely()
              ? RandomInts.randomIntBetween(random(), 1, 500)
              : RandomInts.randomIntBetween(random(), 1, 5);
      data[i] = new byte[fieldCount][];
      for (int j = 0; j < fieldCount; ++j) {
        final int length = rarely() ? random().nextInt(1000) : random().nextInt(10);
        final int max = rarely() ? 256 : 2;
        data[i][j] = randomByteArray(length, max);
      }
    }

    final FieldType type = new FieldType(StringField.TYPE_STORED);
    type.setIndexed(false);
    type.freeze();
    IntField id = new IntField("id", 0, Store.YES);
    for (int i = 0; i < data.length; ++i) {
      Document doc = new Document();
      doc.add(id);
      id.setIntValue(i);
      for (int j = 0; j < data[i].length; ++j) {
        Field f = new Field("bytes" + j, data[i][j], type);
        doc.add(f);
      }
      iw.w.addDocument(doc);
      if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
        iw.w.close();
        // test merging against a non-compressing codec
        if (iwConf.getCodec() == otherCodec) {
          iwConf.setCodec(Codec.getDefault());
        } else {
          iwConf.setCodec(otherCodec);
        }
        iw = new RandomIndexWriter(random(), dir, iwConf.clone());
      }
    }

    for (int i = 0; i < 10; ++i) {
      final int min = random().nextInt(data.length);
      final int max = min + random().nextInt(20);
      iw.deleteDocuments(NumericRangeQuery.newIntRange("id", min, max, true, false));
    }

    iw.forceMerge(2); // force merges with deletions

    iw.commit();

    final DirectoryReader ir = DirectoryReader.open(dir);
    assertTrue(ir.numDocs() > 0);
    int numDocs = 0;
    for (int i = 0; i < ir.maxDoc(); ++i) {
      final Document doc = ir.document(i);
      if (doc == null) {
        continue;
      }
      ++numDocs;
      final int docId = doc.getField("id").numericValue().intValue();
      assertEquals(data[docId].length + 1, doc.getFields().size());
      for (int j = 0; j < data[docId].length; ++j) {
        final byte[] arr = data[docId][j];
        final BytesRef arr2Ref = doc.getBinaryValue("bytes" + j);
        final byte[] arr2 =
            Arrays.copyOfRange(arr2Ref.bytes, arr2Ref.offset, arr2Ref.offset + arr2Ref.length);
        assertArrayEquals(arr, arr2);
      }
    }
    assertTrue(ir.numDocs() <= numDocs);
    ir.close();

    iw.deleteAll();
    iw.commit();
    iw.forceMerge(1);

    iw.close();
    dir.close();
  }
  @Override
  protected void before() throws Exception {
    // enable this by default, for IDE consistency with ant tests (as its the default from ant)
    // TODO: really should be in solr base classes, but some extend LTC directly.
    // we do this in beforeClass, because some tests currently disable it
    restoreProperties.put("solr.directoryFactory", System.getProperty("solr.directoryFactory"));
    if (System.getProperty("solr.directoryFactory") == null) {
      System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory");
    }

    // Restore more Solr properties.
    restoreProperties.put("solr.solr.home", System.getProperty("solr.solr.home"));
    restoreProperties.put("solr.data.dir", System.getProperty("solr.data.dir"));

    // if verbose: print some debugging stuff about which codecs are loaded.
    if (VERBOSE) {
      Set<String> codecs = Codec.availableCodecs();
      for (String codec : codecs) {
        System.out.println(
            "Loaded codec: '" + codec + "': " + Codec.forName(codec).getClass().getName());
      }

      Set<String> postingsFormats = PostingsFormat.availablePostingsFormats();
      for (String postingsFormat : postingsFormats) {
        System.out.println(
            "Loaded postingsFormat: '"
                + postingsFormat
                + "': "
                + PostingsFormat.forName(postingsFormat).getClass().getName());
      }
    }

    savedInfoStream = InfoStream.getDefault();
    final Random random = RandomizedContext.current().getRandom();
    final boolean v = random.nextBoolean();
    if (INFOSTREAM) {
      InfoStream.setDefault(new ThreadNameFixingPrintStreamInfoStream(System.out));
    } else if (v) {
      InfoStream.setDefault(new NullInfoStream());
    }

    Class<?> targetClass = RandomizedContext.current().getTargetClass();
    avoidCodecs = new HashSet<String>();
    if (targetClass.isAnnotationPresent(SuppressCodecs.class)) {
      SuppressCodecs a = targetClass.getAnnotation(SuppressCodecs.class);
      avoidCodecs.addAll(Arrays.asList(a.value()));
    }

    // set back to default
    LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = false;
    LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = false;

    savedCodec = Codec.getDefault();
    int randomVal = random.nextInt(10);
    if ("Lucene3x".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && "random".equals(TEST_POSTINGSFORMAT)
            && "random".equals(TEST_DOCVALUESFORMAT)
            && randomVal == 3
            && !shouldAvoidCodec("Lucene3x"))) { // preflex-only setup
      codec = Codec.forName("Lucene3x");
      assert (codec instanceof PreFlexRWCodec)
          : "fix your classpath to have tests-framework.jar before lucene-core.jar";
      LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true;
    } else if ("Lucene40".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && "random".equals(TEST_POSTINGSFORMAT)
            && randomVal == 0
            && !shouldAvoidCodec("Lucene40"))) { // 4.0 setup
      codec = Codec.forName("Lucene40");
      LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
      assert codec instanceof Lucene40RWCodec
          : "fix your classpath to have tests-framework.jar before lucene-core.jar";
      assert (PostingsFormat.forName("Lucene40") instanceof Lucene40RWPostingsFormat)
          : "fix your classpath to have tests-framework.jar before lucene-core.jar";
    } else if ("Lucene41".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && "random".equals(TEST_POSTINGSFORMAT)
            && "random".equals(TEST_DOCVALUESFORMAT)
            && randomVal == 1
            && !shouldAvoidCodec("Lucene41"))) {
      codec = Codec.forName("Lucene41");
      LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
      assert codec instanceof Lucene41RWCodec
          : "fix your classpath to have tests-framework.jar before lucene-core.jar";
    } else if ("Lucene42".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && "random".equals(TEST_POSTINGSFORMAT)
            && "random".equals(TEST_DOCVALUESFORMAT)
            && randomVal == 2
            && !shouldAvoidCodec("Lucene42"))) {
      codec = Codec.forName("Lucene42");
      LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
      assert codec instanceof Lucene42RWCodec
          : "fix your classpath to have tests-framework.jar before lucene-core.jar";
    } else if ("Lucene45".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && "random".equals(TEST_POSTINGSFORMAT)
            && "random".equals(TEST_DOCVALUESFORMAT)
            && randomVal == 5
            && !shouldAvoidCodec("Lucene45"))) {
      codec = Codec.forName("Lucene45");
      LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE = true;
      assert codec instanceof Lucene45RWCodec
          : "fix your classpath to have tests-framework.jar before lucene-core.jar";
    } else if (("random".equals(TEST_POSTINGSFORMAT) == false)
        || ("random".equals(TEST_DOCVALUESFORMAT) == false)) {
      // the user wired postings or DV: this is messy
      // refactor into RandomCodec....

      final PostingsFormat format;
      if ("random".equals(TEST_POSTINGSFORMAT)) {
        format = PostingsFormat.forName("Lucene41");
      } else {
        format = PostingsFormat.forName(TEST_POSTINGSFORMAT);
      }

      final DocValuesFormat dvFormat;
      if ("random".equals(TEST_DOCVALUESFORMAT)) {
        dvFormat = DocValuesFormat.forName("Lucene45");
      } else {
        dvFormat = DocValuesFormat.forName(TEST_DOCVALUESFORMAT);
      }

      codec =
          new Lucene46Codec() {
            @Override
            public PostingsFormat getPostingsFormatForField(String field) {
              return format;
            }

            @Override
            public DocValuesFormat getDocValuesFormatForField(String field) {
              return dvFormat;
            }

            @Override
            public String toString() {
              return super.toString() + ": " + format.toString() + ", " + dvFormat.toString();
            }
          };
    } else if ("SimpleText".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && randomVal == 9
            && LuceneTestCase.rarely(random)
            && !shouldAvoidCodec("SimpleText"))) {
      codec = new SimpleTextCodec();
    } else if ("Appending".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC) && randomVal == 8 && !shouldAvoidCodec("Appending"))) {
      codec = new AppendingRWCodec();
      LuceneTestCase.OLD_FORMAT_IMPERSONATION_IS_ACTIVE =
          true; // this is really just Lucene40 with some minor changes
    } else if ("CheapBastard".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC)
            && randomVal == 8
            && !shouldAvoidCodec("CheapBastard")
            && !shouldAvoidCodec("Lucene41"))) {
      // we also avoid this codec if Lucene41 is avoided, since thats the postings format it uses.
      codec = new CheapBastardCodec();
    } else if ("Asserting".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Asserting"))) {
      codec = new AssertingCodec();
    } else if ("Compressing".equals(TEST_CODEC)
        || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Compressing"))) {
      codec = CompressingCodec.randomInstance(random);
    } else if (!"random".equals(TEST_CODEC)) {
      codec = Codec.forName(TEST_CODEC);
    } else if ("random".equals(TEST_POSTINGSFORMAT)) {
      codec = new RandomCodec(random, avoidCodecs);
    } else {
      assert false;
    }
    Codec.setDefault(codec);

    // Initialize locale/ timezone.
    String testLocale = System.getProperty("tests.locale", "random");
    String testTimeZone = System.getProperty("tests.timezone", "random");

    // Always pick a random one for consistency (whether tests.locale was specified or not).
    savedLocale = Locale.getDefault();
    Locale randomLocale = randomLocale(random);
    locale = testLocale.equals("random") ? randomLocale : localeForName(testLocale);
    Locale.setDefault(locale);

    // TimeZone.getDefault will set user.timezone to the default timezone of the user's locale.
    // So store the original property value and restore it at end.
    restoreProperties.put("user.timezone", System.getProperty("user.timezone"));
    savedTimeZone = TimeZone.getDefault();
    TimeZone randomTimeZone = randomTimeZone(random());
    timeZone = testTimeZone.equals("random") ? randomTimeZone : TimeZone.getTimeZone(testTimeZone);
    TimeZone.setDefault(timeZone);
    similarity =
        random().nextBoolean() ? new DefaultSimilarity() : new RandomSimilarityProvider(random());

    // Check codec restrictions once at class level.
    try {
      checkCodecRestrictions(codec);
    } catch (AssumptionViolatedException e) {
      System.err.println(
          "NOTE: "
              + e.getMessage()
              + " Suppressed codecs: "
              + Arrays.toString(avoidCodecs.toArray()));
      throw e;
    }
  }
  public void testMerge() throws IOException {
    final Codec codec = Codec.getDefault();
    final SegmentInfo si =
        new SegmentInfo(
            mergedDir,
            Version.LATEST,
            mergedSegment,
            -1,
            false,
            codec,
            Collections.emptyMap(),
            StringHelper.randomId(),
            new HashMap<>());

    SegmentMerger merger =
        new SegmentMerger(
            Arrays.<CodecReader>asList(reader1, reader2),
            si,
            InfoStream.getDefault(),
            mergedDir,
            new FieldInfos.FieldNumbers(),
            newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1))));
    MergeState mergeState = merger.merge();
    int docsMerged = mergeState.segmentInfo.maxDoc();
    assertTrue(docsMerged == 2);
    // Should be able to open a new SegmentReader against the new directory
    SegmentReader mergedReader =
        new SegmentReader(
            new SegmentCommitInfo(mergeState.segmentInfo, 0, -1L, -1L, -1L),
            newIOContext(random()));
    assertTrue(mergedReader != null);
    assertTrue(mergedReader.numDocs() == 2);
    Document newDoc1 = mergedReader.document(0);
    assertTrue(newDoc1 != null);
    // There are 2 unstored fields on the document
    assertTrue(
        DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
    Document newDoc2 = mergedReader.document(1);
    assertTrue(newDoc2 != null);
    assertTrue(
        DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());

    PostingsEnum termDocs =
        TestUtil.docs(
            random(), mergedReader, DocHelper.TEXT_FIELD_2_KEY, new BytesRef("field"), null, 0);
    assertTrue(termDocs != null);
    assertTrue(termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

    int tvCount = 0;
    for (FieldInfo fieldInfo : mergedReader.getFieldInfos()) {
      if (fieldInfo.hasVectors()) {
        tvCount++;
      }
    }

    // System.out.println("stored size: " + stored.size());
    assertEquals("We do not have 3 fields that were indexed with term vector", 3, tvCount);

    Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
    assertNotNull(vector);
    assertEquals(3, vector.size());
    TermsEnum termsEnum = vector.iterator();

    int i = 0;
    while (termsEnum.next() != null) {
      String term = termsEnum.term().utf8ToString();
      int freq = (int) termsEnum.totalTermFreq();
      // System.out.println("Term: " + term + " Freq: " + freq);
      assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
      assertTrue(DocHelper.FIELD_2_FREQS[i] == freq);
      i++;
    }

    TestSegmentReader.checkNorms(mergedReader);
    mergedReader.close();
  }