public void testRandomReads() throws IOException { int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = newBytesReference(length); StreamInput streamInput = pbr.streamInput(); BytesRefBuilder target = new BytesRefBuilder(); while (target.length() < pbr.length()) { switch (randomIntBetween(0, 10)) { case 6: case 5: target.append(new BytesRef(new byte[] {streamInput.readByte()})); break; case 4: case 3: BytesRef bytesRef = streamInput.readBytesRef(scaledRandomIntBetween(1, pbr.length() - target.length())); target.append(bytesRef); break; default: byte[] buffer = new byte[scaledRandomIntBetween(1, pbr.length() - target.length())]; int offset = scaledRandomIntBetween(0, buffer.length - 1); int read = streamInput.read(buffer, offset, buffer.length - offset); target.append(new BytesRef(buffer, offset, read)); break; } } assertEquals(pbr.length(), target.length()); BytesRef targetBytes = target.get(); assertArrayEquals( pbr.toBytes(), Arrays.copyOfRange(targetBytes.bytes, targetBytes.offset, targetBytes.length)); }
public void testWriteStreamableList() throws IOException { final int size = randomIntBetween(0, 5); final List<TestStreamable> expected = new ArrayList<>(size); for (int i = 0; i < size; ++i) { expected.add(new TestStreamable(randomBoolean())); } final BytesStreamOutput out = new BytesStreamOutput(); out.writeStreamableList(expected); final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); final List<TestStreamable> loaded = in.readStreamableList(TestStreamable::new); assertThat(loaded, hasSize(expected.size())); for (int i = 0; i < expected.size(); ++i) { assertEquals(expected.get(i).value, loaded.get(i).value); } assertEquals(0, in.available()); in.close(); out.close(); }
public void testWriteableReaderReturnsWrongName() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry( Collections.singletonList( new NamedWriteableRegistry.Entry( BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> new TestNamedWriteable(in) { @Override public String getWriteableName() { return "intentionally-broken"; } }))); TestNamedWriteable namedWriteableIn = new TestNamedWriteable( randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); AssertionError e = expectThrows(AssertionError.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); assertThat( e.getMessage(), endsWith( " claims to have a different name [intentionally-broken] than it was read from [test-named-writeable].")); }
@Override public Mapper parse(ParseContext context) throws IOException { QueryShardContext queryShardContext = new QueryShardContext(this.queryShardContext); if (context.doc().getField(queryBuilderField.name()) != null) { // If a percolator query has been defined in an array object then multiple percolator queries // could be provided. In order to prevent this we fail if we try to parse more than one query // for the current document. throw new IllegalArgumentException("a document can only contain one percolator query"); } XContentParser parser = context.parser(); QueryBuilder queryBuilder = parseQueryBuilder(queryShardContext.newParseContext(parser), parser.getTokenLocation()); verifyQuery(queryBuilder); // Fetching of terms, shapes and indexed scripts happen during this rewrite: queryBuilder = queryBuilder.rewrite(queryShardContext); try (XContentBuilder builder = XContentFactory.contentBuilder(QUERY_BUILDER_CONTENT_TYPE)) { queryBuilder.toXContent(builder, new MapParams(Collections.emptyMap())); builder.flush(); byte[] queryBuilderAsBytes = BytesReference.toBytes(builder.bytes()); context .doc() .add( new Field( queryBuilderField.name(), queryBuilderAsBytes, queryBuilderField.fieldType())); } Query query = toQuery(queryShardContext, mapUnmappedFieldAsString, queryBuilder); processQuery(query, context); return null; }
// we ignore this test for now since all existing callers of BytesStreamOutput happily // call bytes() after close(). @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/12620") public void testAccessAfterClose() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); // immediately close out.close(); assertEquals(-1, out.size()); assertEquals(-1, out.position()); // writing a single byte must fail try { out.writeByte((byte) 0); fail("expected IllegalStateException: stream closed"); } catch (IllegalStateException iex1) { // expected } // writing in bulk must fail try { out.writeBytes(new byte[0], 0, 0); fail("expected IllegalStateException: stream closed"); } catch (IllegalStateException iex1) { // expected } // toByteArray() must fail try { BytesReference.toBytes(out.bytes()); fail("expected IllegalStateException: stream closed"); } catch (IllegalStateException iex1) { // expected } }
public void testNamedWriteableUnknownNamedWriteable() throws IOException { NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.register( BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); BytesStreamOutput out = new BytesStreamOutput(); out.writeNamedWriteable( new NamedWriteable() { @Override public String getWriteableName() { return "unknown"; } @Override public void writeTo(StreamOutput out) throws IOException {} }); StreamInput in = new NamedWriteableAwareStreamInput( StreamInput.wrap(BytesReference.toBytes(out.bytes())), namedWriteableRegistry); try { // no named writeable registered with given name under test category, can write but cannot // read it back in.readNamedWriteable(BaseNamedWriteable.class); fail("read should have failed"); } catch (IllegalArgumentException e) { assertThat( e.getMessage(), equalTo("unknown named writeable [" + BaseNamedWriteable.class.getName() + "][unknown]")); } }
public void testStreamInputBulkReadWithOffset() throws IOException { final int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = newBytesReference(length); StreamInput si = pbr.streamInput(); assertNotNull(si); // read a bunch of single bytes one by one int offset = randomIntBetween(1, length / 2); for (int i = 0; i < offset; i++) { assertEquals(si.available(), length - i); assertEquals(pbr.get(i), si.readByte()); } // now do NOT reset the stream - keep the stream's offset! // buffer to compare remaining bytes against bulk read byte[] pbrBytesWithOffset = Arrays.copyOfRange(pbr.toBytes(), offset, length); // randomized target buffer to ensure no stale slots byte[] targetBytes = new byte[pbrBytesWithOffset.length]; random().nextBytes(targetBytes); // bulk-read all si.readFully(targetBytes); assertArrayEquals(pbrBytesWithOffset, targetBytes); assertEquals(si.available(), 0); }
public void testSliceStreamInput() throws IOException { int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = newBytesReference(length); // test stream input over slice (upper half of original) int sliceOffset = randomIntBetween(1, length / 2); int sliceLength = length - sliceOffset; BytesReference slice = pbr.slice(sliceOffset, sliceLength); StreamInput sliceInput = slice.streamInput(); assertEquals(sliceInput.available(), sliceLength); // single reads assertEquals(slice.get(0), sliceInput.readByte()); assertEquals(slice.get(1), sliceInput.readByte()); assertEquals(slice.get(2), sliceInput.readByte()); assertEquals(sliceInput.available(), sliceLength - 3); // reset the slice stream for bulk reading sliceInput.reset(); assertEquals(sliceInput.available(), sliceLength); // bulk read byte[] sliceBytes = new byte[sliceLength]; sliceInput.readFully(sliceBytes); assertEquals(sliceInput.available(), 0); // compare slice content with upper half of original byte[] pbrSliceBytes = Arrays.copyOfRange(pbr.toBytes(), sliceOffset, length); assertArrayEquals(pbrSliceBytes, sliceBytes); // compare slice bytes with bytes read from slice via streamInput :D byte[] sliceToBytes = slice.toBytes(); assertEquals(sliceBytes.length, sliceToBytes.length); assertArrayEquals(sliceBytes, sliceToBytes); sliceInput.reset(); assertEquals(sliceInput.available(), sliceLength); byte[] buffer = new byte[sliceLength + scaledRandomIntBetween(1, 100)]; int offset = scaledRandomIntBetween(0, Math.max(1, buffer.length - sliceLength - 1)); int read = sliceInput.read(buffer, offset, sliceLength / 2); assertEquals(sliceInput.available(), sliceLength - read); sliceInput.read(buffer, offset + read, sliceLength - read); assertArrayEquals(sliceBytes, Arrays.copyOfRange(buffer, offset, offset + sliceLength)); assertEquals(sliceInput.available(), 0); }
public void testWriteToOutputStream() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * 4); BytesReference pbr = newBytesReference(length); BytesStreamOutput out = new BytesStreamOutput(); pbr.writeTo(out); assertEquals(pbr.length(), out.size()); assertArrayEquals(pbr.toBytes(), out.bytes().toBytes()); out.close(); }
public void testSliceEquals() { int length = randomIntBetween(100, PAGE_SIZE * randomIntBetween(2, 5)); ByteArray ba1 = bigarrays.newByteArray(length, false); BytesReference pbr = new PagedBytesReference(bigarrays, ba1, length); // test equality of slices int sliceFrom = randomIntBetween(0, pbr.length()); int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom); BytesReference slice1 = pbr.slice(sliceFrom, sliceLength); BytesReference slice2 = pbr.slice(sliceFrom, sliceLength); assertArrayEquals(slice1.toBytes(), slice2.toBytes()); // test a slice with same offset but different length, // unless randomized testing gave us a 0-length slice. if (sliceLength > 0) { BytesReference slice3 = pbr.slice(sliceFrom, sliceLength / 2); assertFalse(Arrays.equals(slice1.toBytes(), slice3.toBytes())); } }
public void testHashCode() throws IOException { // empty content must have hash 1 (JDK compat) BytesReference pbr = newBytesReference(0); assertEquals(Arrays.hashCode(BytesRef.EMPTY_BYTES), pbr.hashCode()); // test with content pbr = newBytesReference(randomIntBetween(0, PAGE_SIZE * randomIntBetween(2, 5))); int jdkHash = Arrays.hashCode(pbr.toBytes()); int pbrHash = pbr.hashCode(); assertEquals(jdkHash, pbrHash); // test hashes of slices int sliceFrom = randomIntBetween(0, pbr.length()); int sliceLength = randomIntBetween(pbr.length() - sliceFrom, pbr.length() - sliceFrom); BytesReference slice = pbr.slice(sliceFrom, sliceLength); int sliceJdkHash = Arrays.hashCode(slice.toBytes()); int sliceHash = slice.hashCode(); assertEquals(sliceJdkHash, sliceHash); }
public void testToBytes() throws IOException { int[] sizes = { 0, randomInt(PAGE_SIZE), PAGE_SIZE, randomIntBetween(2, PAGE_SIZE * randomIntBetween(2, 5)) }; for (int i = 0; i < sizes.length; i++) { BytesReference pbr = newBytesReference(sizes[i]); byte[] bytes = pbr.toBytes(); assertEquals(sizes[i], bytes.length); } }
/** The source of the document if exists. */ public byte[] source() { if (source == null) { return null; } if (sourceAsBytes != null) { return sourceAsBytes; } this.sourceAsBytes = BytesReference.toBytes(sourceRef()); return this.sourceAsBytes; }
public void testSingleShortPageBulkWrite() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); // first bulk-write empty array: should not change anything int expectedSize = 0; byte[] expectedData = randomizedByteArrayWithSize(expectedSize); out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); // bulk-write again with actual bytes expectedSize = 10; expectedData = randomizedByteArrayWithSize(expectedSize); out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); }
public void testIterator() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); BytesRefIterator iterator = pbr.iterator(); BytesRef ref; BytesRefBuilder builder = new BytesRefBuilder(); while ((ref = iterator.next()) != null) { builder.append(ref); } assertArrayEquals(pbr.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); }
public void testSliceWriteToOutputStream() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 5)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(1, length / 2); int sliceLength = length - sliceOffset; BytesReference slice = pbr.slice(sliceOffset, sliceLength); BytesStreamOutput sliceOut = new BytesStreamOutput(sliceLength); slice.writeTo(sliceOut); assertEquals(slice.length(), sliceOut.size()); assertArrayEquals(slice.toBytes(), sliceOut.bytes().toBytes()); sliceOut.close(); }
public void testNamedWriteableNotSupportedWithoutWrapping() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); TestNamedWriteable testNamedWriteable = new TestNamedWriteable("test1", "test2"); out.writeNamedWriteable(testNamedWriteable); StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); try { in.readNamedWriteable(BaseNamedWriteable.class); fail("Expected UnsupportedOperationException"); } catch (UnsupportedOperationException e) { assertThat(e.getMessage(), is("can't read named writeable from StreamInput")); } }
public void testSingleFullPageBulkWrite() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); int expectedSize = BigArrays.BYTE_PAGE_SIZE; byte[] expectedData = randomizedByteArrayWithSize(expectedSize); // write in bulk out.writeBytes(expectedData); assertEquals(expectedSize, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); }
public void testOptionalWriteableReaderReturnsNull() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); out.writeOptionalWriteable( new TestNamedWriteable( randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10))); StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); IOException e = expectThrows( IOException.class, () -> in.readOptionalWriteable((StreamInput ignored) -> null)); assertThat( e.getMessage(), endsWith( "] returned null which is not allowed and probably means it screwed up the stream.")); }
public void testSingleByte() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); assertEquals(0, out.size()); int expectedSize = 1; byte[] expectedData = randomizedByteArrayWithSize(expectedSize); // write single byte out.writeByte(expectedData[0]); assertEquals(expectedSize, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); }
public void testSliceIterator() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length()); int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); BytesRefIterator iterator = slice.iterator(); BytesRef ref = null; BytesRefBuilder builder = new BytesRefBuilder(); while ((ref = iterator.next()) != null) { builder.append(ref); } assertArrayEquals(slice.toBytes(), BytesRef.deepCopyOf(builder.toBytesRef()).bytes); }
public void testSeek() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); int position = 0; assertEquals(position, out.position()); out.seek(position += 10); out.seek(position += BigArrays.BYTE_PAGE_SIZE); out.seek(position += BigArrays.BYTE_PAGE_SIZE + 10); out.seek(position += BigArrays.BYTE_PAGE_SIZE * 2); assertEquals(position, out.position()); assertEquals(position, BytesReference.toBytes(out.bytes()).length); out.close(); }
public void testStreamInput() throws IOException { int length = randomIntBetween(10, scaledRandomIntBetween(PAGE_SIZE * 2, PAGE_SIZE * 20)); BytesReference pbr = newBytesReference(length); StreamInput si = pbr.streamInput(); assertNotNull(si); // read single bytes one by one assertEquals(pbr.get(0), si.readByte()); assertEquals(pbr.get(1), si.readByte()); assertEquals(pbr.get(2), si.readByte()); // reset the stream for bulk reading si.reset(); // buffer for bulk reads byte[] origBuf = new byte[length]; random().nextBytes(origBuf); byte[] targetBuf = Arrays.copyOf(origBuf, origBuf.length); // bulk-read 0 bytes: must not modify buffer si.readBytes(targetBuf, 0, 0); assertEquals(origBuf[0], targetBuf[0]); si.reset(); // read a few few bytes as ints int bytesToRead = randomIntBetween(1, length / 2); for (int i = 0; i < bytesToRead; i++) { int b = si.read(); assertEquals(pbr.get(i) & 0xff, b); } si.reset(); // bulk-read all si.readFully(targetBuf); assertArrayEquals(pbr.toBytes(), targetBuf); // continuing to read should now fail with EOFException try { si.readByte(); fail("expected EOF"); } catch (EOFException | IndexOutOfBoundsException eof) { // yay } // try to read more than the stream contains si.reset(); expectThrows(IndexOutOfBoundsException.class, () -> si.readBytes(targetBuf, 0, length * 2)); }
public void testSingleShortPage() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); int expectedSize = 10; byte[] expectedData = randomizedByteArrayWithSize(expectedSize); // write byte-by-byte for (int i = 0; i < expectedSize; i++) { out.writeByte(expectedData[i]); } assertEquals(expectedSize, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); }
public void testSliceCopyBytesArray() throws IOException { int length = randomIntBetween(10, PAGE_SIZE * randomIntBetween(2, 8)); BytesReference pbr = newBytesReference(length); int sliceOffset = randomIntBetween(0, pbr.length()); int sliceLength = randomIntBetween(pbr.length() - sliceOffset, pbr.length() - sliceOffset); BytesReference slice = pbr.slice(sliceOffset, sliceLength); BytesArray ba1 = slice.copyBytesArray(); BytesArray ba2 = slice.copyBytesArray(); assertNotNull(ba1); assertNotNull(ba2); assertNotSame(ba1.array(), ba2.array()); assertArrayEquals(slice.toBytes(), ba1.array()); assertArrayEquals(slice.toBytes(), ba2.array()); assertArrayEquals(ba1.array(), ba2.array()); }
public void testNamedWriteable() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.register( BaseNamedWriteable.class, TestNamedWriteable.NAME, TestNamedWriteable::new); TestNamedWriteable namedWriteableIn = new TestNamedWriteable( randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); BaseNamedWriteable namedWriteableOut = in.readNamedWriteable(BaseNamedWriteable.class); assertEquals(namedWriteableOut, namedWriteableIn); assertEquals(in.available(), 0); }
public void testSingleFullPageBulkWriteWithOffset() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); int initialOffset = 10; int additionalLength = BigArrays.BYTE_PAGE_SIZE; byte[] expectedData = randomizedByteArrayWithSize(initialOffset + additionalLength); // first create initial offset out.writeBytes(expectedData, 0, initialOffset); assertEquals(initialOffset, out.size()); // now write the rest - more than fits into the remaining first page out.writeBytes(expectedData, initialOffset, additionalLength); assertEquals(expectedData.length, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); }
public void testSingleFullPageBulkWriteWithOffsetCrossover() throws Exception { BytesStreamOutput out = new BytesStreamOutput(); int initialOffset = 10; int additionalLength = BigArrays.BYTE_PAGE_SIZE * 2; byte[] expectedData = randomizedByteArrayWithSize(initialOffset + additionalLength); out.writeBytes(expectedData, 0, initialOffset); assertEquals(initialOffset, out.size()); // now write the rest - more than fits into the remaining page + a full page after // that, // ie. we cross over into a third out.writeBytes(expectedData, initialOffset, additionalLength); assertEquals(expectedData.length, out.size()); assertArrayEquals(expectedData, BytesReference.toBytes(out.bytes())); out.close(); }
public void testNamedWriteableReaderReturnsNull() throws IOException { BytesStreamOutput out = new BytesStreamOutput(); NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.register( BaseNamedWriteable.class, TestNamedWriteable.NAME, (StreamInput in) -> null); TestNamedWriteable namedWriteableIn = new TestNamedWriteable( randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); out.writeNamedWriteable(namedWriteableIn); byte[] bytes = BytesReference.toBytes(out.bytes()); StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(bytes), namedWriteableRegistry); assertEquals(in.available(), bytes.length); IOException e = expectThrows(IOException.class, () -> in.readNamedWriteable(BaseNamedWriteable.class)); assertThat( e.getMessage(), endsWith( "] returned null which is not allowed and probably means it screwed up the stream.")); }
public void testWriteMapOfLists() throws IOException { final int size = randomIntBetween(0, 5); final Map<String, List<String>> expected = new HashMap<>(size); for (int i = 0; i < size; ++i) { int listSize = randomIntBetween(0, 5); List<String> list = new ArrayList<>(listSize); for (int j = 0; j < listSize; ++j) { list.add(randomAsciiOfLength(5)); } expected.put(randomAsciiOfLength(2), list); } final BytesStreamOutput out = new BytesStreamOutput(); out.writeMapOfLists(expected); final StreamInput in = StreamInput.wrap(BytesReference.toBytes(out.bytes())); final Map<String, List<String>> loaded = in.readMapOfLists(); assertThat(loaded.size(), equalTo(expected.size())); for (Map.Entry<String, List<String>> entry : expected.entrySet()) { assertThat(loaded.containsKey(entry.getKey()), equalTo(true)); List<String> loadedList = loaded.get(entry.getKey()); assertThat(loadedList, hasSize(entry.getValue().size())); for (int i = 0; i < loadedList.size(); ++i) { assertEquals(entry.getValue().get(i), loadedList.get(i)); } } assertEquals(0, in.available()); in.close(); out.close(); }