@Test
public void percentileTestSpecific()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
Aggregations result =
query(String.format("SELECT PERCENTILES(age,25.0,75.0) x FROM %s/account", TEST_INDEX));
Percentiles percentiles = result.get("x");
Assert.assertTrue(Math.abs(percentiles.percentile(25.0) - 25.0) < 0.001);
Assert.assertTrue(Math.abs(percentiles.percentile(75.0) - 35.0) < 0.001);
}
@Test
public void extendedStatsTest()
throws IOException, SqlParseException, SQLFeatureNotSupportedException {
Aggregations result =
query(String.format("SELECT EXTENDED_STATS(age) FROM %s/account", TEST_INDEX));
ExtendedStats stats = result.get("EXTENDED_STATS(age)");
Assert.assertEquals(1000, stats.getCount());
assertThat(stats.getMin(), equalTo(20.0));
assertThat(stats.getMax(), equalTo(40.0));
assertThat(stats.getAvg(), equalTo(30.171));
assertThat(stats.getSum(), equalTo(30171.0));
assertThat(stats.getSumOfSquares(), equalTo(946393.0));
Assert.assertTrue(Math.abs(stats.getStdDeviation() - 6.008640362012022) < 0.0001);
Assert.assertTrue(Math.abs(stats.getVariance() - 36.10375899999996) < 0.0001);
}
private void readIndexInputFullyWithRandomSeeks(IndexInput indexInput) throws IOException {
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long pos = 0;
while (pos < indexInput.length()) {
assertEquals(pos, indexInput.getFilePointer());
int op = random().nextInt(5);
if (op == 0) {
int shift = 100 - randomIntBetween(0, 200);
pos = Math.min(indexInput.length() - 1, Math.max(0, pos + shift));
indexInput.seek(pos);
} else if (op == 1) {
indexInput.readByte();
pos++;
} else {
int min = (int) Math.min(indexInput.length() - pos, ref.bytes.length);
indexInput.readBytes(ref.bytes, ref.offset, min);
pos += min;
}
}
}
private void put(File target, long size) throws IOException {
byte[] buf = "Hello, world\n".getBytes();
long rest = size;
target.getParentFile().mkdirs();
OutputStream out = new FileOutputStream(target);
try {
OutputStream bufferred = new BufferedOutputStream(out);
while (rest > 0) {
int count = (int) Math.min(buf.length, rest);
bufferred.write(buf, 0, count);
rest -= count;
}
bufferred.close();
} finally {
out.close();
}
}
private void corruptFile(Directory dir, String fileIn, String fileOut) throws IOException {
IndexInput input = dir.openInput(fileIn, IOContext.READONCE);
IndexOutput output = dir.createOutput(fileOut, IOContext.DEFAULT);
long len = input.length();
byte[] b = new byte[1024];
long broken = randomInt((int) len);
long pos = 0;
while (pos < len) {
int min = (int) Math.min(input.length() - pos, b.length);
input.readBytes(b, 0, min);
if (broken >= pos && broken < pos + min) {
// Flip one byte
int flipPos = (int) (broken - pos);
b[flipPos] = (byte) (b[flipPos] ^ 42);
}
output.writeBytes(b, min);
pos += min;
}
IOUtils.close(input, output);
}
@Test
public void testVerifyingIndexOutput() throws IOException {
Directory dir = newDirectory();
IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
int iters = scaledRandomIntBetween(10, 100);
for (int i = 0; i < iters; i++) {
BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
}
CodecUtil.writeFooter(output);
output.close();
IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
indexInput.seek(0);
BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
long length = indexInput.length();
IndexOutput verifyingOutput =
new Store.LuceneVerifyingIndexOutput(
new StoreFileMetaData("foo1.bar", length, checksum),
dir.createOutput("foo1.bar", IOContext.DEFAULT));
while (length > 0) {
if (random().nextInt(10) == 0) {
verifyingOutput.writeByte(indexInput.readByte());
length--;
} else {
int min = (int) Math.min(length, ref.bytes.length);
indexInput.readBytes(ref.bytes, ref.offset, min);
verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
length -= min;
}
}
Store.verify(verifyingOutput);
verifyingOutput.writeByte((byte) 0x0);
try {
Store.verify(verifyingOutput);
fail("should be a corrupted index");
} catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
// ok
}
IOUtils.close(indexInput, verifyingOutput, dir);
}
/** Test method for {@link GroupElement#toRadix16(byte[])}. */
@Test
public void testToRadix16() {
assertThat(GroupElement.toRadix16(BYTES_ZERO), is(RADIX16_ZERO));
assertThat(GroupElement.toRadix16(BYTES_ONE), is(RADIX16_ONE));
assertThat(GroupElement.toRadix16(BYTES_42), is(RADIX16_42));
byte[] from1234567890 = GroupElement.toRadix16(BYTES_1234567890);
int total = 0;
for (int i = 0; i < from1234567890.length; i++) {
assertThat(from1234567890[i], is(greaterThanOrEqualTo((byte) -8)));
assertThat(from1234567890[i], is(lessThanOrEqualTo((byte) 8)));
total += from1234567890[i] * Math.pow(16, i);
}
assertThat(total, is(1234567890));
byte[] pkrR16 = GroupElement.toRadix16(BYTES_PKR);
for (int i = 0; i < pkrR16.length; i++) {
assertThat(pkrR16[i], is(greaterThanOrEqualTo((byte) -8)));
assertThat(pkrR16[i], is(lessThanOrEqualTo((byte) 8)));
}
}
@Test
public void updateMappingConcurrently() throws Throwable {
// Test that we can concurrently update different indexes and types.
int shardNo = Math.max(5, cluster().size());
prepareCreate("test1").setSettings("index.number_of_shards", shardNo).execute().actionGet();
prepareCreate("test2").setSettings("index.number_of_shards", shardNo).execute().actionGet();
// This is important. The test assumes all nodes are aware of all indices. Due to initializing
// shard throttling
// not all shards are allocated with the initial create index. Wait for it..
ensureYellow();
final Throwable[] threadException = new Throwable[1];
final AtomicBoolean stop = new AtomicBoolean(false);
Thread[] threads = new Thread[3];
final CyclicBarrier barrier = new CyclicBarrier(threads.length);
final ArrayList<Client> clientArray = new ArrayList<Client>();
for (Client c : clients()) {
clientArray.add(c);
}
for (int j = 0; j < threads.length; j++) {
threads[j] =
new Thread(
new Runnable() {
@SuppressWarnings("unchecked")
@Override
public void run() {
try {
barrier.await();
for (int i = 0; i < 100; i++) {
if (stop.get()) {
return;
}
Client client1 = clientArray.get(i % clientArray.size());
Client client2 = clientArray.get((i + 1) % clientArray.size());
String indexName = i % 2 == 0 ? "test2" : "test1";
String typeName = "type" + (i % 10);
String fieldName = Thread.currentThread().getName() + "_" + i;
PutMappingResponse response =
client1
.admin()
.indices()
.preparePutMapping(indexName)
.setType(typeName)
.setSource(
JsonXContent.contentBuilder()
.startObject()
.startObject(typeName)
.startObject("properties")
.startObject(fieldName)
.field("type", "string")
.endObject()
.endObject()
.endObject()
.endObject())
.get();
assertThat(response.isAcknowledged(), equalTo(true));
GetMappingsResponse getMappingResponse =
client2.admin().indices().prepareGetMappings(indexName).get();
ImmutableOpenMap<String, MappingMetaData> mappings =
getMappingResponse.getMappings().get(indexName);
assertThat(mappings.containsKey(typeName), equalTo(true));
assertThat(
((Map<String, Object>)
mappings.get(typeName).getSourceAsMap().get("properties"))
.keySet(),
Matchers.hasItem(fieldName));
}
} catch (Throwable t) {
threadException[0] = t;
stop.set(true);
}
}
});
threads[j].setName("t_" + j);
threads[j].start();
}
for (Thread t : threads) t.join();
if (threadException[0] != null) {
throw threadException[0];
}
}
public void writeRaw(ByteBuffer buf, int numBytes) throws IOException {
int len = Math.min(numBytes, buf.remaining());
byte arr[] = new byte[len];
buf.get(arr, 0, len);
out.write(arr);
}
private static int random(int n) {
return Math.abs(rnd.nextInt()) % n;
}