/**
* Regression test for KUDU-1226. Calls to session.flush() concurrent with AUTO_FLUSH_BACKGROUND
* can end up giving ConvertBatchToListOfResponsesCB a list with nulls if a tablet was already
* flushed. Only happens with multiple tablets.
*/
@Test(timeout = 10000)
public void testConcurrentFlushes() throws Exception {
String tableName = TABLE_NAME_PREFIX + "-testConcurrentFlushes";
CreateTableOptions builder = new CreateTableOptions();
int numTablets = 4;
int numRowsPerTablet = 100;
// Create a 4 tablets table split on 1000, 2000, and 3000.
for (int i = 1; i < numTablets; i++) {
PartialRow split = basicSchema.newPartialRow();
split.addInt(0, i * numRowsPerTablet);
builder.addSplitRow(split);
}
table = createTable(tableName, basicSchema, builder);
// Configure the session to background flush as often as it can (every 1ms).
KuduSession session = syncClient.newSession();
session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
session.setFlushInterval(1);
// Fill each tablet in parallel 1 by 1 then flush. Without the fix this would quickly get an
// NPE.
for (int i = 0; i < numRowsPerTablet; i++) {
for (int j = 0; j < numTablets; j++) {
session.apply(createInsert(i + (numRowsPerTablet * j)));
}
session.flush();
}
}
@Test
public void test() throws Exception {
createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
KuduTableOutputFormat output = new KuduTableOutputFormat();
Configuration conf = new Configuration();
conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, TABLE_NAME);
output.setConf(conf);
String multitonKey = conf.get(KuduTableOutputFormat.MULTITON_KEY);
KuduTable table = KuduTableOutputFormat.getKuduTable(multitonKey);
assertNotNull(table);
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, 1);
row.addInt(1, 2);
row.addInt(2, 3);
row.addString(3, "a string");
row.addBoolean(4, true);
RecordWriter<NullWritable, Operation> rw = output.getRecordWriter(null);
rw.write(NullWritable.get(), insert);
rw.close(null);
AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
assertEquals(1, countRowsInScan(builder.build()));
}
@Test(timeout = 100000)
public void testBatchWithSameRow() throws Exception {
String tableName = TABLE_NAME_PREFIX + "-testBatchWithSameRow";
table = createTable(tableName, basicSchema, new CreateTableOptions());
KuduSession session = syncClient.newSession();
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
// Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end,
// while also clearing the cache between each batch half the time. The delete is added here
// so that a misplaced update would fail if it happens later than its delete.
for (int i = 0; i < 25; i++) {
session.apply(createInsert(i));
for (int j = 0; j < 50; j++) {
Update update = table.newUpdate();
PartialRow row = update.getRow();
row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000);
session.apply(update);
}
Delete del = table.newDelete();
PartialRow row = del.getRow();
row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
session.apply(del);
session.flush();
if (i % 2 == 0) {
client.emptyTabletsCacheForTable(table.getTableId());
}
}
assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
}
private void setUpdateValue(String mid, String[] fields, String[] fieldsValue, PartialRow row) {
row.addString("mid", mid);
int i = 0;
for (; i < fields.length - 1; i++) {
row.addString(fields[i], fieldsValue[i]);
}
row.addString(fields[i], fieldsValue[i + 1]);
}
protected Insert createBasicSchemaInsert(KuduTable table, int key) {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, key);
row.addInt(1, 2);
row.addInt(2, 3);
row.addString(3, "a string");
row.addBoolean(4, true);
return insert;
}
protected static KuduTable createFourTabletsTableWithNineRows(String tableName) throws Exception {
CreateTableBuilder builder = new CreateTableBuilder();
PartialRow splitRow = basicSchema.newPartialRow();
for (int i : KEYS) {
splitRow.addInt(0, i);
builder.addSplitRow(splitRow);
}
KuduTable table = createTable(tableName, basicSchema, builder);
AsyncKuduSession session = client.newSession();
// create a table with on empty tablet and 3 tablets of 3 rows each
for (int key1 : KEYS) {
for (int key2 = 1; key2 <= 3; key2++) {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, key1 + key2);
row.addInt(1, key1);
row.addInt(2, key2);
row.addString(3, "a string");
row.addBoolean(4, true);
session.apply(insert).join(DEFAULT_SLEEP);
}
}
session.close().join(DEFAULT_SLEEP);
return table;
}
@Test(timeout = 10000)
public void testOverWritingValues() throws Exception {
String tableName = TABLE_NAME_PREFIX + "-OverridingValues";
table = createTable(tableName, basicSchema, null);
KuduSession session = syncClient.newSession();
Insert insert = createInsert(0);
PartialRow row = insert.getRow();
// Overwrite all the normal columns.
int magicNumber = 9999;
row.addInt(1, magicNumber);
row.addInt(2, magicNumber);
row.addBoolean(4, false);
// Spam the string column since it's backed by an array.
for (int i = 0; i <= magicNumber; i++) {
row.addString(3, i + "");
}
// We're supposed to keep a constant size.
assertEquals(5, row.getVarLengthData().size());
session.apply(insert);
KuduScanner scanner = syncClient.newScannerBuilder(table).build();
RowResult rr = scanner.nextRows().next();
assertEquals(magicNumber, rr.getInt(1));
assertEquals(magicNumber, rr.getInt(2));
assertEquals(magicNumber + "", rr.getString(3));
assertEquals(false, rr.getBoolean(4));
// Test setting a value post-apply.
try {
row.addInt(1, 0);
fail("Row should be frozen and throw");
} catch (IllegalStateException ex) {
// Ok.
}
}
/**
* 设置Kudu Row
*
* @param mid
* @param fields
* @param fieldsValue
* @param row
*/
private void setInsertValue(String mid, String[] fields, String[] fieldsValue, PartialRow row) {
row.addString("mid", mid);
for (int i = 0; i < fields.length; i++) {
row.addString(fields[i], fieldsValue[i]);
}
}
/**
* Add an upper bound (exclusive) primary key for the scan. If any bound is already added, this
* bound is intersected with that one.
*
* @param partialRow a partial row with specified key columns
* @return this instance
*/
public S exclusiveUpperBound(PartialRow partialRow) {
return exclusiveUpperBoundRaw(partialRow.encodePrimaryKey());
}
/**
* Add a lower bound (inclusive) primary key for the scan. If any bound is already added, this
* bound is intersected with that one.
*
* @param partialRow a partial row with specified key columns
* @return this instance
*/
public S lowerBound(PartialRow partialRow) {
return lowerBoundRaw(partialRow.encodePrimaryKey());
}
