/**
* 单纯插入
*
* @param tablename
* @param mid
* @param fieldsValue
*/
private void insert(String tablename, String mid, String[] fieldsValue) {
checkSession();
KuduTable table;
if (Constants.UPUSERS_ATTR_TABLE.equals(tablename)) {
table = table_attr;
} else {
table = table_days;
}
try {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
setInsertValue(mid, Constants.DAYS_FIELDS, fieldsValue, row);
OperationResponse rsInsert = session.apply(insert);
if (rsInsert.hasRowError()) {
System.out.println(
"=======================================INSERT DATA:"
+ mid
+ "---------------"
+ Arrays.toString(fieldsValue)
+ rsInsert.getRowError().getMessage());
}
} catch (Exception e) {
collector.reportError(e);
}
}
@Test(timeout = 100000)
public void testBatchWithSameRow() throws Exception {
String tableName = TABLE_NAME_PREFIX + "-testBatchWithSameRow";
table = createTable(tableName, basicSchema, new CreateTableOptions());
KuduSession session = syncClient.newSession();
session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
// Insert 25 rows, one per batch, along with 50 updates for each, and a delete at the end,
// while also clearing the cache between each batch half the time. The delete is added here
// so that a misplaced update would fail if it happens later than its delete.
for (int i = 0; i < 25; i++) {
session.apply(createInsert(i));
for (int j = 0; j < 50; j++) {
Update update = table.newUpdate();
PartialRow row = update.getRow();
row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
row.addInt(basicSchema.getColumnByIndex(1).getName(), 1000);
session.apply(update);
}
Delete del = table.newDelete();
PartialRow row = del.getRow();
row.addInt(basicSchema.getColumnByIndex(0).getName(), i);
session.apply(del);
session.flush();
if (i % 2 == 0) {
client.emptyTabletsCacheForTable(table.getTableId());
}
}
assertEquals(0, countRowsInScan(client.newScannerBuilder(table).build()));
}
protected static KuduTable createFourTabletsTableWithNineRows(String tableName) throws Exception {
CreateTableBuilder builder = new CreateTableBuilder();
PartialRow splitRow = basicSchema.newPartialRow();
for (int i : KEYS) {
splitRow.addInt(0, i);
builder.addSplitRow(splitRow);
}
KuduTable table = createTable(tableName, basicSchema, builder);
AsyncKuduSession session = client.newSession();
// create a table with on empty tablet and 3 tablets of 3 rows each
for (int key1 : KEYS) {
for (int key2 = 1; key2 <= 3; key2++) {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, key1 + key2);
row.addInt(1, key1);
row.addInt(2, key2);
row.addString(3, "a string");
row.addBoolean(4, true);
session.apply(insert).join(DEFAULT_SLEEP);
}
}
session.close().join(DEFAULT_SLEEP);
return table;
}
@Test
public void test() throws Exception {
createTable(TABLE_NAME, getBasicSchema(), getBasicCreateTableOptions());
KuduTableOutputFormat output = new KuduTableOutputFormat();
Configuration conf = new Configuration();
conf.set(KuduTableOutputFormat.MASTER_ADDRESSES_KEY, getMasterAddresses());
conf.set(KuduTableOutputFormat.OUTPUT_TABLE_KEY, TABLE_NAME);
output.setConf(conf);
String multitonKey = conf.get(KuduTableOutputFormat.MULTITON_KEY);
KuduTable table = KuduTableOutputFormat.getKuduTable(multitonKey);
assertNotNull(table);
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, 1);
row.addInt(1, 2);
row.addInt(2, 3);
row.addString(3, "a string");
row.addBoolean(4, true);
RecordWriter<NullWritable, Operation> rw = output.getRecordWriter(null);
rw.write(NullWritable.get(), insert);
rw.close(null);
AsyncKuduScanner.AsyncKuduScannerBuilder builder = client.newScannerBuilder(table);
assertEquals(1, countRowsInScan(builder.build()));
}
/**
* 执行插入更新操作
*
* @param tablename
* @param mid
* @param fieldsValue
*/
private void insertUpdate(String tablename, String mid, String[] fieldsValue) {
checkSession();
KuduTable table;
if (Constants.UPUSERS_ATTR_TABLE.equals(tablename)) {
table = table_attr;
} else {
table = table_days;
}
try {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
setInsertValue(mid, Constants.ATTR_INSERT_FIELDS, fieldsValue, row);
OperationResponse rsInsert = session.apply(insert);
if (rsInsert.hasRowError()) {
if ("key already present".equals(rsInsert.getRowError().getMessage())) {
Update update = table.newUpdate();
PartialRow urow = update.getRow();
setUpdateValue(mid, Constants.ATTR_UPDATE_FIELDS, fieldsValue, urow);
OperationResponse rsUpdate = session.apply(update);
if (rsUpdate.hasRowError()) {
System.out.println(
"=======================================ERROR UPDATE :" + rsUpdate.getRowError());
} else {
System.out.println(
"=======================================UPDATE DATA:"
+ mid
+ ":"
+ Arrays.toString(fieldsValue));
}
} else {
System.out.println(
"=======================================ERROR INSERT :" + rsInsert.getRowError());
}
} else {
System.out.println(
"=======================================INSERT DATA:"
+ mid
+ ":"
+ Arrays.toString(fieldsValue));
}
} catch (Exception e) {
collector.reportError(e);
}
}
protected Insert createBasicSchemaInsert(KuduTable table, int key) {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, key);
row.addInt(1, 2);
row.addInt(2, 3);
row.addString(3, "a string");
row.addBoolean(4, true);
return insert;
}
/**
* Helper method to easily kill a tablet server that serves the given table's only tablet's
* leader. The currently running test case will be failed if there's more than one tablet, if the
* tablet has no leader after some retries, or if the tablet server was already killed.
*
* <p>This method is thread-safe.
*
* @param table a KuduTable which will get its single tablet's leader killed.
* @throws Exception
*/
protected static void killTabletLeader(KuduTable table) throws Exception {
LocatedTablet.Replica leader = null;
DeadlineTracker deadlineTracker = new DeadlineTracker();
deadlineTracker.setDeadline(DEFAULT_SLEEP);
while (leader == null) {
if (deadlineTracker.timedOut()) {
fail("Timed out while trying to find a leader for this table: " + table.getName());
}
List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
if (tablets.isEmpty() || tablets.size() > 1) {
fail(
"Currently only support killing leaders for tables containing 1 tablet, table "
+ table.getName()
+ " has "
+ tablets.size());
}
LocatedTablet tablet = tablets.get(0);
if (tablet.getReplicas().size() == 1) {
fail("Table " + table.getName() + " only has 1 tablet, please enable replication");
}
leader = tablet.getLeaderReplica();
if (leader == null) {
LOG.info(
"Sleeping while waiting for a tablet LEADER to arise, currently slept "
+ deadlineTracker.getElapsedMillis()
+ "ms");
Thread.sleep(50);
}
}
Integer port = leader.getRpcPort();
Process ts = TABLET_SERVERS.get(port);
if (ts == null) {
// The TS is already dead, good.
return;
}
LOG.info("Killing server at port " + port);
ts.destroy();
ts.waitFor();
TABLET_SERVERS.remove(port);
}