@Test
public void testBulkTableUpdateCommandUpdateRecordToExistingRow() throws Exception {
FileSystem fileSystem = miniCluster.getFileSystem();
Path root = new Path(fileSystem.getUri() + "/");
String tableName = "testBulkTableUpdateCommandUpdateRecordToExistingRow";
Iface client = getClient();
Path mrIncWorkingPath = new Path(new Path(root, "working"), tableName);
creatTable(tableName, new Path(root, "tables"), true, mrIncWorkingPath.toString());
String rowId = "row1";
String recordId = "record1";
addRow(client, tableName, rowId, recordId, "value1");
generateData(mrIncWorkingPath.toString(), rowId, recordId, "value2");
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value1", column.getValue());
}
BulkTableUpdateCommand bulkTableUpdateCommand = new BulkTableUpdateCommand();
bulkTableUpdateCommand.setAutoLoad(true);
bulkTableUpdateCommand.setTable(tableName);
bulkTableUpdateCommand.setWaitForDataBeVisible(true);
bulkTableUpdateCommand.addExtraConfig(conf);
assertEquals(0, (int) bulkTableUpdateCommand.run(getClient()));
TableStats tableStats = client.tableStats(tableName);
assertEquals(1, tableStats.getRowCount());
assertEquals(1, tableStats.getRecordCount());
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value2", column.getValue());
}
}
@Test
public void testDriverAddSingleRecordToExistingRow() throws Exception {
FileSystem fileSystem = miniCluster.getFileSystem();
Path root = new Path(fileSystem.getUri() + "/");
String tableName = "testDriverAddSingleRecordToExistingRow";
Iface client = getClient();
creatTable(tableName, new Path(root, "tables"), true);
addRow(client, tableName, "row1", "record1", "value1");
Driver driver = new Driver();
driver.setConf(conf);
String mrIncWorkingPathStr = new Path(root, "working").toString();
generateData(mrIncWorkingPathStr);
String outputPathStr = new Path(root, "output").toString();
String blurZkConnection = miniCluster.getZkConnectionString();
assertEquals(
0,
driver.run(
new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));
client.loadData(tableName, outputPathStr);
waitUntilAllImportsAreCompleted(client, tableName);
TableStats tableStats = client.tableStats(tableName);
assertEquals(1, tableStats.getRowCount());
assertEquals(2, tableStats.getRecordCount());
}
@Before
public void ensureCleanTables() throws Exception {
setupConfigIfNeeded();
Iface client = Config.getClient();
List<String> tableList = client.tableList();
if (!tableList.isEmpty()) {
for (String table : tableList) {
client.disableTable(table);
client.removeTable(table, true);
}
}
}
private void runSearch(int start, int fetch) throws SQLException {
currentStart = start;
currentFetch = fetch;
BlurQuery blurQuery = new BlurQuery();
blurQuery.minimumNumberOfResults = Long.MAX_VALUE;
blurQuery.maxQueryTime = Long.MAX_VALUE;
blurQuery.uuid = UUID.randomUUID().toString();
blurQuery.fetch = fetch;
blurQuery.start = start;
blurQuery.query = new Query();
blurQuery.query.query = parser.getWhere();
blurQuery.query.rowQuery = false;
try {
schema = client.schema(tableName);
} catch (BlurException e) {
e.printStackTrace();
throw new SQLException(e);
} catch (TException e) {
e.printStackTrace();
throw new SQLException(e);
}
selector = new Selector();
setupSelector(selector, schema, columnNames);
selector.recordOnly = !blurQuery.query.rowQuery;
Map<String, Map<String, ColumnDefinition>> columnFamilies = schema.getFamilies();
Map<String, ColumnDefinition> cfSet = columnFamilies.get(columnFamily);
columnFamilies.clear();
columnFamilies.put(columnFamily, cfSet);
blurResultSetMetaData = new BlurResultSetMetaData(columnNames, columnFamilies);
try {
results = client.query(tableName, blurQuery);
} catch (BlurException e) {
e.printStackTrace();
throw new SQLException(e);
} catch (TException e) {
e.printStackTrace();
throw new SQLException(e);
}
if (results.totalResults > 0) {
size = results.results.size();
}
totalResults = results.totalResults;
}
private CommandDescriptor getCommandDescriptor(Iface client, String commandName)
throws BlurException, TException {
List<CommandDescriptor> listInstalledCommands = client.listInstalledCommands();
for (CommandDescriptor commandDescriptor : listInstalledCommands) {
if (commandDescriptor.getCommandName().equals(commandName)) {
return commandDescriptor;
}
}
return null;
}
private void waitUntilAllImportsAreCompleted(Iface client, String tableName)
throws BlurException, TException, InterruptedException {
while (true) {
Thread.sleep(1000);
TableStats tableStats = client.tableStats(tableName);
if (tableStats.getSegmentImportInProgressCount() == 0
&& tableStats.getSegmentImportPendingCount() == 0) {
return;
}
}
}
private void addRow(Iface client, String tableName, String rowId, String recordId, String value)
throws BlurException, TException {
List<RecordMutation> recordMutations = new ArrayList<RecordMutation>();
List<Column> columns = new ArrayList<Column>();
columns.add(new Column("col0", value));
Record record = new Record(recordId, "fam0", columns);
recordMutations.add(new RecordMutation(RecordMutationType.REPLACE_ENTIRE_RECORD, record));
RowMutation rowMutation =
new RowMutation(tableName, rowId, RowMutationType.REPLACE_ROW, recordMutations);
client.mutate(rowMutation);
}
private void creatTable(String tableName, Path tables, boolean fastDisable, String workingPath)
throws BlurException, TException {
Path tablePath = new Path(tables, tableName);
Iface client = getClient();
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setTableUri(tablePath.toString());
tableDescriptor.setName(tableName);
tableDescriptor.setShardCount(2);
tableDescriptor.putToTableProperties(
BlurConstants.BLUR_TABLE_DISABLE_FAST_DIR, Boolean.toString(fastDisable));
if (workingPath != null) {
tableDescriptor.putToTableProperties(
BlurConstants.BLUR_BULK_UPDATE_WORKING_PATH, workingPath);
}
client.createTable(tableDescriptor);
ColumnDefinition colDef = new ColumnDefinition();
colDef.setFamily("fam0");
colDef.setColumnName("col0");
colDef.setFieldType("string");
client.addColumnDefinition(tableName, colDef);
}
@SuppressWarnings({"unchecked", "rawtypes"})
@Test
public void testGetTableSummaries() throws Exception {
Iface client = Config.getClient();
TableDescriptor td = new TableDescriptor();
td.setShardCount(11);
td.setTableUri("file://" + TABLE_PATH + "/tableUnitTable");
td.setCluster("default");
td.setName("tableUnitTable");
td.setEnabled(true);
client.createTable(td);
Map<String, List> data = TableUtil.getTableSummaries();
assertEquals(1, data.get("tables").size());
assertEquals(0l, ((Map<String, Object>) data.get("tables").get(0)).get("rows"));
assertEquals(0l, ((Map<String, Object>) data.get("tables").get(0)).get("records"));
assertEquals("default", ((Map<String, Object>) data.get("tables").get(0)).get("cluster"));
assertEquals("tableUnitTable", ((Map<String, Object>) data.get("tables").get(0)).get("name"));
assertEquals(
0,
((List<String>) ((Map<String, Object>) data.get("tables").get(0)).get("families")).size());
}
@Override
public boolean next() throws SQLException {
if (displayRows != null && displayRowsPosition + 1 < displayRows.size()) {
overallRowPosition++;
displayRowsPosition++;
return true;
} else if (resultPosition + 1 < size) {
INNER:
while (resultPosition + 1 < size) {
displayRows.clear();
resultPosition++;
displayRowsPosition = 0;
overallRowPosition++;
final BlurResult result = results.results.get(resultPosition);
try {
FetchRecordResult recordResult = result.getFetchResult().getRecordResult();
selector.setRowId(recordResult.getRowid());
selector.setRecordId(recordResult.getRecord().getRecordId());
fetchResult = client.fetchRow(tableName, selector);
Record record = fetchResult.recordResult.record;
if (!record.family.equals(columnFamily)) {
continue INNER;
}
String rowId = fetchResult.recordResult.rowid;
displayRows.add(
addColumns(result.getScore(), result.getLocationId(), rowId, record.family, record));
return true;
} catch (Exception e) {
e.printStackTrace();
throw new SQLException(e);
}
}
return next();
} else if (overallRowPosition < totalResults) {
currentStart += currentFetch;
runSearch(currentStart, currentFetch);
displayRowsPosition = 0;
resultPosition = -1;
return next();
}
return false;
}
@Test
public void testDriverUpdateRecordToExistingRow() throws Exception {
FileSystem fileSystem = miniCluster.getFileSystem();
Path root = new Path(fileSystem.getUri() + "/");
String tableName = "testDriverUpdateRecordToExistingRow";
Iface client = getClient();
creatTable(tableName, new Path(root, "tables"), true);
String rowId = "row1";
String recordId = "record1";
addRow(client, tableName, rowId, recordId, "value1");
Driver driver = new Driver();
driver.setConf(conf);
String mrIncWorkingPathStr = new Path(root, "working").toString();
generateData(mrIncWorkingPathStr, rowId, recordId, "value2");
String outputPathStr = new Path(root, "output").toString();
String blurZkConnection = miniCluster.getZkConnectionString();
assertEquals(
0,
driver.run(
new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value1", column.getValue());
}
client.loadData(tableName, outputPathStr);
waitUntilAllImportsAreCompleted(client, tableName);
TableStats tableStats = client.tableStats(tableName);
assertEquals(1, tableStats.getRowCount());
assertEquals(1, tableStats.getRecordCount());
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value2", column.getValue());
}
}