@Test
public void testDriverAddSingleRecordToExistingRow() throws Exception {
FileSystem fileSystem = miniCluster.getFileSystem();
Path root = new Path(fileSystem.getUri() + "/");
String tableName = "testDriverAddSingleRecordToExistingRow";
Iface client = getClient();
creatTable(tableName, new Path(root, "tables"), true);
addRow(client, tableName, "row1", "record1", "value1");
Driver driver = new Driver();
driver.setConf(conf);
String mrIncWorkingPathStr = new Path(root, "working").toString();
generateData(mrIncWorkingPathStr);
String outputPathStr = new Path(root, "output").toString();
String blurZkConnection = miniCluster.getZkConnectionString();
assertEquals(
0,
driver.run(
new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));
client.loadData(tableName, outputPathStr);
waitUntilAllImportsAreCompleted(client, tableName);
TableStats tableStats = client.tableStats(tableName);
assertEquals(1, tableStats.getRowCount());
assertEquals(2, tableStats.getRecordCount());
}
@BeforeClass
public static void setupTest() throws Exception {
setupJavaHome();
File file = new File("./target/tmp/BlurInputFormatTest_tmp");
String pathStr = file.getAbsoluteFile().toURI().toString();
System.setProperty("test.build.data", pathStr + "/data");
System.setProperty("hadoop.log.dir", pathStr + "/hadoop_log");
miniCluster = new MiniCluster();
miniCluster.startBlurCluster(pathStr + "/blur", 2, 2);
miniCluster.startMrMiniCluster();
conf = miniCluster.getMRConfiguration();
BufferStore.initNewBuffer(128, 128 * 128);
}
@AfterClass
public static void teardown() throws IOException {
if (miniCluster != null) {
miniCluster.stopMrMiniCluster();
}
rm(new File("build"));
}
@Test
public void testBulkTableUpdateCommandUpdateRecordToExistingRow() throws Exception {
FileSystem fileSystem = miniCluster.getFileSystem();
Path root = new Path(fileSystem.getUri() + "/");
String tableName = "testBulkTableUpdateCommandUpdateRecordToExistingRow";
Iface client = getClient();
Path mrIncWorkingPath = new Path(new Path(root, "working"), tableName);
creatTable(tableName, new Path(root, "tables"), true, mrIncWorkingPath.toString());
String rowId = "row1";
String recordId = "record1";
addRow(client, tableName, rowId, recordId, "value1");
generateData(mrIncWorkingPath.toString(), rowId, recordId, "value2");
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value1", column.getValue());
}
BulkTableUpdateCommand bulkTableUpdateCommand = new BulkTableUpdateCommand();
bulkTableUpdateCommand.setAutoLoad(true);
bulkTableUpdateCommand.setTable(tableName);
bulkTableUpdateCommand.setWaitForDataBeVisible(true);
bulkTableUpdateCommand.addExtraConfig(conf);
assertEquals(0, (int) bulkTableUpdateCommand.run(getClient()));
TableStats tableStats = client.tableStats(tableName);
assertEquals(1, tableStats.getRowCount());
assertEquals(1, tableStats.getRecordCount());
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value2", column.getValue());
}
}
private static void createTable(String tableName) throws BlurException, TException, IOException {
Blur.Iface client = client();
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setName(tableName);
tableDescriptor.setShardCount(5);
tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
client.createTable(tableDescriptor);
List<String> tableList = client.tableList();
assertTrue(tableList.contains(tableName));
}
private SolrServer createServerAndTableWithSimpleTestDoc(String table)
throws BlurException, TException, IOException, SolrServerException {
createTable(table);
SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
SolrInputDocument doc;
doc = createSimpleTestDoc();
server.add(doc);
return server;
}
@BeforeClass
public static void startCluster() throws IOException {
GCWatcher.init(0.60);
LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
testDirectory.mkdirs();
Path directory = new Path(testDirectory.getPath());
FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
FsAction userAction = dirPermissions.getUserAction();
FsAction groupAction = dirPermissions.getGroupAction();
FsAction otherAction = dirPermissions.getOtherAction();
StringBuilder builder = new StringBuilder();
builder.append(userAction.ordinal());
builder.append(groupAction.ordinal());
builder.append(otherAction.ordinal());
String dirPermissionNum = builder.toString();
System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
testDirectory.delete();
miniCluster = new MiniCluster();
miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true);
connectionStr = miniCluster.getControllerConnectionStr();
}
private void generateData(String mrIncWorkingPathStr, String rowId, String recordId, String value)
throws IOException {
Path path = new Path(new Path(mrIncWorkingPathStr), "new");
Writer writer =
new SequenceFile.Writer(
miniCluster.getFileSystem(),
conf,
new Path(path, UUID.randomUUID().toString()),
Text.class,
BlurRecord.class);
BlurRecord blurRecord = new BlurRecord();
blurRecord.setRowId(rowId);
blurRecord.setRecordId(recordId);
blurRecord.setFamily("fam0");
blurRecord.addColumn("col0", value);
writer.append(new Text(rowId), blurRecord);
writer.close();
}
public SolrServer create() throws Exception {
createTable(tableName);
SolrServer server =
new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), tableName);
for (int i = 0; i < rows; i++) {
SolrInputDocument parent = new SolrInputDocument();
parent.addField(BlurConstants.ROW_ID, i);
for (int j = 0; j < recordsPerRow; j++) {
SolrInputDocument child = new SolrInputDocument();
child.addField(BlurConstants.RECORD_ID, j);
for (String colName : columns) {
child.addField(colName, "value" + i + "-" + j);
}
parent.addChildDocument(child);
}
server.add(parent);
}
return server;
}
@Test
public void docShouldBeDiscoverableWithMultiValuedFields()
throws SolrServerException, IOException, BlurException, TException {
String table = "docShouldBeDiscoverableWithMultiValuedFields";
createTable(table);
SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
SolrInputDocument doc = new SolrInputDocument();
doc.addField("rowid", "1");
SolrInputDocument child = new SolrInputDocument();
child.addField("recordid", "1");
child.addField("fam.value", "123");
child.addField("fam.value", "124");
doc.addChildDocument(child);
server.add(doc);
assertTotalResults(table, "fam.value:123", 1l);
assertTotalResults(table, "fam.value:124", 1l);
assertTotalResults(table, "fam.value:justincase", 0l);
removeTable(table);
}
@AfterClass
public static void shutdownCluster() {
miniCluster.shutdownBlurCluster();
}
private Iface getClient() {
return BlurClient.getClientFromZooKeeperConnectionStr(miniCluster.getZkConnectionString());
}
@Test
public void testDriverUpdateRecordToExistingRow() throws Exception {
FileSystem fileSystem = miniCluster.getFileSystem();
Path root = new Path(fileSystem.getUri() + "/");
String tableName = "testDriverUpdateRecordToExistingRow";
Iface client = getClient();
creatTable(tableName, new Path(root, "tables"), true);
String rowId = "row1";
String recordId = "record1";
addRow(client, tableName, rowId, recordId, "value1");
Driver driver = new Driver();
driver.setConf(conf);
String mrIncWorkingPathStr = new Path(root, "working").toString();
generateData(mrIncWorkingPathStr, rowId, recordId, "value2");
String outputPathStr = new Path(root, "output").toString();
String blurZkConnection = miniCluster.getZkConnectionString();
assertEquals(
0,
driver.run(
new String[] {tableName, mrIncWorkingPathStr, outputPathStr, blurZkConnection, "1"}));
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value1", column.getValue());
}
client.loadData(tableName, outputPathStr);
waitUntilAllImportsAreCompleted(client, tableName);
TableStats tableStats = client.tableStats(tableName);
assertEquals(1, tableStats.getRowCount());
assertEquals(1, tableStats.getRecordCount());
{
Selector selector = new Selector();
selector.setRowId(rowId);
FetchResult fetchRow = client.fetchRow(tableName, selector);
Row row = fetchRow.getRowResult().getRow();
assertEquals(rowId, row.getId());
List<Record> records = row.getRecords();
assertEquals(1, records.size());
Record record = records.get(0);
assertEquals(recordId, record.getRecordId());
List<Column> columns = record.getColumns();
assertEquals(1, columns.size());
Column column = columns.get(0);
assertEquals("col0", column.getName());
assertEquals("value2", column.getValue());
}
}