/**
* Creates a record with a blob using a Repository obtained via LilyClient. This verifies that the
* blobs work remotely and it was able to retrieve the blob stores config from ZooKeeper.
*/
@Test
public void testBlob() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
// Obtain a repository
Repository repository = client.getRepository();
String NS = "org.lilyproject.client.test";
// Create a blob field type and record type
TypeManager typeManager = repository.getTypeManager();
ValueType blobType = typeManager.getValueType("BLOB");
FieldType blobFieldType =
typeManager.newFieldType(blobType, new QName(NS, "data"), Scope.VERSIONED);
blobFieldType = typeManager.createFieldType(blobFieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "file"));
recordType.addFieldTypeEntry(blobFieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Upload a blob that, based upon the current default config, should end up in HBase
// (> 5000 bytes and < 200000 bytes)
byte[] data = makeBlobData(10000);
Blob blob = new Blob("application/octet-stream", (long) data.length, null);
OutputStream blobStream = repository.getOutputStream(blob);
IOUtils.copy(new ByteArrayInputStream(data), blobStream);
blobStream.close();
assertTrue(blob.getValue() != null);
// Create a record with this blob
Record record = repository.newRecord();
record.setRecordType(new QName(NS, "file"));
record.setField(new QName(NS, "data"), blob);
record = repository.create(record);
}
@Test
public void testIndexerApi() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
final Repository repository = client.getRepository();
// Create a field type and record type
String NS = "org.lilyproject.client.test";
TypeManager typeManager = repository.getTypeManager();
FieldType fieldType =
typeManager.newFieldType("STRING", new QName(NS, "indexfield"), Scope.VERSIONED);
fieldType = typeManager.createFieldType(fieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "indexrt"));
recordType.addFieldTypeEntry(fieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Create a record
Record record = repository.newRecord();
record.setId(repository.getIdGenerator().newRecordId("indexrecord"));
record.setRecordType(recordType.getName());
record.setField(fieldType.getName(), "value");
repository.create(record);
// explicitly index the record (if this succeeds, the test succeeded to verify that we can
// access the indexer through lily-client)
client.getIndexer().index(record.getId());
}
/** Test scanners: scanners work directly on HBase, also remotely. */
@Test
public void testScanners() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
// Obtain a repository
Repository repository = client.getRepository();
IdGenerator idGenerator = repository.getIdGenerator();
String NS = "org.lilyproject.client.test";
// Create a field type and record type
TypeManager typeManager = repository.getTypeManager();
FieldType fieldType =
typeManager.newFieldType("STRING", new QName(NS, "scanfield"), Scope.VERSIONED);
fieldType = typeManager.createFieldType(fieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "scanrt"));
recordType.addFieldTypeEntry(fieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Create some records
for (int i = 0; i < 10; i++) {
Record record = repository.newRecord();
record.setId(repository.getIdGenerator().newRecordId("A" + i));
record.setRecordType(new QName(NS, "scanrt"));
record.setField(new QName(NS, "scanfield"), "value " + i);
repository.create(record);
}
// Do a scan
RecordScan scan = new RecordScan();
scan.setStartRecordId(idGenerator.newRecordId("A"));
scan.setStopRecordId(idGenerator.newRecordId("B"));
RecordScanner scanner = repository.getScanner(scan);
int i = 0;
while (scanner.next() != null) {
i++;
}
assertEquals("Number of scanned records", 10, i);
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
lilyProxy = new LilyProxy();
InputStream is = BatchBuildTest.class.getResourceAsStream("solrschema.xml");
byte[] solrSchema = IOUtils.toByteArray(is);
IOUtils.closeQuietly(is);
lilyProxy.start(solrSchema);
solrProxy = lilyProxy.getSolrProxy();
solrServer = solrProxy.getSolrServer();
lilyServerProxy = lilyProxy.getLilyServerProxy();
lilyClient = lilyServerProxy.getClient();
repository = lilyClient.getRepository();
typeManager = repository.getTypeManager();
FieldType ft1 =
typeManager.createFieldType(
"STRING", new QName("batchindex-test", "field1"), Scope.NON_VERSIONED);
FieldType ft2 =
typeManager.createFieldType(
"LINK", new QName("batchindex-test", "linkField"), Scope.NON_VERSIONED);
typeManager
.recordTypeBuilder()
.defaultNamespace("batchindex-test")
.name("rt1")
.fieldEntry()
.use(ft1)
.add()
.fieldEntry()
.use(ft2)
.add()
.create();
model = lilyServerProxy.getIndexerModel();
is = BatchBuildTest.class.getResourceAsStream("indexerconf.xml");
byte[] indexerConfiguration = IOUtils.toByteArray(is);
IOUtils.closeQuietly(is);
IndexDefinition index = model.newIndex(INDEX_NAME);
Map<String, String> solrShards = new HashMap<String, String>();
solrShards.put("shard1", "http://localhost:8983/solr");
index.setSolrShards(solrShards);
index.setConfiguration(indexerConfiguration);
index.setUpdateState(IndexUpdateState.DO_NOT_SUBSCRIBE);
model.addIndex(index);
}
@Test
public void testOne() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
//
// Create some records
//
Repository repository = client.getRepository();
TypeManager typeManager = repository.getTypeManager();
IdGenerator idGenerator = repository.getIdGenerator();
FieldType ft1 =
typeManager.createFieldType("STRING", new QName("test", "field1"), Scope.NON_VERSIONED);
FieldType ft2 =
typeManager.createFieldType("LINK", new QName("test", "field2"), Scope.NON_VERSIONED);
RecordType rt1 =
typeManager
.recordTypeBuilder()
.defaultNamespace("test")
.name("rt1")
.fieldEntry()
.use(ft1)
.add()
.fieldEntry()
.use(ft2)
.add()
.create();
for (int i = 0; i < 300; i++) {
repository
.recordBuilder()
.recordType(rt1.getName())
.field(ft1.getName(), "foo bar bar")
.field(ft2.getName(), new Link(idGenerator.newRecordId()))
.create();
}
Assert.assertTrue(
"Processing messages took too long", lilyProxy.waitSepEventsProcessed(60000L));
//
// Count number of records in each region
//
for (String tableName : TABLE_NAMES) {
HTable table = new HTable(lilyProxy.getHBaseProxy().getConf(), tableName);
for (HRegionInfo regionInfo : table.getRegionsInfo().keySet()) {
Scan scan = new Scan();
scan.setStartRow(regionInfo.getStartKey());
scan.setStopRow(regionInfo.getEndKey());
ResultScanner scanner = table.getScanner(scan);
int count = 0;
for (Result result : scanner) {
// System.out.println("result = " + Arrays.toString(result.getRow()));
count++;
}
assertTrue(
String.format(
"Number of records in region '%s' is %d, expected between 60 and 140, "
+ "start key is '%s', end key is '%s'",
regionInfo.getRegionNameAsString(),
count,
Bytes.toStringBinary(regionInfo.getStartKey()),
Bytes.toStringBinary(regionInfo.getEndKey())),
count >= 60 && count <= 140);
}
}
}
@Test
public void testGetHostnames() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
Set<String> hosts = client.getLilyHostnames();
assertEquals(1, hosts.size());
}
