@Test
public void testIndexerApi() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
final Repository repository = client.getRepository();
// Create a field type and record type
String NS = "org.lilyproject.client.test";
TypeManager typeManager = repository.getTypeManager();
FieldType fieldType =
typeManager.newFieldType("STRING", new QName(NS, "indexfield"), Scope.VERSIONED);
fieldType = typeManager.createFieldType(fieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "indexrt"));
recordType.addFieldTypeEntry(fieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Create a record
Record record = repository.newRecord();
record.setId(repository.getIdGenerator().newRecordId("indexrecord"));
record.setRecordType(recordType.getName());
record.setField(fieldType.getName(), "value");
repository.create(record);
// explicitly index the record (if this succeeds, the test succeeded to verify that we can
// access the indexer through lily-client)
client.getIndexer().index(record.getId());
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
repoSetup.setupCore();
repoSetup.setupRepository();
repositoryManager = repoSetup.getRepositoryManager();
repository = repoSetup.getRepositoryManager().getRepository(Table.RECORD.name);
TypeManager typeManager = repository.getTypeManager();
typeManager.createFieldType("STRING", new QName("ns", "stringField"), Scope.NON_VERSIONED);
typeManager.createFieldType(
"LIST<STRING>", new QName("ns", "stringListField"), Scope.NON_VERSIONED);
typeManager.createFieldType("LONG", new QName("ns", "longField"), Scope.NON_VERSIONED);
typeManager
.recordTypeBuilder()
.defaultNamespace("ns")
.name("rt")
.fieldEntry()
.name("stringField")
.add()
.fieldEntry()
.name("stringListField")
.add()
.fieldEntry()
.name("longField")
.add()
.create();
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
lilyProxy = new LilyProxy();
InputStream is = BatchBuildTest.class.getResourceAsStream("solrschema.xml");
byte[] solrSchema = IOUtils.toByteArray(is);
IOUtils.closeQuietly(is);
lilyProxy.start(solrSchema);
solrProxy = lilyProxy.getSolrProxy();
solrServer = solrProxy.getSolrServer();
lilyServerProxy = lilyProxy.getLilyServerProxy();
lilyClient = lilyServerProxy.getClient();
repository = lilyClient.getRepository();
typeManager = repository.getTypeManager();
FieldType ft1 =
typeManager.createFieldType(
"STRING", new QName("batchindex-test", "field1"), Scope.NON_VERSIONED);
FieldType ft2 =
typeManager.createFieldType(
"LINK", new QName("batchindex-test", "linkField"), Scope.NON_VERSIONED);
typeManager
.recordTypeBuilder()
.defaultNamespace("batchindex-test")
.name("rt1")
.fieldEntry()
.use(ft1)
.add()
.fieldEntry()
.use(ft2)
.add()
.create();
model = lilyServerProxy.getIndexerModel();
is = BatchBuildTest.class.getResourceAsStream("indexerconf.xml");
byte[] indexerConfiguration = IOUtils.toByteArray(is);
IOUtils.closeQuietly(is);
IndexDefinition index = model.newIndex(INDEX_NAME);
Map<String, String> solrShards = new HashMap<String, String>();
solrShards.put("shard1", "http://localhost:8983/solr");
index.setSolrShards(solrShards);
index.setConfiguration(indexerConfiguration);
index.setUpdateState(IndexUpdateState.DO_NOT_SUBSCRIBE);
model.addIndex(index);
}
public Record read(RecordId recordId, Long version, QName... fieldNames)
throws RepositoryException, InterruptedException {
FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
List<FieldType> fields = getFieldTypesFromNames(fieldTypes, fieldNames);
return read(recordId, version, fields, fieldTypes);
}
public List<Record> readVersions(RecordId recordId, List<Long> versions, QName... fieldNames)
throws RepositoryException, InterruptedException {
ArgumentValidator.notNull(recordId, "recordId");
ArgumentValidator.notNull(versions, "versions");
if (versions.isEmpty()) return new ArrayList<Record>();
Collections.sort(versions);
FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
List<FieldType> fields = getFieldTypesFromNames(fieldTypes, fieldNames);
Long lowestRequestedVersion = versions.get(0);
Long highestRequestedVersion = versions.get(versions.size() - 1);
int numberOfVersionsToRetrieve = (int) (highestRequestedVersion - lowestRequestedVersion + 1);
Result result = getRow(recordId, highestRequestedVersion, numberOfVersionsToRetrieve, fields);
Long latestVersion = recdec.getLatestVersion(result);
// Drop the versions that are higher than the latestVersion
List<Long> validVersions = new ArrayList<Long>();
for (Long version : versions) {
if (version > latestVersion) break;
validVersions.add(version);
}
return recdec.decodeRecords(recordId, validVersions, result, fieldTypes);
}
@Override
public BlobAccess getBlob(RecordId recordId, Long version, QName fieldName, int... indexes)
throws RepositoryException, InterruptedException {
Record record = read(recordId, version, fieldName);
FieldType fieldType = typeManager.getFieldTypeByName(fieldName);
return blobManager.getBlobAccess(record, fieldName, fieldType, indexes);
}
public List<Record> readVersions(
RecordId recordId, Long fromVersion, Long toVersion, QName... fieldNames)
throws RepositoryException, InterruptedException {
ArgumentValidator.notNull(recordId, "recordId");
ArgumentValidator.notNull(fromVersion, "fromVersion");
ArgumentValidator.notNull(toVersion, "toVersion");
if (fromVersion > toVersion) {
throw new IllegalArgumentException(
"fromVersion '"
+ fromVersion
+ "' must be smaller or equal to toVersion '"
+ toVersion
+ "'");
}
FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
List<FieldType> fields = getFieldTypesFromNames(fieldTypes, fieldNames);
int numberOfVersionsToRetrieve = (int) (toVersion - fromVersion + 1);
Result result = getRow(recordId, toVersion, numberOfVersionsToRetrieve, fields);
if (fromVersion < 1L) fromVersion = 1L; // Put the fromVersion to a sensible value
Long latestVersion = recdec.getLatestVersion(result);
if (latestVersion < toVersion)
toVersion = latestVersion; // Limit the toVersion to the highest possible version
List<Long> versionsToRead = new ArrayList<Long>();
for (long version = fromVersion; version <= toVersion; version++) {
versionsToRead.add(version);
}
return recdec.decodeRecords(recordId, versionsToRead, result, fieldTypes);
}
public IdRecord readWithIds(RecordId recordId, Long version, List<SchemaId> fieldIds)
throws RepositoryException, InterruptedException {
FieldTypes fieldTypes = typeManager.getFieldTypesSnapshot();
List<FieldType> fields = getFieldTypesFromIds(fieldIds, fieldTypes);
return readWithIds(recordId, version, fields, fieldTypes);
}
@Test
public void testClearDerefMap() throws Exception {
DerefMap derefMap =
DerefMapHbaseImpl.create(
INDEX_NAME, lilyProxy.getHBaseProxy().getConf(), null, repository.getIdGenerator());
Record linkedRecord =
repository
.recordBuilder()
.id("deref-test-linkedrecord")
.recordType(rt1.getName())
.field(ft1.getName(), "deref test linkedrecord")
.create();
Record record =
repository
.recordBuilder()
.id("deref-test-main")
.recordType(rt1.getName())
.field(ft1.getName(), "deref test main")
.field(ft2.getName(), new Link(linkedRecord.getId()))
.create();
SchemaId vtag = typeManager.getFieldTypeByName(VersionTag.LAST).getId();
DependantRecordIdsIterator it = null;
try {
it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
assertTrue(!it.hasNext());
} finally {
it.close();
}
setBatchIndexConf(
getResourceAsByteArray("batchIndexConf-testClearDerefmap-false.json"), null, false);
buildAndCommit();
QueryResponse response = solrServer.query(new SolrQuery("field1:deref\\ test\\ main"));
assertEquals(1, response.getResults().size());
try {
it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
assertTrue(it.hasNext());
} finally {
it.close();
}
setBatchIndexConf(
null, getResourceAsByteArray("batchIndexConf-testClearDerefmap-true.json"), true);
waitForIndexAndCommit(BUILD_TIMEOUT);
try {
it = derefMap.findDependantsOf(linkedRecord.getId(), ft1.getId(), vtag);
assertTrue(!it.hasNext());
} finally {
it.close();
}
}
public SolrInputDocument build() throws InterruptedException, RepositoryException {
solrDoc.setField("lily.id", recordId.toString());
solrDoc.setField("lily.table", table);
solrDoc.setField("lily.key", key);
solrDoc.setField("lily.vtagId", vtag.toString());
solrDoc.setField("lily.vtag", typeManager.getFieldTypeById(vtag).getName().getName());
solrDoc.setField("lily.version", version);
return solrDoc;
}
private FieldType getFieldType(QName qname)
throws IndexerConfException, InterruptedException, RepositoryException {
if (systemFields.isSystemField(qname)) {
return systemFields.get(qname);
}
return typeManager.getFieldTypeByName(qname);
}
/** Test scanners: scanners work directly on HBase, also remotely. */
@Test
public void testScanners() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
// Obtain a repository
Repository repository = client.getRepository();
IdGenerator idGenerator = repository.getIdGenerator();
String NS = "org.lilyproject.client.test";
// Create a field type and record type
TypeManager typeManager = repository.getTypeManager();
FieldType fieldType =
typeManager.newFieldType("STRING", new QName(NS, "scanfield"), Scope.VERSIONED);
fieldType = typeManager.createFieldType(fieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "scanrt"));
recordType.addFieldTypeEntry(fieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Create some records
for (int i = 0; i < 10; i++) {
Record record = repository.newRecord();
record.setId(repository.getIdGenerator().newRecordId("A" + i));
record.setRecordType(new QName(NS, "scanrt"));
record.setField(new QName(NS, "scanfield"), "value " + i);
repository.create(record);
}
// Do a scan
RecordScan scan = new RecordScan();
scan.setStartRecordId(idGenerator.newRecordId("A"));
scan.setStopRecordId(idGenerator.newRecordId("B"));
RecordScanner scanner = repository.getScanner(scan);
int i = 0;
while (scanner.next() != null) {
i++;
}
assertEquals("Number of scanned records", 10, i);
}
/**
* Creates a record with a blob using a Repository obtained via LilyClient. This verifies that the
* blobs work remotely and it was able to retrieve the blob stores config from ZooKeeper.
*/
@Test
public void testBlob() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
// Obtain a repository
Repository repository = client.getRepository();
String NS = "org.lilyproject.client.test";
// Create a blob field type and record type
TypeManager typeManager = repository.getTypeManager();
ValueType blobType = typeManager.getValueType("BLOB");
FieldType blobFieldType =
typeManager.newFieldType(blobType, new QName(NS, "data"), Scope.VERSIONED);
blobFieldType = typeManager.createFieldType(blobFieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "file"));
recordType.addFieldTypeEntry(blobFieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Upload a blob that, based upon the current default config, should end up in HBase
// (> 5000 bytes and < 200000 bytes)
byte[] data = makeBlobData(10000);
Blob blob = new Blob("application/octet-stream", (long) data.length, null);
OutputStream blobStream = repository.getOutputStream(blob);
IOUtils.copy(new ByteArrayInputStream(data), blobStream);
blobStream.close();
assertTrue(blob.getValue() != null);
// Create a record with this blob
Record record = repository.newRecord();
record.setRecordType(new QName(NS, "file"));
record.setField(new QName(NS, "data"), blob);
record = repository.create(record);
}
private Set<SchemaId> parseVersionTags(String vtagsSpec)
throws IndexerConfException, InterruptedException {
Set<SchemaId> vtags = new HashSet<SchemaId>();
if (vtagsSpec == null) return vtags;
for (String tag : COMMA_SPLITTER.split(vtagsSpec)) {
try {
vtags.add(typeManager.getFieldTypeByName(VersionTag.qname(tag)).getId());
} catch (FieldTypeNotFoundException e) {
throw new IndexerConfException("unknown vtag used in indexer configuration: " + tag);
} catch (RepositoryException e) {
throw new IndexerConfException("error loading field type for vtag: " + tag, e);
}
}
return vtags;
}
public ListValueType(TypeManager typeManager, String typeParams)
throws RepositoryException, InterruptedException {
ArgumentValidator.notNull(typeParams, "typeParams");
this.fullName = NAME + "<" + typeParams + ">";
this.valueType = typeManager.getValueType(typeParams);
}
@Before
public void setup() throws Exception {
this.ft1 = typeManager.getFieldTypeByName(new QName("batchindex-test", "field1"));
this.ft2 = typeManager.getFieldTypeByName(new QName("batchindex-test", "linkField"));
this.rt1 = typeManager.getRecordTypeByName(new QName("batchindex-test", "rt1"), null);
}
@Override
public InputStream getInputStream(Record record, QName fieldName, int... indexes)
throws RepositoryException, InterruptedException {
FieldType fieldType = typeManager.getFieldTypeByName(fieldName);
return blobManager.getBlobAccess(record, fieldName, fieldType, indexes).getInputStream();
}
@Test
public void testOne() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
//
// Create some records
//
Repository repository = client.getRepository();
TypeManager typeManager = repository.getTypeManager();
IdGenerator idGenerator = repository.getIdGenerator();
FieldType ft1 =
typeManager.createFieldType("STRING", new QName("test", "field1"), Scope.NON_VERSIONED);
FieldType ft2 =
typeManager.createFieldType("LINK", new QName("test", "field2"), Scope.NON_VERSIONED);
RecordType rt1 =
typeManager
.recordTypeBuilder()
.defaultNamespace("test")
.name("rt1")
.fieldEntry()
.use(ft1)
.add()
.fieldEntry()
.use(ft2)
.add()
.create();
for (int i = 0; i < 300; i++) {
repository
.recordBuilder()
.recordType(rt1.getName())
.field(ft1.getName(), "foo bar bar")
.field(ft2.getName(), new Link(idGenerator.newRecordId()))
.create();
}
Assert.assertTrue(
"Processing messages took too long", lilyProxy.waitSepEventsProcessed(60000L));
//
// Count number of records in each region
//
for (String tableName : TABLE_NAMES) {
HTable table = new HTable(lilyProxy.getHBaseProxy().getConf(), tableName);
for (HRegionInfo regionInfo : table.getRegionsInfo().keySet()) {
Scan scan = new Scan();
scan.setStartRow(regionInfo.getStartKey());
scan.setStopRow(regionInfo.getEndKey());
ResultScanner scanner = table.getScanner(scan);
int count = 0;
for (Result result : scanner) {
// System.out.println("result = " + Arrays.toString(result.getRow()));
count++;
}
assertTrue(
String.format(
"Number of records in region '%s' is %d, expected between 60 and 140, "
+ "start key is '%s', end key is '%s'",
regionInfo.getRegionNameAsString(),
count,
Bytes.toStringBinary(regionInfo.getStartKey()),
Bytes.toStringBinary(regionInfo.getEndKey())),
count >= 60 && count <= 140);
}
}
}
private ResultScanner createHBaseResultScanner(RecordScan scan)
throws RepositoryException, InterruptedException {
Scan hbaseScan = new Scan();
hbaseScan.setMaxVersions(1);
if (scan.getRawStartRecordId() != null) {
hbaseScan.setStartRow(scan.getRawStartRecordId());
} else if (scan.getStartRecordId() != null) {
hbaseScan.setStartRow(scan.getStartRecordId().toBytes());
}
if (scan.getRawStopRecordId() != null) {
hbaseScan.setStopRow(scan.getRawStopRecordId());
} else if (scan.getStopRecordId() != null) {
hbaseScan.setStopRow(scan.getStopRecordId().toBytes());
}
// Filters
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
// filter out deleted records
filterList.addFilter(REAL_RECORDS_FILTER);
// add user's filter
if (scan.getRecordFilter() != null) {
Filter filter = filterFactory.createHBaseFilter(scan.getRecordFilter(), this, filterFactory);
filterList.addFilter(filter);
}
hbaseScan.setFilter(filterList);
hbaseScan.setCaching(scan.getCaching());
hbaseScan.setCacheBlocks(scan.getCacheBlocks());
ReturnFields returnFields = scan.getReturnFields();
if (returnFields != null && returnFields.getType() != ReturnFields.Type.ALL) {
RecordDecoder.addSystemColumnsToScan(hbaseScan);
switch (returnFields.getType()) {
case ENUM:
for (QName field : returnFields.getFields()) {
FieldTypeImpl fieldType = (FieldTypeImpl) typeManager.getFieldTypeByName(field);
hbaseScan.addColumn(RecordCf.DATA.bytes, fieldType.getQualifier());
}
break;
case NONE:
// nothing to add
break;
default:
throw new RuntimeException("Unrecognized ReturnFields type: " + returnFields.getType());
}
} else {
hbaseScan.addFamily(RecordCf.DATA.bytes);
}
ResultScanner hbaseScanner;
try {
hbaseScanner = recordTable.getScanner(hbaseScan);
} catch (IOException e) {
throw new RecordException("Error creating scanner", e);
}
return hbaseScanner;
}
