@Test
public void testScanRecordFilterList() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
RecordId recordId = idGenerator.newRecordId("foo");
RecordScan scan = new RecordScan();
RecordFilterList filterList = new RecordFilterList(RecordFilterList.Operator.MUST_PASS_ONE);
filterList.addFilter(new RecordIdPrefixFilter(recordId));
filterList.addFilter(new RecordTypeFilter(new QName("ns", "stringField")));
scan.setRecordFilter(filterList);
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordFilterList);
RecordFilterList parsedFilterList = filterList;
assertTrue(parsedFilterList.getFilters().get(0) instanceof RecordIdPrefixFilter);
assertTrue(parsedFilterList.getFilters().get(1) instanceof RecordTypeFilter);
assertEquals(RecordFilterList.Operator.MUST_PASS_ONE, parsedFilterList.getOperator());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals(
"org.lilyproject.repository.api.filter.RecordFilterList",
node.get("recordFilter").get("@class").getTextValue());
assertTrue(node.get("recordFilter").get("filters").isArray());
assertEquals(2, node.get("recordFilter").get("filters").size());
assertEquals(
"org.lilyproject.repository.api.filter.RecordIdPrefixFilter",
node.get("recordFilter").get("filters").get(0).get("@class").getTextValue());
}
@Test
public void testFromString() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
ValueType valueType = mock(ValueType.class);
when(valueType.getBaseName()).thenReturn("STRING");
assertEquals("foo", FieldValueStringConverter.fromString("foo", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("INTEGER");
assertEquals(
new Integer(123), FieldValueStringConverter.fromString("123", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("LONG");
assertEquals(
new Long(12345), FieldValueStringConverter.fromString("12345", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("DOUBLE");
assertEquals(
new Double(12345.6),
FieldValueStringConverter.fromString("12345.6", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("DECIMAL");
assertEquals(
new BigDecimal("12345.12345"),
FieldValueStringConverter.fromString("12345.12345", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("URI");
assertEquals(
new URI("http://www.ngdata.com/"),
FieldValueStringConverter.fromString("http://www.ngdata.com/", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("BOOLEAN");
assertEquals(
Boolean.TRUE, FieldValueStringConverter.fromString("true", valueType, idGenerator));
assertEquals(
Boolean.FALSE, FieldValueStringConverter.fromString("false", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("LINK");
assertEquals(
new Link(idGenerator.newRecordId("foobar")),
FieldValueStringConverter.fromString("USER.foobar", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("DATE");
assertEquals(
new LocalDate(2012, 6, 28),
FieldValueStringConverter.fromString("2012-06-28", valueType, idGenerator));
when(valueType.getBaseName()).thenReturn("DATETIME");
assertEquals(
new DateTime(2012, 6, 28, 5, 36, 0, 0, DateTimeZone.UTC),
FieldValueStringConverter.fromString("2012-06-28T05:36Z", valueType, idGenerator));
}
@Test
public void testScanRecordId() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
RecordScan scan = new RecordScan();
scan.setStartRecordId(idGenerator.newRecordId());
scan.setStopRecordId(idGenerator.newRecordId("foo"));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertEquals(scan.getStartRecordId(), parsedScan.getStartRecordId());
assertEquals(scan.getStopRecordId(), parsedScan.getStopRecordId());
// RecordId's should be simply string properties in json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals(scan.getStartRecordId().toString(), node.get("startRecordId").getTextValue());
assertEquals("USER.foo", node.get("stopRecordId").getTextValue());
}
/** Test scanners: scanners work directly on HBase, also remotely. */
@Test
public void testScanners() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
// Obtain a repository
Repository repository = client.getRepository();
IdGenerator idGenerator = repository.getIdGenerator();
String NS = "org.lilyproject.client.test";
// Create a field type and record type
TypeManager typeManager = repository.getTypeManager();
FieldType fieldType =
typeManager.newFieldType("STRING", new QName(NS, "scanfield"), Scope.VERSIONED);
fieldType = typeManager.createFieldType(fieldType);
RecordType recordType = typeManager.newRecordType(new QName(NS, "scanrt"));
recordType.addFieldTypeEntry(fieldType.getId(), true);
recordType = typeManager.createRecordType(recordType);
// Create some records
for (int i = 0; i < 10; i++) {
Record record = repository.newRecord();
record.setId(repository.getIdGenerator().newRecordId("A" + i));
record.setRecordType(new QName(NS, "scanrt"));
record.setField(new QName(NS, "scanfield"), "value " + i);
repository.create(record);
}
// Do a scan
RecordScan scan = new RecordScan();
scan.setStartRecordId(idGenerator.newRecordId("A"));
scan.setStopRecordId(idGenerator.newRecordId("B"));
RecordScanner scanner = repository.getScanner(scan);
int i = 0;
while (scanner.next() != null) {
i++;
}
assertEquals("Number of scanned records", 10, i);
}
public IndexRecordFilterData(JsonParser jp, IdGenerator idGenerator) throws IOException {
JsonToken current = jp.getCurrentToken();
if (current != JsonToken.START_OBJECT) {
throw new RuntimeException("Not a JSON object.");
}
while (jp.nextToken() != JsonToken.END_OBJECT) {
String fieldName = jp.getCurrentName();
current = jp.nextToken(); // move from field name to field value
if (fieldName.equals("old")) {
oldRecordExists = jp.getBooleanValue();
} else if (fieldName.equals("new")) {
newRecordExists = jp.getBooleanValue();
} else if (fieldName.equals("newRecordType")) {
newRecordType = idGenerator.getSchemaId(jp.getBinaryValue());
} else if (fieldName.equals("oldRecordType")) {
oldRecordType = idGenerator.getSchemaId(jp.getBinaryValue());
} else if (fieldName.equals("includeSubscriptions")) {
includeSubscriptions = jp.getBooleanValue();
} else if (fieldName.equals("fields")) {
if (current != JsonToken.START_ARRAY) {
throw new RuntimeException("updatedFields is not a JSON array");
}
fieldChanges = new ArrayList<FieldChange>();
while (jp.nextToken() != JsonToken.END_ARRAY) {
fieldChanges.add(new FieldChange(jp, idGenerator));
}
} else if (fieldName.equals("subscriptions")) {
if (current != JsonToken.START_ARRAY) {
throw new RuntimeException("subscriptions is not a JSON array");
}
indexSubscriptionIds = Sets.newHashSet();
while (jp.nextToken() != JsonToken.END_ARRAY) {
indexSubscriptionIds.add(jp.getText());
}
}
}
}
@Test
public void testScanRecordIdPrefixFilter() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
RecordId recordId = idGenerator.newRecordId("foo");
RecordScan scan = new RecordScan();
scan.setRecordFilter(new RecordIdPrefixFilter(recordId));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordIdPrefixFilter);
assertEquals(recordId, ((RecordIdPrefixFilter) parsedScan.getRecordFilter()).getRecordId());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals(
"org.lilyproject.repository.api.filter.RecordIdPrefixFilter",
node.get("recordFilter").get("@class").getTextValue());
assertEquals("USER.foo", node.get("recordFilter").get("recordId").getTextValue());
}
@Test
public void testScanRecordVariantFilter() throws Exception {
IdGenerator idGenerator = new IdGeneratorImpl();
final Map<String, String> variantProperties = new HashMap<String, String>();
variantProperties.put("lang", "en");
variantProperties.put("branch", null);
RecordId recordId = idGenerator.newRecordId("foo");
RecordScan scan = new RecordScan();
scan.setRecordFilter(new RecordVariantFilter(recordId, variantProperties));
byte[] data = scanToBytes(scan);
RecordScan parsedScan = scanFromBytes(data);
assertNotNull(parsedScan.getRecordFilter());
assertTrue(parsedScan.getRecordFilter() instanceof RecordVariantFilter);
assertEquals(
recordId.getMaster(),
((RecordVariantFilter) parsedScan.getRecordFilter()).getMasterRecordId());
assertEquals(
variantProperties,
((RecordVariantFilter) parsedScan.getRecordFilter()).getVariantProperties());
// Check json
JsonNode node = new ObjectMapper().readTree(data);
assertEquals(
"org.lilyproject.repository.api.filter.RecordVariantFilter",
node.get("recordFilter").get("@class").getTextValue());
assertEquals("USER.foo", node.get("recordFilter").get("recordId").getTextValue());
assertEquals(
"en", node.get("recordFilter").get("variantProperties").get("lang").getTextValue());
assertEquals(
null, node.get("recordFilter").get("variantProperties").get("branch").getTextValue());
}
public FieldChange(JsonParser jp, IdGenerator idGenerator) throws IOException {
JsonToken current = jp.getCurrentToken();
if (current != JsonToken.START_OBJECT) {
throw new RuntimeException("Not a JSON object.");
}
while (jp.nextToken() != JsonToken.END_OBJECT) {
String fieldName = jp.getCurrentName();
current = jp.nextToken(); // move from field name to field value
if (fieldName.equals("id")) {
this.id = idGenerator.getSchemaId(jp.getBinaryValue());
} else if (fieldName.equals("old")) {
oldValue = jp.getBinaryValue();
} else if (fieldName.equals("new")) {
newValue = jp.getBinaryValue();
}
}
}
public void addLink(RecordId target, SchemaId fieldTypeId) {
addLink(idGenerator.newAbsoluteRecordId(Table.RECORD.name, target), fieldTypeId);
}
@Test
public void testOne() throws Exception {
LilyClient client = lilyProxy.getLilyServerProxy().getClient();
//
// Create some records
//
Repository repository = client.getRepository();
TypeManager typeManager = repository.getTypeManager();
IdGenerator idGenerator = repository.getIdGenerator();
FieldType ft1 =
typeManager.createFieldType("STRING", new QName("test", "field1"), Scope.NON_VERSIONED);
FieldType ft2 =
typeManager.createFieldType("LINK", new QName("test", "field2"), Scope.NON_VERSIONED);
RecordType rt1 =
typeManager
.recordTypeBuilder()
.defaultNamespace("test")
.name("rt1")
.fieldEntry()
.use(ft1)
.add()
.fieldEntry()
.use(ft2)
.add()
.create();
for (int i = 0; i < 300; i++) {
repository
.recordBuilder()
.recordType(rt1.getName())
.field(ft1.getName(), "foo bar bar")
.field(ft2.getName(), new Link(idGenerator.newRecordId()))
.create();
}
Assert.assertTrue(
"Processing messages took too long", lilyProxy.waitSepEventsProcessed(60000L));
//
// Count number of records in each region
//
for (String tableName : TABLE_NAMES) {
HTable table = new HTable(lilyProxy.getHBaseProxy().getConf(), tableName);
for (HRegionInfo regionInfo : table.getRegionsInfo().keySet()) {
Scan scan = new Scan();
scan.setStartRow(regionInfo.getStartKey());
scan.setStopRow(regionInfo.getEndKey());
ResultScanner scanner = table.getScanner(scan);
int count = 0;
for (Result result : scanner) {
// System.out.println("result = " + Arrays.toString(result.getRow()));
count++;
}
assertTrue(
String.format(
"Number of records in region '%s' is %d, expected between 60 and 140, "
+ "start key is '%s', end key is '%s'",
regionInfo.getRegionNameAsString(),
count,
Bytes.toStringBinary(regionInfo.getStartKey()),
Bytes.toStringBinary(regionInfo.getEndKey())),
count >= 60 && count <= 140);
}
}
}
/** Creates a record event from the json data supplied as bytes. */
public RecordEvent(byte[] data, IdGenerator idGenerator) throws IOException {
// Using streaming JSON parsing for performance. We expect the JSON to be correct, validation
// is absent/minimal.
JsonParser jp = JsonFormat.JSON_FACTORY.createJsonParser(data);
JsonToken current;
current = jp.nextToken();
if (current != JsonToken.START_OBJECT) {
throw new RuntimeException("Not a JSON object.");
}
while (jp.nextToken() != JsonToken.END_OBJECT) {
String fieldName = jp.getCurrentName();
current = jp.nextToken(); // move from field name to field value
if (fieldName.equals("type")) {
String messageType = jp.getText();
if (messageType.equals(Type.CREATE.getName())) {
type = Type.CREATE;
} else if (messageType.equals(Type.DELETE.getName())) {
type = Type.DELETE;
} else if (messageType.equals(Type.UPDATE.getName())) {
type = Type.UPDATE;
} else if (messageType.equals(Type.INDEX.getName())) {
type = Type.INDEX;
} else {
throw new RuntimeException("Unexpected kind of message type: " + messageType);
}
} else if (fieldName.equals("tableName")) {
this.tableName = jp.getText();
} else if (fieldName.equals("versionCreated")) {
versionCreated = jp.getLongValue();
} else if (fieldName.equals("versionUpdated")) {
versionUpdated = jp.getLongValue();
} else if (fieldName.equals("recordTypeChanged")) {
recordTypeChanged = jp.getBooleanValue();
} else if (fieldName.equals("updatedFields")) {
if (current != JsonToken.START_ARRAY) {
throw new RuntimeException("updatedFields is not a JSON array");
}
while (jp.nextToken() != JsonToken.END_ARRAY) {
addUpdatedField(idGenerator.getSchemaId(jp.getBinaryValue()));
}
} else if (fieldName.equals("vtagsToIndex")) {
if (current != JsonToken.START_ARRAY) {
throw new RuntimeException("vtagsToIndex is not a JSON array");
}
while (jp.nextToken() != JsonToken.END_ARRAY) {
addVTagToIndex(idGenerator.getSchemaId(jp.getBinaryValue()));
}
} else if (fieldName.equals("attributes")) {
if (current != JsonToken.START_OBJECT) {
throw new RuntimeException("Attributes is not a JSON object");
}
this.attributes = new HashMap<String, String>();
while (jp.nextToken() != JsonToken.END_OBJECT) {
String key = jp.getCurrentName();
String value = jp.getText();
attributes.put(key, value);
}
} else if (fieldName.equals("indexFilterData")) {
this.indexRecordFilterData = new IndexRecordFilterData(jp, idGenerator);
}
}
}
