@Test
public void testMultipleArguments() throws Exception {
final Kiji kiji = getKiji();
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FORMATTED_RKF);
new InstanceBuilder(kiji)
.withTable(layout.getName(), layout)
.withRow("dummy", "str1", "str2", 1, 2L)
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withValue(2L, "string-value2")
.withRow("dummy", "str1", "str2", 1)
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy", "str1", "str2")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy", "str1")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.build();
final KijiTableLayout layoutTwo = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
kiji.createTable(layoutTwo.getDesc());
final KijiTable table = kiji.openTable(layout.getName());
try {
final KijiTable tableTwo = kiji.openTable(layoutTwo.getName());
try {
assertEquals(
BaseTool.SUCCESS,
runTool(new LsTool(), table.getURI().toString(), tableTwo.getURI().toString()));
assertEquals(9, mToolOutputLines.length);
assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), kiji.getURI().toString()));
assertEquals(2, mToolOutputLines.length);
assertEquals(
BaseTool.SUCCESS,
runTool(new LsTool(), kiji.getURI().toString(), table.getURI().toString()));
assertEquals(3, mToolOutputLines.length);
} finally {
tableTwo.release();
}
} finally {
table.release();
}
}
private <T> T populateFromRow(
EntitySpec<T> spec, T entity, long startTime, long endTime, Object... entityIdComponents)
throws IOException {
// TODO: Use a pool of tables and/or table readers
final KijiTable table = mKiji.openTable(spec.getTableName());
try {
final KijiTableReader reader = table.openTableReader();
try {
final KijiDataRequestBuilder builder = KijiDataRequest.builder();
builder.withTimeRange(startTime, endTime);
spec.populateColumnRequests(builder);
final KijiDataRequest dataRequest = builder.build();
final EntityId entityId = table.getEntityId(entityIdComponents);
final KijiRowData row = reader.get(entityId, dataRequest);
try {
return spec.populateEntityFromRow(entity, row);
} catch (IllegalAccessException iae) {
throw new RuntimeException(iae);
}
} finally {
reader.close();
}
} finally {
table.release();
}
}
/** A test to ensure that policies can mask the key value stores of their producers. */
@Test
public void testKVMasking() throws IOException {
// Create a freshness policy that knows where to find the text file backed kv-store.
KijiFreshnessPolicy policy =
new ShadowingFreshening("file:" + new File(getLocalTempDir(), KV_FILENAME));
// Install a freshness policy.
KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
try {
manager.registerFreshener(
"user",
new KijiColumnName("info", "name"),
policy,
new UnconfiguredScoreFunction(),
Collections.<String, String>emptyMap(),
true,
false);
} finally {
manager.close();
}
final KijiTable userTable = getKiji().openTable("user");
try {
final FreshKijiTableReader reader =
FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
try {
// Read from the table to ensure that the user name is updated.
KijiRowData data =
reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
assertEquals("Old Gumbie Cat", data.getMostRecentValue("info", "name").toString());
} finally {
reader.close();
}
} finally {
userTable.release();
}
}
/**
* Gets a set of input splits for a MapReduce job running over a Kiji table. One split is created
* per region in the input Kiji table.
*
* @param configuration of the job using the splits. The configuration should specify the input
* Kiji table being used, through the configuration variable {@link
* KijiConfKeys#KIJI_INPUT_TABLE_URI}.
* @param numSplits desired for the job. This framework hint is ignored by this method.
* @return an array of input splits to be operated on in the MapReduce job.
* @throws IOException if an I/O error occurs while communicating with HBase to determine the
* regions in the Kiji table.
*/
@Override
public InputSplit[] getSplits(JobConf configuration, int numSplits) throws IOException {
final String uriString =
Preconditions.checkNotNull(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI));
final KijiURI inputTableURI = KijiURI.newBuilder(uriString).build();
final Kiji kiji = Kiji.Factory.open(inputTableURI, configuration);
try {
final KijiTable table = kiji.openTable(inputTableURI.getTable());
try {
final HTableInterface htable = HBaseKijiTable.downcast(table).getHTable();
final List<InputSplit> splits = Lists.newArrayList();
for (KijiRegion region : table.getRegions()) {
final byte[] startKey = region.getStartKey();
// TODO(KIJIMR-65): For now pick the first available location (ie. region server), if any.
final String location =
region.getLocations().isEmpty() ? null : region.getLocations().iterator().next();
final TableSplit tableSplit =
new TableSplit(htable.getTableName(), startKey, region.getEndKey(), location);
splits.add(new KijiTableSplit(tableSplit));
}
return splits.toArray(new InputSplit[0]);
} finally {
table.release();
}
} finally {
kiji.release();
}
}
@Test
public void testTableNoFamilies() throws Exception {
final Kiji kiji =
new InstanceBuilder(getKiji())
.withTable(KijiTableLayouts.getLayout(KijiTableLayouts.NOFAMILY))
.build();
final KijiTable table = kiji.openTable("nofamily");
try {
assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
} finally {
table.release();
}
}
/** A test to make sure that producers run inside of freshening can access key value stores. */
@Test
public void testSimpleKVStore() throws IOException {
final String path = new Path("file:" + new File(getLocalTempDir(), KV_FILENAME)).toString();
final Map<String, String> params = Maps.newHashMap();
params.put(SimpleKVScoreFunction.PARAMETER_KEY, path);
// Install a freshness policy.
KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
try {
manager.registerFreshener(
"user",
new KijiColumnName("info", "name"),
AlwaysFreshen.class.getName(),
SimpleKVScoreFunction.class.getName(),
params,
true,
false,
false);
} finally {
manager.close();
}
final KijiTable userTable = getKiji().openTable("user");
try {
final FreshKijiTableReader reader =
FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
try {
// Read from the table to ensure that the user name is updated.
KijiRowData data =
reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
assertEquals("Railway Cat", data.getMostRecentValue("info", "name").toString());
} finally {
reader.close();
}
} finally {
userTable.release();
}
}
@After
public final void teardownTestBulkImporter() throws Exception {
mReader.close();
mTable.release();
}
/**
* Initializes a new specification for an Entity from an annotated Java class.
*
* @param klass Annotated Java class to derive an entity specification from.
* @param kiji Kiji instance where to fetch entities from.
* @throws IOException on I/O error.
*/
public EntitySpec(Class<T> klass, Kiji kiji) throws IOException {
mClass = klass;
final KijiEntity entity = klass.getAnnotation(KijiEntity.class);
Preconditions.checkArgument(
entity != null, "Class '{}' has no @KijiEntity annotation.", klass);
mTableName = entity.table();
final KijiTable table = kiji.openTable(mTableName);
try {
final KijiTableLayout layout = table.getLayout();
// TODO: Support deprecated RowKeyFormat?
final RowKeyFormat2 rowKeyFormat = (RowKeyFormat2) layout.getDesc().getKeysFormat();
final Map<String, RowKeyComponent> rkcMap = Maps.newHashMap();
final Map<String, Integer> rkcIndexMap = Maps.newHashMap();
for (int index = 0; index < rowKeyFormat.getComponents().size(); ++index) {
final RowKeyComponent rkc = rowKeyFormat.getComponents().get(index);
rkcMap.put(rkc.getName(), rkc);
rkcIndexMap.put(rkc.getName(), index);
}
mRowKeyComponentMap = ImmutableMap.copyOf(rkcMap);
mRowKeyComponentIndexMap = ImmutableMap.copyOf(rkcIndexMap);
// --------------------------------------------------------------------
// Parse fields with annotations from the entity class:
final List<Field> columnFields = Lists.newArrayList();
final List<Field> entityIdFields = Lists.newArrayList();
for (final Field field : mClass.getDeclaredFields()) {
final KijiColumn column = field.getAnnotation(KijiColumn.class);
final EntityIdField eidField = field.getAnnotation(EntityIdField.class);
if ((column != null) && (eidField != null)) {
throw new IllegalArgumentException(
String.format(
"Field '%s' cannot have both @KijiColumn and @EntityIdField annotations.",
field));
} else if (column != null) {
LOG.debug("Validating column field '{}'.", field);
field.setAccessible(true);
columnFields.add(field);
final FamilyLayout flayout = layout.getFamilyMap().get(column.family());
Preconditions.checkArgument(
flayout != null,
"Field '%s' maps to non-existing family '%s' from table '%s'.",
field,
column.family(),
mTableName);
if (column.qualifier().isEmpty()) {
// Request for a map-type family:
Preconditions.checkArgument(
flayout.isMapType(),
"Field '%s' maps to family '%s' from table '%s' which is not a map-type family.",
field,
column.family(),
mTableName);
// Validate field type:
if (column.pageSize() > 0) {
Preconditions.checkArgument(
MapFamilyVersionIterator.class.isAssignableFrom(field.getType()),
"Fields mapped to map-type family with paging enabled must be "
+ "MapFamilyVersionIterator, got '{}'.",
field.getType());
} else {
// TODO Validate type when no paging enabled on map-type family.
}
} else {
// Request for a fully-qualified column:
final ColumnLayout clayout = flayout.getColumnMap().get(column.qualifier());
Preconditions.checkArgument(
flayout != null,
"Field '%s' maps to non-existing column '%s:%s' from table '%s'.",
field,
column.family(),
column.qualifier(),
mTableName);
// Validate field type:
if (column.pageSize() > 0) {
Preconditions.checkArgument(
ColumnVersionIterator.class.isAssignableFrom(field.getType()),
"Fields mapped to column with paging enabled must be "
+ "ColumnVersionIterator, got '{}'.",
field.getType());
} else {
// TODO Validate type when no paging enabled on the column.
}
}
} else if (eidField != null) {
LOG.debug("Validating entity ID field '{}'.", field);
field.setAccessible(true);
entityIdFields.add(field);
final RowKeyComponent rkc = mRowKeyComponentMap.get(eidField.component());
Preconditions.checkArgument(
rkc != null,
"Field '%s' maps to unknown entity ID component '%s'.",
field,
eidField.component());
} else {
LOG.debug("Ignoring field '{}' with no annotation.", field);
}
}
mColumnFields = ImmutableList.copyOf(columnFields);
mEntityIdFields = ImmutableList.copyOf(entityIdFields);
} finally {
table.release();
}
}