@Test
public void testBuilder() throws Exception {
final KijiTableLayout layout =
KijiTableLayout.newLayout(KijiTableLayouts.getLayout(KijiTableLayouts.SIMPLE));
final Kiji kiji =
new InstanceBuilder()
.withTable("table", layout)
.withRow("row1")
.withFamily("family")
.withQualifier("column")
.withValue(1, "foo1")
.withValue(2, "foo2")
.withRow("row2")
.withFamily("family")
.withQualifier("column")
.withValue(100, "foo3")
.build();
final KijiTable table = kiji.openTable("table");
final KijiTableReader reader = table.openTableReader();
// Verify the first row.
final KijiDataRequest req = KijiDataRequest.create("family", "column");
final KijiRowData row1 = reader.get(table.getEntityId("row1"), req);
assertEquals("foo2", row1.getValue("family", "column", 2).toString());
// Verify the second row.
final KijiRowData row2 = reader.get(table.getEntityId("row2"), req);
assertEquals("foo3", row2.getValue("family", "column", 100).toString());
ResourceUtils.closeOrLog(reader);
ResourceUtils.releaseOrLog(table);
ResourceUtils.releaseOrLog(kiji);
}
/** A test to ensure that policies can mask the key value stores of their producers. */
@Test
public void testKVMasking() throws IOException {
// Create a freshness policy that knows where to find the text file backed kv-store.
KijiFreshnessPolicy policy =
new ShadowingFreshening("file:" + new File(getLocalTempDir(), KV_FILENAME));
// Install a freshness policy.
KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
try {
manager.registerFreshener(
"user",
new KijiColumnName("info", "name"),
policy,
new UnconfiguredScoreFunction(),
Collections.<String, String>emptyMap(),
true,
false);
} finally {
manager.close();
}
final KijiTable userTable = getKiji().openTable("user");
try {
final FreshKijiTableReader reader =
FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
try {
// Read from the table to ensure that the user name is updated.
KijiRowData data =
reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
assertEquals("Old Gumbie Cat", data.getMostRecentValue("info", "name").toString());
} finally {
reader.close();
}
} finally {
userTable.release();
}
}
private final void populateFieldFromFullyQualifiedColumn(
T entity, Field field, KijiColumn column, KijiRowData row)
throws IOException, IllegalAccessException {
if (column.maxVersions() == 1) {
// Field represents a single value from a fully-qualified column:
LOG.debug(
"Populating field '{}' from column '{}:{}'.",
field,
column.family(),
column.qualifier());
KijiCell<?> cell = row.getMostRecentCell(column.family(), column.qualifier());
if (cell == null) return;
Object value = cell.getData();
if (field.getType() == KijiCell.class) {
value = cell;
} else if (field.getType() == String.class && value != null) {
value = value.toString();
}
// If there is no cell for a field with a primitive type, use the default value:
if ((null == value) && field.getType().isPrimitive()) {
value = Defaults.defaultValue(field.getType());
}
field.set(entity, value);
} else {
// Field represents a time-series from a fully-qualified column:
if (column.pageSize() > 0) {
final ColumnVersionIterator<?> iterator =
new ColumnVersionIterator<Object>(
row, column.family(), column.qualifier(), column.pageSize());
field.set(entity, iterator);
} else {
Object value = null;
if (field.getType() == KijiCellValueIterator.class) {
value =
new KijiCellValueIterator<Object>(
row.iterator(column.family(), column.qualifier()));
} else if (field.getType() == TimeSeries.class) {
final TimeSeries<Object> timeseries = new TimeSeries<Object>();
for (final KijiCell<Object> cell :
row.asIterable(column.family(), column.qualifier())) {
timeseries.put(cell.getTimestamp(), cell.getData());
}
value = timeseries;
}
field.set(entity, value);
}
}
}
/** {@inheritDoc} */
@Override
public void produce(KijiRowData input, ProducerContext context) throws IOException {
if (!input.containsColumn("info", "email")) {
// This user doesn't have an email address.
return;
}
String email = input.getMostRecentValue("info", "email").toString();
int atSymbol = email.indexOf("@");
if (atSymbol < 0) {
// Couldn't find the '@' in the email address. Give up.
return;
}
String domain = email.substring(atSymbol + 1);
context.put(domain);
}
/**
* Populates an entity from a row.
*
* @param entity Entity object to populate from a row.
* @param row Kiji row to populate the entity from.
* @return the populated entity.
* @throws IllegalAccessException
* @throws IOException
*/
public T populateEntityFromRow(T entity, KijiRowData row)
throws IllegalAccessException, IOException {
// Populate fields from the row columns:
for (final Field field : mColumnFields) {
final KijiColumn column = field.getAnnotation(KijiColumn.class);
Preconditions.checkState(column != null);
if (column.qualifier().isEmpty()) {
// Field is populated from a map-type family:
populateFieldFromMapTypeFamily(entity, field, column, row);
} else {
// Field is populated from a fully-qualified column:
populateFieldFromFullyQualifiedColumn(entity, field, column, row);
}
}
// Populate fields from the row entity ID:
for (final Field field : mEntityIdFields) {
final EntityIdField eidField = field.getAnnotation(EntityIdField.class);
Preconditions.checkState(eidField != null);
final int index = mRowKeyComponentIndexMap.get(eidField.component());
field.set(entity, row.getEntityId().getComponentByIndex(index));
}
return entity;
}
/**
* Populates the specified key and value with the next key-value pair read from the input split.
*
* @param key instance to populate with the next key read.
* @param value instance to popualte with the next value read.
* @return <code>true</code> if a new key-value was read, <code>false</code> if we have reached
* the end of the input split.
*/
@Override
public boolean next(KijiKey key, KijiValue value) {
if (mIterator.hasNext()) {
// Read the next row and store it in the provided key/value pair.
final KijiRowData row = mIterator.next();
if (null != key) {
key.set(row.getEntityId());
}
if (null != value) {
value.set(row);
}
return true;
} else {
return false;
}
}
/** A test to make sure that producers run inside of freshening can access key value stores. */
@Test
public void testSimpleKVStore() throws IOException {
final String path = new Path("file:" + new File(getLocalTempDir(), KV_FILENAME)).toString();
final Map<String, String> params = Maps.newHashMap();
params.put(SimpleKVScoreFunction.PARAMETER_KEY, path);
// Install a freshness policy.
KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
try {
manager.registerFreshener(
"user",
new KijiColumnName("info", "name"),
AlwaysFreshen.class.getName(),
SimpleKVScoreFunction.class.getName(),
params,
true,
false,
false);
} finally {
manager.close();
}
final KijiTable userTable = getKiji().openTable("user");
try {
final FreshKijiTableReader reader =
FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
try {
// Read from the table to ensure that the user name is updated.
KijiRowData data =
reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
assertEquals("Railway Cat", data.getMostRecentValue("info", "name").toString());
} finally {
reader.close();
}
} finally {
userTable.release();
}
}
@Test
public void testKVStoreInIsFresh() throws IOException {
// Create a freshness policy that knows where to find the text file backed kv-store.
KijiFreshnessPolicy policy =
new KVStoreInIsFreshPolicy("file:" + new File(getLocalTempDir(), KV_FILENAME));
// Install a freshness policy.
KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
try {
manager.registerFreshener(
"user",
new KijiColumnName("info", "name"),
policy,
new UnconfiguredScoreFunction(),
Collections.<String, String>emptyMap(),
true,
false);
} finally {
manager.close();
}
KijiTable userTable = null;
FreshKijiTableReader freshReader = null;
try {
userTable = getKiji().openTable("user");
freshReader =
FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
// Read from the table to ensure that the user name is updated.
KijiRowData data =
freshReader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
// IsFresh should have returned true, so nothing should be written.
assertEquals("Felis", data.getMostRecentValue("info", "name").toString());
} finally {
ResourceUtils.closeOrLog(freshReader);
ResourceUtils.releaseOrLog(userTable);
}
}
private final void populateFieldFromMapTypeFamily(
T entity, Field field, KijiColumn column, KijiRowData row)
throws IOException, IllegalAccessException {
LOG.debug("Populating field '{}' from map-type family '{}'.", field, column.family());
if (column.pageSize() > 0) {
// Field is a closeable iterator of map-family entries (qualifier, timestamp, value).
LOG.debug(
"Populating field '{}' from paging-enabled map-type family '{}'.",
field,
column.family());
final MapFamilyVersionIterator<?> iterator =
new MapFamilyVersionIterator<Object>(
row, column.family(), column.pageSize(), column.pageSize());
field.set(entity, iterator);
} else if (column.maxVersions() == 1) {
// Field is a map: qualifier -> single value:
LOG.debug("Populating single version map field '{}'.", field);
Object value = null;
if (field.getType() == KijiCellIterator.class) {
Iterator<KijiCell<Object>> it = row.iterator(column.family());
value = new KijiCellIterator<Object>(it);
} else if (field.getType() == MapTypeCell.class) {
value = new MapTypeCell<Object>(row.getMostRecentCells(column.family()));
} else if (field.getType() == MapTypeValue.class) {
value = new MapTypeValue<Object>(row.getMostRecentValues(column.family()));
}
field.set(entity, value);
} else {
// Field is a map: qualifier -> time-series
LOG.debug("Populating map field '{}'.", field);
Object value = null;
if (field.getType() == KijiCellIterator.class) {
Iterator<KijiCell<Object>> it = row.iterator(column.family());
value = new KijiCellIterator<Object>(it);
} else if (field.getType() == TSMapTypeValue.class) {
// TODO: ARGH. This is awful.
TSMapTypeValue<Object> tsValues = new TSMapTypeValue<Object>();
for (String s : row.getQualifiers(column.family())) {
final TimeSeries<Object> timeseries = new TimeSeries<Object>();
for (final KijiCell<Object> cell : row.asIterable(column.family(), s)) {
timeseries.put(cell.getTimestamp(), cell.getData());
}
tsValues.put(s, timeseries);
}
value = tsValues;
}
field.set(entity, value);
}
}
@Override
public void gather(KijiRowData kijiRowData, GathererContext gathererContext) throws IOException {
rows++;
if (rows % 5000 == 0) {
LOG.info("Processed row: " + rows);
}
try {
// After 2005, before the future plus 10 minutes to allow for clock misalignments
checkLong(
kijiRowData.getMostRecentCell("data", "match_id").getTimestamp(),
"start_time",
1104537600,
System.currentTimeMillis() + 60000);
for (String team : TEAMS) {
Integer towerStatus = kijiRowData.getMostRecentValue("data", team + "_towers_status");
checkInt(towerStatus, team + "_tower_status", 0, ((Double) Math.pow(2, 11)).intValue());
}
for (String team : TEAMS) {
Integer raxStatus = kijiRowData.getMostRecentValue("data", team + "_barracks_status");
checkInt(raxStatus, team + "_barracks_status", 0, ((Double) Math.pow(2, 6)).intValue());
}
checkInt(
(Integer) kijiRowData.getMostRecentValue("data", "human_players"),
"human_players",
0,
10);
DotaValues.LobbyType.fromInt((Integer) kijiRowData.getMostRecentValue("data", "lobby_type"));
DotaValues.GameMode.fromInt((Integer) kijiRowData.getMostRecentValue("data", "game_mode"));
for (String s :
new String[] {"cluster", "season", "duration", "negative_votes", "positive_votes"}) {
Integer n = kijiRowData.getMostRecentValue("data", s);
checkInt(n, s, 0, Integer.MAX_VALUE / 2);
}
checkInt(
(Integer) kijiRowData.getMostRecentValue("data", "league_id"),
"league_id",
0,
Integer.MAX_VALUE);
Players players = kijiRowData.getMostRecentValue("data", "player_data");
for (Player player : players.getPlayers()) {
Integer n = player.getAccountId();
if (n != null) {
if (n != -1) {
checkInt(n, "account_id", 0, Integer.MAX_VALUE);
}
}
DotaValues.LeaverStatus.fromInt(player.getLeaverStatus());
checkInt(player.getAssists(), "assists", 0, 1000);
checkInt(player.getDeaths(), "deaths", 0, 1000);
checkInt(player.getDenies(), "denies", 0, 5000);
checkInt(player.getGold(), "gold", 0, 10000);
checkInt(player.getGoldSpent(), "gold_spent", 0, 1000000);
checkInt(player.getHeroDamage(), "hero_damage");
checkInt(player.getHeroHealing(), "hero_healing");
checkInt(player.getHeroId(), "hero_id", 0, 200);
checkInt(player.getKills(), "kills", 0, 1000);
checkInt(player.getLastHits(), "last_hits", 0, 10000);
checkInt(player.getPlayerSlot(), "player_slot", 0, 256);
checkInt(player.getLevel(), "level", 0, 25);
checkDouble(player.getGoldPerMinute(), "gold_per_minute", 0, 200000);
checkDouble(player.getExpPerMinute(), "exp_per_minute", 0, 200000);
List<AbilityUpgrade> aus = player.getAbilityUpgrades();
checkInt(aus.size(), "ability upgrade size", 0, 25);
for (AbilityUpgrade au : aus) {
checkInt(au.getLevel(), "ability upgrade level", 0, 25);
checkInt(au.getTime(), "ability upgrade time", 0, 1000000);
checkInt(au.getAbilityId(), "ability id");
}
}
} catch (RuntimeException re) {
gathererContext.write(new Text(re.getMessage()), ONE);
}
}
/** {@inheritDoc} */
@Override
protected void map(KijiRowData input, Context mapContext) throws IOException {
mProducerContext.setEntityId(input.getEntityId());
mProducer.produce(input, mProducerContext);
mapContext.getCounter(JobHistoryCounters.PRODUCER_ROWS_PROCESSED).increment(1);
}
