/**
* Generates a split for a given table.
*
* @param tableURI URI of the Kiji table to split.
* @param nsplits Number of splits.
* @param conf Base Hadoop configuration used to open the Kiji instance.
* @return a list of split start keys, as HFileKeyValue (with no value, just the keys).
* @throws IOException on I/O error.
*/
private static List<HFileKeyValue> makeTableKeySplit(
KijiURI tableURI, int nsplits, Configuration conf) throws IOException {
final Kiji kiji = Kiji.Factory.open(tableURI, conf);
try {
final KijiTable table = kiji.openTable(tableURI.getTable());
try {
if (NUM_SPLITS_AUTO == nsplits) {
final List<HFileKeyValue> startKeys = Lists.newArrayList();
for (KijiRegion region : table.getRegions()) {
startKeys.add(HFileKeyValue.createFromRowKey(region.getStartKey()));
}
return startKeys;
} else {
switch (KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat())) {
case RAW:
{
// The user has explicitly specified how many HFiles to create, but this is not
// possible when row key hashing is disabled.
throw new JobConfigurationException(
String.format(
"Table '%s' has row key hashing disabled, so the number of HFile splits must be"
+ "determined by the number of HRegions in the HTable. "
+ "Use an HFileMapReduceJobOutput constructor that enables auto splitting.",
table.getName()));
}
case FORMATTED:
case HASH:
case HASH_PREFIX:
{
// Those cases are supported:
break;
}
default:
throw new RuntimeException(
"Unhandled row key encoding: "
+ KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat()));
}
return generateEvenStartKeys(nsplits);
}
} finally {
ResourceUtils.releaseOrLog(table);
}
} finally {
ResourceUtils.releaseOrLog(kiji);
}
}
/**
* Initializes a new table-wide record writer.
*
* @param oformat KijiHFileOutputFormat this writer is built from.
* @param context Context of the task.
* @throws IOException on I/O error.
*/
public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context)
throws IOException {
mContext = Preconditions.checkNotNull(context);
mConf = mContext.getConfiguration();
mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis());
mLatestTimestampBytes = toBytes(mLatestTimestamp);
mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION);
mFileSystem = mOutputDir.getFileSystem(mConf);
mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build();
final Kiji kiji = Kiji.Factory.open(mTableURI, mConf);
final KijiTable table = kiji.openTable(mTableURI.getTable());
mLayout = table.getLayout();
ResourceUtils.releaseOrLog(table);
ResourceUtils.releaseOrLog(kiji);
}
/**
* Initializes a new specification for an Entity from an annotated Java class.
*
* @param klass Annotated Java class to derive an entity specification from.
* @param kiji Kiji instance where to fetch entities from.
* @throws IOException on I/O error.
*/
public EntitySpec(Class<T> klass, Kiji kiji) throws IOException {
mClass = klass;
final KijiEntity entity = klass.getAnnotation(KijiEntity.class);
Preconditions.checkArgument(
entity != null, "Class '{}' has no @KijiEntity annotation.", klass);
mTableName = entity.table();
final KijiTable table = kiji.openTable(mTableName);
try {
final KijiTableLayout layout = table.getLayout();
// TODO: Support deprecated RowKeyFormat?
final RowKeyFormat2 rowKeyFormat = (RowKeyFormat2) layout.getDesc().getKeysFormat();
final Map<String, RowKeyComponent> rkcMap = Maps.newHashMap();
final Map<String, Integer> rkcIndexMap = Maps.newHashMap();
for (int index = 0; index < rowKeyFormat.getComponents().size(); ++index) {
final RowKeyComponent rkc = rowKeyFormat.getComponents().get(index);
rkcMap.put(rkc.getName(), rkc);
rkcIndexMap.put(rkc.getName(), index);
}
mRowKeyComponentMap = ImmutableMap.copyOf(rkcMap);
mRowKeyComponentIndexMap = ImmutableMap.copyOf(rkcIndexMap);
// --------------------------------------------------------------------
// Parse fields with annotations from the entity class:
final List<Field> columnFields = Lists.newArrayList();
final List<Field> entityIdFields = Lists.newArrayList();
for (final Field field : mClass.getDeclaredFields()) {
final KijiColumn column = field.getAnnotation(KijiColumn.class);
final EntityIdField eidField = field.getAnnotation(EntityIdField.class);
if ((column != null) && (eidField != null)) {
throw new IllegalArgumentException(
String.format(
"Field '%s' cannot have both @KijiColumn and @EntityIdField annotations.",
field));
} else if (column != null) {
LOG.debug("Validating column field '{}'.", field);
field.setAccessible(true);
columnFields.add(field);
final FamilyLayout flayout = layout.getFamilyMap().get(column.family());
Preconditions.checkArgument(
flayout != null,
"Field '%s' maps to non-existing family '%s' from table '%s'.",
field,
column.family(),
mTableName);
if (column.qualifier().isEmpty()) {
// Request for a map-type family:
Preconditions.checkArgument(
flayout.isMapType(),
"Field '%s' maps to family '%s' from table '%s' which is not a map-type family.",
field,
column.family(),
mTableName);
// Validate field type:
if (column.pageSize() > 0) {
Preconditions.checkArgument(
MapFamilyVersionIterator.class.isAssignableFrom(field.getType()),
"Fields mapped to map-type family with paging enabled must be "
+ "MapFamilyVersionIterator, got '{}'.",
field.getType());
} else {
// TODO Validate type when no paging enabled on map-type family.
}
} else {
// Request for a fully-qualified column:
final ColumnLayout clayout = flayout.getColumnMap().get(column.qualifier());
Preconditions.checkArgument(
flayout != null,
"Field '%s' maps to non-existing column '%s:%s' from table '%s'.",
field,
column.family(),
column.qualifier(),
mTableName);
// Validate field type:
if (column.pageSize() > 0) {
Preconditions.checkArgument(
ColumnVersionIterator.class.isAssignableFrom(field.getType()),
"Fields mapped to column with paging enabled must be "
+ "ColumnVersionIterator, got '{}'.",
field.getType());
} else {
// TODO Validate type when no paging enabled on the column.
}
}
} else if (eidField != null) {
LOG.debug("Validating entity ID field '{}'.", field);
field.setAccessible(true);
entityIdFields.add(field);
final RowKeyComponent rkc = mRowKeyComponentMap.get(eidField.component());
Preconditions.checkArgument(
rkc != null,
"Field '%s' maps to unknown entity ID component '%s'.",
field,
eidField.component());
} else {
LOG.debug("Ignoring field '{}' with no annotation.", field);
}
}
mColumnFields = ImmutableList.copyOf(columnFields);
mEntityIdFields = ImmutableList.copyOf(entityIdFields);
} finally {
table.release();
}
}
