/**
* Releases all the resources used by this Kiji instance.
*
* @throws IOException on I/O error.
*/
private void close() throws IOException {
final State oldState = mState.getAndSet(State.CLOSED);
Preconditions.checkState(
oldState == State.OPEN, "Cannot close Kiji instance %s in state %s.", this, oldState);
LOG.debug("Closing {}.", this);
if (mMonitor != null) {
try {
mMonitor.unregisterInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
} catch (KeeperException ke) {
// Unrecoverable ZooKeeper error:
throw new IOException(ke);
}
mMonitor.close();
}
if (mZKClient != null) {
mZKClient.release();
}
ResourceUtils.closeOrLog(mMetaTable);
ResourceUtils.closeOrLog(mSystemTable);
ResourceUtils.closeOrLog(mSchemaTable);
ResourceUtils.closeOrLog(mSecurityManager);
ResourceUtils.closeOrLog(mAdmin);
mSchemaTable = null;
mMetaTable = null;
mAdmin = null;
mSecurityManager = null;
LOG.debug("{} closed.", this);
}
@Test
public void testMultipleArguments() throws Exception {
final Kiji kiji = getKiji();
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FORMATTED_RKF);
new InstanceBuilder(kiji)
.withTable(layout.getName(), layout)
.withRow("dummy", "str1", "str2", 1, 2L)
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withValue(2L, "string-value2")
.withRow("dummy", "str1", "str2", 1)
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy", "str1", "str2")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy", "str1")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.build();
final KijiTable table = kiji.openTable(layout.getName());
final KijiTableLayout layoutTwo = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
kiji.createTable(layoutTwo.getDesc());
final KijiTable tableTwo = kiji.openTable(layoutTwo.getName());
try {
assertEquals(
BaseTool.SUCCESS,
runTool(new LsTool(), table.getURI().toString(), tableTwo.getURI().toString()));
assertEquals(9, mToolOutputLines.length);
assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), kiji.getURI().toString()));
assertEquals(2, mToolOutputLines.length);
assertEquals(
BaseTool.SUCCESS,
runTool(new LsTool(), kiji.getURI().toString(), table.getURI().toString()));
assertEquals(3, mToolOutputLines.length);
// assertEquals(2, mToolOutputLines.length);
} finally {
ResourceUtils.releaseOrLog(table);
ResourceUtils.releaseOrLog(tableTwo);
}
}
/**
* Release all resources used by this record reader.
*
* @throws IOException if there is an error closing the resources.
*/
@Override
public void close() throws IOException {
ResourceUtils.closeOrLog(mScanner);
ResourceUtils.closeOrLog(mReader);
ResourceUtils.releaseOrLog(mTable);
ResourceUtils.releaseOrLog(mKiji);
mIterator = null;
mScanner = null;
mReader = null;
mTable = null;
mKiji = null;
mSplit = null;
}
/**
* Generates a split for a given table.
*
* @param tableURI URI of the Kiji table to split.
* @param nsplits Number of splits.
* @param conf Base Hadoop configuration used to open the Kiji instance.
* @return a list of split start keys, as HFileKeyValue (with no value, just the keys).
* @throws IOException on I/O error.
*/
private static List<HFileKeyValue> makeTableKeySplit(
KijiURI tableURI, int nsplits, Configuration conf) throws IOException {
final Kiji kiji = Kiji.Factory.open(tableURI, conf);
try {
final KijiTable table = kiji.openTable(tableURI.getTable());
try {
if (NUM_SPLITS_AUTO == nsplits) {
final List<HFileKeyValue> startKeys = Lists.newArrayList();
for (KijiRegion region : table.getRegions()) {
startKeys.add(HFileKeyValue.createFromRowKey(region.getStartKey()));
}
return startKeys;
} else {
switch (KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat())) {
case RAW:
{
// The user has explicitly specified how many HFiles to create, but this is not
// possible when row key hashing is disabled.
throw new JobConfigurationException(
String.format(
"Table '%s' has row key hashing disabled, so the number of HFile splits must be"
+ "determined by the number of HRegions in the HTable. "
+ "Use an HFileMapReduceJobOutput constructor that enables auto splitting.",
table.getName()));
}
case FORMATTED:
case HASH:
case HASH_PREFIX:
{
// Those cases are supported:
break;
}
default:
throw new RuntimeException(
"Unhandled row key encoding: "
+ KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat()));
}
return generateEvenStartKeys(nsplits);
}
} finally {
ResourceUtils.releaseOrLog(table);
}
} finally {
ResourceUtils.releaseOrLog(kiji);
}
}
/**
* Creates a temporary JSON file with the specified layout.
*
* @param desc Layout descriptor.
* @return Temporary JSON file containing the specified layout.
* @throws IOException on I/O error.
*/
public static File getTempFile(TableLayoutDesc desc) throws IOException {
final File layoutFile = File.createTempFile("layout-" + desc.getName(), "json");
layoutFile.deleteOnExit();
final OutputStream fos = new FileOutputStream(layoutFile);
IOUtils.write(ToJson.toJsonString(desc), fos);
ResourceUtils.closeOrLog(fos);
return layoutFile;
}
/**
* Initializes a new table-wide record writer.
*
* @param oformat KijiHFileOutputFormat this writer is built from.
* @param context Context of the task.
* @throws IOException on I/O error.
*/
public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context)
throws IOException {
mContext = Preconditions.checkNotNull(context);
mConf = mContext.getConfiguration();
mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis());
mLatestTimestampBytes = toBytes(mLatestTimestamp);
mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION);
mFileSystem = mOutputDir.getFileSystem(mConf);
mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build();
final Kiji kiji = Kiji.Factory.open(mTableURI, mConf);
final KijiTable table = kiji.openTable(mTableURI.getTable());
mLayout = table.getLayout();
ResourceUtils.releaseOrLog(table);
ResourceUtils.releaseOrLog(kiji);
}
/**
* Loads a table layout descriptor from a JSON resource.
*
* @param resourcePath Path of the resource to load the JSON descriptor from.
* @return the decoded TableLayoutDesc.
* @throws IOException on I/O error.
*/
public static TableLayoutDesc getLayout(String resourcePath) throws IOException {
final InputStream istream =
KijiTableLayouts.class.getClassLoader().getResourceAsStream(resourcePath);
try {
final String json = IOUtils.toString(istream);
return (TableLayoutDesc) FromJson.fromJsonString(json, TableLayoutDesc.SCHEMA$);
} finally {
ResourceUtils.closeOrLog(istream);
}
}
/**
* Closes an HFile writer.
*
* @param hfileWriter The writer to close.
* @throws IOException If there is an error.
*/
private void closeWriter(HFile.Writer hfileWriter) throws IOException {
LOG.info("Closing HFile " + hfileWriter.getPath());
// Write file metadata:
hfileWriter.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY, toBytes(mLatestTimestamp));
final String taskAttemptID = mContext.getTaskAttemptID().toString();
hfileWriter.appendFileInfo(StoreFile.BULKLOAD_TASK_KEY, toBytes(taskAttemptID));
hfileWriter.appendFileInfo(StoreFile.MAJOR_COMPACTION_KEY, toBytes(true));
ResourceUtils.closeOrLog(hfileWriter);
}
@Test
public void testKijiLsStartAndLimitRow() throws Exception {
final Kiji kiji = getKiji();
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
kiji.createTable(layout.getDesc());
final KijiTable table = kiji.openTable(layout.getName());
try {
assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
// TODO: Validate output
} finally {
ResourceUtils.releaseOrLog(table);
}
}
@Test
public void testTableColumns() throws Exception {
final Kiji kiji = getKiji();
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.SIMPLE);
kiji.createTable(layout.getDesc());
final KijiTable table = kiji.openTable(layout.getName());
try {
// Table is empty:
assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
assertEquals(1, mToolOutputLines.length);
assertTrue(mToolOutputLines[0].contains("family:column"));
} finally {
ResourceUtils.releaseOrLog(table);
}
}
@Test
public void testKVStoreInIsFresh() throws IOException {
// Create a freshness policy that knows where to find the text file backed kv-store.
KijiFreshnessPolicy policy =
new KVStoreInIsFreshPolicy("file:" + new File(getLocalTempDir(), KV_FILENAME));
// Install a freshness policy.
KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
try {
manager.registerFreshener(
"user",
new KijiColumnName("info", "name"),
policy,
new UnconfiguredScoreFunction(),
Collections.<String, String>emptyMap(),
true,
false);
} finally {
manager.close();
}
KijiTable userTable = null;
FreshKijiTableReader freshReader = null;
try {
userTable = getKiji().openTable("user");
freshReader =
FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
// Read from the table to ensure that the user name is updated.
KijiRowData data =
freshReader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
// IsFresh should have returned true, so nothing should be written.
assertEquals("Felis", data.getMostRecentValue("info", "name").toString());
} finally {
ResourceUtils.closeOrLog(freshReader);
ResourceUtils.releaseOrLog(userTable);
}
}
@Test
public void testFormattedRowKey() throws Exception {
final Kiji kiji = getKiji();
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FORMATTED_RKF);
new InstanceBuilder(kiji)
.withTable(layout.getName(), layout)
.withRow("dummy", "str1", "str2", 1, 2L)
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withValue(2L, "string-value2")
.withRow("dummy", "str1", "str2", 1)
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy", "str1", "str2")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy", "str1")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.withRow("dummy")
.withFamily("family")
.withQualifier("column")
.withValue(1L, "string-value")
.build();
final KijiTable table = kiji.openTable(layout.getName());
try {
assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
} finally {
ResourceUtils.releaseOrLog(table);
}
}