@Test
public void testRetrieveAll() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen());
manager.storePolicy("user", "info:visits", TestProducer.class, new NeverFreshen());
LOG.info(KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString());
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=retrieve-all"));
assertEquals(
"Freshness policy attached to column: info:visits\n"
+ " Freshness policy class: org.kiji.scoring.lib.NeverFreshen\n"
+ " Freshness policy state: \n"
+ " Producer class: org.kiji.scoring.tools.TestFreshTool$TestProducer\n"
+ "Freshness policy attached to column: info:name\n"
+ " Freshness policy class: org.kiji.scoring.lib.NeverFreshen\n"
+ " Freshness policy state: \n"
+ " Producer class: org.kiji.scoring.tools.TestFreshTool$TestProducer\n",
mToolOutputStr);
}
/**
* Gets a set of input splits for a MapReduce job running over a Kiji table. One split is created
* per region in the input Kiji table.
*
* @param configuration of the job using the splits. The configuration should specify the input
* Kiji table being used, through the configuration variable {@link
* KijiConfKeys#KIJI_INPUT_TABLE_URI}.
* @param numSplits desired for the job. This framework hint is ignored by this method.
* @return an array of input splits to be operated on in the MapReduce job.
* @throws IOException if an I/O error occurs while communicating with HBase to determine the
* regions in the Kiji table.
*/
@Override
public InputSplit[] getSplits(JobConf configuration, int numSplits) throws IOException {
final String uriString =
Preconditions.checkNotNull(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI));
final KijiURI inputTableURI = KijiURI.newBuilder(uriString).build();
final Kiji kiji = Kiji.Factory.open(inputTableURI, configuration);
try {
final KijiTable table = kiji.openTable(inputTableURI.getTable());
try {
final HTableInterface htable = HBaseKijiTable.downcast(table).getHTable();
final List<InputSplit> splits = Lists.newArrayList();
for (KijiRegion region : table.getRegions()) {
final byte[] startKey = region.getStartKey();
// TODO(KIJIMR-65): For now pick the first available location (ie. region server), if any.
final String location =
region.getLocations().isEmpty() ? null : region.getLocations().iterator().next();
final TableSplit tableSplit =
new TableSplit(htable.getTableName(), startKey, region.getEndKey(), location);
splits.add(new KijiTableSplit(tableSplit));
}
return splits.toArray(new InputSplit[0]);
} finally {
table.release();
}
} finally {
kiji.release();
}
}
@Test
public void testListIntances() throws Exception {
final Kiji kiji = getKiji();
final KijiURI hbaseURI = KijiURI.newBuilder(kiji.getURI()).withInstanceName(null).build();
final LsTool ls = new LsTool();
assertEquals(BaseTool.SUCCESS, runTool(ls, "--kiji=" + hbaseURI));
assertEquals(1, mToolOutputLines.length);
assertEquals(kiji.getURI(), KijiURI.newBuilder(mToolOutputLines[0]).build());
}
@Test
public void testListInstances() throws Exception {
final Kiji kiji = getKiji();
final KijiURI hbaseURI = KijiURI.newBuilder(kiji.getURI()).withInstanceName(null).build();
final LsTool ls = new LsTool();
assertEquals(BaseTool.SUCCESS, runTool(ls, hbaseURI.toString()));
final Set<String> instances = Sets.newHashSet(mToolOutputLines);
assertTrue(instances.contains(kiji.getURI().toString()));
}
/**
* Extracts the ID of the fake HBase from a Kiji URI.
*
* @param uri URI to extract a fake HBase ID from.
* @return the fake HBase ID, if any, or null.
*/
private static String getFakeHBaseID(KijiURI uri) {
if (uri.getZookeeperQuorum().size() != 1) {
return null;
}
final String zkHost = uri.getZookeeperQuorum().get(0);
if (!zkHost.startsWith(FAKE_HBASE_ID_PREFIX)) {
return null;
}
return zkHost.substring(FAKE_HBASE_ID_PREFIX.length());
}
/**
* Creates a Kiji table in an HBase instance, without checking for validation compatibility and
* without applying permissions.
*
* @param tableLayout The initial layout of the table (with unassigned column ids).
* @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1
* regions created. Pass null to specify the default single region.
* @throws IOException on I/O error.
* @throws KijiAlreadyExistsException if the table already exists.
*/
private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys)
throws IOException {
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();
// This will validate the layout and may throw an InvalidLayoutException.
final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout);
if (getMetaTable().tableExists(tableLayout.getName())) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
if (tableLayout.getKeysFormat() instanceof RowKeyFormat) {
LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'.");
}
getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout);
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
// system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications.
// Invariant: ZooKeeper hold the most recent layout of the table.
LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI);
try {
final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient);
try {
final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId());
monitor.notifyNewTableLayout(tableURI, layoutId, -1);
} finally {
monitor.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
}
try {
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor desc =
translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout);
LOG.debug("Creating HBase table '{}'.", desc.getNameAsString());
if (null != splitKeys) {
getHBaseAdmin().createTable(desc, splitKeys);
} else {
getHBaseAdmin().createTable(desc);
}
} catch (TableExistsException tee) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
}
/**
* {@inheritDoc}
*
* @throws IOException when instance in configuration can not be opened and closed.
*/
@Override
public final void run(final KijiRESTConfiguration configuration, final Environment environment)
throws IOException {
final KijiURI clusterURI = KijiURI.newBuilder(configuration.getClusterURI()).build();
// Load specified instances and health checks for each.
final Set<KijiURI> instances =
InstanceUtil.getInstances(
clusterURI,
new InstancesMapTo<KijiURI>() {
public KijiURI apply(KijiURI instanceURI) throws IOException {
InstanceUtil.openAndCloseInstance(instanceURI);
LOG.info("Loading instance {} upon startup.", instanceURI.toOrderedString());
environment.addHealthCheck(new InstanceHealthCheck(instanceURI));
return instanceURI;
}
});
final ManagedKijiClient managedKijiClient = new ManagedKijiClient(instances);
environment.manage(managedKijiClient);
// Remove all built-in Dropwizard ExceptionHandler.
// Always depend on custom ones.
// Inspired by Jeremy Whitlock's suggestion on thoughtspark.org.
Set<Object> jerseyResources = environment.getJerseyResourceConfig().getSingletons();
Iterator<Object> jerseyResourcesIterator = jerseyResources.iterator();
while (jerseyResourcesIterator.hasNext()) {
Object jerseyResource = jerseyResourcesIterator.next();
if (jerseyResource instanceof ExceptionMapper
&& jerseyResource.getClass().getName().startsWith("com.yammer.dropwizard.jersey")) {
jerseyResourcesIterator.remove();
}
}
// Update instances periodically.
final RefreshInstances instanceRefresher = new RefreshInstances(clusterURI, managedKijiClient);
ScheduledExecutorService scheduler =
environment.managedScheduledExecutorService("instance_refresh_scheduler", 1);
scheduler.scheduleAtFixedRate(
instanceRefresher,
INSTANCE_REFRESH_PERIOD_MINUTES, // Start a period from now.
INSTANCE_REFRESH_PERIOD_MINUTES,
MINUTES);
// Load admin task to manually update instances.
environment.addTask(new RefreshInstancesTask(instanceRefresher));
// Load resources.
for (KijiRestPlugin plugin : Lookups.get(KijiRestPlugin.class)) {
LOG.info("Loading plugin {}", plugin.getClass());
plugin.install(managedKijiClient, configuration, environment);
}
// Allow global CORS filter. CORS off by default.
if (configuration.getCORS()) {
environment.addFilter(
CrossOriginFilter.class, configuration.getHttpConfiguration().getRootPath());
LOG.info("Global cross-origin resource sharing is allowed.");
}
}
/**
* Initializes a new table-wide record writer.
*
* @param oformat KijiHFileOutputFormat this writer is built from.
* @param context Context of the task.
* @throws IOException on I/O error.
*/
public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context)
throws IOException {
mContext = Preconditions.checkNotNull(context);
mConf = mContext.getConfiguration();
mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis());
mLatestTimestampBytes = toBytes(mLatestTimestamp);
mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION);
mFileSystem = mOutputDir.getFileSystem(mConf);
mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build();
final Kiji kiji = Kiji.Factory.open(mTableURI, mConf);
final KijiTable table = kiji.openTable(mTableURI.getTable());
mLayout = table.getLayout();
ResourceUtils.releaseOrLog(table);
ResourceUtils.releaseOrLog(kiji);
}
@Test
public void testValidate() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen());
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=validate-all"));
KijiFreshnessPolicyRecord record =
KijiFreshnessPolicyRecord.newBuilder()
.setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString())
.setProducerClass(TestProducer.class.getName())
.setFreshnessPolicyClass(AlwaysFreshen.class.getName())
.setFreshnessPolicyState("")
.build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final EncoderFactory encoderFactory = EncoderFactory.get();
Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null);
final DatumWriter<KijiFreshnessPolicyRecord> recordWriter =
new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$);
recordWriter.write(record, encoder);
getKiji()
.getMetaTable()
.putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray());
assertEquals(
BaseTool.FAILURE,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=validate-all"));
assertEquals(
"Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]);
assertEquals(
"NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not "
+ "contain family: columnName",
mToolOutputLines[2]);
}
@Test
public void testRetrieveAllEmpty() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=retrieve-all"));
assertEquals(
"There are no freshness policies attached to columns in table: user", mToolOutputStr);
}
@Test
public void testIllegalRegister() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI())
.withTableName("user")
.withColumnNames(Lists.newArrayList("info:name"))
.build()
.toString(),
"--do=register",
"--policy-class=org.kiji..badname",
"--policy-state={\"shelfLife\":10}",
"--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer",
"--interactive=false");
assertEquals(
"BAD_POLICY_NAME: java.lang.IllegalArgumentException: Policy class name: "
+ "org.kiji..badname is not a valid Java class identifier.",
mToolOutputLines[0]);
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI())
.withTableName("user")
.withColumnNames(Lists.newArrayList("info:name"))
.build()
.toString(),
"--do=register",
"--policy-class=org.kiji.class",
"--policy-state={\"shelfLife\":10}",
"--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer.",
"--interactive=false");
assertEquals(
"BAD_PRODUCER_NAME: java.lang.IllegalArgumentException: Producer class name: org."
+ "kiji.scoring.tools.TestFreshTool$TestProducer. is not a valid Java class identifier.",
mToolOutputLines[0]);
}
/**
* Creates a new record reader that scans over a subset of rows from a Kiji table. The record
* reader will scan over rows in the table specified in the provided input split, subject to row
* limits specified in the data request serialized into the specified configuration.
*
* @param split for the MapReduce task that will use this record reader. The split specifies a
* subset of rows from a Kiji table.
* @param configuration for the MapReduce job using this record reader. The configuration should
* specify the input Kiji table through the configuration variable {@link
* KijiConfKeys#KIJI_INPUT_TABLE_URI} and a serialized {@link KijiDataRequest} through the
* configuration variable {@link KijiConfKeys#KIJI_INPUT_DATA_REQUEST}.
* @throws IOException if there is a problem constructing the record reader and opening the
* resources it requires.
*/
public KijiTableRecordReader(InputSplit split, Configuration configuration) throws IOException {
// Get data request from the job configuration.
final String dataRequestB64 = configuration.get(KijiConfKeys.KIJI_INPUT_DATA_REQUEST);
Preconditions.checkNotNull(dataRequestB64, "Missing data request in job configuration.");
final byte[] dataRequestBytes = Base64.decodeBase64(Bytes.toBytes(dataRequestB64));
mDataRequest = (KijiDataRequest) SerializationUtils.deserialize(dataRequestBytes);
// Open connections to Kiji.
assert split instanceof KijiTableSplit;
mSplit = (KijiTableSplit) split;
final KijiURI inputURI =
KijiURI.newBuilder(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI)).build();
final KijiScannerOptions scannerOptions =
new KijiScannerOptions()
.setStartRow(new HBaseEntityId(mSplit.getStartRow()))
.setStopRow(new HBaseEntityId(mSplit.getEndRow()));
mKiji = Kiji.Factory.open(inputURI, configuration);
mTable = mKiji.openTable(inputURI.getTable());
mReader = mTable.openTableReader();
mScanner = mReader.getScanner(mDataRequest, scannerOptions);
mIterator = mScanner.iterator();
}
@Test
public void testCreateTableWithInvalidSchemaClassInLayout() throws Exception {
final TableLayoutDesc layout = KijiTableLayouts.getLayout(KijiTableLayouts.INVALID_SCHEMA);
final File layoutFile = getTempLayoutFile(layout);
final KijiURI tableURI =
KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();
assertEquals(
BaseTool.FAILURE,
runTool(new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile));
assertEquals(2, mToolOutputLines.length);
assertTrue(
mToolOutputLines[1].startsWith(
"Error: Schema with type 'class' must be a valid Java identifier."));
}
@Test
public void testUnregisterAll() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen());
manager.storePolicy("user", "info:visits", TestProducer.class, new NeverFreshen());
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=unregister-all"));
assertEquals(0, manager.retrievePolicies("user").size());
assertEquals("All freshness policies removed from table: user", mToolOutputStr);
}
/** {@inheritDoc} */
@Override
public boolean equals(Object obj) {
if (null == obj) {
return false;
}
if (obj == this) {
return true;
}
if (!getClass().equals(obj.getClass())) {
return false;
}
final Kiji other = (Kiji) obj;
// Equal if the two instances have the same URI:
return mURI.equals(other.getURI());
}
/**
* Generates a split for a given table.
*
* @param tableURI URI of the Kiji table to split.
* @param nsplits Number of splits.
* @param conf Base Hadoop configuration used to open the Kiji instance.
* @return a list of split start keys, as HFileKeyValue (with no value, just the keys).
* @throws IOException on I/O error.
*/
private static List<HFileKeyValue> makeTableKeySplit(
KijiURI tableURI, int nsplits, Configuration conf) throws IOException {
final Kiji kiji = Kiji.Factory.open(tableURI, conf);
try {
final KijiTable table = kiji.openTable(tableURI.getTable());
try {
if (NUM_SPLITS_AUTO == nsplits) {
final List<HFileKeyValue> startKeys = Lists.newArrayList();
for (KijiRegion region : table.getRegions()) {
startKeys.add(HFileKeyValue.createFromRowKey(region.getStartKey()));
}
return startKeys;
} else {
switch (KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat())) {
case RAW:
{
// The user has explicitly specified how many HFiles to create, but this is not
// possible when row key hashing is disabled.
throw new JobConfigurationException(
String.format(
"Table '%s' has row key hashing disabled, so the number of HFile splits must be"
+ "determined by the number of HRegions in the HTable. "
+ "Use an HFileMapReduceJobOutput constructor that enables auto splitting.",
table.getName()));
}
case FORMATTED:
case HASH:
case HASH_PREFIX:
{
// Those cases are supported:
break;
}
default:
throw new RuntimeException(
"Unhandled row key encoding: "
+ KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat()));
}
return generateEvenStartKeys(nsplits);
}
} finally {
ResourceUtils.releaseOrLog(table);
}
} finally {
ResourceUtils.releaseOrLog(kiji);
}
}
public GatherReduceDriverTemplate(String outputFilePath, String inputTableName) {
PropertyConfigurator.configure(
this.getClass().getResourceAsStream(Constants.LOG4J_PROPERTIES_FILE_PATH));
try {
Configuration hBaseConfiguration =
HBaseConfiguration.addHbaseResources(new Configuration(true));
hBaseConfiguration.set("mapred.textoutputformat.separator", "|");
hBaseConfiguration.setInt("hbase.client.scanner.caching", 1000);
KijiURI tableUri =
KijiURI.newBuilder(String.format("kiji://.env/default/%s", inputTableName)).build();
String additionalJarsPath = "";
try {
additionalJarsPath =
InetAddress.getLocalHost().getHostName().equals("master")
? GlobalConstants.ADDTIONAL_JARS_PATH_KIJI_CLUSTER
: GlobalConstants.ADDTIONAL_JARS_PATH_BENTO;
} catch (UnknownHostException unknownHostException) {
logger.error(unknownHostException);
unknownHostException.printStackTrace();
System.exit(-1);
}
this.mapReduceJob =
KijiGatherJobBuilder.create()
.withConf(hBaseConfiguration)
.withGatherer(GathererTemplate.class)
.withReducer(ReducerTemplate.class)
.withInputTable(tableUri)
.withOutput(new TextMapReduceJobOutput(new Path(outputFilePath), 1))
.addJarDirectory(new Path(additionalJarsPath))
.build();
} catch (IOException ioException) {
logger.error("IO Exception while configuring MapReduce Job", ioException);
System.exit(1);
} catch (Exception unknownException) {
logger.error("Unknown Exception while configuring MapReduce Job", unknownException);
System.exit(1);
}
}
@Test
public void testRetrieveEmpty() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI())
.withTableName("user")
.withColumnNames(Lists.newArrayList("info:name"))
.build()
.toString(),
"--do=retrieve"));
assertEquals(
"There is no freshness policy attached to column: info:name in table: user",
mToolOutputStr);
}
@Test
public void testUnregister() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen());
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI())
.withTableName("user")
.withColumnNames(Lists.newArrayList("info:name"))
.build()
.toString(),
"--do=unregister"));
assertEquals(null, manager.retrievePolicy("user", "info:name"));
assertEquals("Freshness policy removed from column: info:name in table user", mToolOutputStr);
}
/** {@inheritDoc} */
@Override
public void deleteTable(String tableName) throws IOException {
final State state = mState.get();
Preconditions.checkState(
state == State.OPEN, "Cannot delete table in Kiji instance %s in state %s.", this, state);
// Delete from HBase.
String hbaseTable =
KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName).toString();
getHBaseAdmin().disableTable(hbaseTable);
getHBaseAdmin().deleteTable(hbaseTable);
// Delete from the meta table.
getMetaTable().deleteTable(tableName);
// If the table persists immediately after deletion attempt, then give up.
if (getHBaseAdmin().tableExists(hbaseTable)) {
LOG.warn("HBase table " + hbaseTable + " survives deletion attempt. Giving up...");
}
}
@Test
public void testCreateHashedTableWithNumRegions() throws Exception {
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
final File layoutFile = getTempLayoutFile(layout);
final KijiURI tableURI =
KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();
assertEquals(
BaseTool.SUCCESS,
runTool(
new CreateTableTool(),
"--table=" + tableURI,
"--layout=" + layoutFile,
"--num-regions=" + 2,
"--debug"));
assertEquals(2, mToolOutputLines.length);
assertTrue(mToolOutputLines[0].startsWith("Parsing table layout: "));
assertTrue(mToolOutputLines[1].startsWith("Creating Kiji table"));
}
/** {@inheritDoc} */
@Deprecated
@Override
public void createTable(String tableName, KijiTableLayout tableLayout, byte[][] splitKeys)
throws IOException {
if (getMetaTable().tableExists(tableName)) {
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
if (!tableName.equals(tableLayout.getName())) {
throw new RuntimeException(
String.format(
"Table name from layout descriptor '%s' does match table name '%s'.",
tableLayout.getName(), tableName));
}
createTable(tableLayout.getDesc(), splitKeys);
}
@Test
public void testCreateUnhashedTableWithNumRegions() throws Exception {
final KijiTableLayout layout =
KijiTableLayout.newLayout(KijiTableLayouts.getFooUnhashedTestLayout());
final File layoutFile = getTempLayoutFile(layout);
final KijiURI tableURI =
KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();
assertEquals(
BaseTool.FAILURE,
runTool(
new CreateTableTool(),
"--table=" + tableURI,
"--layout=" + layoutFile,
"--num-regions=4"));
assertEquals(3, mToolOutputLines.length);
assertTrue(
mToolOutputLines[2].startsWith(
"Error: May not use numRegions > 1 if row key hashing is disabled in the layout"));
}
@Test
public void testCreateHashedTableWithSplitKeys() throws Exception {
final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
final File layoutFile = getTempLayoutFile(layout);
final KijiURI tableURI =
KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();
final String splitKeyFile =
getClass().getClassLoader().getResource(REGION_SPLIT_KEY_FILE).getPath();
assertEquals(
BaseTool.FAILURE,
runTool(
new CreateTableTool(),
"--table=" + tableURI,
"--layout=" + layoutFile,
"--split-key-file=file://" + splitKeyFile));
assertEquals(3, mToolOutputLines.length);
assertTrue(
mToolOutputLines[2].startsWith(
"Error: Row key hashing is enabled for the table. Use --num-regions=N instead."));
}
@Test
public void testCreateUnhashedTableWithSplitKeys() throws Exception {
final KijiTableLayout layout =
KijiTableLayout.newLayout(KijiTableLayouts.getFooUnhashedTestLayout());
final File layoutFile = getTempLayoutFile(layout);
final KijiURI tableURI =
KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();
final String splitKeyFile =
getClass().getClassLoader().getResource(REGION_SPLIT_KEY_FILE).getPath();
assertEquals(
BaseTool.SUCCESS,
runTool(
new CreateTableTool(),
"--table=" + tableURI,
"--layout=" + layoutFile,
"--split-key-file=file://" + splitKeyFile,
"--debug"));
assertEquals(2, mToolOutputLines.length);
assertTrue(mToolOutputLines[0].startsWith("Parsing table layout: "));
assertTrue(mToolOutputLines[1].startsWith("Creating Kiji table"));
}
/** {@inheritDoc} */
@Override
public void createTable(TableLayoutDesc tableLayout, byte[][] splitKeys) throws IOException {
final State state = mState.get();
Preconditions.checkState(
state == State.OPEN, "Cannot create table in Kiji instance %s in state %s.", this, state);
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();
LOG.debug("Creating Kiji table '{}'.", tableURI);
ensureValidationCompatibility(tableLayout);
// If security is enabled, apply the permissions to the new table.
if (isSecurityEnabled()) {
getSecurityManager().lock();
try {
createTableUnchecked(tableLayout, splitKeys);
getSecurityManager().applyPermissionsToNewTable(tableURI);
} finally {
getSecurityManager().unlock();
}
} else {
createTableUnchecked(tableLayout, splitKeys);
}
}
@Test
public void testRegister() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI())
.withTableName("user")
.withColumnNames(Lists.newArrayList("info:name"))
.build()
.toString(),
"--do=register",
"--policy-class=org.kiji.scoring.lib.ShelfLife",
"--policy-state={\"shelfLife\":10}",
"--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer"));
final KijiFreshnessPolicyRecord record =
KijiFreshnessPolicyRecord.newBuilder()
.setRecordVersion("policyrecord-0.1.0")
.setProducerClass(TestProducer.class.getName())
.setFreshnessPolicyClass(
Class.forName("org.kiji.scoring.lib.ShelfLife")
.asSubclass(KijiFreshnessPolicy.class)
.getName())
.setFreshnessPolicyState("{\"shelfLife\":10}")
.build();
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
assertEquals(record, manager.retrievePolicy("user", "info:name"));
assertEquals(
"Freshness policy: org.kiji.scoring.lib.ShelfLife with state: {\"shelfLife\":10} "
+ "and producer: org.kiji.scoring.tools.TestFreshTool$TestProducer\n"
+ "attached to column: info:name in table: user",
mToolOutputStr);
// Test another ordering for arguments
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
"--do=register",
KijiURI.newBuilder(getKiji().getURI())
.withTableName("user")
.withColumnNames(Lists.newArrayList("info:visits"))
.build()
.toString(),
"--policy-state={\"shelfLife\":10}",
"--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer",
"--policy-class=org.kiji.scoring.lib.ShelfLife"));
final KijiFreshnessPolicyRecord record2 =
KijiFreshnessPolicyRecord.newBuilder()
.setRecordVersion("policyrecord-0.1.0")
.setProducerClass(TestProducer.class.getName())
.setFreshnessPolicyClass(
Class.forName("org.kiji.scoring.lib.ShelfLife")
.asSubclass(KijiFreshnessPolicy.class)
.getName())
.setFreshnessPolicyState("{\"shelfLife\":10}")
.build();
assertEquals(record2, manager.retrievePolicy("user", "info:visits"));
assertEquals(
"Freshness policy: org.kiji.scoring.lib.ShelfLife with state: {\"shelfLife\":10} "
+ "and producer: org.kiji.scoring.tools.TestFreshTool$TestProducer\n"
+ "attached to column: info:visits in table: user",
mToolOutputStr);
}
/** {@inheritDoc} */
@Override
public KijiTableLayout modifyTableLayout(
TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException {
final State state = mState.get();
Preconditions.checkState(
state == State.OPEN,
"Cannot modify table layout in Kiji instance %s in state %s.",
this,
state);
Preconditions.checkNotNull(update);
ensureValidationCompatibility(update);
if (dryRun && (null == printStream)) {
printStream = System.out;
}
final KijiMetaTable metaTable = getMetaTable();
final String tableName = update.getName();
// Throws a KijiTableNotFoundException if there is no table.
metaTable.getTableLayout(tableName);
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
LOG.debug("Applying layout update {} on table {}", update, tableURI);
KijiTableLayout newLayout = null;
if (dryRun) {
// Process column ids and perform validation, but don't actually update the meta table.
final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1);
final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0);
newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout);
} else {
// Actually set it.
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
try {
// Use ZooKeeper to inform all watchers that a new table layout is available.
final HBaseTableLayoutUpdater updater =
new HBaseTableLayoutUpdater(this, tableURI, update);
try {
updater.update();
newLayout = updater.getNewLayout();
} finally {
updater.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
} else {
// System versions before system-2.0 do not enforce table layout update consistency or
// validation.
newLayout = metaTable.updateTableLayout(tableName, update);
}
}
Preconditions.checkState(newLayout != null);
if (dryRun) {
printStream.println("This table layout is valid.");
}
LOG.debug("Computing new HBase schema");
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor newTableDescriptor =
translator.toHTableDescriptor(mURI.getInstance(), newLayout);
LOG.debug("Reading existing HBase schema");
final KijiManagedHBaseTableName hbaseTableName =
KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName);
HTableDescriptor currentTableDescriptor = null;
byte[] tableNameAsBytes = hbaseTableName.toBytes();
try {
currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes);
} catch (TableNotFoundException tnfe) {
if (!dryRun) {
throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception.
}
}
if (currentTableDescriptor == null) {
if (dryRun) {
printStream.println("Would create new table: " + tableName);
currentTableDescriptor =
HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName);
} else {
throw new RuntimeException(
"Table " + hbaseTableName.getKijiTableName() + " does not exist");
}
}
LOG.debug("Existing table descriptor: {}", currentTableDescriptor);
LOG.debug("New table descriptor: {}", newTableDescriptor);
LOG.debug("Checking for differences between the new HBase schema and the existing one");
final HTableDescriptorComparator comparator = new HTableDescriptorComparator();
if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) {
LOG.debug("HBase schemas are the same. No need to change HBase schema");
if (dryRun) {
printStream.println("This layout does not require any physical table schema changes.");
}
} else {
LOG.debug("HBase schema must be changed, but no columns will be deleted");
if (dryRun) {
printStream.println("Changes caused by this table layout:");
} else {
LOG.debug("Disabling HBase table");
getHBaseAdmin().disableTable(hbaseTableName.toString());
}
for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) {
final String columnName = Bytes.toString(newColumnDescriptor.getName());
final ColumnId columnId = ColumnId.fromString(columnName);
final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId);
final HColumnDescriptor currentColumnDescriptor =
currentTableDescriptor.getFamily(newColumnDescriptor.getName());
if (null == currentColumnDescriptor) {
if (dryRun) {
printStream.println(" Creating new locality group: " + lgName);
} else {
LOG.debug("Creating new column " + columnName);
getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor);
}
} else if (!newColumnDescriptor.equals(currentColumnDescriptor)) {
if (dryRun) {
printStream.println(" Modifying locality group: " + lgName);
} else {
LOG.debug("Modifying column " + columnName);
getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor);
}
} else {
LOG.debug("No changes needed for column " + columnName);
}
}
if (dryRun) {
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
printStream.printf(
" Changing max_filesize from %d to %d: %n",
currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize());
}
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
printStream.printf(
" Changing memstore_flushsize from %d to %d: %n",
currentTableDescriptor.getMemStoreFlushSize(),
newTableDescriptor.getMemStoreFlushSize());
}
} else {
LOG.debug("Modifying table descriptor");
getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor);
}
if (!dryRun) {
LOG.debug("Re-enabling HBase table");
getHBaseAdmin().enableTable(hbaseTableName.toString());
}
}
return newLayout;
}
/**
* Creates a new <code>HBaseKiji</code> instance.
*
* <p>Should only be used by Kiji.Factory.open().
*
* <p>Caller does not need to use retain(), but must call release() when done with it.
*
* @param kijiURI the KijiURI.
* @param conf Hadoop Configuration. Deep copied internally.
* @param tableFactory HTableInterface factory.
* @param lockFactory Factory for locks.
* @throws IOException on I/O error.
*/
HBaseKiji(
KijiURI kijiURI,
Configuration conf,
HTableInterfaceFactory tableFactory,
LockFactory lockFactory)
throws IOException {
mConstructorStack = CLEANUP_LOG.isDebugEnabled() ? Debug.getStackTrace() : null;
// Deep copy the configuration.
mConf = new Configuration(conf);
// Validate arguments.
mHTableFactory = Preconditions.checkNotNull(tableFactory);
mLockFactory = Preconditions.checkNotNull(lockFactory);
mURI = Preconditions.checkNotNull(kijiURI);
// Configure the ZooKeeper quorum:
mConf.setStrings("hbase.zookeeper.quorum", mURI.getZookeeperQuorum().toArray(new String[0]));
mConf.setInt("hbase.zookeeper.property.clientPort", mURI.getZookeeperClientPort());
// Check for an instance name.
Preconditions.checkArgument(
mURI.getInstance() != null, "KijiURI '%s' does not specify a Kiji instance name.", mURI);
if (LOG.isDebugEnabled()) {
Debug.logConfiguration(mConf);
LOG.debug(
"Opening kiji instance '{}'"
+ " with client software version '{}'"
+ " and client data version '{}'.",
mURI,
VersionInfo.getSoftwareVersion(),
VersionInfo.getClientDataVersion());
}
// Load these lazily.
mSchemaTable = null;
mMetaTable = null;
mSecurityManager = null;
mSystemTable = new HBaseSystemTable(mURI, mConf, mHTableFactory);
mRetainCount.set(1);
final State oldState = mState.getAndSet(State.OPEN);
Preconditions.checkState(
oldState == State.UNINITIALIZED, "Cannot open Kiji instance in state %s.", oldState);
LOG.debug("Kiji instance '{}' is now opened.", mURI);
mSystemVersion = mSystemTable.getDataVersion();
LOG.debug("Kiji instance '{}' has data version '{}'.", mURI, mSystemVersion);
// Make sure the data version for the client matches the cluster.
LOG.debug("Validating version for Kiji instance '{}'.", mURI);
try {
VersionInfo.validateVersion(this);
} catch (IOException ioe) {
// If an IOException occurred the object will not be constructed so need to clean it up.
close();
throw ioe;
} catch (KijiNotInstalledException kie) {
// Some clients handle this unchecked Exception so do the same here.
close();
throw kie;
}
// TODO(SCHEMA-491) Share ZooKeeperClient instances when possible.
if (mSystemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) >= 0) {
// system-2.0 clients must connect to ZooKeeper:
// - to register themselves as table users;
// - to receive table layout updates.
mZKClient = HBaseFactory.Provider.get().getZooKeeperClient(mURI);
try {
mMonitor = new ZooKeeperMonitor(mZKClient);
mMonitor.registerInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
} catch (KeeperException ke) {
// Unrecoverable KeeperException:
throw new IOException(ke);
}
} else {
// system-1.x clients do not need a ZooKeeper connection.
mZKClient = null;
mMonitor = null;
}
}
/** {@inheritDoc} */
@Override
public int hashCode() {
return mURI.hashCode();
}