/**
* Creates a Kiji table in an HBase instance, without checking for validation compatibility and
* without applying permissions.
*
* @param tableLayout The initial layout of the table (with unassigned column ids).
* @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1
* regions created. Pass null to specify the default single region.
* @throws IOException on I/O error.
* @throws KijiAlreadyExistsException if the table already exists.
*/
private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys)
throws IOException {
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();
// This will validate the layout and may throw an InvalidLayoutException.
final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout);
if (getMetaTable().tableExists(tableLayout.getName())) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
if (tableLayout.getKeysFormat() instanceof RowKeyFormat) {
LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'.");
}
getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout);
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
// system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications.
// Invariant: ZooKeeper hold the most recent layout of the table.
LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI);
try {
final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient);
try {
final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId());
monitor.notifyNewTableLayout(tableURI, layoutId, -1);
} finally {
monitor.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
}
try {
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor desc =
translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout);
LOG.debug("Creating HBase table '{}'.", desc.getNameAsString());
if (null != splitKeys) {
getHBaseAdmin().createTable(desc, splitKeys);
} else {
getHBaseAdmin().createTable(desc);
}
} catch (TableExistsException tee) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
}
/** {@inheritDoc} */
@Override
public void deleteTable(String tableName) throws IOException {
final State state = mState.get();
Preconditions.checkState(
state == State.OPEN, "Cannot delete table in Kiji instance %s in state %s.", this, state);
// Delete from HBase.
String hbaseTable =
KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName).toString();
getHBaseAdmin().disableTable(hbaseTable);
getHBaseAdmin().deleteTable(hbaseTable);
// Delete from the meta table.
getMetaTable().deleteTable(tableName);
// If the table persists immediately after deletion attempt, then give up.
if (getHBaseAdmin().tableExists(hbaseTable)) {
LOG.warn("HBase table " + hbaseTable + " survives deletion attempt. Giving up...");
}
}
/** {@inheritDoc} */
@Override
public KijiTableLayout modifyTableLayout(
TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException {
final State state = mState.get();
Preconditions.checkState(
state == State.OPEN,
"Cannot modify table layout in Kiji instance %s in state %s.",
this,
state);
Preconditions.checkNotNull(update);
ensureValidationCompatibility(update);
if (dryRun && (null == printStream)) {
printStream = System.out;
}
final KijiMetaTable metaTable = getMetaTable();
final String tableName = update.getName();
// Throws a KijiTableNotFoundException if there is no table.
metaTable.getTableLayout(tableName);
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
LOG.debug("Applying layout update {} on table {}", update, tableURI);
KijiTableLayout newLayout = null;
if (dryRun) {
// Process column ids and perform validation, but don't actually update the meta table.
final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1);
final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0);
newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout);
} else {
// Actually set it.
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
try {
// Use ZooKeeper to inform all watchers that a new table layout is available.
final HBaseTableLayoutUpdater updater =
new HBaseTableLayoutUpdater(this, tableURI, update);
try {
updater.update();
newLayout = updater.getNewLayout();
} finally {
updater.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
} else {
// System versions before system-2.0 do not enforce table layout update consistency or
// validation.
newLayout = metaTable.updateTableLayout(tableName, update);
}
}
Preconditions.checkState(newLayout != null);
if (dryRun) {
printStream.println("This table layout is valid.");
}
LOG.debug("Computing new HBase schema");
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor newTableDescriptor =
translator.toHTableDescriptor(mURI.getInstance(), newLayout);
LOG.debug("Reading existing HBase schema");
final KijiManagedHBaseTableName hbaseTableName =
KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName);
HTableDescriptor currentTableDescriptor = null;
byte[] tableNameAsBytes = hbaseTableName.toBytes();
try {
currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes);
} catch (TableNotFoundException tnfe) {
if (!dryRun) {
throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception.
}
}
if (currentTableDescriptor == null) {
if (dryRun) {
printStream.println("Would create new table: " + tableName);
currentTableDescriptor =
HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName);
} else {
throw new RuntimeException(
"Table " + hbaseTableName.getKijiTableName() + " does not exist");
}
}
LOG.debug("Existing table descriptor: {}", currentTableDescriptor);
LOG.debug("New table descriptor: {}", newTableDescriptor);
LOG.debug("Checking for differences between the new HBase schema and the existing one");
final HTableDescriptorComparator comparator = new HTableDescriptorComparator();
if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) {
LOG.debug("HBase schemas are the same. No need to change HBase schema");
if (dryRun) {
printStream.println("This layout does not require any physical table schema changes.");
}
} else {
LOG.debug("HBase schema must be changed, but no columns will be deleted");
if (dryRun) {
printStream.println("Changes caused by this table layout:");
} else {
LOG.debug("Disabling HBase table");
getHBaseAdmin().disableTable(hbaseTableName.toString());
}
for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) {
final String columnName = Bytes.toString(newColumnDescriptor.getName());
final ColumnId columnId = ColumnId.fromString(columnName);
final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId);
final HColumnDescriptor currentColumnDescriptor =
currentTableDescriptor.getFamily(newColumnDescriptor.getName());
if (null == currentColumnDescriptor) {
if (dryRun) {
printStream.println(" Creating new locality group: " + lgName);
} else {
LOG.debug("Creating new column " + columnName);
getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor);
}
} else if (!newColumnDescriptor.equals(currentColumnDescriptor)) {
if (dryRun) {
printStream.println(" Modifying locality group: " + lgName);
} else {
LOG.debug("Modifying column " + columnName);
getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor);
}
} else {
LOG.debug("No changes needed for column " + columnName);
}
}
if (dryRun) {
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
printStream.printf(
" Changing max_filesize from %d to %d: %n",
currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize());
}
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
printStream.printf(
" Changing memstore_flushsize from %d to %d: %n",
currentTableDescriptor.getMemStoreFlushSize(),
newTableDescriptor.getMemStoreFlushSize());
}
} else {
LOG.debug("Modifying table descriptor");
getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor);
}
if (!dryRun) {
LOG.debug("Re-enabling HBase table");
getHBaseAdmin().enableTable(hbaseTableName.toString());
}
}
return newLayout;
}
/**
* Creates a new <code>HBaseKiji</code> instance.
*
* <p>Should only be used by Kiji.Factory.open().
*
* <p>Caller does not need to use retain(), but must call release() when done with it.
*
* @param kijiURI the KijiURI.
* @param conf Hadoop Configuration. Deep copied internally.
* @param tableFactory HTableInterface factory.
* @param lockFactory Factory for locks.
* @throws IOException on I/O error.
*/
HBaseKiji(
KijiURI kijiURI,
Configuration conf,
HTableInterfaceFactory tableFactory,
LockFactory lockFactory)
throws IOException {
mConstructorStack = CLEANUP_LOG.isDebugEnabled() ? Debug.getStackTrace() : null;
// Deep copy the configuration.
mConf = new Configuration(conf);
// Validate arguments.
mHTableFactory = Preconditions.checkNotNull(tableFactory);
mLockFactory = Preconditions.checkNotNull(lockFactory);
mURI = Preconditions.checkNotNull(kijiURI);
// Configure the ZooKeeper quorum:
mConf.setStrings("hbase.zookeeper.quorum", mURI.getZookeeperQuorum().toArray(new String[0]));
mConf.setInt("hbase.zookeeper.property.clientPort", mURI.getZookeeperClientPort());
// Check for an instance name.
Preconditions.checkArgument(
mURI.getInstance() != null, "KijiURI '%s' does not specify a Kiji instance name.", mURI);
if (LOG.isDebugEnabled()) {
Debug.logConfiguration(mConf);
LOG.debug(
"Opening kiji instance '{}'"
+ " with client software version '{}'"
+ " and client data version '{}'.",
mURI,
VersionInfo.getSoftwareVersion(),
VersionInfo.getClientDataVersion());
}
// Load these lazily.
mSchemaTable = null;
mMetaTable = null;
mSecurityManager = null;
mSystemTable = new HBaseSystemTable(mURI, mConf, mHTableFactory);
mRetainCount.set(1);
final State oldState = mState.getAndSet(State.OPEN);
Preconditions.checkState(
oldState == State.UNINITIALIZED, "Cannot open Kiji instance in state %s.", oldState);
LOG.debug("Kiji instance '{}' is now opened.", mURI);
mSystemVersion = mSystemTable.getDataVersion();
LOG.debug("Kiji instance '{}' has data version '{}'.", mURI, mSystemVersion);
// Make sure the data version for the client matches the cluster.
LOG.debug("Validating version for Kiji instance '{}'.", mURI);
try {
VersionInfo.validateVersion(this);
} catch (IOException ioe) {
// If an IOException occurred the object will not be constructed so need to clean it up.
close();
throw ioe;
} catch (KijiNotInstalledException kie) {
// Some clients handle this unchecked Exception so do the same here.
close();
throw kie;
}
// TODO(SCHEMA-491) Share ZooKeeperClient instances when possible.
if (mSystemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) >= 0) {
// system-2.0 clients must connect to ZooKeeper:
// - to register themselves as table users;
// - to receive table layout updates.
mZKClient = HBaseFactory.Provider.get().getZooKeeperClient(mURI);
try {
mMonitor = new ZooKeeperMonitor(mZKClient);
mMonitor.registerInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
} catch (KeeperException ke) {
// Unrecoverable KeeperException:
throw new IOException(ke);
}
} else {
// system-1.x clients do not need a ZooKeeper connection.
mZKClient = null;
mMonitor = null;
}
}
