/**
* Ensures that a table is not created or modified to enable layout validation without the
* requisite system version.
*
* <p>Throws an exception if a table layout has validation enabled, but the overall instance data
* version is too low to support table layout validation.
*
* <p>Table layouts with layout version <tt>layout-1.3.0</tt> or higher must be applied to systems
* with data version <tt>system-2.0</tt> or higher. A layout of 1.3 or above in system-1.0
* environment will trigger an exception in this method.
*
* <p>Older layout versions may be applied in <tt>system-1.0</tt> or <tt>system-2.0</tt>
* environments; such layouts are ignored by this method.
*
* @param layout the table layout for which to ensure compatibility.
* @throws IOException in case of an error reading from the system table.
* @throws InvalidLayoutException if the layout and system versions are incompatible.
*/
private void ensureValidationCompatibility(TableLayoutDesc layout) throws IOException {
final ProtocolVersion layoutVersion = ProtocolVersion.parse(layout.getVersion());
final ProtocolVersion systemVersion = getSystemTable().getDataVersion();
if ((layoutVersion.compareTo(Versions.LAYOUT_VALIDATION_VERSION) >= 0)
&& (systemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) < 0)) {
throw new InvalidLayoutException(
String.format(
"Layout version: %s not supported by system version: %s",
layoutVersion, systemVersion));
}
}
/**
* Releases all the resources used by this Kiji instance.
*
* @throws IOException on I/O error.
*/
private void close() throws IOException {
final State oldState = mState.getAndSet(State.CLOSED);
Preconditions.checkState(
oldState == State.OPEN, "Cannot close Kiji instance %s in state %s.", this, oldState);
LOG.debug("Closing {}.", this);
if (mMonitor != null) {
try {
mMonitor.unregisterInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
} catch (KeeperException ke) {
// Unrecoverable ZooKeeper error:
throw new IOException(ke);
}
mMonitor.close();
}
if (mZKClient != null) {
mZKClient.release();
}
ResourceUtils.closeOrLog(mMetaTable);
ResourceUtils.closeOrLog(mSystemTable);
ResourceUtils.closeOrLog(mSchemaTable);
ResourceUtils.closeOrLog(mSecurityManager);
ResourceUtils.closeOrLog(mAdmin);
mSchemaTable = null;
mMetaTable = null;
mAdmin = null;
mSecurityManager = null;
LOG.debug("{} closed.", this);
}
/**
* Creates a Kiji table in an HBase instance, without checking for validation compatibility and
* without applying permissions.
*
* @param tableLayout The initial layout of the table (with unassigned column ids).
* @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1
* regions created. Pass null to specify the default single region.
* @throws IOException on I/O error.
* @throws KijiAlreadyExistsException if the table already exists.
*/
private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys)
throws IOException {
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();
// This will validate the layout and may throw an InvalidLayoutException.
final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout);
if (getMetaTable().tableExists(tableLayout.getName())) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
if (tableLayout.getKeysFormat() instanceof RowKeyFormat) {
LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'.");
}
getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout);
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
// system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications.
// Invariant: ZooKeeper hold the most recent layout of the table.
LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI);
try {
final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient);
try {
final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId());
monitor.notifyNewTableLayout(tableURI, layoutId, -1);
} finally {
monitor.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
}
try {
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor desc =
translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout);
LOG.debug("Creating HBase table '{}'.", desc.getNameAsString());
if (null != splitKeys) {
getHBaseAdmin().createTable(desc, splitKeys);
} else {
getHBaseAdmin().createTable(desc);
}
} catch (TableExistsException tee) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
}
@Test
public void testValidate() throws Exception {
getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen());
assertEquals(
BaseTool.SUCCESS,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=validate-all"));
KijiFreshnessPolicyRecord record =
KijiFreshnessPolicyRecord.newBuilder()
.setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString())
.setProducerClass(TestProducer.class.getName())
.setFreshnessPolicyClass(AlwaysFreshen.class.getName())
.setFreshnessPolicyState("")
.build();
final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
final EncoderFactory encoderFactory = EncoderFactory.get();
Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null);
final DatumWriter<KijiFreshnessPolicyRecord> recordWriter =
new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$);
recordWriter.write(record, encoder);
getKiji()
.getMetaTable()
.putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray());
assertEquals(
BaseTool.FAILURE,
runTool(
new FreshTool(),
KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
"--do=validate-all"));
assertEquals(
"Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]);
assertEquals(
"NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not "
+ "contain family: columnName",
mToolOutputLines[2]);
}
/**
* Validates a new table layout against a reference layout for mutual compatibility.
*
* @param reference the reference layout against which to validate.
* @param layout the new layout to validate.
* @throws IOException in case of an IO Error reading from the schema table. Throws
* InvalidLayoutException if the layouts are incompatible.
*/
public void validate(KijiTableLayout reference, KijiTableLayout layout) throws IOException {
final ProtocolVersion layoutVersion = ProtocolVersion.parse(layout.getDesc().getVersion());
if (layoutVersion.compareTo(Versions.LAYOUT_VALIDATION_VERSION) < 0) {
// Layout versions older than layout-1.3.0 do not require validation
return;
}
// Accumulator for error messages which will be used to create an exception if errors occur.
final List<String> incompatabilityMessages = Lists.newArrayList();
// Iterate through all families/columns in the new layout,
// find a potential matching reference family/column,
// and validate the reader/writer schema sets.
// If no matching family/column exists in the reference layout the newly create column is valid.
for (FamilyLayout flayout : layout.getFamilies()) {
final ColumnId lgid = flayout.getLocalityGroup().getId();
LocalityGroupLayout refLGLayout = null;
if (reference != null) {
// If there is a reference layout, check for a locality group matching the ID of the LG for
// this family. Locality Group IDs should not change between layouts.
final String refLGName = reference.getLocalityGroupIdNameMap().get(lgid);
if (refLGName != null) {
// If there is a matching reference LG get its layout by name.
refLGLayout = reference.getLocalityGroupMap().get(refLGName);
}
}
// The ColumnId of the FamilyLayout from the table layout. Also matches the FamilyLayout for
// this family in the reference layout if present.
final ColumnId familyId = flayout.getId();
if (flayout.isMapType()) {
// If the family is map-type, get the CellSchema for all values in the family.
final CellSchema cellSchema = flayout.getDesc().getMapSchema();
FamilyLayout refFamilyLayout = null;
if (refLGLayout != null) {
// If there is a matching reference LG, check for the existence of this family.
final String refFamilyName = refLGLayout.getFamilyIdNameMap().get(familyId);
if (refFamilyName != null) {
refFamilyLayout = refLGLayout.getFamilyMap().get(refFamilyName);
}
}
if (refFamilyLayout != null) {
if (refFamilyLayout.isMapType()) {
// If the FamilyLayout from both table layouts are map type, compare their CellSchemas.
final CellSchema refCellSchema = refFamilyLayout.getDesc().getMapSchema();
incompatabilityMessages.addAll(
addColumnNamestoIncompatibilityMessages(
flayout.getName(), null, validateCellSchema(refCellSchema, cellSchema)));
} else if (refFamilyLayout.isGroupType()) {
// If the FamilyLayout changed from group-type to map-type between table layout versions
// that is an incompatible change.
incompatabilityMessages.add(
String.format(
"Family: %s changed from group-type to map-type.", refFamilyLayout.getName()));
} else {
throw new InternalKijiError(
String.format(
"Family: %s is neither map-type nor group-type.", refFamilyLayout.getName()));
}
} else {
// If the reference FamilyLayout is null this indicates a new family, which is inherently
// compatible, but we still have to validate that the new readers and writers are
// internally compatible.
incompatabilityMessages.addAll(
addColumnNamestoIncompatibilityMessages(
flayout.getName(), null, validateCellSchema(null, cellSchema)));
}
} else if (flayout.isGroupType()) {
// Check for a matching family from the reference layout.
FamilyLayout refFamilyLayout = null;
if (refLGLayout != null) {
final String refFamilyName = refLGLayout.getFamilyIdNameMap().get(familyId);
if (refFamilyName != null) {
refFamilyLayout = refLGLayout.getFamilyMap().get(refFamilyName);
}
}
if (refFamilyLayout != null) {
if (refFamilyLayout.isGroupType()) {
// If there is a matching reference family and it is the same family type, iterate
// through the columns checking schema compatibility. Only checks columns from the new
// layout because removed columns are inherently valid.
for (ColumnLayout columnLayout : flayout.getColumns()) {
final CellSchema cellSchema = columnLayout.getDesc().getColumnSchema();
final String refColumnName =
refFamilyLayout.getColumnIdNameMap().get(columnLayout.getId());
ColumnLayout refColumnLayout = null;
if (refColumnName != null) {
// If there is a column from the reference layout with the same column ID, get its
// layout.
refColumnLayout = refFamilyLayout.getColumnMap().get(refColumnName);
}
// If there is a column from the reference layout with the same column ID, get its
// CellSchema.
final CellSchema refCellSchema =
(refColumnLayout == null) ? null : refColumnLayout.getDesc().getColumnSchema();
// If there is no matching column, refCellSchema will be null and this will only test
// that the new reader and writer schemas are internally compatible.
incompatabilityMessages.addAll(
addColumnNamestoIncompatibilityMessages(
flayout.getName(),
columnLayout.getName(),
validateCellSchema(refCellSchema, cellSchema)));
}
} else if (refFamilyLayout.isMapType()) {
// If the FamilyLayout changed from map-type to group-type between table layout versions
// that is an incompatible change.
incompatabilityMessages.add(
String.format(
"Family: %s changed from map-type to group-type.", refFamilyLayout.getName()));
} else {
throw new InternalKijiError(
String.format(
"Family: %s is neither map-type nor group-type.", refFamilyLayout.getName()));
}
} else {
// If the reference FamilyLayout is null this indicates a new family, which is inherently
// compatible, but we still have to validate that the new readers and writers are
// internally compatible.
for (ColumnLayout columnLayout : flayout.getColumns()) {
final CellSchema cellSchema = columnLayout.getDesc().getColumnSchema();
incompatabilityMessages.addAll(
addColumnNamestoIncompatibilityMessages(
flayout.getName(),
columnLayout.getName(),
validateCellSchema(null, cellSchema)));
}
}
} else {
throw new InternalKijiError(
String.format("Family: %s is neither map-type nor group-type.", flayout.getName()));
}
}
// If there were any incompatibility errors, throw an exception.
if (incompatabilityMessages.size() != 0) {
throw new InvalidLayoutSchemaException(incompatabilityMessages);
}
}
/** {@inheritDoc} */
@Override
public KijiTableLayout modifyTableLayout(
TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException {
final State state = mState.get();
Preconditions.checkState(
state == State.OPEN,
"Cannot modify table layout in Kiji instance %s in state %s.",
this,
state);
Preconditions.checkNotNull(update);
ensureValidationCompatibility(update);
if (dryRun && (null == printStream)) {
printStream = System.out;
}
final KijiMetaTable metaTable = getMetaTable();
final String tableName = update.getName();
// Throws a KijiTableNotFoundException if there is no table.
metaTable.getTableLayout(tableName);
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
LOG.debug("Applying layout update {} on table {}", update, tableURI);
KijiTableLayout newLayout = null;
if (dryRun) {
// Process column ids and perform validation, but don't actually update the meta table.
final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1);
final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0);
newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout);
} else {
// Actually set it.
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
try {
// Use ZooKeeper to inform all watchers that a new table layout is available.
final HBaseTableLayoutUpdater updater =
new HBaseTableLayoutUpdater(this, tableURI, update);
try {
updater.update();
newLayout = updater.getNewLayout();
} finally {
updater.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
} else {
// System versions before system-2.0 do not enforce table layout update consistency or
// validation.
newLayout = metaTable.updateTableLayout(tableName, update);
}
}
Preconditions.checkState(newLayout != null);
if (dryRun) {
printStream.println("This table layout is valid.");
}
LOG.debug("Computing new HBase schema");
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor newTableDescriptor =
translator.toHTableDescriptor(mURI.getInstance(), newLayout);
LOG.debug("Reading existing HBase schema");
final KijiManagedHBaseTableName hbaseTableName =
KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName);
HTableDescriptor currentTableDescriptor = null;
byte[] tableNameAsBytes = hbaseTableName.toBytes();
try {
currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes);
} catch (TableNotFoundException tnfe) {
if (!dryRun) {
throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception.
}
}
if (currentTableDescriptor == null) {
if (dryRun) {
printStream.println("Would create new table: " + tableName);
currentTableDescriptor =
HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName);
} else {
throw new RuntimeException(
"Table " + hbaseTableName.getKijiTableName() + " does not exist");
}
}
LOG.debug("Existing table descriptor: {}", currentTableDescriptor);
LOG.debug("New table descriptor: {}", newTableDescriptor);
LOG.debug("Checking for differences between the new HBase schema and the existing one");
final HTableDescriptorComparator comparator = new HTableDescriptorComparator();
if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) {
LOG.debug("HBase schemas are the same. No need to change HBase schema");
if (dryRun) {
printStream.println("This layout does not require any physical table schema changes.");
}
} else {
LOG.debug("HBase schema must be changed, but no columns will be deleted");
if (dryRun) {
printStream.println("Changes caused by this table layout:");
} else {
LOG.debug("Disabling HBase table");
getHBaseAdmin().disableTable(hbaseTableName.toString());
}
for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) {
final String columnName = Bytes.toString(newColumnDescriptor.getName());
final ColumnId columnId = ColumnId.fromString(columnName);
final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId);
final HColumnDescriptor currentColumnDescriptor =
currentTableDescriptor.getFamily(newColumnDescriptor.getName());
if (null == currentColumnDescriptor) {
if (dryRun) {
printStream.println(" Creating new locality group: " + lgName);
} else {
LOG.debug("Creating new column " + columnName);
getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor);
}
} else if (!newColumnDescriptor.equals(currentColumnDescriptor)) {
if (dryRun) {
printStream.println(" Modifying locality group: " + lgName);
} else {
LOG.debug("Modifying column " + columnName);
getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor);
}
} else {
LOG.debug("No changes needed for column " + columnName);
}
}
if (dryRun) {
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
printStream.printf(
" Changing max_filesize from %d to %d: %n",
currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize());
}
if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
printStream.printf(
" Changing memstore_flushsize from %d to %d: %n",
currentTableDescriptor.getMemStoreFlushSize(),
newTableDescriptor.getMemStoreFlushSize());
}
} else {
LOG.debug("Modifying table descriptor");
getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor);
}
if (!dryRun) {
LOG.debug("Re-enabling HBase table");
getHBaseAdmin().enableTable(hbaseTableName.toString());
}
}
return newLayout;
}
/**
* Creates a new <code>HBaseKiji</code> instance.
*
* <p>Should only be used by Kiji.Factory.open().
*
* <p>Caller does not need to use retain(), but must call release() when done with it.
*
* @param kijiURI the KijiURI.
* @param conf Hadoop Configuration. Deep copied internally.
* @param tableFactory HTableInterface factory.
* @param lockFactory Factory for locks.
* @throws IOException on I/O error.
*/
HBaseKiji(
KijiURI kijiURI,
Configuration conf,
HTableInterfaceFactory tableFactory,
LockFactory lockFactory)
throws IOException {
mConstructorStack = CLEANUP_LOG.isDebugEnabled() ? Debug.getStackTrace() : null;
// Deep copy the configuration.
mConf = new Configuration(conf);
// Validate arguments.
mHTableFactory = Preconditions.checkNotNull(tableFactory);
mLockFactory = Preconditions.checkNotNull(lockFactory);
mURI = Preconditions.checkNotNull(kijiURI);
// Configure the ZooKeeper quorum:
mConf.setStrings("hbase.zookeeper.quorum", mURI.getZookeeperQuorum().toArray(new String[0]));
mConf.setInt("hbase.zookeeper.property.clientPort", mURI.getZookeeperClientPort());
// Check for an instance name.
Preconditions.checkArgument(
mURI.getInstance() != null, "KijiURI '%s' does not specify a Kiji instance name.", mURI);
if (LOG.isDebugEnabled()) {
Debug.logConfiguration(mConf);
LOG.debug(
"Opening kiji instance '{}'"
+ " with client software version '{}'"
+ " and client data version '{}'.",
mURI,
VersionInfo.getSoftwareVersion(),
VersionInfo.getClientDataVersion());
}
// Load these lazily.
mSchemaTable = null;
mMetaTable = null;
mSecurityManager = null;
mSystemTable = new HBaseSystemTable(mURI, mConf, mHTableFactory);
mRetainCount.set(1);
final State oldState = mState.getAndSet(State.OPEN);
Preconditions.checkState(
oldState == State.UNINITIALIZED, "Cannot open Kiji instance in state %s.", oldState);
LOG.debug("Kiji instance '{}' is now opened.", mURI);
mSystemVersion = mSystemTable.getDataVersion();
LOG.debug("Kiji instance '{}' has data version '{}'.", mURI, mSystemVersion);
// Make sure the data version for the client matches the cluster.
LOG.debug("Validating version for Kiji instance '{}'.", mURI);
try {
VersionInfo.validateVersion(this);
} catch (IOException ioe) {
// If an IOException occurred the object will not be constructed so need to clean it up.
close();
throw ioe;
} catch (KijiNotInstalledException kie) {
// Some clients handle this unchecked Exception so do the same here.
close();
throw kie;
}
// TODO(SCHEMA-491) Share ZooKeeperClient instances when possible.
if (mSystemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) >= 0) {
// system-2.0 clients must connect to ZooKeeper:
// - to register themselves as table users;
// - to receive table layout updates.
mZKClient = HBaseFactory.Provider.get().getZooKeeperClient(mURI);
try {
mMonitor = new ZooKeeperMonitor(mZKClient);
mMonitor.registerInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
} catch (KeeperException ke) {
// Unrecoverable KeeperException:
throw new IOException(ke);
}
} else {
// system-1.x clients do not need a ZooKeeper connection.
mZKClient = null;
mMonitor = null;
}
}