public MutationState createTable(CreateTableStatement statement, byte[][] splits)
throws SQLException {
PTableType tableType = statement.getTableType();
boolean isView = tableType == PTableType.VIEW;
if (isView && !statement.getProps().isEmpty()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_TABLE_CONFIG)
.build()
.buildException();
}
connection.rollback();
boolean wasAutoCommit = connection.getAutoCommit();
try {
connection.setAutoCommit(false);
TableName tableNameNode = statement.getTableName();
String schemaName = tableNameNode.getSchemaName();
String tableName = tableNameNode.getTableName();
PrimaryKeyConstraint pkConstraint = statement.getPrimaryKeyConstraint();
String pkName = null;
Set<String> pkColumns = Collections.<String>emptySet();
Iterator<String> pkColumnsIterator = Iterators.emptyIterator();
if (pkConstraint != null) {
pkColumns = pkConstraint.getColumnNames();
pkColumnsIterator = pkColumns.iterator();
pkName = pkConstraint.getName();
}
List<ColumnDef> colDefs = statement.getColumnDefs();
List<PColumn> columns = Lists.newArrayListWithExpectedSize(colDefs.size());
PreparedStatement colUpsert = connection.prepareStatement(INSERT_COLUMN);
int columnOrdinal = 0;
Map<String, PName> familyNames = Maps.newLinkedHashMap();
boolean isPK = false;
for (ColumnDef colDef : colDefs) {
if (colDef.isPK()) {
if (isPK) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_ALREADY_EXISTS)
.setColumnName(colDef.getColumnDefName().getColumnName().getName())
.build()
.buildException();
}
isPK = true;
}
PColumn column = newColumn(columnOrdinal++, colDef, pkConstraint);
if (SchemaUtil.isPKColumn(column)) {
// TODO: remove this constraint?
if (!pkColumns.isEmpty()
&& !column.getName().getString().equals(pkColumnsIterator.next())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_OUT_OF_ORDER)
.setSchemaName(schemaName)
.setTableName(tableName)
.setColumnName(column.getName().getString())
.build()
.buildException();
}
}
columns.add(column);
if (colDef.getDataType() == PDataType.BINARY && colDefs.size() > 1) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.BINARY_IN_ROW_KEY)
.setSchemaName(schemaName)
.setTableName(tableName)
.setColumnName(column.getName().getString())
.build()
.buildException();
}
if (column.getFamilyName() != null) {
familyNames.put(column.getFamilyName().getString(), column.getFamilyName());
}
}
if (!isPK && pkColumns.isEmpty()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_MISSING)
.setSchemaName(schemaName)
.setTableName(tableName)
.build()
.buildException();
}
List<Pair<byte[], Map<String, Object>>> familyPropList =
Lists.newArrayListWithExpectedSize(familyNames.size());
Map<String, Object> commonFamilyProps = Collections.emptyMap();
Map<String, Object> tableProps = Collections.emptyMap();
if (!statement.getProps().isEmpty()) {
if (statement.isView()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES)
.build()
.buildException();
}
for (String familyName : statement.getProps().keySet()) {
if (!familyName.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) {
if (familyNames.get(familyName) == null) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.PROPERTIES_FOR_FAMILY)
.setFamilyName(familyName)
.build()
.buildException();
}
}
}
commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size());
Collection<Pair<String, Object>> props =
statement.getProps().get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY);
// Somewhat hacky way of determining if property is for HColumnDescriptor or
// HTableDescriptor
HColumnDescriptor defaultDescriptor =
new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES);
for (Pair<String, Object> prop : props) {
if (defaultDescriptor.getValue(prop.getFirst()) != null) {
commonFamilyProps.put(prop.getFirst(), prop.getSecond());
} else {
tableProps.put(prop.getFirst(), prop.getSecond());
}
}
}
for (PName familyName : familyNames.values()) {
Collection<Pair<String, Object>> props = statement.getProps().get(familyName.getString());
if (props.isEmpty()) {
familyPropList.add(
new Pair<byte[], Map<String, Object>>(familyName.getBytes(), commonFamilyProps));
} else {
Map<String, Object> combinedFamilyProps =
Maps.newHashMapWithExpectedSize(props.size() + commonFamilyProps.size());
combinedFamilyProps.putAll(commonFamilyProps);
for (Pair<String, Object> prop : props) {
combinedFamilyProps.put(prop.getFirst(), prop.getSecond());
}
familyPropList.add(
new Pair<byte[], Map<String, Object>>(familyName.getBytes(), combinedFamilyProps));
}
}
// Bootstrapping for our SYSTEM.TABLE that creates itself before it exists
if (tableType == PTableType.SYSTEM) {
PTable table =
new PTableImpl(
new PNameImpl(tableName),
tableType,
MetaDataProtocol.MIN_TABLE_TIMESTAMP,
0,
QueryConstants.SYSTEM_TABLE_PK_NAME,
null,
columns);
connection.addTable(schemaName, table);
}
for (PColumn column : columns) {
addColumnMutation(schemaName, tableName, column, colUpsert);
}
Integer saltBucketNum = (Integer) tableProps.remove(PhoenixDatabaseMetaData.SALT_BUCKETS);
if (saltBucketNum != null
&& (saltBucketNum <= 0 || saltBucketNum > SaltingUtil.MAX_BUCKET_NUM)) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_BUCKET_NUM)
.build()
.buildException();
}
PreparedStatement tableUpsert = connection.prepareStatement(CREATE_TABLE);
tableUpsert.setString(1, schemaName);
tableUpsert.setString(2, tableName);
tableUpsert.setString(3, tableType.getSerializedValue());
tableUpsert.setInt(4, 0);
tableUpsert.setInt(5, columnOrdinal);
if (saltBucketNum != null) {
tableUpsert.setInt(6, saltBucketNum);
} else {
tableUpsert.setNull(6, Types.INTEGER);
}
tableUpsert.setString(7, pkName);
tableUpsert.execute();
final List<Mutation> tableMetaData = connection.getMutationState().toMutations();
connection.rollback();
MetaDataMutationResult result =
connection
.getQueryServices()
.createTable(tableMetaData, isView, tableProps, familyPropList, splits);
MutationCode code = result.getMutationCode();
switch (code) {
case TABLE_ALREADY_EXISTS:
connection.addTable(schemaName, result.getTable());
if (!statement.ifNotExists()) {
throw new TableAlreadyExistsException(schemaName, tableName);
}
break;
case NEWER_TABLE_FOUND:
// TODO: add table if in result?
throw new NewerTableAlreadyExistsException(schemaName, tableName);
case UNALLOWED_TABLE_MUTATION:
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MUTATE_TABLE)
.setSchemaName(schemaName)
.setTableName(tableName)
.build()
.buildException();
default:
PTable table =
new PTableImpl(
new PNameImpl(tableName),
tableType,
result.getMutationTime(),
0,
pkName,
saltBucketNum,
columns);
connection.addTable(schemaName, table);
if (tableType == PTableType.USER) {
connection.setAutoCommit(true);
// Delete everything in the column. You'll still be able to do queries at earlier
// timestamps
Long scn = connection.getSCN();
long ts = (scn == null ? result.getMutationTime() : scn);
PSchema schema =
new PSchemaImpl(
schemaName,
ImmutableMap.<String, PTable>of(table.getName().getString(), table));
TableRef tableRef = new TableRef(null, table, schema, ts);
byte[] emptyCF = SchemaUtil.getEmptyColumnFamily(table.getColumnFamilies());
MutationPlan plan =
new PostDDLCompiler(connection).compile(tableRef, emptyCF, null, ts);
return connection.getQueryServices().updateData(plan);
}
break;
}
return new MutationState(0, connection);
} finally {
connection.setAutoCommit(wasAutoCommit);
}
}
@Test
public void testProperties() throws Exception {
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement()
.execute(
"CREATE TABLE NON_TX_TABLE1(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR) TTL=1000");
conn.createStatement().execute("CREATE INDEX idx1 ON NON_TX_TABLE1(a.v, b.v) TTL=1000");
conn.createStatement()
.execute("CREATE INDEX idx2 ON NON_TX_TABLE1(c.v) INCLUDE (a.v, b.v) TTL=1000");
conn.createStatement().execute("ALTER TABLE NON_TX_TABLE1 SET TRANSACTIONAL=true");
HTableDescriptor desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("NON_TX_TABLE1"));
for (HColumnDescriptor colDesc : desc.getFamilies()) {
assertEquals(
QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(1000, colDesc.getTimeToLive());
assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
}
desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("IDX1"));
for (HColumnDescriptor colDesc : desc.getFamilies()) {
assertEquals(
QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(1000, colDesc.getTimeToLive());
assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
}
desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("IDX2"));
for (HColumnDescriptor colDesc : desc.getFamilies()) {
assertEquals(
QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(1000, colDesc.getTimeToLive());
assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
}
conn.createStatement()
.execute(
"CREATE TABLE NON_TX_TABLE2(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TRANSACTIONAL=true, VERSIONS=10");
desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
for (HColumnDescriptor colDesc : desc.getFamilies()) {
assertEquals(10, colDesc.getMaxVersions());
assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
assertEquals(null, colDesc.getValue(TxConstants.PROPERTY_TTL));
}
conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TTL=1000");
desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
for (HColumnDescriptor colDesc : desc.getFamilies()) {
assertEquals(10, colDesc.getMaxVersions());
assertEquals(1000, colDesc.getTimeToLive());
assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
}
conn.createStatement()
.execute(
"CREATE TABLE NON_TX_TABLE3(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
conn.createStatement()
.execute("ALTER TABLE NON_TX_TABLE3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20");
desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("NON_TX_TABLE3"));
assertEquals(
QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL,
desc.getFamily(Bytes.toBytes("A")).getMaxVersions());
assertEquals(10, desc.getFamily(Bytes.toBytes("B")).getMaxVersions());
assertEquals(20, desc.getFamily(Bytes.toBytes("C")).getMaxVersions());
conn.createStatement()
.execute(
"CREATE TABLE NON_TX_TABLE4(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
try {
conn.createStatement()
.execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, VERSIONS=1");
fail();
} catch (SQLException e) {
assertEquals(
SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(),
e.getErrorCode());
}
try {
conn.createStatement()
.execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, b.VERSIONS=1");
fail();
} catch (SQLException e) {
assertEquals(
SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(),
e.getErrorCode());
}
conn.createStatement()
.execute(
"CREATE TABLE TX_TABLE1(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true");
desc =
conn.unwrap(PhoenixConnection.class)
.getQueryServices()
.getTableDescriptor(Bytes.toBytes("TX_TABLE1"));
for (HColumnDescriptor colDesc : desc.getFamilies()) {
assertEquals(
QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
}
}
/**
* Verify schema modification takes.
*
* @throws IOException
* @throws InterruptedException
*/
@Test(timeout = 300000)
public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
final TableName tableName = TableName.valueOf("changeTableSchemaOnline");
TEST_UTIL
.getMiniHBaseCluster()
.getMaster()
.getConfiguration()
.setBoolean("hbase.online.schema.update.enable", true);
HTableDescriptor[] tables = admin.listTables();
int numTables = tables.length;
TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
tables = this.admin.listTables();
assertEquals(numTables + 1, tables.length);
// FIRST, do htabledescriptor changes.
HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
// Make a copy and assert copy is good.
HTableDescriptor copy = new HTableDescriptor(htd);
assertTrue(htd.equals(copy));
// Now amend the copy. Introduce differences.
long newFlushSize = htd.getMemStoreFlushSize() / 2;
if (newFlushSize <= 0) {
newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
}
copy.setMemStoreFlushSize(newFlushSize);
final String key = "anyoldkey";
assertTrue(htd.getValue(key) == null);
copy.setValue(key, key);
boolean expectedException = false;
try {
admin.modifyTable(tableName, copy);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertFalse(expectedException);
HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
assertFalse(htd.equals(modifiedHtd));
assertTrue(copy.equals(modifiedHtd));
assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
assertEquals(key, modifiedHtd.getValue(key));
// Now work on column family changes.
int countOfFamilies = modifiedHtd.getFamilies().size();
assertTrue(countOfFamilies > 0);
HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
int maxversions = hcd.getMaxVersions();
final int newMaxVersions = maxversions + 1;
hcd.setMaxVersions(newMaxVersions);
final byte[] hcdName = hcd.getName();
expectedException = false;
try {
this.admin.modifyColumn(tableName, hcd);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertFalse(expectedException);
modifiedHtd = this.admin.getTableDescriptor(tableName);
HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
// Try adding a column
assertFalse(this.admin.isTableDisabled(tableName));
final String xtracolName = "xtracol";
HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
xtracol.setValue(xtracolName, xtracolName);
expectedException = false;
try {
this.admin.addColumn(tableName, xtracol);
} catch (TableNotDisabledException re) {
expectedException = true;
}
// Add column should work even if the table is enabled
assertFalse(expectedException);
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd != null);
assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
// Delete the just-added column.
this.admin.deleteColumn(tableName, xtracol.getName());
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd == null);
// Delete the table
this.admin.disableTable(tableName);
this.admin.deleteTable(tableName);
this.admin.listTables();
assertFalse(this.admin.tableExists(tableName));
}
/**
* Verify schema modification takes.
*
* @throws IOException
*/
@Test
public void testChangeTableSchema() throws IOException {
final byte[] tableName = Bytes.toBytes("changeTableSchema");
HTableDescriptor[] tables = admin.listTables();
int numTables = tables.length;
TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
tables = this.admin.listTables();
assertEquals(numTables + 1, tables.length);
// FIRST, do htabledescriptor changes.
HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
// Make a copy and assert copy is good.
HTableDescriptor copy = new HTableDescriptor(htd);
assertTrue(htd.equals(copy));
// Now amend the copy. Introduce differences.
long newFlushSize = htd.getMemStoreFlushSize() / 2;
copy.setMemStoreFlushSize(newFlushSize);
final String key = "anyoldkey";
assertTrue(htd.getValue(key) == null);
copy.setValue(key, key);
boolean expectedException = false;
try {
this.admin.modifyTable(tableName, copy);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertTrue(expectedException);
this.admin.disableTable(tableName);
assertTrue(this.admin.isTableDisabled(tableName));
modifyTable(tableName, copy);
HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
// Assert returned modifiedhcd is same as the copy.
assertFalse(htd.equals(modifiedHtd));
assertTrue(copy.equals(modifiedHtd));
assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
assertEquals(key, modifiedHtd.getValue(key));
// Reenable table to test it fails if not disabled.
this.admin.enableTable(tableName);
assertFalse(this.admin.isTableDisabled(tableName));
// Now work on column family changes.
int countOfFamilies = modifiedHtd.getFamilies().size();
assertTrue(countOfFamilies > 0);
HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
int maxversions = hcd.getMaxVersions();
final int newMaxVersions = maxversions + 1;
hcd.setMaxVersions(newMaxVersions);
final byte[] hcdName = hcd.getName();
expectedException = false;
try {
this.admin.modifyColumn(tableName, hcd);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertTrue(expectedException);
this.admin.disableTable(tableName);
assertTrue(this.admin.isTableDisabled(tableName));
// Modify Column is synchronous
this.admin.modifyColumn(tableName, hcd);
modifiedHtd = this.admin.getTableDescriptor(tableName);
HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());
// Try adding a column
// Reenable table to test it fails if not disabled.
this.admin.enableTable(tableName);
assertFalse(this.admin.isTableDisabled(tableName));
final String xtracolName = "xtracol";
HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
xtracol.setValue(xtracolName, xtracolName);
try {
this.admin.addColumn(tableName, xtracol);
} catch (TableNotDisabledException re) {
expectedException = true;
}
assertTrue(expectedException);
this.admin.disableTable(tableName);
assertTrue(this.admin.isTableDisabled(tableName));
this.admin.addColumn(tableName, xtracol);
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd != null);
assertTrue(hcd.getValue(xtracolName).equals(xtracolName));
// Delete the just-added column.
this.admin.deleteColumn(tableName, xtracol.getName());
modifiedHtd = this.admin.getTableDescriptor(tableName);
hcd = modifiedHtd.getFamily(xtracol.getName());
assertTrue(hcd == null);
// Delete the table
this.admin.deleteTable(tableName);
this.admin.listTables();
assertFalse(this.admin.tableExists(tableName));
}
