public MutationState createTable(CreateTableStatement statement, byte[][] splits) throws SQLException { PTableType tableType = statement.getTableType(); boolean isView = tableType == PTableType.VIEW; if (isView && !statement.getProps().isEmpty()) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_TABLE_CONFIG) .build() .buildException(); } connection.rollback(); boolean wasAutoCommit = connection.getAutoCommit(); try { connection.setAutoCommit(false); TableName tableNameNode = statement.getTableName(); String schemaName = tableNameNode.getSchemaName(); String tableName = tableNameNode.getTableName(); PrimaryKeyConstraint pkConstraint = statement.getPrimaryKeyConstraint(); String pkName = null; Set<String> pkColumns = Collections.<String>emptySet(); Iterator<String> pkColumnsIterator = Iterators.emptyIterator(); if (pkConstraint != null) { pkColumns = pkConstraint.getColumnNames(); pkColumnsIterator = pkColumns.iterator(); pkName = pkConstraint.getName(); } List<ColumnDef> colDefs = statement.getColumnDefs(); List<PColumn> columns = Lists.newArrayListWithExpectedSize(colDefs.size()); PreparedStatement colUpsert = connection.prepareStatement(INSERT_COLUMN); int columnOrdinal = 0; Map<String, PName> familyNames = Maps.newLinkedHashMap(); boolean isPK = false; for (ColumnDef colDef : colDefs) { if (colDef.isPK()) { if (isPK) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_ALREADY_EXISTS) .setColumnName(colDef.getColumnDefName().getColumnName().getName()) .build() .buildException(); } isPK = true; } PColumn column = newColumn(columnOrdinal++, colDef, pkConstraint); if (SchemaUtil.isPKColumn(column)) { // TODO: remove this constraint? if (!pkColumns.isEmpty() && !column.getName().getString().equals(pkColumnsIterator.next())) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_OUT_OF_ORDER) .setSchemaName(schemaName) .setTableName(tableName) .setColumnName(column.getName().getString()) .build() .buildException(); } } columns.add(column); if (colDef.getDataType() == PDataType.BINARY && colDefs.size() > 1) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.BINARY_IN_ROW_KEY) .setSchemaName(schemaName) .setTableName(tableName) .setColumnName(column.getName().getString()) .build() .buildException(); } if (column.getFamilyName() != null) { familyNames.put(column.getFamilyName().getString(), column.getFamilyName()); } } if (!isPK && pkColumns.isEmpty()) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.PRIMARY_KEY_MISSING) .setSchemaName(schemaName) .setTableName(tableName) .build() .buildException(); } List<Pair<byte[], Map<String, Object>>> familyPropList = Lists.newArrayListWithExpectedSize(familyNames.size()); Map<String, Object> commonFamilyProps = Collections.emptyMap(); Map<String, Object> tableProps = Collections.emptyMap(); if (!statement.getProps().isEmpty()) { if (statement.isView()) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.VIEW_WITH_PROPERTIES) .build() .buildException(); } for (String familyName : statement.getProps().keySet()) { if (!familyName.equals(QueryConstants.ALL_FAMILY_PROPERTIES_KEY)) { if (familyNames.get(familyName) == null) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.PROPERTIES_FOR_FAMILY) .setFamilyName(familyName) .build() .buildException(); } } } commonFamilyProps = Maps.newHashMapWithExpectedSize(statement.getProps().size()); tableProps = Maps.newHashMapWithExpectedSize(statement.getProps().size()); Collection<Pair<String, Object>> props = statement.getProps().get(QueryConstants.ALL_FAMILY_PROPERTIES_KEY); // Somewhat hacky way of determining if property is for HColumnDescriptor or // HTableDescriptor HColumnDescriptor defaultDescriptor = new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES); for (Pair<String, Object> prop : props) { if (defaultDescriptor.getValue(prop.getFirst()) != null) { commonFamilyProps.put(prop.getFirst(), prop.getSecond()); } else { tableProps.put(prop.getFirst(), prop.getSecond()); } } } for (PName familyName : familyNames.values()) { Collection<Pair<String, Object>> props = statement.getProps().get(familyName.getString()); if (props.isEmpty()) { familyPropList.add( new Pair<byte[], Map<String, Object>>(familyName.getBytes(), commonFamilyProps)); } else { Map<String, Object> combinedFamilyProps = Maps.newHashMapWithExpectedSize(props.size() + commonFamilyProps.size()); combinedFamilyProps.putAll(commonFamilyProps); for (Pair<String, Object> prop : props) { combinedFamilyProps.put(prop.getFirst(), prop.getSecond()); } familyPropList.add( new Pair<byte[], Map<String, Object>>(familyName.getBytes(), combinedFamilyProps)); } } // Bootstrapping for our SYSTEM.TABLE that creates itself before it exists if (tableType == PTableType.SYSTEM) { PTable table = new PTableImpl( new PNameImpl(tableName), tableType, MetaDataProtocol.MIN_TABLE_TIMESTAMP, 0, QueryConstants.SYSTEM_TABLE_PK_NAME, null, columns); connection.addTable(schemaName, table); } for (PColumn column : columns) { addColumnMutation(schemaName, tableName, column, colUpsert); } Integer saltBucketNum = (Integer) tableProps.remove(PhoenixDatabaseMetaData.SALT_BUCKETS); if (saltBucketNum != null && (saltBucketNum <= 0 || saltBucketNum > SaltingUtil.MAX_BUCKET_NUM)) { throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_BUCKET_NUM) .build() .buildException(); } PreparedStatement tableUpsert = connection.prepareStatement(CREATE_TABLE); tableUpsert.setString(1, schemaName); tableUpsert.setString(2, tableName); tableUpsert.setString(3, tableType.getSerializedValue()); tableUpsert.setInt(4, 0); tableUpsert.setInt(5, columnOrdinal); if (saltBucketNum != null) { tableUpsert.setInt(6, saltBucketNum); } else { tableUpsert.setNull(6, Types.INTEGER); } tableUpsert.setString(7, pkName); tableUpsert.execute(); final List<Mutation> tableMetaData = connection.getMutationState().toMutations(); connection.rollback(); MetaDataMutationResult result = connection .getQueryServices() .createTable(tableMetaData, isView, tableProps, familyPropList, splits); MutationCode code = result.getMutationCode(); switch (code) { case TABLE_ALREADY_EXISTS: connection.addTable(schemaName, result.getTable()); if (!statement.ifNotExists()) { throw new TableAlreadyExistsException(schemaName, tableName); } break; case NEWER_TABLE_FOUND: // TODO: add table if in result? throw new NewerTableAlreadyExistsException(schemaName, tableName); case UNALLOWED_TABLE_MUTATION: throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_MUTATE_TABLE) .setSchemaName(schemaName) .setTableName(tableName) .build() .buildException(); default: PTable table = new PTableImpl( new PNameImpl(tableName), tableType, result.getMutationTime(), 0, pkName, saltBucketNum, columns); connection.addTable(schemaName, table); if (tableType == PTableType.USER) { connection.setAutoCommit(true); // Delete everything in the column. You'll still be able to do queries at earlier // timestamps Long scn = connection.getSCN(); long ts = (scn == null ? result.getMutationTime() : scn); PSchema schema = new PSchemaImpl( schemaName, ImmutableMap.<String, PTable>of(table.getName().getString(), table)); TableRef tableRef = new TableRef(null, table, schema, ts); byte[] emptyCF = SchemaUtil.getEmptyColumnFamily(table.getColumnFamilies()); MutationPlan plan = new PostDDLCompiler(connection).compile(tableRef, emptyCF, null, ts); return connection.getQueryServices().updateData(plan); } break; } return new MutationState(0, connection); } finally { connection.setAutoCommit(wasAutoCommit); } }
@Test public void testProperties() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); conn.createStatement() .execute( "CREATE TABLE NON_TX_TABLE1(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR) TTL=1000"); conn.createStatement().execute("CREATE INDEX idx1 ON NON_TX_TABLE1(a.v, b.v) TTL=1000"); conn.createStatement() .execute("CREATE INDEX idx2 ON NON_TX_TABLE1(c.v) INCLUDE (a.v, b.v) TTL=1000"); conn.createStatement().execute("ALTER TABLE NON_TX_TABLE1 SET TRANSACTIONAL=true"); HTableDescriptor desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE1")); for (HColumnDescriptor colDesc : desc.getFamilies()) { assertEquals( QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions()); assertEquals(1000, colDesc.getTimeToLive()); assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL))); } desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("IDX1")); for (HColumnDescriptor colDesc : desc.getFamilies()) { assertEquals( QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions()); assertEquals(1000, colDesc.getTimeToLive()); assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL))); } desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("IDX2")); for (HColumnDescriptor colDesc : desc.getFamilies()) { assertEquals( QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions()); assertEquals(1000, colDesc.getTimeToLive()); assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL))); } conn.createStatement() .execute( "CREATE TABLE NON_TX_TABLE2(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)"); conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TRANSACTIONAL=true, VERSIONS=10"); desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2")); for (HColumnDescriptor colDesc : desc.getFamilies()) { assertEquals(10, colDesc.getMaxVersions()); assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive()); assertEquals(null, colDesc.getValue(TxConstants.PROPERTY_TTL)); } conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TTL=1000"); desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2")); for (HColumnDescriptor colDesc : desc.getFamilies()) { assertEquals(10, colDesc.getMaxVersions()); assertEquals(1000, colDesc.getTimeToLive()); assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL))); } conn.createStatement() .execute( "CREATE TABLE NON_TX_TABLE3(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)"); conn.createStatement() .execute("ALTER TABLE NON_TX_TABLE3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20"); desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE3")); assertEquals( QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, desc.getFamily(Bytes.toBytes("A")).getMaxVersions()); assertEquals(10, desc.getFamily(Bytes.toBytes("B")).getMaxVersions()); assertEquals(20, desc.getFamily(Bytes.toBytes("C")).getMaxVersions()); conn.createStatement() .execute( "CREATE TABLE NON_TX_TABLE4(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)"); try { conn.createStatement() .execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, VERSIONS=1"); fail(); } catch (SQLException e) { assertEquals( SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode()); } try { conn.createStatement() .execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, b.VERSIONS=1"); fail(); } catch (SQLException e) { assertEquals( SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(), e.getErrorCode()); } conn.createStatement() .execute( "CREATE TABLE TX_TABLE1(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true"); desc = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("TX_TABLE1")); for (HColumnDescriptor colDesc : desc.getFamilies()) { assertEquals( QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions()); assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive()); assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL))); } }
/** * Verify schema modification takes. * * @throws IOException * @throws InterruptedException */ @Test(timeout = 300000) public void testOnlineChangeTableSchema() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf("changeTableSchemaOnline"); TEST_UTIL .getMiniHBaseCluster() .getMaster() .getConfiguration() .setBoolean("hbase.online.schema.update.enable", true); HTableDescriptor[] tables = admin.listTables(); int numTables = tables.length; TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close(); tables = this.admin.listTables(); assertEquals(numTables + 1, tables.length); // FIRST, do htabledescriptor changes. HTableDescriptor htd = this.admin.getTableDescriptor(tableName); // Make a copy and assert copy is good. HTableDescriptor copy = new HTableDescriptor(htd); assertTrue(htd.equals(copy)); // Now amend the copy. Introduce differences. long newFlushSize = htd.getMemStoreFlushSize() / 2; if (newFlushSize <= 0) { newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2; } copy.setMemStoreFlushSize(newFlushSize); final String key = "anyoldkey"; assertTrue(htd.getValue(key) == null); copy.setValue(key, key); boolean expectedException = false; try { admin.modifyTable(tableName, copy); } catch (TableNotDisabledException re) { expectedException = true; } assertFalse(expectedException); HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName); assertFalse(htd.equals(modifiedHtd)); assertTrue(copy.equals(modifiedHtd)); assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize()); assertEquals(key, modifiedHtd.getValue(key)); // Now work on column family changes. int countOfFamilies = modifiedHtd.getFamilies().size(); assertTrue(countOfFamilies > 0); HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); int maxversions = hcd.getMaxVersions(); final int newMaxVersions = maxversions + 1; hcd.setMaxVersions(newMaxVersions); final byte[] hcdName = hcd.getName(); expectedException = false; try { this.admin.modifyColumn(tableName, hcd); } catch (TableNotDisabledException re) { expectedException = true; } assertFalse(expectedException); modifiedHtd = this.admin.getTableDescriptor(tableName); HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName); assertEquals(newMaxVersions, modifiedHcd.getMaxVersions()); // Try adding a column assertFalse(this.admin.isTableDisabled(tableName)); final String xtracolName = "xtracol"; HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); xtracol.setValue(xtracolName, xtracolName); expectedException = false; try { this.admin.addColumn(tableName, xtracol); } catch (TableNotDisabledException re) { expectedException = true; } // Add column should work even if the table is enabled assertFalse(expectedException); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd != null); assertTrue(hcd.getValue(xtracolName).equals(xtracolName)); // Delete the just-added column. this.admin.deleteColumn(tableName, xtracol.getName()); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd == null); // Delete the table this.admin.disableTable(tableName); this.admin.deleteTable(tableName); this.admin.listTables(); assertFalse(this.admin.tableExists(tableName)); }
/** * Verify schema modification takes. * * @throws IOException */ @Test public void testChangeTableSchema() throws IOException { final byte[] tableName = Bytes.toBytes("changeTableSchema"); HTableDescriptor[] tables = admin.listTables(); int numTables = tables.length; TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY); tables = this.admin.listTables(); assertEquals(numTables + 1, tables.length); // FIRST, do htabledescriptor changes. HTableDescriptor htd = this.admin.getTableDescriptor(tableName); // Make a copy and assert copy is good. HTableDescriptor copy = new HTableDescriptor(htd); assertTrue(htd.equals(copy)); // Now amend the copy. Introduce differences. long newFlushSize = htd.getMemStoreFlushSize() / 2; copy.setMemStoreFlushSize(newFlushSize); final String key = "anyoldkey"; assertTrue(htd.getValue(key) == null); copy.setValue(key, key); boolean expectedException = false; try { this.admin.modifyTable(tableName, copy); } catch (TableNotDisabledException re) { expectedException = true; } assertTrue(expectedException); this.admin.disableTable(tableName); assertTrue(this.admin.isTableDisabled(tableName)); modifyTable(tableName, copy); HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName); // Assert returned modifiedhcd is same as the copy. assertFalse(htd.equals(modifiedHtd)); assertTrue(copy.equals(modifiedHtd)); assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize()); assertEquals(key, modifiedHtd.getValue(key)); // Reenable table to test it fails if not disabled. this.admin.enableTable(tableName); assertFalse(this.admin.isTableDisabled(tableName)); // Now work on column family changes. int countOfFamilies = modifiedHtd.getFamilies().size(); assertTrue(countOfFamilies > 0); HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next(); int maxversions = hcd.getMaxVersions(); final int newMaxVersions = maxversions + 1; hcd.setMaxVersions(newMaxVersions); final byte[] hcdName = hcd.getName(); expectedException = false; try { this.admin.modifyColumn(tableName, hcd); } catch (TableNotDisabledException re) { expectedException = true; } assertTrue(expectedException); this.admin.disableTable(tableName); assertTrue(this.admin.isTableDisabled(tableName)); // Modify Column is synchronous this.admin.modifyColumn(tableName, hcd); modifiedHtd = this.admin.getTableDescriptor(tableName); HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName); assertEquals(newMaxVersions, modifiedHcd.getMaxVersions()); // Try adding a column // Reenable table to test it fails if not disabled. this.admin.enableTable(tableName); assertFalse(this.admin.isTableDisabled(tableName)); final String xtracolName = "xtracol"; HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName); xtracol.setValue(xtracolName, xtracolName); try { this.admin.addColumn(tableName, xtracol); } catch (TableNotDisabledException re) { expectedException = true; } assertTrue(expectedException); this.admin.disableTable(tableName); assertTrue(this.admin.isTableDisabled(tableName)); this.admin.addColumn(tableName, xtracol); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd != null); assertTrue(hcd.getValue(xtracolName).equals(xtracolName)); // Delete the just-added column. this.admin.deleteColumn(tableName, xtracol.getName()); modifiedHtd = this.admin.getTableDescriptor(tableName); hcd = modifiedHtd.getFamily(xtracol.getName()); assertTrue(hcd == null); // Delete the table this.admin.deleteTable(tableName); this.admin.listTables(); assertFalse(this.admin.tableExists(tableName)); }