private void ensureColumnFamilyExists(String tableName, String columnFamily) throws StorageException { HBaseAdmin adm = getAdminInterface(); HTableDescriptor desc = ensureTableExists(tableName); Preconditions.checkNotNull(desc); HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes()); // Create our column family, if necessary if (cf == null) { try { adm.disableTable(tableName); desc.addFamily( new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ)); adm.modifyTable(tableName.getBytes(), desc); try { logger.debug( "Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily); Thread.sleep(1000L); } catch (InterruptedException ie) { throw new TemporaryStorageException(ie); } adm.enableTable(tableName); } catch (TableNotFoundException ee) { logger.error("TableNotFoundException", ee); throw new PermanentStorageException(ee); } catch (org.apache.hadoop.hbase.TableExistsException ee) { logger.debug("Swallowing exception {}", ee); } catch (IOException ee) { throw new TemporaryStorageException(ee); } } else { // check if compression was enabled, if not - enable it if (cf.getCompressionType() == null || cf.getCompressionType() == Compression.Algorithm.NONE) { try { adm.disableTable(tableName); adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ)); adm.enableTable(tableName); } catch (IOException e) { throw new TemporaryStorageException(e); } } } }
@Test(timeout = 180000) public void testRestoreSnapshot() throws Exception { String nsp = prefix + "_testRestoreSnapshot"; NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp) .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10") .build(); ADMIN.createNamespace(nspDesc); assertNotNull("Namespace descriptor found null.", ADMIN.getNamespaceDescriptor(nsp)); TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"); HTableDescriptor tableDescOne = new HTableDescriptor(tableName1); ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4); NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp); assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount()); String snapshot = "snapshot_testRestoreSnapshot"; ADMIN.snapshot(snapshot, tableName1); List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1); Collections.sort(regions); ADMIN.split(tableName1, Bytes.toBytes("JJJ")); Thread.sleep(2000); assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount()); ADMIN.disableTable(tableName1); ADMIN.restoreSnapshot(snapshot); assertEquals("Total regions count should be 4 after restore.", 4, nstate.getRegionCount()); ADMIN.enableTable(tableName1); ADMIN.deleteSnapshot(snapshot); }
/** Apply column family options such as Bloom filters, compression, and data block encoding. */ protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor tableDesc = admin.getTableDescriptor(tableName); LOG.info("Disabling table " + Bytes.toString(tableName)); admin.disableTable(tableName); for (byte[] cf : columnFamilies) { HColumnDescriptor columnDesc = tableDesc.getFamily(cf); boolean isNewCf = columnDesc == null; if (isNewCf) { columnDesc = new HColumnDescriptor(cf); } if (bloomType != null) { columnDesc.setBloomFilterType(bloomType); } if (compressAlgo != null) { columnDesc.setCompressionType(compressAlgo); } if (dataBlockEncodingAlgo != null) { columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo); columnDesc.setEncodeOnDisk(!encodeInCacheOnly); } if (inMemoryCF) { columnDesc.setInMemory(inMemoryCF); } if (isNewCf) { admin.addColumn(tableName, columnDesc); } else { admin.modifyColumn(tableName, columnDesc); } } LOG.info("Enabling table " + Bytes.toString(tableName)); admin.enableTable(tableName); }
public void fixInconsistent() throws IOException { if (ifFix == true) { for (String segFullName : inconsistentHTables) { String[] sepNameList = segFullName.split(","); HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0])); logger.info( "Change the host of htable " + sepNameList[0] + "belonging to cube " + sepNameList[1] + " from " + desc.getValue(IRealizationConstants.HTableTag) + " to " + dstCfg.getMetadataUrlPrefix()); hbaseAdmin.disableTable(sepNameList[0]); desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix()); hbaseAdmin.modifyTable(sepNameList[0], desc); hbaseAdmin.enableTable(sepNameList[0]); } } else { logger.info("------ Inconsistent HTables Needed To Be Fixed ------"); for (String hTable : inconsistentHTables) { String[] sepNameList = hTable.split(","); logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]); } logger.info("----------------------------------------------------"); } }
/** * @param rawAttributeName is the attribute name viewed by applications, it allows multiple * values. For example, secondaryIndex in secondaryIndex$1 and coprocessor in corpcessor$2 * @param indexOfAttribute is of the same raw attribute name, for example 2 in secondary$2 */ static void updateTableAttribute( Configuration conf, byte[] tableName, String rawAttributeName, int indexOfAttribute, boolean ifUpdateorRemove, String value) throws IOException { HBaseAdmin admin = new HBaseAdmin(conf); HTableDescriptor desc = admin.getTableDescriptor(tableName); admin.disableTable(tableName); // System.out.println("TTDEBUG: disable table " + Bytes.toString(tableName)); String coprocessorKey = rawAttributeName + indexOfAttribute; if (!ifUpdateorRemove) { desc.remove(Bytes.toBytes(coprocessorKey)); } else { desc.setValue(coprocessorKey, value); } admin.modifyTable(tableName, desc); // System.out.println("TTDEBUG: modify table " + Bytes.toString(tableName)); admin.enableTable(tableName); // System.out.println("TTDEBUG: enable table " + Bytes.toString(tableName)); HTableDescriptor descNew = admin.getTableDescriptor(tableName); // modify table is asynchronous, has to loop over to check while (!desc.equals(descNew)) { System.err.println( "TTDEBUG: waiting for descriptor to change: from " + descNew + " to " + desc); try { Thread.sleep(500); } catch (InterruptedException ex) { } descNew = admin.getTableDescriptor(tableName); } }
/** * 删除指定表名 * * @param rowKey */ public void deleteTable(byte[] rowKey) { Connection conn = null; HBaseAdmin admin = null; try { conn = ConnectionFactory.createConnection(conf); admin = (HBaseAdmin) conn.getAdmin(); // 在删除一张表前,要使其失效 admin.disableTable(rowKey); admin.deleteTable(rowKey); admin.enableTable(rowKey); } catch (Exception e) { logger.error("HBaseAdmin deleteTable exception, errMsg:{}", e.getMessage()); } finally { try { if (null != admin) { admin.close(); } } catch (IOException e) { logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage()); } try { if (null != conn) { conn.close(); } } catch (Exception e) { logger.error("Connection close exception, errMsg:{}", e.getMessage()); } } }
public static void createTable(Configuration hbaseConf, String tableName) throws IOException { HBaseAdmin admin = new HBaseAdmin(hbaseConf); if (!admin.tableExists(tableName)) { HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName)); HColumnDescriptor datafam = new HColumnDescriptor(HBaseTimestampStorage.TSO_FAMILY); datafam.setMaxVersions(3); desc.addFamily(datafam); admin.createTable(desc); } if (admin.isTableDisabled(tableName)) { admin.enableTable(tableName); } admin.close(); LOG.info("Table {} created successfully", tableName); }
public static void main(String args[]) throws MasterNotRunningException, IOException { // Instantiating configuration class Configuration conf = HBaseConfiguration.create(); // Instantiating HBaseAdmin class HBaseAdmin admin = new HBaseAdmin(conf); // Verifying weather the table is disabled Boolean bool = admin.isTableEnabled("emp"); System.out.println(bool); // Disabling the table using HBaseAdmin object if (!bool) { admin.enableTable("emp"); System.out.println("Table Enabled"); } }
@Override public void setConfigurationProperty(final String key, final String value) throws StorageException { byte[] name = tableName.getBytes(); HTableDescriptor desc = ensureTableExists(tableName); try { HBaseAdmin adm = getAdminInterface(); adm.disableTable(tableName); desc.setValue(key, value); adm.modifyTable(name, desc); adm.enableTable(tableName); } catch (IOException e) { throw new PermanentStorageException(e); } }
@Test(timeout = 180000) public void testRestoreSnapshotQuotaExceed() throws Exception { String nsp = prefix + "_testRestoreSnapshotQuotaExceed"; NamespaceDescriptor nspDesc = NamespaceDescriptor.create(nsp) .addConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "10") .build(); ADMIN.createNamespace(nspDesc); NamespaceDescriptor ndesc = ADMIN.getNamespaceDescriptor(nsp); assertNotNull("Namespace descriptor found null.", ndesc); TableName tableName1 = TableName.valueOf(nsp + TableName.NAMESPACE_DELIM + "table1"); HTableDescriptor tableDescOne = new HTableDescriptor(tableName1); ADMIN.createTable(tableDescOne, Bytes.toBytes("AAA"), Bytes.toBytes("ZZZ"), 4); NamespaceTableAndRegionInfo nstate = getNamespaceState(nsp); assertEquals("Intial region count should be 4.", 4, nstate.getRegionCount()); String snapshot = "snapshot_testRestoreSnapshotQuotaExceed"; ADMIN.snapshot(snapshot, tableName1); List<HRegionInfo> regions = ADMIN.getTableRegions(tableName1); Collections.sort(regions); ADMIN.split(tableName1, Bytes.toBytes("JJJ")); Thread.sleep(2000); assertEquals("Total regions count should be 5.", 5, nstate.getRegionCount()); ndesc.setConfiguration(TableNamespaceManager.KEY_MAX_REGIONS, "2"); ADMIN.modifyNamespace(ndesc); ADMIN.disableTable(tableName1); try { ADMIN.restoreSnapshot(snapshot); fail( "Region quota is exceeded so QuotaExceededException should be thrown but HBaseAdmin" + " wraps IOException into RestoreSnapshotException"); } catch (RestoreSnapshotException ignore) { } ADMIN.enableTable(tableName1); ADMIN.deleteSnapshot(snapshot); }
private void doIncrementalLoadTest(boolean shouldChangeRegions) throws Exception { util = new HBaseTestingUtility(); Configuration conf = util.getConfiguration(); byte[][] splitKeys = generateRandomSplitKeys(4); HBaseAdmin admin = null; try { util.startMiniCluster(); Path testDir = util.getDataTestDirOnTestFS("testLocalMRIncrementalLoad"); admin = util.getHBaseAdmin(); HTable table = util.createTable(TABLE_NAME, FAMILIES, splitKeys); assertEquals("Should start with empty table", 0, util.countRows(table)); int numRegions = -1; try (RegionLocator r = table.getRegionLocator()) { numRegions = r.getStartKeys().length; } assertEquals("Should make 5 regions", numRegions, 5); // Generate the bulk load files util.startMiniMapReduceCluster(); runIncrementalPELoad(conf, table, testDir); // This doesn't write into the table, just makes files assertEquals("HFOF should not touch actual table", 0, util.countRows(table)); // Make sure that a directory was created for every CF int dir = 0; for (FileStatus f : testDir.getFileSystem(conf).listStatus(testDir)) { for (byte[] family : FAMILIES) { if (Bytes.toString(family).equals(f.getPath().getName())) { ++dir; } } } assertEquals("Column family not found in FS.", FAMILIES.length, dir); // handle the split case if (shouldChangeRegions) { LOG.info("Changing regions in table"); admin.disableTable(table.getName()); while (util.getMiniHBaseCluster() .getMaster() .getAssignmentManager() .getRegionStates() .isRegionsInTransition()) { Threads.sleep(200); LOG.info("Waiting on table to finish disabling"); } util.deleteTable(table.getName()); byte[][] newSplitKeys = generateRandomSplitKeys(14); table = util.createTable(TABLE_NAME, FAMILIES, newSplitKeys); while (table.getRegionLocations().size() != 15 || !admin.isTableAvailable(table.getName())) { Thread.sleep(200); LOG.info("Waiting for new region assignment to happen"); } } // Perform the actual load new LoadIncrementalHFiles(conf).doBulkLoad(testDir, table); // Ensure data shows up int expectedRows = NMapInputFormat.getNumMapTasks(conf) * ROWSPERSPLIT; assertEquals( "LoadIncrementalHFiles should put expected data in table", expectedRows, util.countRows(table)); Scan scan = new Scan(); ResultScanner results = table.getScanner(scan); for (Result res : results) { assertEquals(FAMILIES.length, res.rawCells().length); Cell first = res.rawCells()[0]; for (Cell kv : res.rawCells()) { assertTrue(CellUtil.matchingRow(first, kv)); assertTrue(Bytes.equals(CellUtil.cloneValue(first), CellUtil.cloneValue(kv))); } } results.close(); String tableDigestBefore = util.checksumRows(table); // Cause regions to reopen admin.disableTable(TABLE_NAME); while (!admin.isTableDisabled(TABLE_NAME)) { Thread.sleep(200); LOG.info("Waiting for table to disable"); } admin.enableTable(TABLE_NAME); util.waitTableAvailable(TABLE_NAME); assertEquals( "Data should remain after reopening of regions", tableDigestBefore, util.checksumRows(table)); } finally { if (admin != null) admin.close(); util.shutdownMiniMapReduceCluster(); util.shutdownMiniCluster(); } }
@Ignore @Test public void testNonTxToTxTableFailure() throws Exception { Connection conn = DriverManager.getConnection(getUrl()); // Put table in SYSTEM schema to prevent attempts to update the cache after we disable // SYSTEM.CATALOG conn.createStatement() .execute("CREATE TABLE SYSTEM.NON_TX_TABLE(k INTEGER PRIMARY KEY, v VARCHAR)"); conn.createStatement().execute("UPSERT INTO SYSTEM.NON_TX_TABLE VALUES (1)"); conn.commit(); // Reset empty column value to an empty value like it is pre-transactions HTableInterface htable = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE")); Put put = new Put(PInteger.INSTANCE.toBytes(1)); put.add( QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, ByteUtil.EMPTY_BYTE_ARRAY); htable.put(put); HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin(); admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); try { // This will succeed initially in updating the HBase metadata, but then will fail when // the SYSTEM.CATALOG table is attempted to be updated, exercising the code to restore // the coprocessors back to the non transactional ones. conn.createStatement().execute("ALTER TABLE SYSTEM.NON_TX_TABLE SET TRANSACTIONAL=true"); fail(); } catch (SQLException e) { assertTrue( e.getMessage().contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME + " is disabled")); } finally { admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); admin.close(); } ResultSet rs = conn.createStatement().executeQuery("SELECT k FROM SYSTEM.NON_TX_TABLE WHERE v IS NULL"); assertTrue(rs.next()); assertEquals(1, rs.getInt(1)); assertFalse(rs.next()); htable = conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTable(Bytes.toBytes("SYSTEM.NON_TX_TABLE")); assertFalse( htable .getTableDescriptor() .getCoprocessors() .contains(TransactionProcessor.class.getName())); assertEquals( 1, conn.unwrap(PhoenixConnection.class) .getQueryServices() .getTableDescriptor(Bytes.toBytes("SYSTEM.NON_TX_TABLE")) .getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES) .getMaxVersions()); }