@Test
public void testRemoves() throws IOException {
final String name = "testRemoves";
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any detrius laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name);
TableDescriptors htds = new FSTableDescriptors(fs, rootdir);
HTableDescriptor htd = new HTableDescriptor(name);
htds.add(htd);
assertNotNull(htds.remove(htd.getNameAsString()));
assertNull(htds.remove(htd.getNameAsString()));
}
@Override
public void add(HTableDescriptor htd) throws IOException {
if (Bytes.equals(HConstants.ROOT_TABLE_NAME, htd.getName())) {
throw new NotImplementedException();
}
if (Bytes.equals(HConstants.META_TABLE_NAME, htd.getName())) {
throw new NotImplementedException();
}
if (HConstants.HBASE_NON_USER_TABLE_DIRS.contains(htd.getNameAsString())) {
throw new NotImplementedException();
}
if (!this.fsreadonly) updateHTableDescriptor(this.fs, this.rootdir, htd);
long modtime = getTableInfoModtime(this.fs, this.rootdir, htd.getNameAsString());
this.cache.put(htd.getNameAsString(), new TableDescriptorModtime(modtime, htd));
}
/**
* Find all column families that are replicated from this cluster
*
* @return the full list of the replicated column families of this cluster as: tableName, family
* name, replicationType
* <p>Currently replicationType is Global. In the future, more replication types may be
* extended here. For example 1) the replication may only apply to selected peers instead of
* all peers 2) the replicationType may indicate the host Cluster servers as Slave for the
* table:columnFam.
*/
public List<HashMap<String, String>> listReplicated() throws IOException {
List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();
Admin admin = connection.getAdmin();
HTableDescriptor[] tables;
try {
tables = admin.listTables();
} finally {
if (admin != null) admin.close();
}
for (HTableDescriptor table : tables) {
HColumnDescriptor[] columns = table.getColumnFamilies();
String tableName = table.getNameAsString();
for (HColumnDescriptor column : columns) {
if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
// At this moment, the columfam is replicated to all peers
HashMap<String, String> replicationEntry = new HashMap<String, String>();
replicationEntry.put(TNAME, tableName);
replicationEntry.put(CFNAME, column.getNameAsString());
replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
replicationColFams.add(replicationEntry);
}
}
}
return replicationColFams;
}
@Test
public void testReadingHTDFromFS() throws IOException {
final String name = "testReadingHTDFromFS";
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
HTableDescriptor htd = new HTableDescriptor(name);
Path rootdir = UTIL.getDataTestDir(name);
createHTDInFS(fs, rootdir, htd);
HTableDescriptor htd2 =
FSTableDescriptors.getTableDescriptor(fs, rootdir, htd.getNameAsString());
assertTrue(htd.equals(htd2));
}
/**
* Create new HTableDescriptor in HDFS. Happens when we are creating table. If forceCreation is
* true then even if previous table descriptor is present it will be overwritten
*
* @param fs
* @param htableDescriptor
* @param rootdir
* @param forceCreation
* @return True if we successfully created file.
*/
public static boolean createTableDescriptor(
FileSystem fs, Path rootdir, HTableDescriptor htableDescriptor, boolean forceCreation)
throws IOException {
FileStatus status = getTableInfoPath(fs, rootdir, htableDescriptor.getNameAsString());
if (status != null) {
LOG.info("Current tableInfoPath = " + status.getPath());
if (!forceCreation) {
if (fs.exists(status.getPath()) && status.getLen() > 0) {
LOG.info("TableInfo already exists.. Skipping creation");
return false;
}
}
}
Path p =
writeTableDescriptor(
fs,
htableDescriptor,
FSUtils.getTablePath(rootdir, htableDescriptor.getNameAsString()),
status);
return p != null;
}
private void verifyColumns() {
if (AbstractRecordReader.isStarQuery(columns)) {
return;
}
for (SchemaPath column : columns) {
if (!(column.equals(ROW_KEY_PATH)
|| hTableDesc.hasFamily(HBaseUtils.getBytes(column.getRootSegment().getPath())))) {
DrillRuntimeException.format(
"The column family '%s' does not exist in HBase table: %s .",
column.getRootSegment().getPath(), hTableDesc.getNameAsString());
}
}
}
/** @return the set of the regions contained in the table */
private List<HRegionInfo> getTableRegions() throws IOException {
LOG.debug("get table regions: " + tableDir);
FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
if (regionDirs == null) return null;
List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
for (FileStatus regionDir : regionDirs) {
HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir.getPath());
regions.add(hri);
}
LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString());
return regions;
}
/**
* Creates a Kiji table in an HBase instance, without checking for validation compatibility and
* without applying permissions.
*
* @param tableLayout The initial layout of the table (with unassigned column ids).
* @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1
* regions created. Pass null to specify the default single region.
* @throws IOException on I/O error.
* @throws KijiAlreadyExistsException if the table already exists.
*/
private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys)
throws IOException {
final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();
// This will validate the layout and may throw an InvalidLayoutException.
final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout);
if (getMetaTable().tableExists(tableLayout.getName())) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
if (tableLayout.getKeysFormat() instanceof RowKeyFormat) {
LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'.");
}
getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout);
if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
// system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications.
// Invariant: ZooKeeper hold the most recent layout of the table.
LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI);
try {
final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient);
try {
final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId());
monitor.notifyNewTableLayout(tableURI, layoutId, -1);
} finally {
monitor.close();
}
} catch (KeeperException ke) {
throw new IOException(ke);
}
}
try {
final HTableSchemaTranslator translator = new HTableSchemaTranslator();
final HTableDescriptor desc =
translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout);
LOG.debug("Creating HBase table '{}'.", desc.getNameAsString());
if (null != splitKeys) {
getHBaseAdmin().createTable(desc, splitKeys);
} else {
getHBaseAdmin().createTable(desc);
}
} catch (TableExistsException tee) {
throw new KijiAlreadyExistsException(
String.format("Kiji table '%s' already exists.", tableURI), tableURI);
}
}
/**
* Clone region directory content from the snapshot info.
*
* <p>Each region is encoded with the table name, so the cloned region will have a different
* region name.
*
* <p>Instead of copying the hfiles a HFileLink is created.
*
* @param region {@link HRegion} cloned
* @param snapshotRegionInfo
*/
private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
throws IOException {
final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
final String tableName = tableDesc.getNameAsString();
SnapshotReferenceUtil.visitRegionStoreFiles(
fs,
snapshotRegionDir,
new FSVisitor.StoreFileVisitor() {
public void storeFile(final String region, final String family, final String hfile)
throws IOException {
LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
Path familyDir = new Path(regionDir, family);
restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
}
});
}
/**
* 创建一张表
*
* @param table
* @throws ParamIsNullException 参数为空
* @throws TableExistsException 表已存在
*/
public static void createTable(HTableDescriptor table)
throws ParamIsNullException, TableExistsException {
if (null == table) {
throw new ParamIsNullException("参数不能为空");
}
logger.info("create table begin... , table:{}", table.toString());
Connection conn = null;
HBaseAdmin admin = null;
String tableName = table.getNameAsString();
try {
logger.info("获取connection");
conn = ConnectionFactory.createConnection(conf);
logger.info("获取admin");
admin = (HBaseAdmin) conn.getAdmin();
/** 表已存在 */
if (admin.tableExists(Bytes.toBytes(tableName))) {
throw new TableExistsException(tableName);
}
logger.info("create...");
admin.createTable(table);
logger.info("table create success, tableName:{}", tableName);
} catch (IOException e) {
logger.error("table create fail, tableName:{}, errMsg:{}", tableName, e);
} finally {
if (null != admin) {
try {
admin.close();
} catch (IOException e) {
logger.error("HBaseAdmin close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (IOException e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
/**
* Method execute
*
* @param mapping
* @param form
* @param request
* @param response
* @return ActionForward
*/
public ActionForward execute(
ActionMapping mapping,
ActionForm form,
HttpServletRequest request,
HttpServletResponse response) {
// TODO Auto-generated method stub
HMaster master = (HMaster) getServlet().getServletContext().getAttribute(HMaster.MASTER);
tableOperation to = new tableOperation();
authorityOperation ao = new authorityOperation();
Configuration conf = master.getConfiguration();
boolean security = conf.getBoolean("hbase.server.security", false);
HTableDescriptor[] adminTables = null;
Map<byte[], ACLField> userTables = null;
List<String> tableList = new ArrayList<String>();
if (security) {
String username = (String) request.getSession().getAttribute("username");
String usertype = (String) request.getSession().getAttribute("usertype");
try {
if (usertype.equals("admin")) {
adminTables = to.getAllTable(master);
for (HTableDescriptor tableDesc : adminTables) {
tableList.add(tableDesc.getNameAsString());
}
} else {
userTables = ao.getUserVisibleTable(master, username);
for (Map.Entry<byte[], ACLField> entry : userTables.entrySet()) {
tableList.add(Bytes.toString(entry.getKey()));
}
}
} catch (MasterNotRunningException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ZooKeeperConnectionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
try {
adminTables = to.getAllTable(master);
} catch (MasterNotRunningException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ZooKeeperConnectionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
for (HTableDescriptor tableDesc : adminTables) {
tableList.add(tableDesc.getNameAsString());
}
}
request.setAttribute("tableList", tableList);
return mapping.findForward("success");
}
/**
* Deletes a HBase table.
*
* @param admin HBase admin
* @param tableDescriptor descriptor for the table to delete
* @throws IOException
*/
private static void deleteTable(final HBaseAdmin admin, final HTableDescriptor tableDescriptor)
throws IOException {
LOG.info("deleting table: " + tableDescriptor.getNameAsString());
admin.disableTable(tableDescriptor.getName());
admin.deleteTable(tableDescriptor.getName());
}
/**
* Restore region by removing files not in the snapshot and adding the missing ones from the
* snapshot.
*/
private void restoreRegion(HRegionInfo regionInfo) throws IOException {
Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
Map<String, List<String>> snapshotFiles =
SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
String tableName = tableDesc.getNameAsString();
// Restore families present in the table
for (Path familyDir : FSUtils.getFamilyDirs(fs, regionDir)) {
byte[] family = Bytes.toBytes(familyDir.getName());
Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
if (snapshotFamilyFiles != null) {
List<String> hfilesToAdd = new LinkedList<String>();
for (String hfileName : snapshotFamilyFiles) {
if (familyFiles.contains(hfileName)) {
// HFile already present
familyFiles.remove(hfileName);
} else {
// HFile missing
hfilesToAdd.add(hfileName);
}
}
// Restore Missing files
for (String hfileName : hfilesToAdd) {
LOG.trace(
"Adding HFileLink "
+ hfileName
+ " to region="
+ regionInfo.getEncodedName()
+ " table="
+ tableName);
restoreStoreFile(familyDir, regionInfo, hfileName);
}
// Remove hfiles not present in the snapshot
for (String hfileName : familyFiles) {
Path hfile = new Path(familyDir, hfileName);
LOG.trace(
"Removing hfile="
+ hfile
+ " from region="
+ regionInfo.getEncodedName()
+ " table="
+ tableName);
HFileArchiver.archiveStoreFile(fs, regionInfo, conf, tableDir, family, hfile);
}
} else {
// Family doesn't exists in the snapshot
LOG.trace(
"Removing family="
+ Bytes.toString(family)
+ " from region="
+ regionInfo.getEncodedName()
+ " table="
+ tableName);
HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
fs.delete(familyDir, true);
}
}
// Add families not present in the table
for (Map.Entry<String, List<String>> familyEntry : snapshotFiles.entrySet()) {
Path familyDir = new Path(regionDir, familyEntry.getKey());
if (!fs.mkdirs(familyDir)) {
throw new IOException("Unable to create familyDir=" + familyDir);
}
for (String hfileName : familyEntry.getValue()) {
LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
restoreStoreFile(familyDir, regionInfo, hfileName);
}
}
}