/** Test loading into a column family that does not exist. */
  @Test
  public void testNonexistentColumnFamilyLoad() throws Exception {
    String testName = "testNonexistentColumnFamilyLoad";
    byte[][][] hFileRanges =
        new byte[][][] {
          new byte[][] {Bytes.toBytes("aaa"), Bytes.toBytes("ccc")},
          new byte[][] {Bytes.toBytes("ddd"), Bytes.toBytes("ooo")},
        };

    final byte[] TABLE = Bytes.toBytes("mytable_" + testName);
    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(TABLE));
    // set real family name to upper case in purpose to simulate the case that
    // family name in HFiles is invalid
    HColumnDescriptor family =
        new HColumnDescriptor(Bytes.toBytes(new String(FAMILY).toUpperCase()));
    htd.addFamily(family);

    try {
      runTest(testName, htd, BloomType.NONE, true, SPLIT_KEYS, hFileRanges);
      assertTrue("Loading into table with non-existent family should have failed", false);
    } catch (Exception e) {
      assertTrue("IOException expected", e instanceof IOException);
      // further check whether the exception message is correct
      String errMsg = e.getMessage();
      assertTrue(
          "Incorrect exception message, expected message: ["
              + EXPECTED_MSG_FOR_NON_EXISTING_FAMILY
              + "], current message: ["
              + errMsg
              + "]",
          errMsg.contains(EXPECTED_MSG_FOR_NON_EXISTING_FAMILY));
    }
  }
Example #2
0
 /**
  * Set the table's replication switch if the table's replication switch is already not set.
  *
  * @param tableName name of the table
  * @param isRepEnabled is replication switch enable or disable
  * @throws IOException if a remote or network exception occurs
  */
 private void setTableRep(final TableName tableName, boolean isRepEnabled) throws IOException {
   Admin admin = null;
   try {
     admin = this.connection.getAdmin();
     HTableDescriptor htd = admin.getTableDescriptor(tableName);
     if (isTableRepEnabled(htd) ^ isRepEnabled) {
       boolean isOnlineSchemaUpdateEnabled =
           this.connection
               .getConfiguration()
               .getBoolean("hbase.online.schema.update.enable", true);
       if (!isOnlineSchemaUpdateEnabled) {
         admin.disableTable(tableName);
       }
       for (HColumnDescriptor hcd : htd.getFamilies()) {
         hcd.setScope(
             isRepEnabled
                 ? HConstants.REPLICATION_SCOPE_GLOBAL
                 : HConstants.REPLICATION_SCOPE_LOCAL);
       }
       admin.modifyTable(tableName, htd);
       if (!isOnlineSchemaUpdateEnabled) {
         admin.enableTable(tableName);
       }
     }
   } finally {
     if (admin != null) {
       try {
         admin.close();
       } catch (IOException e) {
         LOG.warn("Failed to close admin connection.");
         LOG.debug("Details on failure to close admin connection.", e);
       }
     }
   }
 }
Example #3
0
 /**
  * Sweeps the mob files on one column family. It deletes the unused mob files and merges the small
  * mob files into bigger ones.
  *
  * @param tableName The current table name in string format.
  * @param familyName The column family name.
  * @return 0 if success, 2 if job aborted with an exception, 3 if unable to start due to other
  *     compaction,4 if mr job was unsuccessful
  * @throws IOException
  * @throws InterruptedException
  * @throws ClassNotFoundException
  * @throws KeeperException
  * @throws ServiceException
  */
 int sweepFamily(String tableName, String familyName)
     throws IOException, InterruptedException, ClassNotFoundException, KeeperException,
         ServiceException {
   Configuration conf = getConf();
   // make sure the target HBase exists.
   HBaseAdmin.checkHBaseAvailable(conf);
   Connection connection = ConnectionFactory.createConnection(getConf());
   Admin admin = connection.getAdmin();
   try {
     FileSystem fs = FileSystem.get(conf);
     TableName tn = TableName.valueOf(tableName);
     HTableDescriptor htd = admin.getTableDescriptor(tn);
     HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
     if (family == null || !family.isMobEnabled()) {
       throw new IOException("Column family " + familyName + " is not a MOB column family");
     }
     SweepJob job = new SweepJob(conf, fs);
     // Run the sweeping
     return job.sweep(tn, family);
   } catch (Exception e) {
     System.err.println("Job aborted due to exception " + e);
     return 2; // job failed
   } finally {
     try {
       admin.close();
     } catch (IOException e) {
       System.out.println("Failed to close the HBaseAdmin: " + e.getMessage());
     }
     try {
       connection.close();
     } catch (IOException e) {
       System.out.println("Failed to close the connection: " + e.getMessage());
     }
   }
 }
Example #4
0
  // create 'mytable', {NAME=>'colfam:', COMPRESSION=>'lzo'}
  private synchronized HTableDescriptor createTableDescription(
      String tableName, String[] metrics, int[] max_version) {
    // log.info("entry: "+tableName + ":"+metrics);
    HTableDescriptor td = new HTableDescriptor(tableName);
    try {
      for (int i = 0; i < metrics.length; i++) {
        String colName = metrics[i];
        if (colName == null || colName.length() == 0) {
          log.info("Invalid table schema content, contains empty name column.");
          throw new Exception("Invalid table schema content, contains empty name column.");
        }
        HColumnDescriptor hcd = new HColumnDescriptor(colName);
        hcd.setMaxVersions(max_version[i]);
        // hcd.setBloomFilterType(BloomType.ROWCOL);

        // compress it and require to install LZO
        // hcd.setCompressionType(Compression.Algorithm.GZ);
        td.addFamily(hcd);
      }
      // td.setMaxFileSize(1073741824);

    } catch (Exception e) {
      // log.error(e.fillInStackTrace());
      e.printStackTrace();
    }

    // log.info("exit");
    return td;
  }
Example #5
0
  /**
   * Find all column families that are replicated from this cluster
   *
   * @return the full list of the replicated column families of this cluster as: tableName, family
   *     name, replicationType
   *     <p>Currently replicationType is Global. In the future, more replication types may be
   *     extended here. For example 1) the replication may only apply to selected peers instead of
   *     all peers 2) the replicationType may indicate the host Cluster servers as Slave for the
   *     table:columnFam.
   */
  public List<HashMap<String, String>> listReplicated() throws IOException {
    List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();

    Admin admin = connection.getAdmin();
    HTableDescriptor[] tables;
    try {
      tables = admin.listTables();
    } finally {
      if (admin != null) admin.close();
    }

    for (HTableDescriptor table : tables) {
      HColumnDescriptor[] columns = table.getColumnFamilies();
      String tableName = table.getNameAsString();
      for (HColumnDescriptor column : columns) {
        if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
          // At this moment, the columfam is replicated to all peers
          HashMap<String, String> replicationEntry = new HashMap<String, String>();
          replicationEntry.put(TNAME, tableName);
          replicationEntry.put(CFNAME, column.getNameAsString());
          replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
          replicationColFams.add(replicationEntry);
        }
      }
    }

    return replicationColFams;
  }
 /**
  * Test that if we fail a flush, abort gets set on close.
  *
  * @see <a href="https://issues.apache.org/jira/browse/HBASE-4270">HBASE-4270</a>
  * @throws IOException
  * @throws NodeExistsException
  * @throws KeeperException
  */
 @Test
 public void testFailedFlushAborts() throws IOException, NodeExistsException, KeeperException {
   final Server server = new MockServer(HTU, false);
   final RegionServerServices rss = HTU.createMockRegionServerService();
   HTableDescriptor htd = TEST_HTD;
   final HRegionInfo hri =
       new HRegionInfo(htd.getTableName(), HConstants.EMPTY_END_ROW, HConstants.EMPTY_END_ROW);
   HRegion region = HTU.createLocalHRegion(hri, htd);
   try {
     assertNotNull(region);
     // Spy on the region so can throw exception when close is called.
     HRegion spy = Mockito.spy(region);
     final boolean abort = false;
     Mockito.when(spy.close(abort)).thenThrow(new RuntimeException("Mocked failed close!"));
     // The CloseRegionHandler will try to get an HRegion that corresponds
     // to the passed hri -- so insert the region into the online region Set.
     rss.addToOnlineRegions(spy);
     // Assert the Server is NOT stopped before we call close region.
     assertFalse(server.isStopped());
     CloseRegionHandler handler = new CloseRegionHandler(server, rss, hri, false, false, -1);
     boolean throwable = false;
     try {
       handler.process();
     } catch (Throwable t) {
       throwable = true;
     } finally {
       assertTrue(throwable);
       // Abort calls stop so stopped flag should be set.
       assertTrue(server.isStopped());
     }
   } finally {
     HRegion.closeHRegion(region);
   }
 }
  /**
   * This test assumes a master crash/failure during the meta migration process and attempts to
   * continue the meta migration process when a new master takes over. When a master dies during the
   * meta migration we will have some rows of META.CatalogFamily updated with PB serialization and
   * some still hanging with writable serialization. When the backup master/ or fresh start of
   * master attempts the migration it will encounter some rows of META already updated with new HRI
   * and some still legacy. This test will simulate this scenario and validates that the migration
   * process can safely skip the updated rows and migrate any pending rows at startup.
   *
   * @throws Exception
   */
  @Test
  public void testMasterCrashDuringMetaMigration() throws Exception {
    final byte[] FAMILY = Bytes.toBytes("family");
    HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration");
    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
    htd.addFamily(hcd);
    Configuration conf = TEST_UTIL.getConfiguration();
    // Create 10 New regions.
    createMultiRegionsWithPBSerialization(conf, htd.getName(), 10);
    // Create 10 Legacy regions.
    createMultiRegionsWithWritableSerialization(conf, htd.getName(), 10);
    CatalogTracker ct = TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
    // Erase the current version of root meta for this test.
    undoVersionInRoot(ct);

    MetaReader.fullScanMetaAndPrint(ct);
    LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");

    long numMigratedRows =
        MetaMigrationConvertingToPB.updateMetaIfNecessary(TEST_UTIL.getHBaseCluster().getMaster());
    assertEquals(numMigratedRows, 10);

    // Assert that the flag in ROOT is updated to reflect the correct status
    boolean metaUpdated =
        MetaMigrationConvertingToPB.isMetaTableUpdated(
            TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker());
    assertEquals(true, metaUpdated);

    verifyMetaRowsAreUpdated(ct);

    LOG.info("END testMasterCrashDuringMetaMigration");
  }
 /**
  * Setup a clean table before we start mucking with it.
  *
  * @throws IOException
  * @throws InterruptedException
  * @throws KeeperException
  */
 private Table setupTable(TableName tablename) throws Exception {
   HTableDescriptor desc = new HTableDescriptor(tablename);
   HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
   desc.addFamily(hcd); // If a table has no CF's it doesn't get checked
   TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
   return this.connection.getTable(tablename);
 }
    /**
     * returns true if the specified entry must be replicated. We should always replicate meta
     * operations (e.g. flush) and use the user HTD flag to decide whether or not replicate the
     * memstore.
     */
    private boolean requiresReplication(final TableName tableName, final List<Entry> entries)
        throws IOException {
      // unit-tests may not the TableDescriptors, bypass the check and always replicate
      if (tableDescriptors == null) return true;

      Boolean requiresReplication = memstoreReplicationEnabled.getIfPresent(tableName);
      if (requiresReplication == null) {
        // check if the table requires memstore replication
        // some unit-test drop the table, so we should do a bypass check and always replicate.
        HTableDescriptor htd = tableDescriptors.get(tableName);
        requiresReplication = htd == null || htd.hasRegionMemstoreReplication();
        memstoreReplicationEnabled.put(tableName, requiresReplication);
      }

      // if memstore replication is not required, check the entries.
      // meta edits (e.g. flush) must be always replicated.
      if (!requiresReplication) {
        int skipEdits = 0;
        java.util.Iterator<Entry> it = entries.iterator();
        while (it.hasNext()) {
          Entry entry = it.next();
          if (entry.getEdit().isMetaEdit()) {
            requiresReplication = true;
          } else {
            it.remove();
            skipEdits++;
          }
        }
        skippedEdits.addAndGet(skipEdits);
      }
      return requiresReplication;
    }
Example #10
0
  public static void createTableTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);
      List<byte[]> families = new ArrayList<byte[]>();
      families.add(f0);

      if (conf == null) {
        throw new IOException("conf not set.");
      } else {
        HBaseAdmin admin = new HBaseAdmin(conf);
        if (!admin.tableExists(tableName)) {
          HTableDescriptor tableDesc = new HTableDescriptor(tableName);
          for (byte[] family : families) {
            HColumnDescriptor hc = new HColumnDescriptor(family);
            hc.setMaxVersions(1);
            tableDesc.addFamily(hc);
          }
          admin.createTable(tableDesc);
        } else {
          throw new IOException("table " + new String(tableName) + " already  exists.");
        }
      }
      System.out.println("DONE.");
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
 public void fixInconsistent() throws IOException {
   if (ifFix == true) {
     for (String segFullName : inconsistentHTables) {
       String[] sepNameList = segFullName.split(",");
       HTableDescriptor desc = hbaseAdmin.getTableDescriptor(TableName.valueOf(sepNameList[0]));
       logger.info(
           "Change the host of htable "
               + sepNameList[0]
               + "belonging to cube "
               + sepNameList[1]
               + " from "
               + desc.getValue(IRealizationConstants.HTableTag)
               + " to "
               + dstCfg.getMetadataUrlPrefix());
       hbaseAdmin.disableTable(sepNameList[0]);
       desc.setValue(IRealizationConstants.HTableTag, dstCfg.getMetadataUrlPrefix());
       hbaseAdmin.modifyTable(sepNameList[0], desc);
       hbaseAdmin.enableTable(sepNameList[0]);
     }
   } else {
     logger.info("------ Inconsistent HTables Needed To Be Fixed ------");
     for (String hTable : inconsistentHTables) {
       String[] sepNameList = hTable.split(",");
       logger.info(sepNameList[0] + " belonging to cube " + sepNameList[1]);
     }
     logger.info("----------------------------------------------------");
   }
 }
  private static void createTable() throws Exception {
    try {
      Configuration configuration = HBaseConfiguration.create();
      HBaseAdmin.checkHBaseAvailable(configuration);
      Connection connection = ConnectionFactory.createConnection(configuration);

      // Instantiating HbaseAdmin class
      Admin admin = connection.getAdmin();

      // Instantiating table descriptor class
      HTableDescriptor stockTableDesc =
          new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE));

      // Adding column families to table descriptor
      HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF);
      stockTableDesc.addFamily(stock_0414);

      // Execute the table through admin
      if (!admin.tableExists(stockTableDesc.getTableName())) {
        admin.createTable(stockTableDesc);
        System.out.println("Stock table created !!!");
      }

      // Load hbase-site.xml
      HBaseConfiguration.addHbaseResources(configuration);
    } catch (ServiceException e) {
      log.error("Error occurred while creating HBase tables", e);
      throw new Exception("Error occurred while creating HBase tables", e);
    }
  }
Example #13
0
  /** 根据给定的表明和列族创建一张新表。 */
  @Override
  public Builder add() {
    if (type == 0) {
      try {
        if (tableName.trim().equals("") || tableName == null) {
          Log.error("=== LiMingji Error Info === Error: No tableName");
          return this;
        }
        if (columnFamilies == null || columnFamilies.size() == 0) {
          Log.error("=== LiMingji Error Info === Error: No columnFamilies");
          return this;
        }
        if (hBaseAdmin.tableExists(tableName)) {
          Log.info("=== LiMingji Debug Info === Table already exits");
          return this;
        } else {
          HTableDescriptor tableDesc = new HTableDescriptor(tableName);
          for (String column : columnFamilies) {
            tableDesc.addFamily(new HColumnDescriptor(column));
          }
          columnFamilies.clear();
          // preDistribute region
          if (split != null) hBaseAdmin.createTable(tableDesc, split);

          hBaseAdmin.createTable(tableDesc);
          Log.info("=== LiMingji Debug Info === Table has been created. ");
        }
      } catch (IOException e) {
        Log.info("=== LiMingji Debug Info === deleteTable@DTL IOException");
        e.printStackTrace();
      }
    }
    return this;
  }
 /**
  * @param rawAttributeName is the attribute name viewed by applications, it allows multiple
  *     values. For example, secondaryIndex in secondaryIndex$1 and coprocessor in corpcessor$2
  * @param indexOfAttribute is of the same raw attribute name, for example 2 in secondary$2
  */
 static void updateTableAttribute(
     Configuration conf,
     byte[] tableName,
     String rawAttributeName,
     int indexOfAttribute,
     boolean ifUpdateorRemove,
     String value)
     throws IOException {
   HBaseAdmin admin = new HBaseAdmin(conf);
   HTableDescriptor desc = admin.getTableDescriptor(tableName);
   admin.disableTable(tableName);
   //        System.out.println("TTDEBUG: disable table " + Bytes.toString(tableName));
   String coprocessorKey = rawAttributeName + indexOfAttribute;
   if (!ifUpdateorRemove) {
     desc.remove(Bytes.toBytes(coprocessorKey));
   } else {
     desc.setValue(coprocessorKey, value);
   }
   admin.modifyTable(tableName, desc);
   //        System.out.println("TTDEBUG: modify table " + Bytes.toString(tableName));
   admin.enableTable(tableName);
   //        System.out.println("TTDEBUG: enable table " + Bytes.toString(tableName));
   HTableDescriptor descNew = admin.getTableDescriptor(tableName);
   // modify table is asynchronous, has to loop over to check
   while (!desc.equals(descNew)) {
     System.err.println(
         "TTDEBUG: waiting for descriptor to change: from " + descNew + " to " + desc);
     try {
       Thread.sleep(500);
     } catch (InterruptedException ex) {
     }
     descNew = admin.getTableDescriptor(tableName);
   }
 }
Example #15
0
 protected void createTable(String tableName) throws Exception {
   HTableDescriptor td = new HTableDescriptor(tableName.getBytes());
   HColumnDescriptor cd = new HColumnDescriptor(TEST_TABLE_CF.getBytes());
   td.addFamily(cd);
   admin.createTable(td);
   LOG.info(tableName + " table is successfully created.");
 }
Example #16
0
  @Test
  public void testHundredsOfTable() throws IOException {
    final int times = 100;
    HColumnDescriptor fam1 = new HColumnDescriptor("fam1");
    HColumnDescriptor fam2 = new HColumnDescriptor("fam2");
    HColumnDescriptor fam3 = new HColumnDescriptor("fam3");

    for (int i = 0; i < times; i++) {
      HTableDescriptor htd = new HTableDescriptor("table" + i);
      htd.addFamily(fam1);
      htd.addFamily(fam2);
      htd.addFamily(fam3);
      this.admin.createTable(htd);
    }

    for (int i = 0; i < times; i++) {
      String tableName = "table" + i;
      this.admin.disableTable(tableName);
      byte[] tableNameBytes = Bytes.toBytes(tableName);
      assertTrue(this.admin.isTableDisabled(tableNameBytes));
      this.admin.enableTable(tableName);
      assertFalse(this.admin.isTableDisabled(tableNameBytes));
      this.admin.disableTable(tableName);
      assertTrue(this.admin.isTableDisabled(tableNameBytes));
      this.admin.deleteTable(tableName);
    }
  }
  @Test(timeout = 30000)
  public void testCreateDeleteTable() throws IOException {
    // Create table then get the single region for our new table.
    HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
    hdt.setRegionReplication(NB_SERVERS);
    hdt.addCoprocessor(SlowMeCopro.class.getName());
    Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());

    Put p = new Put(row);
    p.add(f, row, row);
    table.put(p);

    Get g = new Get(row);
    Result r = table.get(g);
    Assert.assertFalse(r.isStale());

    try {
      // But if we ask for stale we will get it
      SlowMeCopro.cdl.set(new CountDownLatch(1));
      g = new Get(row);
      g.setConsistency(Consistency.TIMELINE);
      r = table.get(g);
      Assert.assertTrue(r.isStale());
      SlowMeCopro.cdl.get().countDown();
    } finally {
      SlowMeCopro.cdl.get().countDown();
      SlowMeCopro.sleepTime.set(0);
    }

    HTU.getHBaseAdmin().disableTable(hdt.getTableName());
    HTU.deleteTable(hdt.getTableName());
  }
Example #18
0
 /** Apply column family options such as Bloom filters, compression, and data block encoding. */
 protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
     throws IOException {
   HBaseAdmin admin = new HBaseAdmin(conf);
   HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
   LOG.info("Disabling table " + Bytes.toString(tableName));
   admin.disableTable(tableName);
   for (byte[] cf : columnFamilies) {
     HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
     boolean isNewCf = columnDesc == null;
     if (isNewCf) {
       columnDesc = new HColumnDescriptor(cf);
     }
     if (bloomType != null) {
       columnDesc.setBloomFilterType(bloomType);
     }
     if (compressAlgo != null) {
       columnDesc.setCompressionType(compressAlgo);
     }
     if (dataBlockEncodingAlgo != null) {
       columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
       columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
     }
     if (inMemoryCF) {
       columnDesc.setInMemory(inMemoryCF);
     }
     if (isNewCf) {
       admin.addColumn(tableName, columnDesc);
     } else {
       admin.modifyColumn(tableName, columnDesc);
     }
   }
   LOG.info("Enabling table " + Bytes.toString(tableName));
   admin.enableTable(tableName);
 }
  @Test
  public void testMetaMigration() throws Exception {
    LOG.info("Starting testMetaMigration");
    final byte[] FAMILY = Bytes.toBytes("family");
    HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
    htd.addFamily(hcd);
    Configuration conf = TEST_UTIL.getConfiguration();
    byte[][] regionNames =
        new byte[][] {
          HConstants.EMPTY_START_ROW, Bytes.toBytes("region_a"), Bytes.toBytes("region_b")
        };
    createMultiRegionsWithWritableSerialization(conf, htd.getName(), regionNames);
    CatalogTracker ct = TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
    // Erase the current version of root meta for this test.
    undoVersionInRoot(ct);
    MetaReader.fullScanMetaAndPrint(ct);
    LOG.info("Meta Print completed.testMetaMigration");

    long numMigratedRows =
        MetaMigrationConvertingToPB.updateMeta(TEST_UTIL.getHBaseCluster().getMaster());
    MetaReader.fullScanMetaAndPrint(ct);

    // Should be one entry only and it should be for the table we just added.
    assertEquals(regionNames.length, numMigratedRows);

    // Assert that the flag in ROOT is updated to reflect the correct status
    boolean metaUpdated =
        MetaMigrationConvertingToPB.isMetaTableUpdated(
            TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker());
    assertEquals(true, metaUpdated);
    verifyMetaRowsAreUpdated(ct);
  }
 /*
  * (non-Javadoc)
  *
  * @see nl.gridline.zieook.data.hbase.TableOperations#getCurrentDescriptor(java.lang.String)
  */
 @Override
 public HTableDescriptor getCurrentDescriptor(String name) {
   HTableDescriptor descriptor = new HTableDescriptor(getTableName());
   HColumnDescriptor visited = new HColumnDescriptor(VISITED_COLUMN);
   descriptor.addFamily(visited);
   return descriptor;
 }
  public void createTable(String tableName, List<String> ColumnFamilies) {
    HBaseAdmin admin = null;
    try {
      admin = new HBaseAdmin(conf);
      HTableDescriptor tableDescriptor = new HTableDescriptor(Bytes.toBytes(tableName));

      for (String columnFamily : ColumnFamilies) {
        HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamily);
        tableDescriptor.addFamily(columnDescriptor);
      }
      admin.createTable(tableDescriptor);
      admin.close();
    } catch (TableExistsException e) {
      System.out.println("Table already exist:" + tableName);
      try {
        admin.close();
      } catch (IOException e1) {
        System.out.println("Error occurred while cloing the HBaseAdmin conneciton:" + e1);
      }
    } catch (MasterNotRunningException e) {
      throw new RuntimeException("HBase master not running, table creation failed.");
    } catch (ZooKeeperConnectionException e) {
      throw new RuntimeException("Zookeeper not running, table creation failed.");
    } catch (IOException e) {
      throw new RuntimeException("IO error, table creation failed.");
    }
  }
  @Test(timeout = 60000)
  public void testRollbackAndDoubleExecutionOffline() throws Exception {
    final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
    final String familyName = "cf2";
    final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();

    // create the table
    HRegionInfo[] regions =
        MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
    UTIL.getAdmin().disableTable(tableName);

    ProcedureTestingUtility.waitNoProcedureRunning(procExec);
    ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

    HTableDescriptor htd = new HTableDescriptor(UTIL.getAdmin().getTableDescriptor(tableName));
    boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
    htd.setCompactionEnabled(newCompactionEnableOption);
    htd.addFamily(new HColumnDescriptor(familyName));
    htd.setRegionReplication(3);

    // Start the Modify procedure && kill the executor
    long procId =
        procExec.submitProcedure(new ModifyTableProcedure(procExec.getEnvironment(), htd));

    // Restart the executor and rollback the step twice
    int numberOfSteps = 1; // failing at pre operation
    MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps);

    // cf2 should not be present
    MasterProcedureTestingUtility.validateTableCreation(
        UTIL.getHBaseCluster().getMaster(), tableName, regions, "cf1");
  }
Example #23
0
  public static void main(String[] args) throws IOException {

    // Instantiating configuration class
    Configuration con = HBaseConfiguration.create();

    // Instantiating HbaseAdmin class
    HBaseAdmin admin = new HBaseAdmin(con);

    // Instantiating table descriptor class
    HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf("emp_nibin"));

    // Adding column families to table descriptor
    tableDescriptor.addFamily(new HColumnDescriptor("personal_nibin"));
    tableDescriptor.addFamily(new HColumnDescriptor("professional_nibin"));

    // Execute the table through admin
    admin.createTable(tableDescriptor);
    System.out.println(" Table created ");
    System.out.println("listing the table now....");

    HTableDescriptor[] tables = admin.listTables();

    for (int i = 0; i < tables.length; i++) {
      System.out.println(tables[i].getNameAsString());
    }
  }
Example #24
0
 /** @return The HTableDescriptor to create or alter the table */
 protected HTableDescriptor getHTableDescriptor() {
   HTableDescriptor hTableDescriptor = new HTableDescriptor(tableKey.get());
   for (HColumnDescriptor family : this.columnFamilies) {
     hTableDescriptor.addFamily(family);
   }
   return hTableDescriptor;
 }
  public HBaseBlobStoreAccess(HBaseTableFactory tableFactory, boolean clientMode)
      throws IOException, InterruptedException {
    HTableDescriptor tableDescriptor = new HTableDescriptor(BLOB_TABLE);
    tableDescriptor.addFamily(new HColumnDescriptor(BLOBS_COLUMN_FAMILY));

    table = tableFactory.getTable(tableDescriptor, !clientMode);
  }
  public static boolean createTableOrOverwrite(String tableName, String[] columnFamily) {
    if (tableName == null && columnFamily == null) {
      return false;
    }
    Connection connection = null;
    try {
      connection = ConnectionFactory.createConnection(config);
      Admin admin = connection.getAdmin();
      if (admin.tableExists(TableName.valueOf(tableName))) {
        admin.disableTable(TableName.valueOf(tableName));
        admin.deleteTable(TableName.valueOf(tableName));
      }
      HTableDescriptor table = new HTableDescriptor(TableName.valueOf(tableName));

      for (String cf : columnFamily) {
        table.addFamily(new HColumnDescriptor(cf));
      }

      admin.createTable(table);
      System.out.println("create table successfully.");
      return true;
    } catch (IOException e) {
      e.printStackTrace();
      return false;
    } finally {
      try {
        connection.close();
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }
  }
  public void init() {
    GlobalIndexAdmin admin = null;
    try {
      admin = new GlobalIndexAdmin(conf);
      @SuppressWarnings("deprecation")
      HTableDescriptor table = new HTableDescriptor(tableName);
      HColumnDescriptor f1 = new HColumnDescriptor("cf");
      table.addFamily(f1);
      if (!admin.isTableExists(new String(tableName).getBytes())) {
        admin.createTable(table);
        LOG.info(tableName + " is not exists");
      } else {
        LOG.info(tableName + " is exists");
      }

      SecondaryIndex index = SecondaryIndexUtil.createSecondaryIndexFromPattern(C1_INDEX);
      admin.addGlobalIndex(Bytes.toBytes(tableName), index, Bytes.toBytes(indexName), null, false);

    } catch (Exception e) {
      LOG.error(e.getMessage(), e);
    } finally {
      if (null != admin) {
        try {
          admin.close();
        } catch (IOException e) {
          LOG.error("GlobalIndexAdmin close error");
        }
      }
    }
  }
  /**
   * Convert a <table> node from the xml into an HTableDescriptor (with its column families)
   */
  private HTableDescriptor getTable(Node tableNode) {
    NamedNodeMap tableAttributes = tableNode.getAttributes();
    String tableName = tableAttributes.getNamedItem(TABLE_NAME_ATTRIBUTE).getNodeValue();
    HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
    for (int x = 0; x < tableAttributes.getLength(); x++) {
      Node attr = tableAttributes.item(x);
      if (!attr.getNodeName().equalsIgnoreCase(TABLE_NAME_ATTRIBUTE)) { // skip name, already got it
        setAttributeValue(tableDescriptor, attr.getNodeName(), attr.getNodeValue());
      }
    }

    applyMissingTableDefaults(tableDescriptor);

    // parse the column families
    NodeList tableChildren = tableNode.getChildNodes();
    for (int x = 0; x < tableChildren.getLength(); x++) {
      Node tableChild = tableChildren.item(x);
      if (tableChild.getNodeName().equals(COLUMN_FAMILIES_ELEMENT)) {
        for (HColumnDescriptor family : getColumnFamilies(tableChild)) {
          tableDescriptor.addFamily(family);
        }
      }
    }

    // push this entire subtree of the xml file into the table metadata as the table's schema
    tableDescriptor.setValue(FULL_SCHEMA_PROPERTY, getFullXML(tableNode));

    validateTableDefinition(tableDescriptor);

    return tableDescriptor;
  }
Example #29
0
  public static void main(String[] args)
      throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
    Configuration conf = HBaseConfiguration.create();
    conf.set(
        "hbase.zookeeper.quorum", "192.168.10.163:2181,192.168.10.164:2181,192.168.10.165:2181");

    // admin用户
    HBaseAdmin admin = new HBaseAdmin(conf);
    // table
    HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("people"));
    // info列簇
    HColumnDescriptor hcd_info = new HColumnDescriptor("info");
    hcd_info.setMaxVersions(3);
    // data列簇
    HColumnDescriptor hcd_data = new HColumnDescriptor("data");

    // 将列簇添加到htable中
    htd.addFamily(hcd_info);
    htd.addFamily(hcd_data);

    // 创建表
    admin.createTable(htd);

    // 关闭连接
    admin.close();
  }
 @After
 public void tearDown() throws Exception {
   ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
   for (HTableDescriptor htd : UTIL.getHBaseAdmin().listTables()) {
     LOG.info("Tear down, remove table=" + htd.getTableName());
     UTIL.deleteTable(htd.getTableName());
   }
 }