@Override
    void perform() throws IOException {
      HTableDescriptor selected = selectTable(disabledTables);
      HColumnDescriptor cfd = selectFamily(selected);
      if (selected == null || cfd == null) {
        return;
      }

      Admin admin = connection.getAdmin();
      try {
        if (selected.getColumnFamilies().length < 2) {
          LOG.info("No enough column families to delete in table " + selected.getTableName());
          return;
        }
        TableName tableName = selected.getTableName();
        LOG.info("Deleting column family: " + cfd + " from table: " + tableName);
        admin.deleteColumnFamily(tableName, cfd.getName());
        // assertion
        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
        Assert.assertFalse(
            "Column family: " + cfd + " was not added", freshTableDesc.hasFamily(cfd.getName()));
        LOG.info("Deleted column family: " + cfd + " from table: " + tableName);
        disabledTables.put(tableName, freshTableDesc);
      } catch (Exception e) {
        LOG.warn("Caught exception in action: " + this.getClass());
        throw e;
      } finally {
        admin.close();
      }
      verifyTables();
    }
  @Test
  public void testSnapshotFailsOnNonExistantTable() throws Exception {
    // make sure we don't fail on listing snapshots
    SnapshotTestingUtils.assertNoSnapshots(admin);
    TableName tableName = TableName.valueOf("_not_a_table");

    // make sure the table doesn't exist
    boolean fail = false;
    do {
      try {
        admin.getTableDescriptor(tableName);
        fail = true;
        LOG.error("Table:" + tableName + " already exists, checking a new name");
        tableName = TableName.valueOf(tableName + "!");
      } catch (TableNotFoundException e) {
        fail = false;
      }
    } while (fail);

    // snapshot the non-existant table
    try {
      admin.snapshot("fail", tableName, SnapshotType.FLUSH);
      fail("Snapshot succeeded even though there is not table.");
    } catch (SnapshotCreationException e) {
      LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
    }
  }
 @Override
 void perform() throws IOException {
   Admin admin = connection.getAdmin();
   try {
     HTableDescriptor htd = createTableDesc();
     TableName tableName = htd.getTableName();
     if (admin.tableExists(tableName)) {
       return;
     }
     String numRegionKey = String.format(NUM_REGIONS_KEY, this.getClass().getSimpleName());
     numRegions = getConf().getInt(numRegionKey, DEFAULT_NUM_REGIONS);
     byte[] startKey = Bytes.toBytes("row-0000000000");
     byte[] endKey = Bytes.toBytes("row-" + Integer.MAX_VALUE);
     LOG.info("Creating table:" + htd);
     admin.createTable(htd, startKey, endKey, numRegions);
     Assert.assertTrue("Table: " + htd + " was not created", admin.tableExists(tableName));
     HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
     enabledTables.put(tableName, freshTableDesc);
     LOG.info("Created table:" + freshTableDesc);
   } catch (Exception e) {
     LOG.warn("Caught exception in action: " + this.getClass());
     throw e;
   } finally {
     admin.close();
   }
   verifyTables();
 }
    @Override
    void perform() throws IOException {
      HTableDescriptor selected = selectTable(disabledTables);
      if (selected == null) {
        return;
      }

      Admin admin = connection.getAdmin();
      try {
        HColumnDescriptor cfd = createFamilyDesc();
        if (selected.hasFamily(cfd.getName())) {
          LOG.info(
              new String(cfd.getName()) + " already exists in table " + selected.getTableName());
          return;
        }
        TableName tableName = selected.getTableName();
        LOG.info("Adding column family: " + cfd + " to table: " + tableName);
        admin.addColumn(tableName, cfd);
        // assertion
        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
        Assert.assertTrue(
            "Column family: " + cfd + " was not added", freshTableDesc.hasFamily(cfd.getName()));
        LOG.info("Added column family: " + cfd + " to table: " + tableName);
        disabledTables.put(tableName, freshTableDesc);
      } catch (Exception e) {
        LOG.warn("Caught exception in action: " + this.getClass());
        throw e;
      } finally {
        admin.close();
      }
      verifyTables();
    }
Пример #5
0
 /**
  * Sweeps the mob files on one column family. It deletes the unused mob files and merges the small
  * mob files into bigger ones.
  *
  * @param tableName The current table name in string format.
  * @param familyName The column family name.
  * @return 0 if success, 2 if job aborted with an exception, 3 if unable to start due to other
  *     compaction,4 if mr job was unsuccessful
  * @throws IOException
  * @throws InterruptedException
  * @throws ClassNotFoundException
  * @throws KeeperException
  * @throws ServiceException
  */
 int sweepFamily(String tableName, String familyName)
     throws IOException, InterruptedException, ClassNotFoundException, KeeperException,
         ServiceException {
   Configuration conf = getConf();
   // make sure the target HBase exists.
   HBaseAdmin.checkHBaseAvailable(conf);
   Connection connection = ConnectionFactory.createConnection(getConf());
   Admin admin = connection.getAdmin();
   try {
     FileSystem fs = FileSystem.get(conf);
     TableName tn = TableName.valueOf(tableName);
     HTableDescriptor htd = admin.getTableDescriptor(tn);
     HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
     if (family == null || !family.isMobEnabled()) {
       throw new IOException("Column family " + familyName + " is not a MOB column family");
     }
     SweepJob job = new SweepJob(conf, fs);
     // Run the sweeping
     return job.sweep(tn, family);
   } catch (Exception e) {
     System.err.println("Job aborted due to exception " + e);
     return 2; // job failed
   } finally {
     try {
       admin.close();
     } catch (IOException e) {
       System.out.println("Failed to close the HBaseAdmin: " + e.getMessage());
     }
     try {
       connection.close();
     } catch (IOException e) {
       System.out.println("Failed to close the connection: " + e.getMessage());
     }
   }
 }
Пример #6
0
 /**
  * Set the table's replication switch if the table's replication switch is already not set.
  *
  * @param tableName name of the table
  * @param isRepEnabled is replication switch enable or disable
  * @throws IOException if a remote or network exception occurs
  */
 private void setTableRep(final TableName tableName, boolean isRepEnabled) throws IOException {
   Admin admin = null;
   try {
     admin = this.connection.getAdmin();
     HTableDescriptor htd = admin.getTableDescriptor(tableName);
     if (isTableRepEnabled(htd) ^ isRepEnabled) {
       boolean isOnlineSchemaUpdateEnabled =
           this.connection
               .getConfiguration()
               .getBoolean("hbase.online.schema.update.enable", true);
       if (!isOnlineSchemaUpdateEnabled) {
         admin.disableTable(tableName);
       }
       for (HColumnDescriptor hcd : htd.getFamilies()) {
         hcd.setScope(
             isRepEnabled
                 ? HConstants.REPLICATION_SCOPE_GLOBAL
                 : HConstants.REPLICATION_SCOPE_LOCAL);
       }
       admin.modifyTable(tableName, htd);
       if (!isOnlineSchemaUpdateEnabled) {
         admin.enableTable(tableName);
       }
     }
   } finally {
     if (admin != null) {
       try {
         admin.close();
       } catch (IOException e) {
         LOG.warn("Failed to close admin connection.");
         LOG.debug("Details on failure to close admin connection.", e);
       }
     }
   }
 }
Пример #7
0
  @Test
  public void createDoubleTest() throws IOException, InterruptedException {
    String testName = "createDoubleTest";
    String nsName = prefix + "_" + testName;
    LOG.info(testName);

    TableName tableName = TableName.valueOf("my_table");
    TableName tableNameFoo = TableName.valueOf(nsName + ":my_table");
    // create namespace and verify
    admin.createNamespace(NamespaceDescriptor.create(nsName).build());
    TEST_UTIL.createTable(tableName, Bytes.toBytes(nsName));
    TEST_UTIL.createTable(tableNameFoo, Bytes.toBytes(nsName));
    assertEquals(2, admin.listTables().length);
    assertNotNull(admin.getTableDescriptor(tableName));
    assertNotNull(admin.getTableDescriptor(tableNameFoo));
    // remove namespace and verify
    admin.disableTable(tableName);
    admin.deleteTable(tableName);
    assertEquals(1, admin.listTables().length);
  }
    @Override
    void perform() throws IOException {
      HTableDescriptor selected = selectTable(disabledTables);
      if (selected == null) {
        return;
      }
      HColumnDescriptor columnDesc = selectFamily(selected);
      if (columnDesc == null) {
        return;
      }

      Admin admin = connection.getAdmin();
      int versions = RandomUtils.nextInt(10) + 3;
      try {
        TableName tableName = selected.getTableName();
        LOG.info(
            "Altering versions of column family: "
                + columnDesc
                + " to: "
                + versions
                + " in table: "
                + tableName);
        columnDesc.setMinVersions(versions);
        columnDesc.setMaxVersions(versions);
        admin.modifyTable(tableName, selected);
        // assertion
        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
        HColumnDescriptor freshColumnDesc = freshTableDesc.getFamily(columnDesc.getName());
        Assert.assertEquals(
            "Column family: " + columnDesc + " was not altered",
            freshColumnDesc.getMaxVersions(),
            versions);
        Assert.assertEquals(
            "Column family: " + freshColumnDesc + " was not altered",
            freshColumnDesc.getMinVersions(),
            versions);
        LOG.info(
            "Altered versions of column family: "
                + columnDesc
                + " to: "
                + versions
                + " in table: "
                + tableName);
        disabledTables.put(tableName, freshTableDesc);
      } catch (Exception e) {
        LOG.warn("Caught exception in action: " + this.getClass());
        throw e;
      } finally {
        admin.close();
      }
      verifyTables();
    }
Пример #9
0
 /**
  * Connect to peer and check the table descriptor on peer:
  *
  * <ol>
  *   <li>Create the same table on peer when not exist.
  *   <li>Throw exception if the table exists on peer cluster but descriptors are not same.
  * </ol>
  *
  * @param tableName name of the table to sync to the peer
  * @param splits table split keys
  * @throws IOException
  */
 private void checkAndSyncTableDescToPeers(final TableName tableName, final byte[][] splits)
     throws IOException {
   List<ReplicationPeer> repPeers = listValidReplicationPeers();
   if (repPeers == null || repPeers.size() <= 0) {
     throw new IllegalArgumentException("Found no peer cluster for replication.");
   }
   for (ReplicationPeer repPeer : repPeers) {
     Configuration peerConf = repPeer.getConfiguration();
     HTableDescriptor htd = null;
     try (Connection conn = ConnectionFactory.createConnection(peerConf);
         Admin admin = this.connection.getAdmin();
         Admin repHBaseAdmin = conn.getAdmin()) {
       htd = admin.getTableDescriptor(tableName);
       HTableDescriptor peerHtd = null;
       if (!repHBaseAdmin.tableExists(tableName)) {
         repHBaseAdmin.createTable(htd, splits);
       } else {
         peerHtd = repHBaseAdmin.getTableDescriptor(tableName);
         if (peerHtd == null) {
           throw new IllegalArgumentException(
               "Failed to get table descriptor for table "
                   + tableName.getNameAsString()
                   + " from peer cluster "
                   + repPeer.getId());
         } else if (!peerHtd.equals(htd)) {
           throw new IllegalArgumentException(
               "Table "
                   + tableName.getNameAsString()
                   + " exists in peer cluster "
                   + repPeer.getId()
                   + ", but the table descriptors are not same when comapred with source cluster."
                   + " Thus can not enable the table's replication switch.");
         }
       }
     }
   }
 }
    @Override
    void perform() throws IOException {
      HTableDescriptor selected = selectTable(disabledTables);
      if (selected == null) {
        return;
      }
      HColumnDescriptor columnDesc = selectFamily(selected);
      if (columnDesc == null) {
        return;
      }

      Admin admin = connection.getAdmin();
      try {
        TableName tableName = selected.getTableName();
        // possible DataBlockEncoding ids
        int[] possibleIds = {0, 2, 3, 4, 6};
        short id = (short) possibleIds[RandomUtils.nextInt(possibleIds.length)];
        LOG.info(
            "Altering encoding of column family: "
                + columnDesc
                + " to: "
                + id
                + " in table: "
                + tableName);
        columnDesc.setDataBlockEncoding(DataBlockEncoding.getEncodingById(id));
        admin.modifyTable(tableName, selected);
        // assertion
        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
        HColumnDescriptor freshColumnDesc = freshTableDesc.getFamily(columnDesc.getName());
        Assert.assertEquals(
            "Encoding of column family: " + columnDesc + " was not altered",
            freshColumnDesc.getDataBlockEncoding().getId(),
            id);
        LOG.info(
            "Altered encoding of column family: "
                + freshColumnDesc
                + " to: "
                + id
                + " in table: "
                + tableName);
        disabledTables.put(tableName, freshTableDesc);
      } catch (Exception e) {
        LOG.warn("Caught exception in action: " + this.getClass());
        throw e;
      } finally {
        admin.close();
      }
      verifyTables();
    }
    // populate tables
    @Override
    void perform() throws IOException {
      HTableDescriptor selected = selectTable(enabledTables);
      if (selected == null) {
        return;
      }

      Admin admin = connection.getAdmin();
      TableName tableName = selected.getTableName();
      try (Table table = connection.getTable(tableName)) {
        ArrayList<HRegionInfo> regionInfos =
            new ArrayList<HRegionInfo>(admin.getTableRegions(selected.getTableName()));
        int numRegions = regionInfos.size();
        // average number of rows to be added per action to each region
        int average_rows = 1;
        int numRows = average_rows * numRegions;
        LOG.info("Adding " + numRows + " rows to table: " + selected);
        for (int i = 0; i < numRows; i++) {
          // nextInt(Integer.MAX_VALUE)) to return positive numbers only
          byte[] rowKey =
              Bytes.toBytes(
                  "row-" + String.format("%010d", RandomUtils.nextInt(Integer.MAX_VALUE)));
          HColumnDescriptor cfd = selectFamily(selected);
          if (cfd == null) {
            return;
          }
          byte[] family = cfd.getName();
          byte[] qualifier = Bytes.toBytes("col-" + RandomUtils.nextInt(Integer.MAX_VALUE) % 10);
          byte[] value = Bytes.toBytes("val-" + RandomStringUtils.randomAlphanumeric(10));
          Put put = new Put(rowKey);
          put.addColumn(family, qualifier, value);
          table.put(put);
        }
        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
        enabledTables.put(tableName, freshTableDesc);
        LOG.info("Added " + numRows + " rows to table: " + selected);
      } catch (Exception e) {
        LOG.warn("Caught exception in action: " + this.getClass());
        throw e;
      } finally {
        admin.close();
      }
      verifyTables();
    }
Пример #12
0
 public void manualTest(String args[]) throws Exception {
   Configuration conf = HBaseConfiguration.create();
   util = new HBaseTestingUtility(conf);
   if ("newtable".equals(args[0])) {
     TableName tname = TableName.valueOf(args[1]);
     byte[][] splitKeys = generateRandomSplitKeys(4);
     try (Table table = util.createTable(tname, FAMILIES, splitKeys)) {}
   } else if ("incremental".equals(args[0])) {
     TableName tname = TableName.valueOf(args[1]);
     try (Connection c = ConnectionFactory.createConnection(conf);
         Admin admin = c.getAdmin();
         RegionLocator regionLocator = c.getRegionLocator(tname)) {
       Path outDir = new Path("incremental-out");
       runIncrementalPELoad(conf, admin.getTableDescriptor(tname), regionLocator, outDir);
     }
   } else {
     throw new RuntimeException("usage: TestHFileOutputFormat2 newtable | incremental");
   }
 }
    @Override
    void perform() throws IOException {

      HTableDescriptor selected = selectTable(disabledTables);
      if (selected == null) {
        return;
      }

      Admin admin = connection.getAdmin();
      try {
        TableName tableName = selected.getTableName();
        LOG.info("Enabling table :" + selected);
        admin.enableTable(tableName);
        Assert.assertTrue(
            "Table: " + selected + " was not enabled", admin.isTableEnabled(tableName));
        HTableDescriptor freshTableDesc = admin.getTableDescriptor(tableName);
        enabledTables.put(tableName, freshTableDesc);
        LOG.info("Enabled table :" + freshTableDesc);
      } catch (Exception e) {
        LOG.warn("Caught exception in action: " + this.getClass());
        // TODO workaround
        // loose restriction for TableNotDisabledException/TableNotEnabledException thrown in sync
        // operations 1) when enable/disable starts, the table state is changed to
        // ENABLING/DISABLING (ZK node in 1.x), which will be further changed to ENABLED/DISABLED
        // once the operation completes 2) if master failover happens in the middle of the
        // enable/disable operation, the new master will try to recover the tables in
        // ENABLING/DISABLING state, as programmed in
        // AssignmentManager#recoverTableInEnablingState() and
        // AssignmentManager#recoverTableInDisablingState()
        // 3) after the new master initialization completes, the procedure tries to re-do the
        // enable/disable operation, which was already done. Ignore those exceptions before
        // change of behaviors of AssignmentManager in presence of PV2
        if (e instanceof TableNotDisabledException) {
          LOG.warn("Caught TableNotDisabledException in action: " + this.getClass());
          e.printStackTrace();
        } else {
          throw e;
        }
      } finally {
        admin.close();
      }
      verifyTables();
    }