Ejemplo n.º 1
0
 /**
  * Sweeps the mob files on one column family. It deletes the unused mob files and merges the small
  * mob files into bigger ones.
  *
  * @param tableName The current table name in string format.
  * @param familyName The column family name.
  * @return 0 if success, 2 if job aborted with an exception, 3 if unable to start due to other
  *     compaction,4 if mr job was unsuccessful
  * @throws IOException
  * @throws InterruptedException
  * @throws ClassNotFoundException
  * @throws KeeperException
  * @throws ServiceException
  */
 int sweepFamily(String tableName, String familyName)
     throws IOException, InterruptedException, ClassNotFoundException, KeeperException,
         ServiceException {
   Configuration conf = getConf();
   // make sure the target HBase exists.
   HBaseAdmin.checkHBaseAvailable(conf);
   Connection connection = ConnectionFactory.createConnection(getConf());
   Admin admin = connection.getAdmin();
   try {
     FileSystem fs = FileSystem.get(conf);
     TableName tn = TableName.valueOf(tableName);
     HTableDescriptor htd = admin.getTableDescriptor(tn);
     HColumnDescriptor family = htd.getFamily(Bytes.toBytes(familyName));
     if (family == null || !family.isMobEnabled()) {
       throw new IOException("Column family " + familyName + " is not a MOB column family");
     }
     SweepJob job = new SweepJob(conf, fs);
     // Run the sweeping
     return job.sweep(tn, family);
   } catch (Exception e) {
     System.err.println("Job aborted due to exception " + e);
     return 2; // job failed
   } finally {
     try {
       admin.close();
     } catch (IOException e) {
       System.out.println("Failed to close the HBaseAdmin: " + e.getMessage());
     }
     try {
       connection.close();
     } catch (IOException e) {
       System.out.println("Failed to close the connection: " + e.getMessage());
     }
   }
 }
Ejemplo n.º 2
0
 /** Apply column family options such as Bloom filters, compression, and data block encoding. */
 protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
     throws IOException {
   HBaseAdmin admin = new HBaseAdmin(conf);
   HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
   LOG.info("Disabling table " + Bytes.toString(tableName));
   admin.disableTable(tableName);
   for (byte[] cf : columnFamilies) {
     HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
     boolean isNewCf = columnDesc == null;
     if (isNewCf) {
       columnDesc = new HColumnDescriptor(cf);
     }
     if (bloomType != null) {
       columnDesc.setBloomFilterType(bloomType);
     }
     if (compressAlgo != null) {
       columnDesc.setCompressionType(compressAlgo);
     }
     if (dataBlockEncodingAlgo != null) {
       columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
       columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
     }
     if (inMemoryCF) {
       columnDesc.setInMemory(inMemoryCF);
     }
     if (isNewCf) {
       admin.addColumn(tableName, columnDesc);
     } else {
       admin.modifyColumn(tableName, columnDesc);
     }
   }
   LOG.info("Enabling table " + Bytes.toString(tableName));
   admin.enableTable(tableName);
 }
Ejemplo n.º 3
0
  @Test
  public void testColumnFamily() throws Exception {
    DatasetProperties props =
        DatasetProperties.builder().add(Table.PROPERTY_COLUMN_FAMILY, "t").build();
    HBaseTableDefinition tableDefinition = new HBaseTableDefinition("foo");
    String tableName = "testcf";
    DatasetSpecification spec = tableDefinition.configure(tableName, props);

    DatasetAdmin admin =
        new HBaseTableAdmin(
            CONTEXT1,
            spec,
            testHBase.getConfiguration(),
            hBaseTableUtil,
            CConfiguration.create(),
            new LocalLocationFactory(tmpFolder.newFolder()));
    admin.create();
    final HBaseTable table =
        new HBaseTable(CONTEXT1, spec, cConf, testHBase.getConfiguration(), hBaseTableUtil);

    TransactionSystemClient txClient = new DetachedTxSystemClient();
    TransactionExecutor executor = new DefaultTransactionExecutor(txClient, table);
    executor.execute(
        new TransactionExecutor.Subroutine() {
          @Override
          public void apply() throws Exception {
            table.put(new Put("row", "column", "testValue"));
          }
        });

    final HBaseTable table2 =
        new HBaseTable(CONTEXT1, spec, cConf, testHBase.getConfiguration(), hBaseTableUtil);
    executor = new DefaultTransactionExecutor(txClient, table2);
    executor.execute(
        new TransactionExecutor.Subroutine() {
          @Override
          public void apply() throws Exception {
            Assert.assertEquals(
                "testValue", table2.get(new Get("row", "column")).getString("column"));
          }
        });

    // Verify the column family name
    HTableDescriptor htd =
        hBaseTableUtil.getHTableDescriptor(
            testHBase.getHBaseAdmin(), TableId.from(CONTEXT1.getNamespaceId(), tableName));
    HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes("t"));
    Assert.assertNotNull(hcd);
    Assert.assertEquals("t", hcd.getNameAsString());
  }
Ejemplo n.º 4
0
  private void ensureColumnFamilyExists(String tableName, String columnFamily)
      throws StorageException {
    HBaseAdmin adm = getAdminInterface();
    HTableDescriptor desc = ensureTableExists(tableName);

    Preconditions.checkNotNull(desc);

    HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes());

    // Create our column family, if necessary
    if (cf == null) {
      try {
        adm.disableTable(tableName);
        desc.addFamily(
            new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ));
        adm.modifyTable(tableName.getBytes(), desc);

        try {
          logger.debug(
              "Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily);
          Thread.sleep(1000L);
        } catch (InterruptedException ie) {
          throw new TemporaryStorageException(ie);
        }

        adm.enableTable(tableName);
      } catch (TableNotFoundException ee) {
        logger.error("TableNotFoundException", ee);
        throw new PermanentStorageException(ee);
      } catch (org.apache.hadoop.hbase.TableExistsException ee) {
        logger.debug("Swallowing exception {}", ee);
      } catch (IOException ee) {
        throw new TemporaryStorageException(ee);
      }
    } else { // check if compression was enabled, if not - enable it
      if (cf.getCompressionType() == null
          || cf.getCompressionType() == Compression.Algorithm.NONE) {
        try {
          adm.disableTable(tableName);

          adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ));

          adm.enableTable(tableName);
        } catch (IOException e) {
          throw new TemporaryStorageException(e);
        }
      }
    }
  }
Ejemplo n.º 5
0
  /**
   * Test that {@link HFileOutputFormat2} RecordWriter uses compression and bloom filter settings
   * from the column family descriptor
   */
  @Ignore("Goes zombie too frequently; needs work. See HBASE-14563")
  @Test
  public void testColumnFamilySettings() throws Exception {
    Configuration conf = new Configuration(this.util.getConfiguration());
    RecordWriter<ImmutableBytesWritable, Cell> writer = null;
    TaskAttemptContext context = null;
    Path dir = util.getDataTestDir("testColumnFamilySettings");

    // Setup table descriptor
    Table table = Mockito.mock(Table.class);
    RegionLocator regionLocator = Mockito.mock(RegionLocator.class);
    HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
    Mockito.doReturn(htd).when(table).getTableDescriptor();
    for (HColumnDescriptor hcd : HBaseTestingUtility.generateColumnDescriptors()) {
      htd.addFamily(hcd);
    }

    // set up the table to return some mock keys
    setupMockStartKeys(regionLocator);

    try {
      // partial map red setup to get an operational writer for testing
      // We turn off the sequence file compression, because DefaultCodec
      // pollutes the GZip codec pool with an incompatible compressor.
      conf.set("io.seqfile.compression.type", "NONE");
      conf.set("hbase.fs.tmp.dir", dir.toString());
      // turn locality off to eliminate getRegionLocation fail-and-retry time when writing kvs
      conf.setBoolean(HFileOutputFormat2.LOCALITY_SENSITIVE_CONF_KEY, false);

      Job job = new Job(conf, "testLocalMRIncrementalLoad");
      job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings"));
      setupRandomGeneratorMapper(job);
      HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator);
      FileOutputFormat.setOutputPath(job, dir);
      context = createTestTaskAttemptContext(job);
      HFileOutputFormat2 hof = new HFileOutputFormat2();
      writer = hof.getRecordWriter(context);

      // write out random rows
      writeRandomKeyValues(writer, context, htd.getFamiliesKeys(), ROWSPERSPLIT);
      writer.close(context);

      // Make sure that a directory was created for every CF
      FileSystem fs = dir.getFileSystem(conf);

      // commit so that the filesystem has one directory per column family
      hof.getOutputCommitter(context).commitTask(context);
      hof.getOutputCommitter(context).commitJob(context);
      FileStatus[] families = FSUtils.listStatus(fs, dir, new FSUtils.FamilyDirFilter(fs));
      assertEquals(htd.getFamilies().size(), families.length);
      for (FileStatus f : families) {
        String familyStr = f.getPath().getName();
        HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes(familyStr));
        // verify that the compression on this file matches the configured
        // compression
        Path dataFilePath = fs.listStatus(f.getPath())[0].getPath();
        Reader reader = HFile.createReader(fs, dataFilePath, new CacheConfig(conf), conf);
        Map<byte[], byte[]> fileInfo = reader.loadFileInfo();

        byte[] bloomFilter = fileInfo.get(StoreFile.BLOOM_FILTER_TYPE_KEY);
        if (bloomFilter == null) bloomFilter = Bytes.toBytes("NONE");
        assertEquals(
            "Incorrect bloom filter used for column family "
                + familyStr
                + "(reader: "
                + reader
                + ")",
            hcd.getBloomFilterType(),
            BloomType.valueOf(Bytes.toString(bloomFilter)));
        assertEquals(
            "Incorrect compression used for column family "
                + familyStr
                + "(reader: "
                + reader
                + ")",
            hcd.getCompressionType(),
            reader.getFileContext().getCompression());
      }
    } finally {
      dir.getFileSystem(conf).delete(dir, true);
    }
  }
Ejemplo n.º 6
0
  /**
   * Verify schema modification takes.
   *
   * @throws IOException
   * @throws InterruptedException
   */
  @Test(timeout = 300000)
  public void testOnlineChangeTableSchema() throws IOException, InterruptedException {
    final TableName tableName = TableName.valueOf("changeTableSchemaOnline");
    TEST_UTIL
        .getMiniHBaseCluster()
        .getMaster()
        .getConfiguration()
        .setBoolean("hbase.online.schema.update.enable", true);
    HTableDescriptor[] tables = admin.listTables();
    int numTables = tables.length;
    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY).close();
    tables = this.admin.listTables();
    assertEquals(numTables + 1, tables.length);

    // FIRST, do htabledescriptor changes.
    HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
    // Make a copy and assert copy is good.
    HTableDescriptor copy = new HTableDescriptor(htd);
    assertTrue(htd.equals(copy));
    // Now amend the copy. Introduce differences.
    long newFlushSize = htd.getMemStoreFlushSize() / 2;
    if (newFlushSize <= 0) {
      newFlushSize = HTableDescriptor.DEFAULT_MEMSTORE_FLUSH_SIZE / 2;
    }
    copy.setMemStoreFlushSize(newFlushSize);
    final String key = "anyoldkey";
    assertTrue(htd.getValue(key) == null);
    copy.setValue(key, key);
    boolean expectedException = false;
    try {
      admin.modifyTable(tableName, copy);
    } catch (TableNotDisabledException re) {
      expectedException = true;
    }
    assertFalse(expectedException);
    HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
    assertFalse(htd.equals(modifiedHtd));
    assertTrue(copy.equals(modifiedHtd));
    assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
    assertEquals(key, modifiedHtd.getValue(key));

    // Now work on column family changes.
    int countOfFamilies = modifiedHtd.getFamilies().size();
    assertTrue(countOfFamilies > 0);
    HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
    int maxversions = hcd.getMaxVersions();
    final int newMaxVersions = maxversions + 1;
    hcd.setMaxVersions(newMaxVersions);
    final byte[] hcdName = hcd.getName();
    expectedException = false;
    try {
      this.admin.modifyColumn(tableName, hcd);
    } catch (TableNotDisabledException re) {
      expectedException = true;
    }
    assertFalse(expectedException);
    modifiedHtd = this.admin.getTableDescriptor(tableName);
    HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
    assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());

    // Try adding a column
    assertFalse(this.admin.isTableDisabled(tableName));
    final String xtracolName = "xtracol";
    HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
    xtracol.setValue(xtracolName, xtracolName);
    expectedException = false;
    try {
      this.admin.addColumn(tableName, xtracol);
    } catch (TableNotDisabledException re) {
      expectedException = true;
    }
    // Add column should work even if the table is enabled
    assertFalse(expectedException);
    modifiedHtd = this.admin.getTableDescriptor(tableName);
    hcd = modifiedHtd.getFamily(xtracol.getName());
    assertTrue(hcd != null);
    assertTrue(hcd.getValue(xtracolName).equals(xtracolName));

    // Delete the just-added column.
    this.admin.deleteColumn(tableName, xtracol.getName());
    modifiedHtd = this.admin.getTableDescriptor(tableName);
    hcd = modifiedHtd.getFamily(xtracol.getName());
    assertTrue(hcd == null);

    // Delete the table
    this.admin.disableTable(tableName);
    this.admin.deleteTable(tableName);
    this.admin.listTables();
    assertFalse(this.admin.tableExists(tableName));
  }
Ejemplo n.º 7
0
  @Test
  public void testProperties() throws Exception {
    Connection conn = DriverManager.getConnection(getUrl());
    conn.createStatement()
        .execute(
            "CREATE TABLE NON_TX_TABLE1(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR) TTL=1000");
    conn.createStatement().execute("CREATE INDEX idx1 ON NON_TX_TABLE1(a.v, b.v) TTL=1000");
    conn.createStatement()
        .execute("CREATE INDEX idx2 ON NON_TX_TABLE1(c.v) INCLUDE (a.v, b.v) TTL=1000");

    conn.createStatement().execute("ALTER TABLE NON_TX_TABLE1 SET TRANSACTIONAL=true");

    HTableDescriptor desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE1"));
    for (HColumnDescriptor colDesc : desc.getFamilies()) {
      assertEquals(
          QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
      assertEquals(1000, colDesc.getTimeToLive());
      assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
    }

    desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("IDX1"));
    for (HColumnDescriptor colDesc : desc.getFamilies()) {
      assertEquals(
          QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
      assertEquals(1000, colDesc.getTimeToLive());
      assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
    }

    desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("IDX2"));
    for (HColumnDescriptor colDesc : desc.getFamilies()) {
      assertEquals(
          QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
      assertEquals(1000, colDesc.getTimeToLive());
      assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
    }

    conn.createStatement()
        .execute(
            "CREATE TABLE NON_TX_TABLE2(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
    conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TRANSACTIONAL=true, VERSIONS=10");
    desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
    for (HColumnDescriptor colDesc : desc.getFamilies()) {
      assertEquals(10, colDesc.getMaxVersions());
      assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
      assertEquals(null, colDesc.getValue(TxConstants.PROPERTY_TTL));
    }
    conn.createStatement().execute("ALTER TABLE NON_TX_TABLE2 SET TTL=1000");
    desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE2"));
    for (HColumnDescriptor colDesc : desc.getFamilies()) {
      assertEquals(10, colDesc.getMaxVersions());
      assertEquals(1000, colDesc.getTimeToLive());
      assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
    }

    conn.createStatement()
        .execute(
            "CREATE TABLE NON_TX_TABLE3(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
    conn.createStatement()
        .execute("ALTER TABLE NON_TX_TABLE3 SET TRANSACTIONAL=true, b.VERSIONS=10, c.VERSIONS=20");
    desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("NON_TX_TABLE3"));
    assertEquals(
        QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL,
        desc.getFamily(Bytes.toBytes("A")).getMaxVersions());
    assertEquals(10, desc.getFamily(Bytes.toBytes("B")).getMaxVersions());
    assertEquals(20, desc.getFamily(Bytes.toBytes("C")).getMaxVersions());

    conn.createStatement()
        .execute(
            "CREATE TABLE NON_TX_TABLE4(k INTEGER PRIMARY KEY, a.v VARCHAR, b.v VARCHAR, c.v VARCHAR)");
    try {
      conn.createStatement()
          .execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, VERSIONS=1");
      fail();
    } catch (SQLException e) {
      assertEquals(
          SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(),
          e.getErrorCode());
    }

    try {
      conn.createStatement()
          .execute("ALTER TABLE NON_TX_TABLE4 SET TRANSACTIONAL=true, b.VERSIONS=1");
      fail();
    } catch (SQLException e) {
      assertEquals(
          SQLExceptionCode.TX_MAX_VERSIONS_MUST_BE_GREATER_THAN_ONE.getErrorCode(),
          e.getErrorCode());
    }

    conn.createStatement()
        .execute(
            "CREATE TABLE TX_TABLE1(k INTEGER PRIMARY KEY, v VARCHAR) TTL=1000, TRANSACTIONAL=true");
    desc =
        conn.unwrap(PhoenixConnection.class)
            .getQueryServices()
            .getTableDescriptor(Bytes.toBytes("TX_TABLE1"));
    for (HColumnDescriptor colDesc : desc.getFamilies()) {
      assertEquals(
          QueryServicesOptions.DEFAULT_MAX_VERSIONS_TRANSACTIONAL, colDesc.getMaxVersions());
      assertEquals(HColumnDescriptor.DEFAULT_TTL, colDesc.getTimeToLive());
      assertEquals(1000, Integer.parseInt(colDesc.getValue(TxConstants.PROPERTY_TTL)));
    }
  }
Ejemplo n.º 8
0
  /** {@inheritDoc} */
  @Override
  public KijiTableLayout modifyTableLayout(
      TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException {
    final State state = mState.get();
    Preconditions.checkState(
        state == State.OPEN,
        "Cannot modify table layout in Kiji instance %s in state %s.",
        this,
        state);
    Preconditions.checkNotNull(update);

    ensureValidationCompatibility(update);

    if (dryRun && (null == printStream)) {
      printStream = System.out;
    }

    final KijiMetaTable metaTable = getMetaTable();

    final String tableName = update.getName();
    // Throws a KijiTableNotFoundException if there is no table.
    metaTable.getTableLayout(tableName);

    final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
    LOG.debug("Applying layout update {} on table {}", update, tableURI);

    KijiTableLayout newLayout = null;

    if (dryRun) {
      // Process column ids and perform validation, but don't actually update the meta table.
      final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1);
      final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0);
      newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout);
    } else {
      // Actually set it.
      if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
        try {
          // Use ZooKeeper to inform all watchers that a new table layout is available.
          final HBaseTableLayoutUpdater updater =
              new HBaseTableLayoutUpdater(this, tableURI, update);
          try {
            updater.update();
            newLayout = updater.getNewLayout();
          } finally {
            updater.close();
          }
        } catch (KeeperException ke) {
          throw new IOException(ke);
        }
      } else {
        // System versions before system-2.0 do not enforce table layout update consistency or
        // validation.
        newLayout = metaTable.updateTableLayout(tableName, update);
      }
    }
    Preconditions.checkState(newLayout != null);

    if (dryRun) {
      printStream.println("This table layout is valid.");
    }

    LOG.debug("Computing new HBase schema");
    final HTableSchemaTranslator translator = new HTableSchemaTranslator();
    final HTableDescriptor newTableDescriptor =
        translator.toHTableDescriptor(mURI.getInstance(), newLayout);

    LOG.debug("Reading existing HBase schema");
    final KijiManagedHBaseTableName hbaseTableName =
        KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName);
    HTableDescriptor currentTableDescriptor = null;
    byte[] tableNameAsBytes = hbaseTableName.toBytes();
    try {
      currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes);
    } catch (TableNotFoundException tnfe) {
      if (!dryRun) {
        throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception.
      }
    }
    if (currentTableDescriptor == null) {
      if (dryRun) {
        printStream.println("Would create new table: " + tableName);
        currentTableDescriptor =
            HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName);
      } else {
        throw new RuntimeException(
            "Table " + hbaseTableName.getKijiTableName() + " does not exist");
      }
    }
    LOG.debug("Existing table descriptor: {}", currentTableDescriptor);
    LOG.debug("New table descriptor: {}", newTableDescriptor);

    LOG.debug("Checking for differences between the new HBase schema and the existing one");
    final HTableDescriptorComparator comparator = new HTableDescriptorComparator();
    if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) {
      LOG.debug("HBase schemas are the same.  No need to change HBase schema");
      if (dryRun) {
        printStream.println("This layout does not require any physical table schema changes.");
      }
    } else {
      LOG.debug("HBase schema must be changed, but no columns will be deleted");

      if (dryRun) {
        printStream.println("Changes caused by this table layout:");
      } else {
        LOG.debug("Disabling HBase table");
        getHBaseAdmin().disableTable(hbaseTableName.toString());
      }

      for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) {
        final String columnName = Bytes.toString(newColumnDescriptor.getName());
        final ColumnId columnId = ColumnId.fromString(columnName);
        final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId);
        final HColumnDescriptor currentColumnDescriptor =
            currentTableDescriptor.getFamily(newColumnDescriptor.getName());
        if (null == currentColumnDescriptor) {
          if (dryRun) {
            printStream.println("  Creating new locality group: " + lgName);
          } else {
            LOG.debug("Creating new column " + columnName);
            getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor);
          }
        } else if (!newColumnDescriptor.equals(currentColumnDescriptor)) {
          if (dryRun) {
            printStream.println("  Modifying locality group: " + lgName);
          } else {
            LOG.debug("Modifying column " + columnName);
            getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor);
          }
        } else {
          LOG.debug("No changes needed for column " + columnName);
        }
      }

      if (dryRun) {
        if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
          printStream.printf(
              "  Changing max_filesize from %d to %d: %n",
              currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize());
        }
        if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
          printStream.printf(
              "  Changing memstore_flushsize from %d to %d: %n",
              currentTableDescriptor.getMemStoreFlushSize(),
              newTableDescriptor.getMemStoreFlushSize());
        }
      } else {
        LOG.debug("Modifying table descriptor");
        getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor);
      }

      if (!dryRun) {
        LOG.debug("Re-enabling HBase table");
        getHBaseAdmin().enableTable(hbaseTableName.toString());
      }
    }

    return newLayout;
  }
Ejemplo n.º 9
0
  /**
   * Verify schema modification takes.
   *
   * @throws IOException
   */
  @Test
  public void testChangeTableSchema() throws IOException {
    final byte[] tableName = Bytes.toBytes("changeTableSchema");
    HTableDescriptor[] tables = admin.listTables();
    int numTables = tables.length;
    TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
    tables = this.admin.listTables();
    assertEquals(numTables + 1, tables.length);

    // FIRST, do htabledescriptor changes.
    HTableDescriptor htd = this.admin.getTableDescriptor(tableName);
    // Make a copy and assert copy is good.
    HTableDescriptor copy = new HTableDescriptor(htd);
    assertTrue(htd.equals(copy));
    // Now amend the copy. Introduce differences.
    long newFlushSize = htd.getMemStoreFlushSize() / 2;
    copy.setMemStoreFlushSize(newFlushSize);
    final String key = "anyoldkey";
    assertTrue(htd.getValue(key) == null);
    copy.setValue(key, key);
    boolean expectedException = false;
    try {
      this.admin.modifyTable(tableName, copy);
    } catch (TableNotDisabledException re) {
      expectedException = true;
    }
    assertTrue(expectedException);
    this.admin.disableTable(tableName);
    assertTrue(this.admin.isTableDisabled(tableName));
    modifyTable(tableName, copy);
    HTableDescriptor modifiedHtd = this.admin.getTableDescriptor(tableName);
    // Assert returned modifiedhcd is same as the copy.
    assertFalse(htd.equals(modifiedHtd));
    assertTrue(copy.equals(modifiedHtd));
    assertEquals(newFlushSize, modifiedHtd.getMemStoreFlushSize());
    assertEquals(key, modifiedHtd.getValue(key));

    // Reenable table to test it fails if not disabled.
    this.admin.enableTable(tableName);
    assertFalse(this.admin.isTableDisabled(tableName));

    // Now work on column family changes.
    int countOfFamilies = modifiedHtd.getFamilies().size();
    assertTrue(countOfFamilies > 0);
    HColumnDescriptor hcd = modifiedHtd.getFamilies().iterator().next();
    int maxversions = hcd.getMaxVersions();
    final int newMaxVersions = maxversions + 1;
    hcd.setMaxVersions(newMaxVersions);
    final byte[] hcdName = hcd.getName();
    expectedException = false;
    try {
      this.admin.modifyColumn(tableName, hcd);
    } catch (TableNotDisabledException re) {
      expectedException = true;
    }
    assertTrue(expectedException);
    this.admin.disableTable(tableName);
    assertTrue(this.admin.isTableDisabled(tableName));
    // Modify Column is synchronous
    this.admin.modifyColumn(tableName, hcd);
    modifiedHtd = this.admin.getTableDescriptor(tableName);
    HColumnDescriptor modifiedHcd = modifiedHtd.getFamily(hcdName);
    assertEquals(newMaxVersions, modifiedHcd.getMaxVersions());

    // Try adding a column
    // Reenable table to test it fails if not disabled.
    this.admin.enableTable(tableName);
    assertFalse(this.admin.isTableDisabled(tableName));
    final String xtracolName = "xtracol";
    HColumnDescriptor xtracol = new HColumnDescriptor(xtracolName);
    xtracol.setValue(xtracolName, xtracolName);
    try {
      this.admin.addColumn(tableName, xtracol);
    } catch (TableNotDisabledException re) {
      expectedException = true;
    }
    assertTrue(expectedException);
    this.admin.disableTable(tableName);
    assertTrue(this.admin.isTableDisabled(tableName));
    this.admin.addColumn(tableName, xtracol);
    modifiedHtd = this.admin.getTableDescriptor(tableName);
    hcd = modifiedHtd.getFamily(xtracol.getName());
    assertTrue(hcd != null);
    assertTrue(hcd.getValue(xtracolName).equals(xtracolName));

    // Delete the just-added column.
    this.admin.deleteColumn(tableName, xtracol.getName());
    modifiedHtd = this.admin.getTableDescriptor(tableName);
    hcd = modifiedHtd.getFamily(xtracol.getName());
    assertTrue(hcd == null);

    // Delete the table
    this.admin.deleteTable(tableName);
    this.admin.listTables();
    assertFalse(this.admin.tableExists(tableName));
  }