Exemple #1
0
  /**
   * Ensures that a table is not created or modified to enable layout validation without the
   * requisite system version.
   *
   * <p>Throws an exception if a table layout has validation enabled, but the overall instance data
   * version is too low to support table layout validation.
   *
   * <p>Table layouts with layout version <tt>layout-1.3.0</tt> or higher must be applied to systems
   * with data version <tt>system-2.0</tt> or higher. A layout of 1.3 or above in system-1.0
   * environment will trigger an exception in this method.
   *
   * <p>Older layout versions may be applied in <tt>system-1.0</tt> or <tt>system-2.0</tt>
   * environments; such layouts are ignored by this method.
   *
   * @param layout the table layout for which to ensure compatibility.
   * @throws IOException in case of an error reading from the system table.
   * @throws InvalidLayoutException if the layout and system versions are incompatible.
   */
  private void ensureValidationCompatibility(TableLayoutDesc layout) throws IOException {
    final ProtocolVersion layoutVersion = ProtocolVersion.parse(layout.getVersion());
    final ProtocolVersion systemVersion = getSystemTable().getDataVersion();

    if ((layoutVersion.compareTo(Versions.LAYOUT_VALIDATION_VERSION) >= 0)
        && (systemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) < 0)) {
      throw new InvalidLayoutException(
          String.format(
              "Layout version: %s not supported by system version: %s",
              layoutVersion, systemVersion));
    }
  }
Exemple #2
0
  /**
   * Releases all the resources used by this Kiji instance.
   *
   * @throws IOException on I/O error.
   */
  private void close() throws IOException {
    final State oldState = mState.getAndSet(State.CLOSED);
    Preconditions.checkState(
        oldState == State.OPEN, "Cannot close Kiji instance %s in state %s.", this, oldState);

    LOG.debug("Closing {}.", this);
    if (mMonitor != null) {
      try {
        mMonitor.unregisterInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
      } catch (KeeperException ke) {
        // Unrecoverable ZooKeeper error:
        throw new IOException(ke);
      }
      mMonitor.close();
    }
    if (mZKClient != null) {
      mZKClient.release();
    }

    ResourceUtils.closeOrLog(mMetaTable);
    ResourceUtils.closeOrLog(mSystemTable);
    ResourceUtils.closeOrLog(mSchemaTable);
    ResourceUtils.closeOrLog(mSecurityManager);
    ResourceUtils.closeOrLog(mAdmin);
    mSchemaTable = null;
    mMetaTable = null;
    mAdmin = null;
    mSecurityManager = null;
    LOG.debug("{} closed.", this);
  }
Exemple #3
0
  /**
   * Creates a Kiji table in an HBase instance, without checking for validation compatibility and
   * without applying permissions.
   *
   * @param tableLayout The initial layout of the table (with unassigned column ids).
   * @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1
   *     regions created. Pass null to specify the default single region.
   * @throws IOException on I/O error.
   * @throws KijiAlreadyExistsException if the table already exists.
   */
  private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys)
      throws IOException {
    final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();

    // This will validate the layout and may throw an InvalidLayoutException.
    final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout);

    if (getMetaTable().tableExists(tableLayout.getName())) {
      throw new KijiAlreadyExistsException(
          String.format("Kiji table '%s' already exists.", tableURI), tableURI);
    }

    if (tableLayout.getKeysFormat() instanceof RowKeyFormat) {
      LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'.");
    }

    getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout);

    if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
      // system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications.
      // Invariant: ZooKeeper hold the most recent layout of the table.
      LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI);
      try {
        final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient);
        try {
          final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId());
          monitor.notifyNewTableLayout(tableURI, layoutId, -1);
        } finally {
          monitor.close();
        }
      } catch (KeeperException ke) {
        throw new IOException(ke);
      }
    }

    try {
      final HTableSchemaTranslator translator = new HTableSchemaTranslator();
      final HTableDescriptor desc =
          translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout);
      LOG.debug("Creating HBase table '{}'.", desc.getNameAsString());
      if (null != splitKeys) {
        getHBaseAdmin().createTable(desc, splitKeys);
      } else {
        getHBaseAdmin().createTable(desc);
      }
    } catch (TableExistsException tee) {
      throw new KijiAlreadyExistsException(
          String.format("Kiji table '%s' already exists.", tableURI), tableURI);
    }
  }
  @Test
  public void testValidate() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen());
    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=validate-all"));

    KijiFreshnessPolicyRecord record =
        KijiFreshnessPolicyRecord.newBuilder()
            .setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString())
            .setProducerClass(TestProducer.class.getName())
            .setFreshnessPolicyClass(AlwaysFreshen.class.getName())
            .setFreshnessPolicyState("")
            .build();

    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final EncoderFactory encoderFactory = EncoderFactory.get();
    Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null);
    final DatumWriter<KijiFreshnessPolicyRecord> recordWriter =
        new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$);
    recordWriter.write(record, encoder);
    getKiji()
        .getMetaTable()
        .putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray());

    assertEquals(
        BaseTool.FAILURE,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=validate-all"));
    assertEquals(
        "Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]);
    assertEquals(
        "NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not "
            + "contain family: columnName",
        mToolOutputLines[2]);
  }
  /**
   * Validates a new table layout against a reference layout for mutual compatibility.
   *
   * @param reference the reference layout against which to validate.
   * @param layout the new layout to validate.
   * @throws IOException in case of an IO Error reading from the schema table. Throws
   *     InvalidLayoutException if the layouts are incompatible.
   */
  public void validate(KijiTableLayout reference, KijiTableLayout layout) throws IOException {

    final ProtocolVersion layoutVersion = ProtocolVersion.parse(layout.getDesc().getVersion());

    if (layoutVersion.compareTo(Versions.LAYOUT_VALIDATION_VERSION) < 0) {
      // Layout versions older than layout-1.3.0 do not require validation
      return;
    }

    // Accumulator for error messages which will be used to create an exception if errors occur.
    final List<String> incompatabilityMessages = Lists.newArrayList();

    // Iterate through all families/columns in the new layout,
    // find a potential matching reference family/column,
    // and validate the reader/writer schema sets.
    // If no matching family/column exists in the reference layout the newly create column is valid.
    for (FamilyLayout flayout : layout.getFamilies()) {
      final ColumnId lgid = flayout.getLocalityGroup().getId();

      LocalityGroupLayout refLGLayout = null;
      if (reference != null) {
        // If there is a reference layout, check for a locality group matching the ID of the LG for
        // this family.  Locality Group IDs should not change between layouts.
        final String refLGName = reference.getLocalityGroupIdNameMap().get(lgid);
        if (refLGName != null) {
          // If there is a matching reference LG get its layout by name.
          refLGLayout = reference.getLocalityGroupMap().get(refLGName);
        }
      }

      // The ColumnId of the FamilyLayout from the table layout.  Also matches the FamilyLayout for
      // this family in the reference layout if present.
      final ColumnId familyId = flayout.getId();

      if (flayout.isMapType()) {
        // If the family is map-type, get the CellSchema for all values in the family.
        final CellSchema cellSchema = flayout.getDesc().getMapSchema();

        FamilyLayout refFamilyLayout = null;
        if (refLGLayout != null) {
          // If there is a matching reference LG, check for the existence of this family.
          final String refFamilyName = refLGLayout.getFamilyIdNameMap().get(familyId);
          if (refFamilyName != null) {
            refFamilyLayout = refLGLayout.getFamilyMap().get(refFamilyName);
          }
        }

        if (refFamilyLayout != null) {
          if (refFamilyLayout.isMapType()) {
            // If the FamilyLayout from both table layouts are map type, compare their CellSchemas.
            final CellSchema refCellSchema = refFamilyLayout.getDesc().getMapSchema();

            incompatabilityMessages.addAll(
                addColumnNamestoIncompatibilityMessages(
                    flayout.getName(), null, validateCellSchema(refCellSchema, cellSchema)));
          } else if (refFamilyLayout.isGroupType()) {
            // If the FamilyLayout changed from group-type to map-type between table layout versions
            // that is an incompatible change.
            incompatabilityMessages.add(
                String.format(
                    "Family: %s changed from group-type to map-type.", refFamilyLayout.getName()));
          } else {
            throw new InternalKijiError(
                String.format(
                    "Family: %s is neither map-type nor group-type.", refFamilyLayout.getName()));
          }
        } else {
          // If the reference FamilyLayout is null this indicates a new family, which is inherently
          // compatible, but we still have to validate that the new readers and writers are
          // internally compatible.
          incompatabilityMessages.addAll(
              addColumnNamestoIncompatibilityMessages(
                  flayout.getName(), null, validateCellSchema(null, cellSchema)));
        }
      } else if (flayout.isGroupType()) {
        // Check for a matching family from the reference layout.
        FamilyLayout refFamilyLayout = null;
        if (refLGLayout != null) {
          final String refFamilyName = refLGLayout.getFamilyIdNameMap().get(familyId);
          if (refFamilyName != null) {
            refFamilyLayout = refLGLayout.getFamilyMap().get(refFamilyName);
          }
        }

        if (refFamilyLayout != null) {
          if (refFamilyLayout.isGroupType()) {
            // If there is a matching reference family and it is the same family type, iterate
            // through the columns checking schema compatibility.  Only checks columns from the new
            // layout because removed columns are inherently valid.
            for (ColumnLayout columnLayout : flayout.getColumns()) {
              final CellSchema cellSchema = columnLayout.getDesc().getColumnSchema();

              final String refColumnName =
                  refFamilyLayout.getColumnIdNameMap().get(columnLayout.getId());
              ColumnLayout refColumnLayout = null;
              if (refColumnName != null) {
                // If there is a column from the reference layout with the same column ID, get its
                // layout.
                refColumnLayout = refFamilyLayout.getColumnMap().get(refColumnName);
              }
              // If there is a column from the reference layout with the same column ID, get its
              // CellSchema.
              final CellSchema refCellSchema =
                  (refColumnLayout == null) ? null : refColumnLayout.getDesc().getColumnSchema();

              // If there is no matching column, refCellSchema will be null and this will only test
              // that the new reader and writer schemas are internally compatible.
              incompatabilityMessages.addAll(
                  addColumnNamestoIncompatibilityMessages(
                      flayout.getName(),
                      columnLayout.getName(),
                      validateCellSchema(refCellSchema, cellSchema)));
            }
          } else if (refFamilyLayout.isMapType()) {
            // If the FamilyLayout changed from map-type to group-type between table layout versions
            // that is an incompatible change.
            incompatabilityMessages.add(
                String.format(
                    "Family: %s changed from map-type to group-type.", refFamilyLayout.getName()));
          } else {
            throw new InternalKijiError(
                String.format(
                    "Family: %s is neither map-type nor group-type.", refFamilyLayout.getName()));
          }
        } else {
          // If the reference FamilyLayout is null this indicates a new family, which is inherently
          // compatible, but we still have to validate that the new readers and writers are
          // internally compatible.
          for (ColumnLayout columnLayout : flayout.getColumns()) {
            final CellSchema cellSchema = columnLayout.getDesc().getColumnSchema();
            incompatabilityMessages.addAll(
                addColumnNamestoIncompatibilityMessages(
                    flayout.getName(),
                    columnLayout.getName(),
                    validateCellSchema(null, cellSchema)));
          }
        }

      } else {
        throw new InternalKijiError(
            String.format("Family: %s is neither map-type nor group-type.", flayout.getName()));
      }
    }

    // If there were any incompatibility errors, throw an exception.
    if (incompatabilityMessages.size() != 0) {
      throw new InvalidLayoutSchemaException(incompatabilityMessages);
    }
  }
Exemple #6
0
  /** {@inheritDoc} */
  @Override
  public KijiTableLayout modifyTableLayout(
      TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException {
    final State state = mState.get();
    Preconditions.checkState(
        state == State.OPEN,
        "Cannot modify table layout in Kiji instance %s in state %s.",
        this,
        state);
    Preconditions.checkNotNull(update);

    ensureValidationCompatibility(update);

    if (dryRun && (null == printStream)) {
      printStream = System.out;
    }

    final KijiMetaTable metaTable = getMetaTable();

    final String tableName = update.getName();
    // Throws a KijiTableNotFoundException if there is no table.
    metaTable.getTableLayout(tableName);

    final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
    LOG.debug("Applying layout update {} on table {}", update, tableURI);

    KijiTableLayout newLayout = null;

    if (dryRun) {
      // Process column ids and perform validation, but don't actually update the meta table.
      final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1);
      final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0);
      newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout);
    } else {
      // Actually set it.
      if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
        try {
          // Use ZooKeeper to inform all watchers that a new table layout is available.
          final HBaseTableLayoutUpdater updater =
              new HBaseTableLayoutUpdater(this, tableURI, update);
          try {
            updater.update();
            newLayout = updater.getNewLayout();
          } finally {
            updater.close();
          }
        } catch (KeeperException ke) {
          throw new IOException(ke);
        }
      } else {
        // System versions before system-2.0 do not enforce table layout update consistency or
        // validation.
        newLayout = metaTable.updateTableLayout(tableName, update);
      }
    }
    Preconditions.checkState(newLayout != null);

    if (dryRun) {
      printStream.println("This table layout is valid.");
    }

    LOG.debug("Computing new HBase schema");
    final HTableSchemaTranslator translator = new HTableSchemaTranslator();
    final HTableDescriptor newTableDescriptor =
        translator.toHTableDescriptor(mURI.getInstance(), newLayout);

    LOG.debug("Reading existing HBase schema");
    final KijiManagedHBaseTableName hbaseTableName =
        KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName);
    HTableDescriptor currentTableDescriptor = null;
    byte[] tableNameAsBytes = hbaseTableName.toBytes();
    try {
      currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes);
    } catch (TableNotFoundException tnfe) {
      if (!dryRun) {
        throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception.
      }
    }
    if (currentTableDescriptor == null) {
      if (dryRun) {
        printStream.println("Would create new table: " + tableName);
        currentTableDescriptor =
            HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName);
      } else {
        throw new RuntimeException(
            "Table " + hbaseTableName.getKijiTableName() + " does not exist");
      }
    }
    LOG.debug("Existing table descriptor: {}", currentTableDescriptor);
    LOG.debug("New table descriptor: {}", newTableDescriptor);

    LOG.debug("Checking for differences between the new HBase schema and the existing one");
    final HTableDescriptorComparator comparator = new HTableDescriptorComparator();
    if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) {
      LOG.debug("HBase schemas are the same.  No need to change HBase schema");
      if (dryRun) {
        printStream.println("This layout does not require any physical table schema changes.");
      }
    } else {
      LOG.debug("HBase schema must be changed, but no columns will be deleted");

      if (dryRun) {
        printStream.println("Changes caused by this table layout:");
      } else {
        LOG.debug("Disabling HBase table");
        getHBaseAdmin().disableTable(hbaseTableName.toString());
      }

      for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) {
        final String columnName = Bytes.toString(newColumnDescriptor.getName());
        final ColumnId columnId = ColumnId.fromString(columnName);
        final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId);
        final HColumnDescriptor currentColumnDescriptor =
            currentTableDescriptor.getFamily(newColumnDescriptor.getName());
        if (null == currentColumnDescriptor) {
          if (dryRun) {
            printStream.println("  Creating new locality group: " + lgName);
          } else {
            LOG.debug("Creating new column " + columnName);
            getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor);
          }
        } else if (!newColumnDescriptor.equals(currentColumnDescriptor)) {
          if (dryRun) {
            printStream.println("  Modifying locality group: " + lgName);
          } else {
            LOG.debug("Modifying column " + columnName);
            getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor);
          }
        } else {
          LOG.debug("No changes needed for column " + columnName);
        }
      }

      if (dryRun) {
        if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
          printStream.printf(
              "  Changing max_filesize from %d to %d: %n",
              currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize());
        }
        if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
          printStream.printf(
              "  Changing memstore_flushsize from %d to %d: %n",
              currentTableDescriptor.getMemStoreFlushSize(),
              newTableDescriptor.getMemStoreFlushSize());
        }
      } else {
        LOG.debug("Modifying table descriptor");
        getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor);
      }

      if (!dryRun) {
        LOG.debug("Re-enabling HBase table");
        getHBaseAdmin().enableTable(hbaseTableName.toString());
      }
    }

    return newLayout;
  }
Exemple #7
0
  /**
   * Creates a new <code>HBaseKiji</code> instance.
   *
   * <p>Should only be used by Kiji.Factory.open().
   *
   * <p>Caller does not need to use retain(), but must call release() when done with it.
   *
   * @param kijiURI the KijiURI.
   * @param conf Hadoop Configuration. Deep copied internally.
   * @param tableFactory HTableInterface factory.
   * @param lockFactory Factory for locks.
   * @throws IOException on I/O error.
   */
  HBaseKiji(
      KijiURI kijiURI,
      Configuration conf,
      HTableInterfaceFactory tableFactory,
      LockFactory lockFactory)
      throws IOException {

    mConstructorStack = CLEANUP_LOG.isDebugEnabled() ? Debug.getStackTrace() : null;

    // Deep copy the configuration.
    mConf = new Configuration(conf);

    // Validate arguments.
    mHTableFactory = Preconditions.checkNotNull(tableFactory);
    mLockFactory = Preconditions.checkNotNull(lockFactory);
    mURI = Preconditions.checkNotNull(kijiURI);

    // Configure the ZooKeeper quorum:
    mConf.setStrings("hbase.zookeeper.quorum", mURI.getZookeeperQuorum().toArray(new String[0]));
    mConf.setInt("hbase.zookeeper.property.clientPort", mURI.getZookeeperClientPort());

    // Check for an instance name.
    Preconditions.checkArgument(
        mURI.getInstance() != null, "KijiURI '%s' does not specify a Kiji instance name.", mURI);

    if (LOG.isDebugEnabled()) {
      Debug.logConfiguration(mConf);
      LOG.debug(
          "Opening kiji instance '{}'"
              + " with client software version '{}'"
              + " and client data version '{}'.",
          mURI,
          VersionInfo.getSoftwareVersion(),
          VersionInfo.getClientDataVersion());
    }

    // Load these lazily.
    mSchemaTable = null;
    mMetaTable = null;
    mSecurityManager = null;

    mSystemTable = new HBaseSystemTable(mURI, mConf, mHTableFactory);

    mRetainCount.set(1);
    final State oldState = mState.getAndSet(State.OPEN);
    Preconditions.checkState(
        oldState == State.UNINITIALIZED, "Cannot open Kiji instance in state %s.", oldState);
    LOG.debug("Kiji instance '{}' is now opened.", mURI);

    mSystemVersion = mSystemTable.getDataVersion();
    LOG.debug("Kiji instance '{}' has data version '{}'.", mURI, mSystemVersion);

    // Make sure the data version for the client matches the cluster.
    LOG.debug("Validating version for Kiji instance '{}'.", mURI);
    try {
      VersionInfo.validateVersion(this);
    } catch (IOException ioe) {
      // If an IOException occurred the object will not be constructed so need to clean it up.
      close();
      throw ioe;
    } catch (KijiNotInstalledException kie) {
      // Some clients handle this unchecked Exception so do the same here.
      close();
      throw kie;
    }

    // TODO(SCHEMA-491) Share ZooKeeperClient instances when possible.
    if (mSystemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) >= 0) {
      // system-2.0 clients must connect to ZooKeeper:
      //  - to register themselves as table users;
      //  - to receive table layout updates.
      mZKClient = HBaseFactory.Provider.get().getZooKeeperClient(mURI);
      try {
        mMonitor = new ZooKeeperMonitor(mZKClient);
        mMonitor.registerInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
      } catch (KeeperException ke) {
        // Unrecoverable KeeperException:
        throw new IOException(ke);
      }
    } else {
      // system-1.x clients do not need a ZooKeeper connection.
      mZKClient = null;
      mMonitor = null;
    }
  }