/**
   * Generates a split for a given table.
   *
   * @param tableURI URI of the Kiji table to split.
   * @param nsplits Number of splits.
   * @param conf Base Hadoop configuration used to open the Kiji instance.
   * @return a list of split start keys, as HFileKeyValue (with no value, just the keys).
   * @throws IOException on I/O error.
   */
  private static List<HFileKeyValue> makeTableKeySplit(
      KijiURI tableURI, int nsplits, Configuration conf) throws IOException {
    final Kiji kiji = Kiji.Factory.open(tableURI, conf);
    try {
      final KijiTable table = kiji.openTable(tableURI.getTable());
      try {
        if (NUM_SPLITS_AUTO == nsplits) {
          final List<HFileKeyValue> startKeys = Lists.newArrayList();
          for (KijiRegion region : table.getRegions()) {
            startKeys.add(HFileKeyValue.createFromRowKey(region.getStartKey()));
          }
          return startKeys;

        } else {
          switch (KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat())) {
            case RAW:
              {
                // The user has explicitly specified how many HFiles to create, but this is not
                // possible when row key hashing is disabled.
                throw new JobConfigurationException(
                    String.format(
                        "Table '%s' has row key hashing disabled, so the number of HFile splits must be"
                            + "determined by the number of HRegions in the HTable. "
                            + "Use an HFileMapReduceJobOutput constructor that enables auto splitting.",
                        table.getName()));
              }
            case FORMATTED:
            case HASH:
            case HASH_PREFIX:
              {
                // Those cases are supported:
                break;
              }
            default:
              throw new RuntimeException(
                  "Unhandled row key encoding: "
                      + KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat()));
          }
          return generateEvenStartKeys(nsplits);
        }
      } finally {
        ResourceUtils.releaseOrLog(table);
      }
    } finally {
      ResourceUtils.releaseOrLog(kiji);
    }
  }
    /**
     * Initializes a new table-wide record writer.
     *
     * @param oformat KijiHFileOutputFormat this writer is built from.
     * @param context Context of the task.
     * @throws IOException on I/O error.
     */
    public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context)
        throws IOException {
      mContext = Preconditions.checkNotNull(context);
      mConf = mContext.getConfiguration();
      mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis());
      mLatestTimestampBytes = toBytes(mLatestTimestamp);

      mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION);
      mFileSystem = mOutputDir.getFileSystem(mConf);

      mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build();

      final Kiji kiji = Kiji.Factory.open(mTableURI, mConf);
      final KijiTable table = kiji.openTable(mTableURI.getTable());
      mLayout = table.getLayout();
      ResourceUtils.releaseOrLog(table);
      ResourceUtils.releaseOrLog(kiji);
    }
예제 #3
0
    /**
     * Initializes a new specification for an Entity from an annotated Java class.
     *
     * @param klass Annotated Java class to derive an entity specification from.
     * @param kiji Kiji instance where to fetch entities from.
     * @throws IOException on I/O error.
     */
    public EntitySpec(Class<T> klass, Kiji kiji) throws IOException {
      mClass = klass;

      final KijiEntity entity = klass.getAnnotation(KijiEntity.class);
      Preconditions.checkArgument(
          entity != null, "Class '{}' has no @KijiEntity annotation.", klass);
      mTableName = entity.table();

      final KijiTable table = kiji.openTable(mTableName);
      try {
        final KijiTableLayout layout = table.getLayout();

        // TODO: Support deprecated RowKeyFormat?
        final RowKeyFormat2 rowKeyFormat = (RowKeyFormat2) layout.getDesc().getKeysFormat();

        final Map<String, RowKeyComponent> rkcMap = Maps.newHashMap();
        final Map<String, Integer> rkcIndexMap = Maps.newHashMap();
        for (int index = 0; index < rowKeyFormat.getComponents().size(); ++index) {
          final RowKeyComponent rkc = rowKeyFormat.getComponents().get(index);
          rkcMap.put(rkc.getName(), rkc);
          rkcIndexMap.put(rkc.getName(), index);
        }
        mRowKeyComponentMap = ImmutableMap.copyOf(rkcMap);
        mRowKeyComponentIndexMap = ImmutableMap.copyOf(rkcIndexMap);

        // --------------------------------------------------------------------
        // Parse fields with annotations from the entity class:

        final List<Field> columnFields = Lists.newArrayList();
        final List<Field> entityIdFields = Lists.newArrayList();

        for (final Field field : mClass.getDeclaredFields()) {
          final KijiColumn column = field.getAnnotation(KijiColumn.class);
          final EntityIdField eidField = field.getAnnotation(EntityIdField.class);

          if ((column != null) && (eidField != null)) {
            throw new IllegalArgumentException(
                String.format(
                    "Field '%s' cannot have both @KijiColumn and @EntityIdField annotations.",
                    field));

          } else if (column != null) {
            LOG.debug("Validating column field '{}'.", field);
            field.setAccessible(true);
            columnFields.add(field);

            final FamilyLayout flayout = layout.getFamilyMap().get(column.family());
            Preconditions.checkArgument(
                flayout != null,
                "Field '%s' maps to non-existing family '%s' from table '%s'.",
                field,
                column.family(),
                mTableName);

            if (column.qualifier().isEmpty()) {
              // Request for a map-type family:
              Preconditions.checkArgument(
                  flayout.isMapType(),
                  "Field '%s' maps to family '%s' from table '%s' which is not a map-type family.",
                  field,
                  column.family(),
                  mTableName);

              // Validate field type:
              if (column.pageSize() > 0) {
                Preconditions.checkArgument(
                    MapFamilyVersionIterator.class.isAssignableFrom(field.getType()),
                    "Fields mapped to map-type family with paging enabled must be "
                        + "MapFamilyVersionIterator, got '{}'.",
                    field.getType());
              } else {
                // TODO Validate type when no paging enabled on map-type family.
              }

            } else {
              // Request for a fully-qualified column:
              final ColumnLayout clayout = flayout.getColumnMap().get(column.qualifier());
              Preconditions.checkArgument(
                  flayout != null,
                  "Field '%s' maps to non-existing column '%s:%s' from table '%s'.",
                  field,
                  column.family(),
                  column.qualifier(),
                  mTableName);

              // Validate field type:
              if (column.pageSize() > 0) {
                Preconditions.checkArgument(
                    ColumnVersionIterator.class.isAssignableFrom(field.getType()),
                    "Fields mapped to column with paging enabled must be "
                        + "ColumnVersionIterator, got '{}'.",
                    field.getType());
              } else {
                // TODO Validate type when no paging enabled on the column.
              }
            }

          } else if (eidField != null) {
            LOG.debug("Validating entity ID field '{}'.", field);
            field.setAccessible(true);
            entityIdFields.add(field);

            final RowKeyComponent rkc = mRowKeyComponentMap.get(eidField.component());
            Preconditions.checkArgument(
                rkc != null,
                "Field '%s' maps to unknown entity ID component '%s'.",
                field,
                eidField.component());

          } else {
            LOG.debug("Ignoring field '{}' with no annotation.", field);
          }
        }

        mColumnFields = ImmutableList.copyOf(columnFields);
        mEntityIdFields = ImmutableList.copyOf(entityIdFields);

      } finally {
        table.release();
      }
    }