Пример #1
0
  @Test
  public void testScanTable() throws Exception {
    final Kiji kiji = getKiji();
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.SIMPLE);
    kiji.createTable(layout.getName(), layout);
    final KijiTable table = kiji.openTable(layout.getName());
    try {
      // Table is empty:
      assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), "--kiji=" + table.getURI()));
      assertEquals(1, mToolOutputLines.length);
      assertTrue(mToolOutputLines[0].startsWith("Scanning kiji table: "));

      new InstanceBuilder(kiji)
          .withTable(table)
          .withRow("hashed")
          .withFamily("family")
          .withQualifier("column")
          .withValue(314L, "value")
          .build();

      // Table has now one row:
      assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), "--kiji=" + table.getURI()));
      assertEquals(3, mToolOutputLines.length);
      assertTrue(mToolOutputLines[0].startsWith("Scanning kiji table: "));
      assertTrue(mToolOutputLines[1].startsWith("entity-id=hbase='"));

    } finally {
      table.close();
    }
  }
Пример #2
0
  private <T> T populateFromRow(
      EntitySpec<T> spec, T entity, long startTime, long endTime, Object... entityIdComponents)
      throws IOException {

    // TODO: Use a pool of tables and/or table readers
    final KijiTable table = mKiji.openTable(spec.getTableName());
    try {
      final KijiTableReader reader = table.openTableReader();
      try {
        final KijiDataRequestBuilder builder = KijiDataRequest.builder();
        builder.withTimeRange(startTime, endTime);
        spec.populateColumnRequests(builder);
        final KijiDataRequest dataRequest = builder.build();
        final EntityId entityId = table.getEntityId(entityIdComponents);
        final KijiRowData row = reader.get(entityId, dataRequest);

        try {
          return spec.populateEntityFromRow(entity, row);
        } catch (IllegalAccessException iae) {
          throw new RuntimeException(iae);
        }

      } finally {
        reader.close();
      }
    } finally {
      table.release();
    }
  }
Пример #3
0
 /** A test to ensure that policies can mask the key value stores of their producers. */
 @Test
 public void testKVMasking() throws IOException {
   // Create a freshness policy that knows where to find the text file backed kv-store.
   KijiFreshnessPolicy policy =
       new ShadowingFreshening("file:" + new File(getLocalTempDir(), KV_FILENAME));
   // Install a freshness policy.
   KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
   try {
     manager.registerFreshener(
         "user",
         new KijiColumnName("info", "name"),
         policy,
         new UnconfiguredScoreFunction(),
         Collections.<String, String>emptyMap(),
         true,
         false);
   } finally {
     manager.close();
   }
   final KijiTable userTable = getKiji().openTable("user");
   try {
     final FreshKijiTableReader reader =
         FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
     try {
       // Read from the table to ensure that the user name is updated.
       KijiRowData data =
           reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
       assertEquals("Old Gumbie Cat", data.getMostRecentValue("info", "name").toString());
     } finally {
       reader.close();
     }
   } finally {
     userTable.release();
   }
 }
  @Test
  public void testBuilder() throws Exception {
    final KijiTableLayout layout =
        KijiTableLayout.newLayout(KijiTableLayouts.getLayout(KijiTableLayouts.SIMPLE));

    final Kiji kiji =
        new InstanceBuilder()
            .withTable("table", layout)
            .withRow("row1")
            .withFamily("family")
            .withQualifier("column")
            .withValue(1, "foo1")
            .withValue(2, "foo2")
            .withRow("row2")
            .withFamily("family")
            .withQualifier("column")
            .withValue(100, "foo3")
            .build();

    final KijiTable table = kiji.openTable("table");
    final KijiTableReader reader = table.openTableReader();

    // Verify the first row.
    final KijiDataRequest req = KijiDataRequest.create("family", "column");
    final KijiRowData row1 = reader.get(table.getEntityId("row1"), req);
    assertEquals("foo2", row1.getValue("family", "column", 2).toString());

    // Verify the second row.
    final KijiRowData row2 = reader.get(table.getEntityId("row2"), req);
    assertEquals("foo3", row2.getValue("family", "column", 100).toString());

    ResourceUtils.closeOrLog(reader);
    ResourceUtils.releaseOrLog(table);
    ResourceUtils.releaseOrLog(kiji);
  }
  /**
   * Gets a set of input splits for a MapReduce job running over a Kiji table. One split is created
   * per region in the input Kiji table.
   *
   * @param configuration of the job using the splits. The configuration should specify the input
   *     Kiji table being used, through the configuration variable {@link
   *     KijiConfKeys#KIJI_INPUT_TABLE_URI}.
   * @param numSplits desired for the job. This framework hint is ignored by this method.
   * @return an array of input splits to be operated on in the MapReduce job.
   * @throws IOException if an I/O error occurs while communicating with HBase to determine the
   *     regions in the Kiji table.
   */
  @Override
  public InputSplit[] getSplits(JobConf configuration, int numSplits) throws IOException {
    final String uriString =
        Preconditions.checkNotNull(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI));
    final KijiURI inputTableURI = KijiURI.newBuilder(uriString).build();
    final Kiji kiji = Kiji.Factory.open(inputTableURI, configuration);
    try {
      final KijiTable table = kiji.openTable(inputTableURI.getTable());
      try {
        final HTableInterface htable = HBaseKijiTable.downcast(table).getHTable();

        final List<InputSplit> splits = Lists.newArrayList();
        for (KijiRegion region : table.getRegions()) {
          final byte[] startKey = region.getStartKey();
          // TODO(KIJIMR-65): For now pick the first available location (ie. region server), if any.
          final String location =
              region.getLocations().isEmpty() ? null : region.getLocations().iterator().next();
          final TableSplit tableSplit =
              new TableSplit(htable.getTableName(), startKey, region.getEndKey(), location);
          splits.add(new KijiTableSplit(tableSplit));
        }
        return splits.toArray(new InputSplit[0]);

      } finally {
        table.release();
      }
    } finally {
      kiji.release();
    }
  }
Пример #6
0
  @Test
  public void testMultipleArguments() throws Exception {
    final Kiji kiji = getKiji();
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FORMATTED_RKF);
    new InstanceBuilder(kiji)
        .withTable(layout.getName(), layout)
        .withRow("dummy", "str1", "str2", 1, 2L)
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withValue(2L, "string-value2")
        .withRow("dummy", "str1", "str2", 1)
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy", "str1", "str2")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy", "str1")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .build();

    final KijiTable table = kiji.openTable(layout.getName());

    final KijiTableLayout layoutTwo = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
    kiji.createTable(layoutTwo.getDesc());
    final KijiTable tableTwo = kiji.openTable(layoutTwo.getName());

    try {
      assertEquals(
          BaseTool.SUCCESS,
          runTool(new LsTool(), table.getURI().toString(), tableTwo.getURI().toString()));
      assertEquals(9, mToolOutputLines.length);
      assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), kiji.getURI().toString()));
      assertEquals(2, mToolOutputLines.length);
      assertEquals(
          BaseTool.SUCCESS,
          runTool(new LsTool(), kiji.getURI().toString(), table.getURI().toString()));
      assertEquals(3, mToolOutputLines.length);
      // assertEquals(2, mToolOutputLines.length);
    } finally {
      ResourceUtils.releaseOrLog(table);
      ResourceUtils.releaseOrLog(tableTwo);
    }
  }
Пример #7
0
 @Test
 public void testTableNoFamilies() throws Exception {
   final Kiji kiji =
       new InstanceBuilder(getKiji())
           .withTable(KijiTableLayouts.getLayout(KijiTableLayouts.NOFAMILY))
           .build();
   final KijiTable table = kiji.openTable("nofamily");
   try {
     assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
   } finally {
     table.release();
   }
 }
Пример #8
0
 @Test
 public void testKijiLsStartAndLimitRow() throws Exception {
   final Kiji kiji = getKiji();
   final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
   kiji.createTable(layout.getDesc());
   final KijiTable table = kiji.openTable(layout.getName());
   try {
     assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
     // TODO: Validate output
   } finally {
     ResourceUtils.releaseOrLog(table);
   }
 }
Пример #9
0
 @Test
 public void testTableColumns() throws Exception {
   final Kiji kiji = getKiji();
   final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.SIMPLE);
   kiji.createTable(layout.getDesc());
   final KijiTable table = kiji.openTable(layout.getName());
   try {
     // Table is empty:
     assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
     assertEquals(1, mToolOutputLines.length);
     assertTrue(mToolOutputLines[0].contains("family:column"));
   } finally {
     ResourceUtils.releaseOrLog(table);
   }
 }
  @Test
  public void testJSONBulkImporter() throws Exception {
    // Prepare input file:
    File inputFile = File.createTempFile("TestJSONImportInput", ".txt", getLocalTempDir());
    TestingResources.writeTextFile(
        inputFile, TestingResources.get(BulkImporterTestUtils.JSON_IMPORT_DATA));

    Configuration conf = getConf();
    conf.set(
        DescribedInputTextBulkImporter.CONF_FILE,
        BulkImporterTestUtils.localResource(BulkImporterTestUtils.FOO_IMPORT_DESCRIPTOR));

    // Run the bulk-import:
    final KijiMapReduceJob job =
        KijiBulkImportJobBuilder.create()
            .withConf(conf)
            .withBulkImporter(JSONBulkImporter.class)
            .withInput(MapReduceJobInputs.newTextMapReduceJobInput(new Path(inputFile.toString())))
            .withOutput(new DirectKijiTableMapReduceJobOutput(mTable.getURI()))
            .build();
    assertTrue(job.run());

    final Counters counters = job.getHadoopJob().getCounters();
    assertEquals(
        3, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_PROCESSED).getValue());
    assertEquals(
        1, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_INCOMPLETE).getValue());
    assertEquals(
        0, counters.findCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_REJECTED).getValue());

    // Validate output:
    final KijiRowScanner scanner = mReader.getScanner(KijiDataRequest.create("info"));
    BulkImporterTestUtils.validateImportedRows(scanner, false);
    scanner.close();
  }
  /**
   * Generates a split for a given table.
   *
   * @param tableURI URI of the Kiji table to split.
   * @param nsplits Number of splits.
   * @param conf Base Hadoop configuration used to open the Kiji instance.
   * @return a list of split start keys, as HFileKeyValue (with no value, just the keys).
   * @throws IOException on I/O error.
   */
  private static List<HFileKeyValue> makeTableKeySplit(
      KijiURI tableURI, int nsplits, Configuration conf) throws IOException {
    final Kiji kiji = Kiji.Factory.open(tableURI, conf);
    try {
      final KijiTable table = kiji.openTable(tableURI.getTable());
      try {
        if (NUM_SPLITS_AUTO == nsplits) {
          final List<HFileKeyValue> startKeys = Lists.newArrayList();
          for (KijiRegion region : table.getRegions()) {
            startKeys.add(HFileKeyValue.createFromRowKey(region.getStartKey()));
          }
          return startKeys;

        } else {
          switch (KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat())) {
            case RAW:
              {
                // The user has explicitly specified how many HFiles to create, but this is not
                // possible when row key hashing is disabled.
                throw new JobConfigurationException(
                    String.format(
                        "Table '%s' has row key hashing disabled, so the number of HFile splits must be"
                            + "determined by the number of HRegions in the HTable. "
                            + "Use an HFileMapReduceJobOutput constructor that enables auto splitting.",
                        table.getName()));
              }
            case FORMATTED:
            case HASH:
            case HASH_PREFIX:
              {
                // Those cases are supported:
                break;
              }
            default:
              throw new RuntimeException(
                  "Unhandled row key encoding: "
                      + KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat()));
          }
          return generateEvenStartKeys(nsplits);
        }
      } finally {
        ResourceUtils.releaseOrLog(table);
      }
    } finally {
      ResourceUtils.releaseOrLog(kiji);
    }
  }
    /**
     * Initializes a new table-wide record writer.
     *
     * @param oformat KijiHFileOutputFormat this writer is built from.
     * @param context Context of the task.
     * @throws IOException on I/O error.
     */
    public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context)
        throws IOException {
      mContext = Preconditions.checkNotNull(context);
      mConf = mContext.getConfiguration();
      mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis());
      mLatestTimestampBytes = toBytes(mLatestTimestamp);

      mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION);
      mFileSystem = mOutputDir.getFileSystem(mConf);

      mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build();

      final Kiji kiji = Kiji.Factory.open(mTableURI, mConf);
      final KijiTable table = kiji.openTable(mTableURI.getTable());
      mLayout = table.getLayout();
      ResourceUtils.releaseOrLog(table);
      ResourceUtils.releaseOrLog(kiji);
    }
  @Before
  public final void setupTestBulkImporter() throws Exception {
    // Get the test table layouts.
    final KijiTableLayout layout = KijiTableLayout.newLayout(KijiMRTestLayouts.getTestLayout());

    // Populate the environment.
    new InstanceBuilder(getKiji()).withTable("test", layout).build();

    // Fill local variables.
    mTable = getKiji().openTable("test");
    mReader = mTable.openTableReader();
  }
Пример #14
0
  @Test
  public void testFormattedRowKey() throws Exception {
    final Kiji kiji = getKiji();
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FORMATTED_RKF);
    new InstanceBuilder(kiji)
        .withTable(layout.getName(), layout)
        .withRow("dummy", "str1", "str2", 1, 2L)
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withValue(2L, "string-value2")
        .withRow("dummy", "str1", "str2", 1)
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy", "str1", "str2")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy", "str1")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .build();

    final KijiTable table = kiji.openTable(layout.getName());
    try {
      assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), "--kiji=" + table.getURI()));
      // TODO: Validate LsTool output
    } finally {
      table.close();
    }
  }
Пример #15
0
  /** A test to make sure that producers run inside of freshening can access key value stores. */
  @Test
  public void testSimpleKVStore() throws IOException {
    final String path = new Path("file:" + new File(getLocalTempDir(), KV_FILENAME)).toString();
    final Map<String, String> params = Maps.newHashMap();
    params.put(SimpleKVScoreFunction.PARAMETER_KEY, path);

    // Install a freshness policy.
    KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    try {
      manager.registerFreshener(
          "user",
          new KijiColumnName("info", "name"),
          AlwaysFreshen.class.getName(),
          SimpleKVScoreFunction.class.getName(),
          params,
          true,
          false,
          false);
    } finally {
      manager.close();
    }
    final KijiTable userTable = getKiji().openTable("user");
    try {
      final FreshKijiTableReader reader =
          FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
      try {
        // Read from the table to ensure that the user name is updated.
        KijiRowData data =
            reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
        assertEquals("Railway Cat", data.getMostRecentValue("info", "name").toString());
      } finally {
        reader.close();
      }
    } finally {
      userTable.release();
    }
  }
Пример #16
0
  @Test
  public void testKVStoreInIsFresh() throws IOException {
    // Create a freshness policy that knows where to find the text file backed kv-store.
    KijiFreshnessPolicy policy =
        new KVStoreInIsFreshPolicy("file:" + new File(getLocalTempDir(), KV_FILENAME));
    // Install a freshness policy.
    KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    try {
      manager.registerFreshener(
          "user",
          new KijiColumnName("info", "name"),
          policy,
          new UnconfiguredScoreFunction(),
          Collections.<String, String>emptyMap(),
          true,
          false);
    } finally {
      manager.close();
    }
    KijiTable userTable = null;
    FreshKijiTableReader freshReader = null;
    try {
      userTable = getKiji().openTable("user");
      freshReader =
          FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();

      // Read from the table to ensure that the user name is updated.
      KijiRowData data =
          freshReader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
      // IsFresh should have returned true, so nothing should be written.
      assertEquals("Felis", data.getMostRecentValue("info", "name").toString());
    } finally {
      ResourceUtils.closeOrLog(freshReader);
      ResourceUtils.releaseOrLog(userTable);
    }
  }
    /**
     * Creates a new record reader that scans over a subset of rows from a Kiji table. The record
     * reader will scan over rows in the table specified in the provided input split, subject to row
     * limits specified in the data request serialized into the specified configuration.
     *
     * @param split for the MapReduce task that will use this record reader. The split specifies a
     *     subset of rows from a Kiji table.
     * @param configuration for the MapReduce job using this record reader. The configuration should
     *     specify the input Kiji table through the configuration variable {@link
     *     KijiConfKeys#KIJI_INPUT_TABLE_URI} and a serialized {@link KijiDataRequest} through the
     *     configuration variable {@link KijiConfKeys#KIJI_INPUT_DATA_REQUEST}.
     * @throws IOException if there is a problem constructing the record reader and opening the
     *     resources it requires.
     */
    public KijiTableRecordReader(InputSplit split, Configuration configuration) throws IOException {
      // Get data request from the job configuration.
      final String dataRequestB64 = configuration.get(KijiConfKeys.KIJI_INPUT_DATA_REQUEST);
      Preconditions.checkNotNull(dataRequestB64, "Missing data request in job configuration.");
      final byte[] dataRequestBytes = Base64.decodeBase64(Bytes.toBytes(dataRequestB64));
      mDataRequest = (KijiDataRequest) SerializationUtils.deserialize(dataRequestBytes);

      // Open connections to Kiji.
      assert split instanceof KijiTableSplit;
      mSplit = (KijiTableSplit) split;

      final KijiURI inputURI =
          KijiURI.newBuilder(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI)).build();
      final KijiScannerOptions scannerOptions =
          new KijiScannerOptions()
              .setStartRow(new HBaseEntityId(mSplit.getStartRow()))
              .setStopRow(new HBaseEntityId(mSplit.getEndRow()));
      mKiji = Kiji.Factory.open(inputURI, configuration);
      mTable = mKiji.openTable(inputURI.getTable());
      mReader = mTable.openTableReader();
      mScanner = mReader.getScanner(mDataRequest, scannerOptions);
      mIterator = mScanner.iterator();
    }
 /** {@inheritDoc} */
 @Override
 protected Kiji getKiji() {
   return mInputTable.getKiji();
 }
 /** {@inheritDoc} */
 @Override
 public Configuration getConf() {
   return mInputTable.getKiji().getConf();
 }
Пример #20
0
    /**
     * Initializes a new specification for an Entity from an annotated Java class.
     *
     * @param klass Annotated Java class to derive an entity specification from.
     * @param kiji Kiji instance where to fetch entities from.
     * @throws IOException on I/O error.
     */
    public EntitySpec(Class<T> klass, Kiji kiji) throws IOException {
      mClass = klass;

      final KijiEntity entity = klass.getAnnotation(KijiEntity.class);
      Preconditions.checkArgument(
          entity != null, "Class '{}' has no @KijiEntity annotation.", klass);
      mTableName = entity.table();

      final KijiTable table = kiji.openTable(mTableName);
      try {
        final KijiTableLayout layout = table.getLayout();

        // TODO: Support deprecated RowKeyFormat?
        final RowKeyFormat2 rowKeyFormat = (RowKeyFormat2) layout.getDesc().getKeysFormat();

        final Map<String, RowKeyComponent> rkcMap = Maps.newHashMap();
        final Map<String, Integer> rkcIndexMap = Maps.newHashMap();
        for (int index = 0; index < rowKeyFormat.getComponents().size(); ++index) {
          final RowKeyComponent rkc = rowKeyFormat.getComponents().get(index);
          rkcMap.put(rkc.getName(), rkc);
          rkcIndexMap.put(rkc.getName(), index);
        }
        mRowKeyComponentMap = ImmutableMap.copyOf(rkcMap);
        mRowKeyComponentIndexMap = ImmutableMap.copyOf(rkcIndexMap);

        // --------------------------------------------------------------------
        // Parse fields with annotations from the entity class:

        final List<Field> columnFields = Lists.newArrayList();
        final List<Field> entityIdFields = Lists.newArrayList();

        for (final Field field : mClass.getDeclaredFields()) {
          final KijiColumn column = field.getAnnotation(KijiColumn.class);
          final EntityIdField eidField = field.getAnnotation(EntityIdField.class);

          if ((column != null) && (eidField != null)) {
            throw new IllegalArgumentException(
                String.format(
                    "Field '%s' cannot have both @KijiColumn and @EntityIdField annotations.",
                    field));

          } else if (column != null) {
            LOG.debug("Validating column field '{}'.", field);
            field.setAccessible(true);
            columnFields.add(field);

            final FamilyLayout flayout = layout.getFamilyMap().get(column.family());
            Preconditions.checkArgument(
                flayout != null,
                "Field '%s' maps to non-existing family '%s' from table '%s'.",
                field,
                column.family(),
                mTableName);

            if (column.qualifier().isEmpty()) {
              // Request for a map-type family:
              Preconditions.checkArgument(
                  flayout.isMapType(),
                  "Field '%s' maps to family '%s' from table '%s' which is not a map-type family.",
                  field,
                  column.family(),
                  mTableName);

              // Validate field type:
              if (column.pageSize() > 0) {
                Preconditions.checkArgument(
                    MapFamilyVersionIterator.class.isAssignableFrom(field.getType()),
                    "Fields mapped to map-type family with paging enabled must be "
                        + "MapFamilyVersionIterator, got '{}'.",
                    field.getType());
              } else {
                // TODO Validate type when no paging enabled on map-type family.
              }

            } else {
              // Request for a fully-qualified column:
              final ColumnLayout clayout = flayout.getColumnMap().get(column.qualifier());
              Preconditions.checkArgument(
                  flayout != null,
                  "Field '%s' maps to non-existing column '%s:%s' from table '%s'.",
                  field,
                  column.family(),
                  column.qualifier(),
                  mTableName);

              // Validate field type:
              if (column.pageSize() > 0) {
                Preconditions.checkArgument(
                    ColumnVersionIterator.class.isAssignableFrom(field.getType()),
                    "Fields mapped to column with paging enabled must be "
                        + "ColumnVersionIterator, got '{}'.",
                    field.getType());
              } else {
                // TODO Validate type when no paging enabled on the column.
              }
            }

          } else if (eidField != null) {
            LOG.debug("Validating entity ID field '{}'.", field);
            field.setAccessible(true);
            entityIdFields.add(field);

            final RowKeyComponent rkc = mRowKeyComponentMap.get(eidField.component());
            Preconditions.checkArgument(
                rkc != null,
                "Field '%s' maps to unknown entity ID component '%s'.",
                field,
                eidField.component());

          } else {
            LOG.debug("Ignoring field '{}' with no annotation.", field);
          }
        }

        mColumnFields = ImmutableList.copyOf(columnFields);
        mEntityIdFields = ImmutableList.copyOf(entityIdFields);

      } finally {
        table.release();
      }
    }
 @After
 public final void teardownTestBulkImporter() throws Exception {
   mReader.close();
   mTable.release();
 }
Пример #22
0
  @Test
  public void testKijiLsStartAndLimitRow() throws Exception {
    final Kiji kiji = getKiji();
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
    final long timestamp = 10L;
    new InstanceBuilder(kiji)
        .withTable(layout.getName(), layout)
        .withRow("*****@*****.**")
        .withFamily("info")
        .withQualifier("email")
        .withValue(timestamp, "*****@*****.**")
        .withQualifier("name")
        .withValue(timestamp, "Garrett Wu")
        .withRow("*****@*****.**")
        .withFamily("info")
        .withQualifier("email")
        .withValue(timestamp, "*****@*****.**")
        .withQualifier("name")
        .withValue(timestamp, "Aaron Kimball")
        .withRow("*****@*****.**")
        .withFamily("info")
        .withQualifier("email")
        .withValue(timestamp, "*****@*****.**")
        .withQualifier("name")
        .withValue(timestamp, "Christophe Bisciglia")
        .withRow("*****@*****.**")
        .withFamily("info")
        .withQualifier("email")
        .withValue(timestamp, "*****@*****.**")
        .withQualifier("name")
        .withValue(timestamp, "Kiyan Ahmadizadeh")
        .withRow("*****@*****.**")
        .withFamily("info")
        .withQualifier("email")
        .withValue(timestamp, "*****@*****.**")
        .withQualifier("name")
        .withValue(timestamp, "John Doe")
        .withRow("*****@*****.**")
        .withFamily("info")
        .withQualifier("email")
        .withValue(timestamp, "*****@*****.**")
        .withQualifier("name")
        .withValue(timestamp, "Jane Doe")
        .build();

    final KijiTable table = kiji.openTable(layout.getName());
    try {
      assertEquals(
          BaseTool.SUCCESS,
          runTool(new LsTool(), "--kiji=" + table.getURI(), "--columns=info:name"));
      // TODO: Validate output

      assertEquals(
          BaseTool.SUCCESS,
          runTool(
              new LsTool(),
              "--kiji=" + table.getURI(),
              "--columns=info:name",
              "--start-row=hex:50000000000000000000000000000000", // after the second row.
              "--limit-row=hex:e0000000000000000000000000000000" // before the last row.
              ));
      // TODO: Validate output
    } finally {
      table.close();
    }
  }
Пример #23
0
  /**
   * Run the entry addition system. Asks the user for values for all fields and then fills them in.
   *
   * @param args Command line arguments; this is expected to be empty.
   * @return Exit status code for the application; 0 indicates success.
   * @throws IOException If an error contacting Kiji occurs.
   * @throws InterruptedException If the process is interrupted while performing I/O.
   */
  @Override
  public int run(String[] args) throws IOException, InterruptedException {
    final ConsolePrompt console = new ConsolePrompt();

    // Interactively prompt the user for the record fields from the console.
    final String first = console.readLine("First name: ");
    final String last = console.readLine("Last name: ");
    final String email = console.readLine("Email address: ");
    final String telephone = console.readLine("Telephone: ");

    final Address addr = new Address();
    addr.setAddr1(console.readLine("Address line 1: "));

    // Optional: apartment.
    final String aptNumStr = console.readLine("Apartment: ");
    if (!aptNumStr.isEmpty()) {
      addr.setApt(aptNumStr);
    }

    // Optional: address line 2.
    addr.setAddr2(console.readLine("Address line 2: "));
    if (addr.getAddr2().length() == 0) {
      addr.setAddr2(null);
    }

    addr.setCity(console.readLine("City: "));
    addr.setState(console.readLine("State: "));
    addr.setZip(Integer.valueOf(console.readLine("Zip: ")));

    Kiji kiji = null;
    KijiTable table = null;
    KijiTableWriter writer = null;
    try {
      // Load HBase configuration before connecting to Kiji.
      setConf(HBaseConfiguration.addHbaseResources(getConf()));

      // Connect to Kiji and open the table.
      kiji =
          Kiji.Factory.open(
              new KijiConfiguration(getConf(), KijiConfiguration.DEFAULT_INSTANCE_NAME));
      table = kiji.openTable(TABLE_NAME);
      writer = table.openTableWriter();

      // Create a row ID with the first and last name.
      final EntityId user = table.getEntityId(first + "," + last);

      // Write the record fields to appropriate table columns in the row.
      // The column names are specified as constants in the Fields.java class.
      final long timestamp = System.currentTimeMillis();
      writer.put(user, Fields.INFO_FAMILY, Fields.FIRST_NAME, timestamp, first);
      writer.put(user, Fields.INFO_FAMILY, Fields.LAST_NAME, timestamp, last);
      writer.put(user, Fields.INFO_FAMILY, Fields.EMAIL, timestamp, email);
      writer.put(user, Fields.INFO_FAMILY, Fields.TELEPHONE, timestamp, telephone);
      writer.put(user, Fields.INFO_FAMILY, Fields.ADDRESS, timestamp, addr);
    } catch (KijiTableNotFoundException e) {
      System.out.println("Could not find Kiji table: " + TABLE_NAME);
      return 1;
    } finally {
      // Safely free up resources by closing in reverse order.
      IOUtils.closeQuietly(writer);
      IOUtils.closeQuietly(table);
      ReferenceCountableUtils.releaseQuietly(kiji);
      IOUtils.closeQuietly(console);
    }

    return 0;
  }