Example #1
0
  @Test
  public void testMultipleArguments() throws Exception {
    final Kiji kiji = getKiji();
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FORMATTED_RKF);
    new InstanceBuilder(kiji)
        .withTable(layout.getName(), layout)
        .withRow("dummy", "str1", "str2", 1, 2L)
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withValue(2L, "string-value2")
        .withRow("dummy", "str1", "str2", 1)
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy", "str1", "str2")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy", "str1")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .withRow("dummy")
        .withFamily("family")
        .withQualifier("column")
        .withValue(1L, "string-value")
        .build();

    final KijiTableLayout layoutTwo = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
    kiji.createTable(layoutTwo.getDesc());

    final KijiTable table = kiji.openTable(layout.getName());
    try {
      final KijiTable tableTwo = kiji.openTable(layoutTwo.getName());
      try {
        assertEquals(
            BaseTool.SUCCESS,
            runTool(new LsTool(), table.getURI().toString(), tableTwo.getURI().toString()));
        assertEquals(9, mToolOutputLines.length);
        assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), kiji.getURI().toString()));
        assertEquals(2, mToolOutputLines.length);
        assertEquals(
            BaseTool.SUCCESS,
            runTool(new LsTool(), kiji.getURI().toString(), table.getURI().toString()));
        assertEquals(3, mToolOutputLines.length);
      } finally {
        tableTwo.release();
      }
    } finally {
      table.release();
    }
  }
Example #2
0
  private <T> T populateFromRow(
      EntitySpec<T> spec, T entity, long startTime, long endTime, Object... entityIdComponents)
      throws IOException {

    // TODO: Use a pool of tables and/or table readers
    final KijiTable table = mKiji.openTable(spec.getTableName());
    try {
      final KijiTableReader reader = table.openTableReader();
      try {
        final KijiDataRequestBuilder builder = KijiDataRequest.builder();
        builder.withTimeRange(startTime, endTime);
        spec.populateColumnRequests(builder);
        final KijiDataRequest dataRequest = builder.build();
        final EntityId entityId = table.getEntityId(entityIdComponents);
        final KijiRowData row = reader.get(entityId, dataRequest);

        try {
          return spec.populateEntityFromRow(entity, row);
        } catch (IllegalAccessException iae) {
          throw new RuntimeException(iae);
        }

      } finally {
        reader.close();
      }
    } finally {
      table.release();
    }
  }
Example #3
0
 /** A test to ensure that policies can mask the key value stores of their producers. */
 @Test
 public void testKVMasking() throws IOException {
   // Create a freshness policy that knows where to find the text file backed kv-store.
   KijiFreshnessPolicy policy =
       new ShadowingFreshening("file:" + new File(getLocalTempDir(), KV_FILENAME));
   // Install a freshness policy.
   KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
   try {
     manager.registerFreshener(
         "user",
         new KijiColumnName("info", "name"),
         policy,
         new UnconfiguredScoreFunction(),
         Collections.<String, String>emptyMap(),
         true,
         false);
   } finally {
     manager.close();
   }
   final KijiTable userTable = getKiji().openTable("user");
   try {
     final FreshKijiTableReader reader =
         FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
     try {
       // Read from the table to ensure that the user name is updated.
       KijiRowData data =
           reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
       assertEquals("Old Gumbie Cat", data.getMostRecentValue("info", "name").toString());
     } finally {
       reader.close();
     }
   } finally {
     userTable.release();
   }
 }
  /**
   * Gets a set of input splits for a MapReduce job running over a Kiji table. One split is created
   * per region in the input Kiji table.
   *
   * @param configuration of the job using the splits. The configuration should specify the input
   *     Kiji table being used, through the configuration variable {@link
   *     KijiConfKeys#KIJI_INPUT_TABLE_URI}.
   * @param numSplits desired for the job. This framework hint is ignored by this method.
   * @return an array of input splits to be operated on in the MapReduce job.
   * @throws IOException if an I/O error occurs while communicating with HBase to determine the
   *     regions in the Kiji table.
   */
  @Override
  public InputSplit[] getSplits(JobConf configuration, int numSplits) throws IOException {
    final String uriString =
        Preconditions.checkNotNull(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI));
    final KijiURI inputTableURI = KijiURI.newBuilder(uriString).build();
    final Kiji kiji = Kiji.Factory.open(inputTableURI, configuration);
    try {
      final KijiTable table = kiji.openTable(inputTableURI.getTable());
      try {
        final HTableInterface htable = HBaseKijiTable.downcast(table).getHTable();

        final List<InputSplit> splits = Lists.newArrayList();
        for (KijiRegion region : table.getRegions()) {
          final byte[] startKey = region.getStartKey();
          // TODO(KIJIMR-65): For now pick the first available location (ie. region server), if any.
          final String location =
              region.getLocations().isEmpty() ? null : region.getLocations().iterator().next();
          final TableSplit tableSplit =
              new TableSplit(htable.getTableName(), startKey, region.getEndKey(), location);
          splits.add(new KijiTableSplit(tableSplit));
        }
        return splits.toArray(new InputSplit[0]);

      } finally {
        table.release();
      }
    } finally {
      kiji.release();
    }
  }
Example #5
0
 @Test
 public void testTableNoFamilies() throws Exception {
   final Kiji kiji =
       new InstanceBuilder(getKiji())
           .withTable(KijiTableLayouts.getLayout(KijiTableLayouts.NOFAMILY))
           .build();
   final KijiTable table = kiji.openTable("nofamily");
   try {
     assertEquals(BaseTool.SUCCESS, runTool(new LsTool(), table.getURI().toString()));
   } finally {
     table.release();
   }
 }
Example #6
0
  /** A test to make sure that producers run inside of freshening can access key value stores. */
  @Test
  public void testSimpleKVStore() throws IOException {
    final String path = new Path("file:" + new File(getLocalTempDir(), KV_FILENAME)).toString();
    final Map<String, String> params = Maps.newHashMap();
    params.put(SimpleKVScoreFunction.PARAMETER_KEY, path);

    // Install a freshness policy.
    KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    try {
      manager.registerFreshener(
          "user",
          new KijiColumnName("info", "name"),
          AlwaysFreshen.class.getName(),
          SimpleKVScoreFunction.class.getName(),
          params,
          true,
          false,
          false);
    } finally {
      manager.close();
    }
    final KijiTable userTable = getKiji().openTable("user");
    try {
      final FreshKijiTableReader reader =
          FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
      try {
        // Read from the table to ensure that the user name is updated.
        KijiRowData data =
            reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
        assertEquals("Railway Cat", data.getMostRecentValue("info", "name").toString());
      } finally {
        reader.close();
      }
    } finally {
      userTable.release();
    }
  }
 @After
 public final void teardownTestBulkImporter() throws Exception {
   mReader.close();
   mTable.release();
 }
Example #8
0
    /**
     * Initializes a new specification for an Entity from an annotated Java class.
     *
     * @param klass Annotated Java class to derive an entity specification from.
     * @param kiji Kiji instance where to fetch entities from.
     * @throws IOException on I/O error.
     */
    public EntitySpec(Class<T> klass, Kiji kiji) throws IOException {
      mClass = klass;

      final KijiEntity entity = klass.getAnnotation(KijiEntity.class);
      Preconditions.checkArgument(
          entity != null, "Class '{}' has no @KijiEntity annotation.", klass);
      mTableName = entity.table();

      final KijiTable table = kiji.openTable(mTableName);
      try {
        final KijiTableLayout layout = table.getLayout();

        // TODO: Support deprecated RowKeyFormat?
        final RowKeyFormat2 rowKeyFormat = (RowKeyFormat2) layout.getDesc().getKeysFormat();

        final Map<String, RowKeyComponent> rkcMap = Maps.newHashMap();
        final Map<String, Integer> rkcIndexMap = Maps.newHashMap();
        for (int index = 0; index < rowKeyFormat.getComponents().size(); ++index) {
          final RowKeyComponent rkc = rowKeyFormat.getComponents().get(index);
          rkcMap.put(rkc.getName(), rkc);
          rkcIndexMap.put(rkc.getName(), index);
        }
        mRowKeyComponentMap = ImmutableMap.copyOf(rkcMap);
        mRowKeyComponentIndexMap = ImmutableMap.copyOf(rkcIndexMap);

        // --------------------------------------------------------------------
        // Parse fields with annotations from the entity class:

        final List<Field> columnFields = Lists.newArrayList();
        final List<Field> entityIdFields = Lists.newArrayList();

        for (final Field field : mClass.getDeclaredFields()) {
          final KijiColumn column = field.getAnnotation(KijiColumn.class);
          final EntityIdField eidField = field.getAnnotation(EntityIdField.class);

          if ((column != null) && (eidField != null)) {
            throw new IllegalArgumentException(
                String.format(
                    "Field '%s' cannot have both @KijiColumn and @EntityIdField annotations.",
                    field));

          } else if (column != null) {
            LOG.debug("Validating column field '{}'.", field);
            field.setAccessible(true);
            columnFields.add(field);

            final FamilyLayout flayout = layout.getFamilyMap().get(column.family());
            Preconditions.checkArgument(
                flayout != null,
                "Field '%s' maps to non-existing family '%s' from table '%s'.",
                field,
                column.family(),
                mTableName);

            if (column.qualifier().isEmpty()) {
              // Request for a map-type family:
              Preconditions.checkArgument(
                  flayout.isMapType(),
                  "Field '%s' maps to family '%s' from table '%s' which is not a map-type family.",
                  field,
                  column.family(),
                  mTableName);

              // Validate field type:
              if (column.pageSize() > 0) {
                Preconditions.checkArgument(
                    MapFamilyVersionIterator.class.isAssignableFrom(field.getType()),
                    "Fields mapped to map-type family with paging enabled must be "
                        + "MapFamilyVersionIterator, got '{}'.",
                    field.getType());
              } else {
                // TODO Validate type when no paging enabled on map-type family.
              }

            } else {
              // Request for a fully-qualified column:
              final ColumnLayout clayout = flayout.getColumnMap().get(column.qualifier());
              Preconditions.checkArgument(
                  flayout != null,
                  "Field '%s' maps to non-existing column '%s:%s' from table '%s'.",
                  field,
                  column.family(),
                  column.qualifier(),
                  mTableName);

              // Validate field type:
              if (column.pageSize() > 0) {
                Preconditions.checkArgument(
                    ColumnVersionIterator.class.isAssignableFrom(field.getType()),
                    "Fields mapped to column with paging enabled must be "
                        + "ColumnVersionIterator, got '{}'.",
                    field.getType());
              } else {
                // TODO Validate type when no paging enabled on the column.
              }
            }

          } else if (eidField != null) {
            LOG.debug("Validating entity ID field '{}'.", field);
            field.setAccessible(true);
            entityIdFields.add(field);

            final RowKeyComponent rkc = mRowKeyComponentMap.get(eidField.component());
            Preconditions.checkArgument(
                rkc != null,
                "Field '%s' maps to unknown entity ID component '%s'.",
                field,
                eidField.component());

          } else {
            LOG.debug("Ignoring field '{}' with no annotation.", field);
          }
        }

        mColumnFields = ImmutableList.copyOf(columnFields);
        mEntityIdFields = ImmutableList.copyOf(entityIdFields);

      } finally {
        table.release();
      }
    }