@Test
  public void testBuilder() throws Exception {
    final KijiTableLayout layout =
        KijiTableLayout.newLayout(KijiTableLayouts.getLayout(KijiTableLayouts.SIMPLE));

    final Kiji kiji =
        new InstanceBuilder()
            .withTable("table", layout)
            .withRow("row1")
            .withFamily("family")
            .withQualifier("column")
            .withValue(1, "foo1")
            .withValue(2, "foo2")
            .withRow("row2")
            .withFamily("family")
            .withQualifier("column")
            .withValue(100, "foo3")
            .build();

    final KijiTable table = kiji.openTable("table");
    final KijiTableReader reader = table.openTableReader();

    // Verify the first row.
    final KijiDataRequest req = KijiDataRequest.create("family", "column");
    final KijiRowData row1 = reader.get(table.getEntityId("row1"), req);
    assertEquals("foo2", row1.getValue("family", "column", 2).toString());

    // Verify the second row.
    final KijiRowData row2 = reader.get(table.getEntityId("row2"), req);
    assertEquals("foo3", row2.getValue("family", "column", 100).toString());

    ResourceUtils.closeOrLog(reader);
    ResourceUtils.releaseOrLog(table);
    ResourceUtils.releaseOrLog(kiji);
  }
Пример #2
0
 /** A test to ensure that policies can mask the key value stores of their producers. */
 @Test
 public void testKVMasking() throws IOException {
   // Create a freshness policy that knows where to find the text file backed kv-store.
   KijiFreshnessPolicy policy =
       new ShadowingFreshening("file:" + new File(getLocalTempDir(), KV_FILENAME));
   // Install a freshness policy.
   KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
   try {
     manager.registerFreshener(
         "user",
         new KijiColumnName("info", "name"),
         policy,
         new UnconfiguredScoreFunction(),
         Collections.<String, String>emptyMap(),
         true,
         false);
   } finally {
     manager.close();
   }
   final KijiTable userTable = getKiji().openTable("user");
   try {
     final FreshKijiTableReader reader =
         FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
     try {
       // Read from the table to ensure that the user name is updated.
       KijiRowData data =
           reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
       assertEquals("Old Gumbie Cat", data.getMostRecentValue("info", "name").toString());
     } finally {
       reader.close();
     }
   } finally {
     userTable.release();
   }
 }
Пример #3
0
    private final void populateFieldFromFullyQualifiedColumn(
        T entity, Field field, KijiColumn column, KijiRowData row)
        throws IOException, IllegalAccessException {

      if (column.maxVersions() == 1) {
        // Field represents a single value from a fully-qualified column:
        LOG.debug(
            "Populating field '{}' from column '{}:{}'.",
            field,
            column.family(),
            column.qualifier());
        KijiCell<?> cell = row.getMostRecentCell(column.family(), column.qualifier());
        if (cell == null) return;
        Object value = cell.getData();

        if (field.getType() == KijiCell.class) {
          value = cell;
        } else if (field.getType() == String.class && value != null) {
          value = value.toString();
        }

        // If there is no cell for a field with a primitive type, use the default value:
        if ((null == value) && field.getType().isPrimitive()) {
          value = Defaults.defaultValue(field.getType());
        }

        field.set(entity, value);
      } else {
        // Field represents a time-series from a fully-qualified column:
        if (column.pageSize() > 0) {
          final ColumnVersionIterator<?> iterator =
              new ColumnVersionIterator<Object>(
                  row, column.family(), column.qualifier(), column.pageSize());
          field.set(entity, iterator);
        } else {
          Object value = null;
          if (field.getType() == KijiCellValueIterator.class) {
            value =
                new KijiCellValueIterator<Object>(
                    row.iterator(column.family(), column.qualifier()));
          } else if (field.getType() == TimeSeries.class) {
            final TimeSeries<Object> timeseries = new TimeSeries<Object>();
            for (final KijiCell<Object> cell :
                row.asIterable(column.family(), column.qualifier())) {
              timeseries.put(cell.getTimestamp(), cell.getData());
            }
            value = timeseries;
          }
          field.set(entity, value);
        }
      }
    }
 /** {@inheritDoc} */
 @Override
 public void produce(KijiRowData input, ProducerContext context) throws IOException {
   if (!input.containsColumn("info", "email")) {
     // This user doesn't have an email address.
     return;
   }
   String email = input.getMostRecentValue("info", "email").toString();
   int atSymbol = email.indexOf("@");
   if (atSymbol < 0) {
     // Couldn't find the '@' in the email address. Give up.
     return;
   }
   String domain = email.substring(atSymbol + 1);
   context.put(domain);
 }
Пример #5
0
    /**
     * Populates an entity from a row.
     *
     * @param entity Entity object to populate from a row.
     * @param row Kiji row to populate the entity from.
     * @return the populated entity.
     * @throws IllegalAccessException
     * @throws IOException
     */
    public T populateEntityFromRow(T entity, KijiRowData row)
        throws IllegalAccessException, IOException {

      // Populate fields from the row columns:
      for (final Field field : mColumnFields) {
        final KijiColumn column = field.getAnnotation(KijiColumn.class);
        Preconditions.checkState(column != null);

        if (column.qualifier().isEmpty()) {
          // Field is populated from a map-type family:
          populateFieldFromMapTypeFamily(entity, field, column, row);

        } else {
          // Field is populated from a fully-qualified column:
          populateFieldFromFullyQualifiedColumn(entity, field, column, row);
        }
      }

      // Populate fields from the row entity ID:
      for (final Field field : mEntityIdFields) {
        final EntityIdField eidField = field.getAnnotation(EntityIdField.class);
        Preconditions.checkState(eidField != null);

        final int index = mRowKeyComponentIndexMap.get(eidField.component());
        field.set(entity, row.getEntityId().getComponentByIndex(index));
      }
      return entity;
    }
 /**
  * Populates the specified key and value with the next key-value pair read from the input split.
  *
  * @param key instance to populate with the next key read.
  * @param value instance to popualte with the next value read.
  * @return <code>true</code> if a new key-value was read, <code>false</code> if we have reached
  *     the end of the input split.
  */
 @Override
 public boolean next(KijiKey key, KijiValue value) {
   if (mIterator.hasNext()) {
     // Read the next row and store it in the provided key/value pair.
     final KijiRowData row = mIterator.next();
     if (null != key) {
       key.set(row.getEntityId());
     }
     if (null != value) {
       value.set(row);
     }
     return true;
   } else {
     return false;
   }
 }
Пример #7
0
  /** A test to make sure that producers run inside of freshening can access key value stores. */
  @Test
  public void testSimpleKVStore() throws IOException {
    final String path = new Path("file:" + new File(getLocalTempDir(), KV_FILENAME)).toString();
    final Map<String, String> params = Maps.newHashMap();
    params.put(SimpleKVScoreFunction.PARAMETER_KEY, path);

    // Install a freshness policy.
    KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    try {
      manager.registerFreshener(
          "user",
          new KijiColumnName("info", "name"),
          AlwaysFreshen.class.getName(),
          SimpleKVScoreFunction.class.getName(),
          params,
          true,
          false,
          false);
    } finally {
      manager.close();
    }
    final KijiTable userTable = getKiji().openTable("user");
    try {
      final FreshKijiTableReader reader =
          FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();
      try {
        // Read from the table to ensure that the user name is updated.
        KijiRowData data =
            reader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
        assertEquals("Railway Cat", data.getMostRecentValue("info", "name").toString());
      } finally {
        reader.close();
      }
    } finally {
      userTable.release();
    }
  }
Пример #8
0
  @Test
  public void testKVStoreInIsFresh() throws IOException {
    // Create a freshness policy that knows where to find the text file backed kv-store.
    KijiFreshnessPolicy policy =
        new KVStoreInIsFreshPolicy("file:" + new File(getLocalTempDir(), KV_FILENAME));
    // Install a freshness policy.
    KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    try {
      manager.registerFreshener(
          "user",
          new KijiColumnName("info", "name"),
          policy,
          new UnconfiguredScoreFunction(),
          Collections.<String, String>emptyMap(),
          true,
          false);
    } finally {
      manager.close();
    }
    KijiTable userTable = null;
    FreshKijiTableReader freshReader = null;
    try {
      userTable = getKiji().openTable("user");
      freshReader =
          FreshKijiTableReader.Builder.create().withTable(userTable).withTimeout(10000).build();

      // Read from the table to ensure that the user name is updated.
      KijiRowData data =
          freshReader.get(userTable.getEntityId("felix"), KijiDataRequest.create("info", "name"));
      // IsFresh should have returned true, so nothing should be written.
      assertEquals("Felis", data.getMostRecentValue("info", "name").toString());
    } finally {
      ResourceUtils.closeOrLog(freshReader);
      ResourceUtils.releaseOrLog(userTable);
    }
  }
Пример #9
0
    private final void populateFieldFromMapTypeFamily(
        T entity, Field field, KijiColumn column, KijiRowData row)
        throws IOException, IllegalAccessException {

      LOG.debug("Populating field '{}' from map-type family '{}'.", field, column.family());

      if (column.pageSize() > 0) {
        // Field is a closeable iterator of map-family entries (qualifier, timestamp, value).
        LOG.debug(
            "Populating field '{}' from paging-enabled map-type family '{}'.",
            field,
            column.family());
        final MapFamilyVersionIterator<?> iterator =
            new MapFamilyVersionIterator<Object>(
                row, column.family(), column.pageSize(), column.pageSize());
        field.set(entity, iterator);

      } else if (column.maxVersions() == 1) {
        // Field is a map: qualifier -> single value:

        LOG.debug("Populating single version map field '{}'.", field);
        Object value = null;

        if (field.getType() == KijiCellIterator.class) {
          Iterator<KijiCell<Object>> it = row.iterator(column.family());
          value = new KijiCellIterator<Object>(it);
        } else if (field.getType() == MapTypeCell.class) {
          value = new MapTypeCell<Object>(row.getMostRecentCells(column.family()));
        } else if (field.getType() == MapTypeValue.class) {
          value = new MapTypeValue<Object>(row.getMostRecentValues(column.family()));
        }
        field.set(entity, value);

      } else {
        // Field is a map: qualifier -> time-series
        LOG.debug("Populating map field '{}'.", field);
        Object value = null;
        if (field.getType() == KijiCellIterator.class) {
          Iterator<KijiCell<Object>> it = row.iterator(column.family());
          value = new KijiCellIterator<Object>(it);
        } else if (field.getType() == TSMapTypeValue.class) {
          // TODO: ARGH. This is awful.
          TSMapTypeValue<Object> tsValues = new TSMapTypeValue<Object>();
          for (String s : row.getQualifiers(column.family())) {
            final TimeSeries<Object> timeseries = new TimeSeries<Object>();
            for (final KijiCell<Object> cell : row.asIterable(column.family(), s)) {
              timeseries.put(cell.getTimestamp(), cell.getData());
            }
            tsValues.put(s, timeseries);
          }
          value = tsValues;
        }
        field.set(entity, value);
      }
    }
Пример #10
0
  @Override
  public void gather(KijiRowData kijiRowData, GathererContext gathererContext) throws IOException {
    rows++;
    if (rows % 5000 == 0) {
      LOG.info("Processed row: " + rows);
    }
    try {
      // After 2005, before the future plus 10 minutes to allow for clock misalignments
      checkLong(
          kijiRowData.getMostRecentCell("data", "match_id").getTimestamp(),
          "start_time",
          1104537600,
          System.currentTimeMillis() + 60000);
      for (String team : TEAMS) {
        Integer towerStatus = kijiRowData.getMostRecentValue("data", team + "_towers_status");
        checkInt(towerStatus, team + "_tower_status", 0, ((Double) Math.pow(2, 11)).intValue());
      }
      for (String team : TEAMS) {
        Integer raxStatus = kijiRowData.getMostRecentValue("data", team + "_barracks_status");
        checkInt(raxStatus, team + "_barracks_status", 0, ((Double) Math.pow(2, 6)).intValue());
      }
      checkInt(
          (Integer) kijiRowData.getMostRecentValue("data", "human_players"),
          "human_players",
          0,
          10);

      DotaValues.LobbyType.fromInt((Integer) kijiRowData.getMostRecentValue("data", "lobby_type"));
      DotaValues.GameMode.fromInt((Integer) kijiRowData.getMostRecentValue("data", "game_mode"));
      for (String s :
          new String[] {"cluster", "season", "duration", "negative_votes", "positive_votes"}) {
        Integer n = kijiRowData.getMostRecentValue("data", s);
        checkInt(n, s, 0, Integer.MAX_VALUE / 2);
      }
      checkInt(
          (Integer) kijiRowData.getMostRecentValue("data", "league_id"),
          "league_id",
          0,
          Integer.MAX_VALUE);
      Players players = kijiRowData.getMostRecentValue("data", "player_data");
      for (Player player : players.getPlayers()) {
        Integer n = player.getAccountId();
        if (n != null) {
          if (n != -1) {
            checkInt(n, "account_id", 0, Integer.MAX_VALUE);
          }
        }
        DotaValues.LeaverStatus.fromInt(player.getLeaverStatus());
        checkInt(player.getAssists(), "assists", 0, 1000);
        checkInt(player.getDeaths(), "deaths", 0, 1000);
        checkInt(player.getDenies(), "denies", 0, 5000);
        checkInt(player.getGold(), "gold", 0, 10000);
        checkInt(player.getGoldSpent(), "gold_spent", 0, 1000000);
        checkInt(player.getHeroDamage(), "hero_damage");
        checkInt(player.getHeroHealing(), "hero_healing");
        checkInt(player.getHeroId(), "hero_id", 0, 200);
        checkInt(player.getKills(), "kills", 0, 1000);
        checkInt(player.getLastHits(), "last_hits", 0, 10000);
        checkInt(player.getPlayerSlot(), "player_slot", 0, 256);
        checkInt(player.getLevel(), "level", 0, 25);
        checkDouble(player.getGoldPerMinute(), "gold_per_minute", 0, 200000);
        checkDouble(player.getExpPerMinute(), "exp_per_minute", 0, 200000);
        List<AbilityUpgrade> aus = player.getAbilityUpgrades();
        checkInt(aus.size(), "ability upgrade size", 0, 25);
        for (AbilityUpgrade au : aus) {
          checkInt(au.getLevel(), "ability upgrade level", 0, 25);
          checkInt(au.getTime(), "ability upgrade time", 0, 1000000);
          checkInt(au.getAbilityId(), "ability id");
        }
      }
    } catch (RuntimeException re) {
      gathererContext.write(new Text(re.getMessage()), ONE);
    }
  }
Пример #11
0
 /** {@inheritDoc} */
 @Override
 protected void map(KijiRowData input, Context mapContext) throws IOException {
   mProducerContext.setEntityId(input.getEntityId());
   mProducer.produce(input, mProducerContext);
   mapContext.getCounter(JobHistoryCounters.PRODUCER_ROWS_PROCESSED).increment(1);
 }