@Test
  public void testRetrieveAll() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen());
    manager.storePolicy("user", "info:visits", TestProducer.class, new NeverFreshen());

    LOG.info(KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString());

    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=retrieve-all"));

    assertEquals(
        "Freshness policy attached to column: info:visits\n"
            + "  Freshness policy class: org.kiji.scoring.lib.NeverFreshen\n"
            + "  Freshness policy state: \n"
            + "  Producer class: org.kiji.scoring.tools.TestFreshTool$TestProducer\n"
            + "Freshness policy attached to column: info:name\n"
            + "  Freshness policy class: org.kiji.scoring.lib.NeverFreshen\n"
            + "  Freshness policy state: \n"
            + "  Producer class: org.kiji.scoring.tools.TestFreshTool$TestProducer\n",
        mToolOutputStr);
  }
  /**
   * Gets a set of input splits for a MapReduce job running over a Kiji table. One split is created
   * per region in the input Kiji table.
   *
   * @param configuration of the job using the splits. The configuration should specify the input
   *     Kiji table being used, through the configuration variable {@link
   *     KijiConfKeys#KIJI_INPUT_TABLE_URI}.
   * @param numSplits desired for the job. This framework hint is ignored by this method.
   * @return an array of input splits to be operated on in the MapReduce job.
   * @throws IOException if an I/O error occurs while communicating with HBase to determine the
   *     regions in the Kiji table.
   */
  @Override
  public InputSplit[] getSplits(JobConf configuration, int numSplits) throws IOException {
    final String uriString =
        Preconditions.checkNotNull(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI));
    final KijiURI inputTableURI = KijiURI.newBuilder(uriString).build();
    final Kiji kiji = Kiji.Factory.open(inputTableURI, configuration);
    try {
      final KijiTable table = kiji.openTable(inputTableURI.getTable());
      try {
        final HTableInterface htable = HBaseKijiTable.downcast(table).getHTable();

        final List<InputSplit> splits = Lists.newArrayList();
        for (KijiRegion region : table.getRegions()) {
          final byte[] startKey = region.getStartKey();
          // TODO(KIJIMR-65): For now pick the first available location (ie. region server), if any.
          final String location =
              region.getLocations().isEmpty() ? null : region.getLocations().iterator().next();
          final TableSplit tableSplit =
              new TableSplit(htable.getTableName(), startKey, region.getEndKey(), location);
          splits.add(new KijiTableSplit(tableSplit));
        }
        return splits.toArray(new InputSplit[0]);

      } finally {
        table.release();
      }
    } finally {
      kiji.release();
    }
  }
Пример #3
0
  @Test
  public void testListIntances() throws Exception {
    final Kiji kiji = getKiji();
    final KijiURI hbaseURI = KijiURI.newBuilder(kiji.getURI()).withInstanceName(null).build();

    final LsTool ls = new LsTool();
    assertEquals(BaseTool.SUCCESS, runTool(ls, "--kiji=" + hbaseURI));
    assertEquals(1, mToolOutputLines.length);
    assertEquals(kiji.getURI(), KijiURI.newBuilder(mToolOutputLines[0]).build());
  }
Пример #4
0
  @Test
  public void testListInstances() throws Exception {
    final Kiji kiji = getKiji();
    final KijiURI hbaseURI = KijiURI.newBuilder(kiji.getURI()).withInstanceName(null).build();

    final LsTool ls = new LsTool();
    assertEquals(BaseTool.SUCCESS, runTool(ls, hbaseURI.toString()));
    final Set<String> instances = Sets.newHashSet(mToolOutputLines);
    assertTrue(instances.contains(kiji.getURI().toString()));
  }
 /**
  * Extracts the ID of the fake HBase from a Kiji URI.
  *
  * @param uri URI to extract a fake HBase ID from.
  * @return the fake HBase ID, if any, or null.
  */
 private static String getFakeHBaseID(KijiURI uri) {
   if (uri.getZookeeperQuorum().size() != 1) {
     return null;
   }
   final String zkHost = uri.getZookeeperQuorum().get(0);
   if (!zkHost.startsWith(FAKE_HBASE_ID_PREFIX)) {
     return null;
   }
   return zkHost.substring(FAKE_HBASE_ID_PREFIX.length());
 }
Пример #6
0
  /**
   * Creates a Kiji table in an HBase instance, without checking for validation compatibility and
   * without applying permissions.
   *
   * @param tableLayout The initial layout of the table (with unassigned column ids).
   * @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1
   *     regions created. Pass null to specify the default single region.
   * @throws IOException on I/O error.
   * @throws KijiAlreadyExistsException if the table already exists.
   */
  private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys)
      throws IOException {
    final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();

    // This will validate the layout and may throw an InvalidLayoutException.
    final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout);

    if (getMetaTable().tableExists(tableLayout.getName())) {
      throw new KijiAlreadyExistsException(
          String.format("Kiji table '%s' already exists.", tableURI), tableURI);
    }

    if (tableLayout.getKeysFormat() instanceof RowKeyFormat) {
      LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'.");
    }

    getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout);

    if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
      // system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications.
      // Invariant: ZooKeeper hold the most recent layout of the table.
      LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI);
      try {
        final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient);
        try {
          final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId());
          monitor.notifyNewTableLayout(tableURI, layoutId, -1);
        } finally {
          monitor.close();
        }
      } catch (KeeperException ke) {
        throw new IOException(ke);
      }
    }

    try {
      final HTableSchemaTranslator translator = new HTableSchemaTranslator();
      final HTableDescriptor desc =
          translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout);
      LOG.debug("Creating HBase table '{}'.", desc.getNameAsString());
      if (null != splitKeys) {
        getHBaseAdmin().createTable(desc, splitKeys);
      } else {
        getHBaseAdmin().createTable(desc);
      }
    } catch (TableExistsException tee) {
      throw new KijiAlreadyExistsException(
          String.format("Kiji table '%s' already exists.", tableURI), tableURI);
    }
  }
Пример #7
0
  /**
   * {@inheritDoc}
   *
   * @throws IOException when instance in configuration can not be opened and closed.
   */
  @Override
  public final void run(final KijiRESTConfiguration configuration, final Environment environment)
      throws IOException {
    final KijiURI clusterURI = KijiURI.newBuilder(configuration.getClusterURI()).build();

    // Load specified instances and health checks for each.
    final Set<KijiURI> instances =
        InstanceUtil.getInstances(
            clusterURI,
            new InstancesMapTo<KijiURI>() {
              public KijiURI apply(KijiURI instanceURI) throws IOException {
                InstanceUtil.openAndCloseInstance(instanceURI);
                LOG.info("Loading instance {} upon startup.", instanceURI.toOrderedString());
                environment.addHealthCheck(new InstanceHealthCheck(instanceURI));
                return instanceURI;
              }
            });

    final ManagedKijiClient managedKijiClient = new ManagedKijiClient(instances);
    environment.manage(managedKijiClient);

    // Remove all built-in Dropwizard ExceptionHandler.
    // Always depend on custom ones.
    // Inspired by Jeremy Whitlock's suggestion on thoughtspark.org.
    Set<Object> jerseyResources = environment.getJerseyResourceConfig().getSingletons();
    Iterator<Object> jerseyResourcesIterator = jerseyResources.iterator();
    while (jerseyResourcesIterator.hasNext()) {
      Object jerseyResource = jerseyResourcesIterator.next();
      if (jerseyResource instanceof ExceptionMapper
          && jerseyResource.getClass().getName().startsWith("com.yammer.dropwizard.jersey")) {
        jerseyResourcesIterator.remove();
      }
    }

    // Update instances periodically.
    final RefreshInstances instanceRefresher = new RefreshInstances(clusterURI, managedKijiClient);
    ScheduledExecutorService scheduler =
        environment.managedScheduledExecutorService("instance_refresh_scheduler", 1);
    scheduler.scheduleAtFixedRate(
        instanceRefresher,
        INSTANCE_REFRESH_PERIOD_MINUTES, // Start a period from now.
        INSTANCE_REFRESH_PERIOD_MINUTES,
        MINUTES);

    // Load admin task to manually update instances.
    environment.addTask(new RefreshInstancesTask(instanceRefresher));

    // Load resources.
    for (KijiRestPlugin plugin : Lookups.get(KijiRestPlugin.class)) {
      LOG.info("Loading plugin {}", plugin.getClass());
      plugin.install(managedKijiClient, configuration, environment);
    }

    // Allow global CORS filter. CORS off by default.
    if (configuration.getCORS()) {
      environment.addFilter(
          CrossOriginFilter.class, configuration.getHttpConfiguration().getRootPath());
      LOG.info("Global cross-origin resource sharing is allowed.");
    }
  }
    /**
     * Initializes a new table-wide record writer.
     *
     * @param oformat KijiHFileOutputFormat this writer is built from.
     * @param context Context of the task.
     * @throws IOException on I/O error.
     */
    public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context)
        throws IOException {
      mContext = Preconditions.checkNotNull(context);
      mConf = mContext.getConfiguration();
      mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis());
      mLatestTimestampBytes = toBytes(mLatestTimestamp);

      mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION);
      mFileSystem = mOutputDir.getFileSystem(mConf);

      mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build();

      final Kiji kiji = Kiji.Factory.open(mTableURI, mConf);
      final KijiTable table = kiji.openTable(mTableURI.getTable());
      mLayout = table.getLayout();
      ResourceUtils.releaseOrLog(table);
      ResourceUtils.releaseOrLog(kiji);
    }
  @Test
  public void testValidate() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen());
    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=validate-all"));

    KijiFreshnessPolicyRecord record =
        KijiFreshnessPolicyRecord.newBuilder()
            .setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString())
            .setProducerClass(TestProducer.class.getName())
            .setFreshnessPolicyClass(AlwaysFreshen.class.getName())
            .setFreshnessPolicyState("")
            .build();

    final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    final EncoderFactory encoderFactory = EncoderFactory.get();
    Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null);
    final DatumWriter<KijiFreshnessPolicyRecord> recordWriter =
        new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$);
    recordWriter.write(record, encoder);
    getKiji()
        .getMetaTable()
        .putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray());

    assertEquals(
        BaseTool.FAILURE,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=validate-all"));
    assertEquals(
        "Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]);
    assertEquals(
        "NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not "
            + "contain family: columnName",
        mToolOutputLines[2]);
  }
Пример #10
0
 @Test
 public void testRetrieveAllEmpty() throws Exception {
   getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
   assertEquals(
       BaseTool.SUCCESS,
       runTool(
           new FreshTool(),
           KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
           "--do=retrieve-all"));
   assertEquals(
       "There are no freshness policies attached to columns in table: user", mToolOutputStr);
 }
Пример #11
0
  @Test
  public void testIllegalRegister() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));

    runTool(
        new FreshTool(),
        KijiURI.newBuilder(getKiji().getURI())
            .withTableName("user")
            .withColumnNames(Lists.newArrayList("info:name"))
            .build()
            .toString(),
        "--do=register",
        "--policy-class=org.kiji..badname",
        "--policy-state={\"shelfLife\":10}",
        "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer",
        "--interactive=false");

    assertEquals(
        "BAD_POLICY_NAME: java.lang.IllegalArgumentException: Policy class name: "
            + "org.kiji..badname is not a valid Java class identifier.",
        mToolOutputLines[0]);

    runTool(
        new FreshTool(),
        KijiURI.newBuilder(getKiji().getURI())
            .withTableName("user")
            .withColumnNames(Lists.newArrayList("info:name"))
            .build()
            .toString(),
        "--do=register",
        "--policy-class=org.kiji.class",
        "--policy-state={\"shelfLife\":10}",
        "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer.",
        "--interactive=false");

    assertEquals(
        "BAD_PRODUCER_NAME: java.lang.IllegalArgumentException: Producer class name: org."
            + "kiji.scoring.tools.TestFreshTool$TestProducer. is not a valid Java class identifier.",
        mToolOutputLines[0]);
  }
    /**
     * Creates a new record reader that scans over a subset of rows from a Kiji table. The record
     * reader will scan over rows in the table specified in the provided input split, subject to row
     * limits specified in the data request serialized into the specified configuration.
     *
     * @param split for the MapReduce task that will use this record reader. The split specifies a
     *     subset of rows from a Kiji table.
     * @param configuration for the MapReduce job using this record reader. The configuration should
     *     specify the input Kiji table through the configuration variable {@link
     *     KijiConfKeys#KIJI_INPUT_TABLE_URI} and a serialized {@link KijiDataRequest} through the
     *     configuration variable {@link KijiConfKeys#KIJI_INPUT_DATA_REQUEST}.
     * @throws IOException if there is a problem constructing the record reader and opening the
     *     resources it requires.
     */
    public KijiTableRecordReader(InputSplit split, Configuration configuration) throws IOException {
      // Get data request from the job configuration.
      final String dataRequestB64 = configuration.get(KijiConfKeys.KIJI_INPUT_DATA_REQUEST);
      Preconditions.checkNotNull(dataRequestB64, "Missing data request in job configuration.");
      final byte[] dataRequestBytes = Base64.decodeBase64(Bytes.toBytes(dataRequestB64));
      mDataRequest = (KijiDataRequest) SerializationUtils.deserialize(dataRequestBytes);

      // Open connections to Kiji.
      assert split instanceof KijiTableSplit;
      mSplit = (KijiTableSplit) split;

      final KijiURI inputURI =
          KijiURI.newBuilder(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI)).build();
      final KijiScannerOptions scannerOptions =
          new KijiScannerOptions()
              .setStartRow(new HBaseEntityId(mSplit.getStartRow()))
              .setStopRow(new HBaseEntityId(mSplit.getEndRow()));
      mKiji = Kiji.Factory.open(inputURI, configuration);
      mTable = mKiji.openTable(inputURI.getTable());
      mReader = mTable.openTableReader();
      mScanner = mReader.getScanner(mDataRequest, scannerOptions);
      mIterator = mScanner.iterator();
    }
Пример #13
0
  @Test
  public void testCreateTableWithInvalidSchemaClassInLayout() throws Exception {
    final TableLayoutDesc layout = KijiTableLayouts.getLayout(KijiTableLayouts.INVALID_SCHEMA);
    final File layoutFile = getTempLayoutFile(layout);
    final KijiURI tableURI =
        KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();

    assertEquals(
        BaseTool.FAILURE,
        runTool(new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile));
    assertEquals(2, mToolOutputLines.length);
    assertTrue(
        mToolOutputLines[1].startsWith(
            "Error: Schema with type 'class' must be a valid Java identifier."));
  }
Пример #14
0
  @Test
  public void testUnregisterAll() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen());
    manager.storePolicy("user", "info:visits", TestProducer.class, new NeverFreshen());

    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(),
            "--do=unregister-all"));
    assertEquals(0, manager.retrievePolicies("user").size());
    assertEquals("All freshness policies removed from table: user", mToolOutputStr);
  }
Пример #15
0
  /** {@inheritDoc} */
  @Override
  public boolean equals(Object obj) {
    if (null == obj) {
      return false;
    }
    if (obj == this) {
      return true;
    }
    if (!getClass().equals(obj.getClass())) {
      return false;
    }
    final Kiji other = (Kiji) obj;

    // Equal if the two instances have the same URI:
    return mURI.equals(other.getURI());
  }
  /**
   * Generates a split for a given table.
   *
   * @param tableURI URI of the Kiji table to split.
   * @param nsplits Number of splits.
   * @param conf Base Hadoop configuration used to open the Kiji instance.
   * @return a list of split start keys, as HFileKeyValue (with no value, just the keys).
   * @throws IOException on I/O error.
   */
  private static List<HFileKeyValue> makeTableKeySplit(
      KijiURI tableURI, int nsplits, Configuration conf) throws IOException {
    final Kiji kiji = Kiji.Factory.open(tableURI, conf);
    try {
      final KijiTable table = kiji.openTable(tableURI.getTable());
      try {
        if (NUM_SPLITS_AUTO == nsplits) {
          final List<HFileKeyValue> startKeys = Lists.newArrayList();
          for (KijiRegion region : table.getRegions()) {
            startKeys.add(HFileKeyValue.createFromRowKey(region.getStartKey()));
          }
          return startKeys;

        } else {
          switch (KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat())) {
            case RAW:
              {
                // The user has explicitly specified how many HFiles to create, but this is not
                // possible when row key hashing is disabled.
                throw new JobConfigurationException(
                    String.format(
                        "Table '%s' has row key hashing disabled, so the number of HFile splits must be"
                            + "determined by the number of HRegions in the HTable. "
                            + "Use an HFileMapReduceJobOutput constructor that enables auto splitting.",
                        table.getName()));
              }
            case FORMATTED:
            case HASH:
            case HASH_PREFIX:
              {
                // Those cases are supported:
                break;
              }
            default:
              throw new RuntimeException(
                  "Unhandled row key encoding: "
                      + KijiTableLayout.getEncoding(table.getLayout().getDesc().getKeysFormat()));
          }
          return generateEvenStartKeys(nsplits);
        }
      } finally {
        ResourceUtils.releaseOrLog(table);
      }
    } finally {
      ResourceUtils.releaseOrLog(kiji);
    }
  }
  public GatherReduceDriverTemplate(String outputFilePath, String inputTableName) {

    PropertyConfigurator.configure(
        this.getClass().getResourceAsStream(Constants.LOG4J_PROPERTIES_FILE_PATH));

    try {
      Configuration hBaseConfiguration =
          HBaseConfiguration.addHbaseResources(new Configuration(true));

      hBaseConfiguration.set("mapred.textoutputformat.separator", "|");
      hBaseConfiguration.setInt("hbase.client.scanner.caching", 1000);

      KijiURI tableUri =
          KijiURI.newBuilder(String.format("kiji://.env/default/%s", inputTableName)).build();

      String additionalJarsPath = "";

      try {
        additionalJarsPath =
            InetAddress.getLocalHost().getHostName().equals("master")
                ? GlobalConstants.ADDTIONAL_JARS_PATH_KIJI_CLUSTER
                : GlobalConstants.ADDTIONAL_JARS_PATH_BENTO;
      } catch (UnknownHostException unknownHostException) {
        logger.error(unknownHostException);
        unknownHostException.printStackTrace();
        System.exit(-1);
      }

      this.mapReduceJob =
          KijiGatherJobBuilder.create()
              .withConf(hBaseConfiguration)
              .withGatherer(GathererTemplate.class)
              .withReducer(ReducerTemplate.class)
              .withInputTable(tableUri)
              .withOutput(new TextMapReduceJobOutput(new Path(outputFilePath), 1))
              .addJarDirectory(new Path(additionalJarsPath))
              .build();

    } catch (IOException ioException) {
      logger.error("IO Exception while configuring MapReduce Job", ioException);
      System.exit(1);
    } catch (Exception unknownException) {
      logger.error("Unknown Exception while configuring MapReduce Job", unknownException);
      System.exit(1);
    }
  }
Пример #18
0
 @Test
 public void testRetrieveEmpty() throws Exception {
   getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
   assertEquals(
       BaseTool.SUCCESS,
       runTool(
           new FreshTool(),
           KijiURI.newBuilder(getKiji().getURI())
               .withTableName("user")
               .withColumnNames(Lists.newArrayList("info:name"))
               .build()
               .toString(),
           "--do=retrieve"));
   assertEquals(
       "There is no freshness policy attached to column: info:name in table: user",
       mToolOutputStr);
 }
Пример #19
0
  @Test
  public void testUnregister() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen());

    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI())
                .withTableName("user")
                .withColumnNames(Lists.newArrayList("info:name"))
                .build()
                .toString(),
            "--do=unregister"));
    assertEquals(null, manager.retrievePolicy("user", "info:name"));
    assertEquals("Freshness policy removed from column: info:name in table user", mToolOutputStr);
  }
Пример #20
0
  /** {@inheritDoc} */
  @Override
  public void deleteTable(String tableName) throws IOException {
    final State state = mState.get();
    Preconditions.checkState(
        state == State.OPEN, "Cannot delete table in Kiji instance %s in state %s.", this, state);
    // Delete from HBase.
    String hbaseTable =
        KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName).toString();
    getHBaseAdmin().disableTable(hbaseTable);
    getHBaseAdmin().deleteTable(hbaseTable);

    // Delete from the meta table.
    getMetaTable().deleteTable(tableName);

    // If the table persists immediately after deletion attempt, then give up.
    if (getHBaseAdmin().tableExists(hbaseTable)) {
      LOG.warn("HBase table " + hbaseTable + " survives deletion attempt. Giving up...");
    }
  }
Пример #21
0
  @Test
  public void testCreateHashedTableWithNumRegions() throws Exception {
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
    final File layoutFile = getTempLayoutFile(layout);
    final KijiURI tableURI =
        KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();

    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new CreateTableTool(),
            "--table=" + tableURI,
            "--layout=" + layoutFile,
            "--num-regions=" + 2,
            "--debug"));
    assertEquals(2, mToolOutputLines.length);
    assertTrue(mToolOutputLines[0].startsWith("Parsing table layout: "));
    assertTrue(mToolOutputLines[1].startsWith("Creating Kiji table"));
  }
Пример #22
0
  /** {@inheritDoc} */
  @Deprecated
  @Override
  public void createTable(String tableName, KijiTableLayout tableLayout, byte[][] splitKeys)
      throws IOException {
    if (getMetaTable().tableExists(tableName)) {
      final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
      throw new KijiAlreadyExistsException(
          String.format("Kiji table '%s' already exists.", tableURI), tableURI);
    }

    if (!tableName.equals(tableLayout.getName())) {
      throw new RuntimeException(
          String.format(
              "Table name from layout descriptor '%s' does match table name '%s'.",
              tableLayout.getName(), tableName));
    }

    createTable(tableLayout.getDesc(), splitKeys);
  }
Пример #23
0
  @Test
  public void testCreateUnhashedTableWithNumRegions() throws Exception {
    final KijiTableLayout layout =
        KijiTableLayout.newLayout(KijiTableLayouts.getFooUnhashedTestLayout());
    final File layoutFile = getTempLayoutFile(layout);
    final KijiURI tableURI =
        KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();

    assertEquals(
        BaseTool.FAILURE,
        runTool(
            new CreateTableTool(),
            "--table=" + tableURI,
            "--layout=" + layoutFile,
            "--num-regions=4"));
    assertEquals(3, mToolOutputLines.length);
    assertTrue(
        mToolOutputLines[2].startsWith(
            "Error: May not use numRegions > 1 if row key hashing is disabled in the layout"));
  }
Пример #24
0
  @Test
  public void testCreateHashedTableWithSplitKeys() throws Exception {
    final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST);
    final File layoutFile = getTempLayoutFile(layout);
    final KijiURI tableURI =
        KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();

    final String splitKeyFile =
        getClass().getClassLoader().getResource(REGION_SPLIT_KEY_FILE).getPath();

    assertEquals(
        BaseTool.FAILURE,
        runTool(
            new CreateTableTool(),
            "--table=" + tableURI,
            "--layout=" + layoutFile,
            "--split-key-file=file://" + splitKeyFile));
    assertEquals(3, mToolOutputLines.length);
    assertTrue(
        mToolOutputLines[2].startsWith(
            "Error: Row key hashing is enabled for the table. Use --num-regions=N instead."));
  }
Пример #25
0
  @Test
  public void testCreateUnhashedTableWithSplitKeys() throws Exception {
    final KijiTableLayout layout =
        KijiTableLayout.newLayout(KijiTableLayouts.getFooUnhashedTestLayout());
    final File layoutFile = getTempLayoutFile(layout);
    final KijiURI tableURI =
        KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build();

    final String splitKeyFile =
        getClass().getClassLoader().getResource(REGION_SPLIT_KEY_FILE).getPath();

    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new CreateTableTool(),
            "--table=" + tableURI,
            "--layout=" + layoutFile,
            "--split-key-file=file://" + splitKeyFile,
            "--debug"));
    assertEquals(2, mToolOutputLines.length);
    assertTrue(mToolOutputLines[0].startsWith("Parsing table layout: "));
    assertTrue(mToolOutputLines[1].startsWith("Creating Kiji table"));
  }
Пример #26
0
  /** {@inheritDoc} */
  @Override
  public void createTable(TableLayoutDesc tableLayout, byte[][] splitKeys) throws IOException {
    final State state = mState.get();
    Preconditions.checkState(
        state == State.OPEN, "Cannot create table in Kiji instance %s in state %s.", this, state);

    final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build();
    LOG.debug("Creating Kiji table '{}'.", tableURI);

    ensureValidationCompatibility(tableLayout);

    // If security is enabled, apply the permissions to the new table.
    if (isSecurityEnabled()) {
      getSecurityManager().lock();
      try {
        createTableUnchecked(tableLayout, splitKeys);
        getSecurityManager().applyPermissionsToNewTable(tableURI);
      } finally {
        getSecurityManager().unlock();
      }
    } else {
      createTableUnchecked(tableLayout, splitKeys);
    }
  }
Пример #27
0
  @Test
  public void testRegister() throws Exception {
    getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST));
    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            KijiURI.newBuilder(getKiji().getURI())
                .withTableName("user")
                .withColumnNames(Lists.newArrayList("info:name"))
                .build()
                .toString(),
            "--do=register",
            "--policy-class=org.kiji.scoring.lib.ShelfLife",
            "--policy-state={\"shelfLife\":10}",
            "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer"));

    final KijiFreshnessPolicyRecord record =
        KijiFreshnessPolicyRecord.newBuilder()
            .setRecordVersion("policyrecord-0.1.0")
            .setProducerClass(TestProducer.class.getName())
            .setFreshnessPolicyClass(
                Class.forName("org.kiji.scoring.lib.ShelfLife")
                    .asSubclass(KijiFreshnessPolicy.class)
                    .getName())
            .setFreshnessPolicyState("{\"shelfLife\":10}")
            .build();
    final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji());
    assertEquals(record, manager.retrievePolicy("user", "info:name"));
    assertEquals(
        "Freshness policy: org.kiji.scoring.lib.ShelfLife with state: {\"shelfLife\":10} "
            + "and producer: org.kiji.scoring.tools.TestFreshTool$TestProducer\n"
            + "attached to column: info:name in table: user",
        mToolOutputStr);

    // Test another ordering for arguments
    assertEquals(
        BaseTool.SUCCESS,
        runTool(
            new FreshTool(),
            "--do=register",
            KijiURI.newBuilder(getKiji().getURI())
                .withTableName("user")
                .withColumnNames(Lists.newArrayList("info:visits"))
                .build()
                .toString(),
            "--policy-state={\"shelfLife\":10}",
            "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer",
            "--policy-class=org.kiji.scoring.lib.ShelfLife"));

    final KijiFreshnessPolicyRecord record2 =
        KijiFreshnessPolicyRecord.newBuilder()
            .setRecordVersion("policyrecord-0.1.0")
            .setProducerClass(TestProducer.class.getName())
            .setFreshnessPolicyClass(
                Class.forName("org.kiji.scoring.lib.ShelfLife")
                    .asSubclass(KijiFreshnessPolicy.class)
                    .getName())
            .setFreshnessPolicyState("{\"shelfLife\":10}")
            .build();
    assertEquals(record2, manager.retrievePolicy("user", "info:visits"));
    assertEquals(
        "Freshness policy: org.kiji.scoring.lib.ShelfLife with state: {\"shelfLife\":10} "
            + "and producer: org.kiji.scoring.tools.TestFreshTool$TestProducer\n"
            + "attached to column: info:visits in table: user",
        mToolOutputStr);
  }
Пример #28
0
  /** {@inheritDoc} */
  @Override
  public KijiTableLayout modifyTableLayout(
      TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException {
    final State state = mState.get();
    Preconditions.checkState(
        state == State.OPEN,
        "Cannot modify table layout in Kiji instance %s in state %s.",
        this,
        state);
    Preconditions.checkNotNull(update);

    ensureValidationCompatibility(update);

    if (dryRun && (null == printStream)) {
      printStream = System.out;
    }

    final KijiMetaTable metaTable = getMetaTable();

    final String tableName = update.getName();
    // Throws a KijiTableNotFoundException if there is no table.
    metaTable.getTableLayout(tableName);

    final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build();
    LOG.debug("Applying layout update {} on table {}", update, tableURI);

    KijiTableLayout newLayout = null;

    if (dryRun) {
      // Process column ids and perform validation, but don't actually update the meta table.
      final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1);
      final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0);
      newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout);
    } else {
      // Actually set it.
      if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) {
        try {
          // Use ZooKeeper to inform all watchers that a new table layout is available.
          final HBaseTableLayoutUpdater updater =
              new HBaseTableLayoutUpdater(this, tableURI, update);
          try {
            updater.update();
            newLayout = updater.getNewLayout();
          } finally {
            updater.close();
          }
        } catch (KeeperException ke) {
          throw new IOException(ke);
        }
      } else {
        // System versions before system-2.0 do not enforce table layout update consistency or
        // validation.
        newLayout = metaTable.updateTableLayout(tableName, update);
      }
    }
    Preconditions.checkState(newLayout != null);

    if (dryRun) {
      printStream.println("This table layout is valid.");
    }

    LOG.debug("Computing new HBase schema");
    final HTableSchemaTranslator translator = new HTableSchemaTranslator();
    final HTableDescriptor newTableDescriptor =
        translator.toHTableDescriptor(mURI.getInstance(), newLayout);

    LOG.debug("Reading existing HBase schema");
    final KijiManagedHBaseTableName hbaseTableName =
        KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName);
    HTableDescriptor currentTableDescriptor = null;
    byte[] tableNameAsBytes = hbaseTableName.toBytes();
    try {
      currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes);
    } catch (TableNotFoundException tnfe) {
      if (!dryRun) {
        throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception.
      }
    }
    if (currentTableDescriptor == null) {
      if (dryRun) {
        printStream.println("Would create new table: " + tableName);
        currentTableDescriptor =
            HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName);
      } else {
        throw new RuntimeException(
            "Table " + hbaseTableName.getKijiTableName() + " does not exist");
      }
    }
    LOG.debug("Existing table descriptor: {}", currentTableDescriptor);
    LOG.debug("New table descriptor: {}", newTableDescriptor);

    LOG.debug("Checking for differences between the new HBase schema and the existing one");
    final HTableDescriptorComparator comparator = new HTableDescriptorComparator();
    if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) {
      LOG.debug("HBase schemas are the same.  No need to change HBase schema");
      if (dryRun) {
        printStream.println("This layout does not require any physical table schema changes.");
      }
    } else {
      LOG.debug("HBase schema must be changed, but no columns will be deleted");

      if (dryRun) {
        printStream.println("Changes caused by this table layout:");
      } else {
        LOG.debug("Disabling HBase table");
        getHBaseAdmin().disableTable(hbaseTableName.toString());
      }

      for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) {
        final String columnName = Bytes.toString(newColumnDescriptor.getName());
        final ColumnId columnId = ColumnId.fromString(columnName);
        final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId);
        final HColumnDescriptor currentColumnDescriptor =
            currentTableDescriptor.getFamily(newColumnDescriptor.getName());
        if (null == currentColumnDescriptor) {
          if (dryRun) {
            printStream.println("  Creating new locality group: " + lgName);
          } else {
            LOG.debug("Creating new column " + columnName);
            getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor);
          }
        } else if (!newColumnDescriptor.equals(currentColumnDescriptor)) {
          if (dryRun) {
            printStream.println("  Modifying locality group: " + lgName);
          } else {
            LOG.debug("Modifying column " + columnName);
            getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor);
          }
        } else {
          LOG.debug("No changes needed for column " + columnName);
        }
      }

      if (dryRun) {
        if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
          printStream.printf(
              "  Changing max_filesize from %d to %d: %n",
              currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize());
        }
        if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) {
          printStream.printf(
              "  Changing memstore_flushsize from %d to %d: %n",
              currentTableDescriptor.getMemStoreFlushSize(),
              newTableDescriptor.getMemStoreFlushSize());
        }
      } else {
        LOG.debug("Modifying table descriptor");
        getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor);
      }

      if (!dryRun) {
        LOG.debug("Re-enabling HBase table");
        getHBaseAdmin().enableTable(hbaseTableName.toString());
      }
    }

    return newLayout;
  }
Пример #29
0
  /**
   * Creates a new <code>HBaseKiji</code> instance.
   *
   * <p>Should only be used by Kiji.Factory.open().
   *
   * <p>Caller does not need to use retain(), but must call release() when done with it.
   *
   * @param kijiURI the KijiURI.
   * @param conf Hadoop Configuration. Deep copied internally.
   * @param tableFactory HTableInterface factory.
   * @param lockFactory Factory for locks.
   * @throws IOException on I/O error.
   */
  HBaseKiji(
      KijiURI kijiURI,
      Configuration conf,
      HTableInterfaceFactory tableFactory,
      LockFactory lockFactory)
      throws IOException {

    mConstructorStack = CLEANUP_LOG.isDebugEnabled() ? Debug.getStackTrace() : null;

    // Deep copy the configuration.
    mConf = new Configuration(conf);

    // Validate arguments.
    mHTableFactory = Preconditions.checkNotNull(tableFactory);
    mLockFactory = Preconditions.checkNotNull(lockFactory);
    mURI = Preconditions.checkNotNull(kijiURI);

    // Configure the ZooKeeper quorum:
    mConf.setStrings("hbase.zookeeper.quorum", mURI.getZookeeperQuorum().toArray(new String[0]));
    mConf.setInt("hbase.zookeeper.property.clientPort", mURI.getZookeeperClientPort());

    // Check for an instance name.
    Preconditions.checkArgument(
        mURI.getInstance() != null, "KijiURI '%s' does not specify a Kiji instance name.", mURI);

    if (LOG.isDebugEnabled()) {
      Debug.logConfiguration(mConf);
      LOG.debug(
          "Opening kiji instance '{}'"
              + " with client software version '{}'"
              + " and client data version '{}'.",
          mURI,
          VersionInfo.getSoftwareVersion(),
          VersionInfo.getClientDataVersion());
    }

    // Load these lazily.
    mSchemaTable = null;
    mMetaTable = null;
    mSecurityManager = null;

    mSystemTable = new HBaseSystemTable(mURI, mConf, mHTableFactory);

    mRetainCount.set(1);
    final State oldState = mState.getAndSet(State.OPEN);
    Preconditions.checkState(
        oldState == State.UNINITIALIZED, "Cannot open Kiji instance in state %s.", oldState);
    LOG.debug("Kiji instance '{}' is now opened.", mURI);

    mSystemVersion = mSystemTable.getDataVersion();
    LOG.debug("Kiji instance '{}' has data version '{}'.", mURI, mSystemVersion);

    // Make sure the data version for the client matches the cluster.
    LOG.debug("Validating version for Kiji instance '{}'.", mURI);
    try {
      VersionInfo.validateVersion(this);
    } catch (IOException ioe) {
      // If an IOException occurred the object will not be constructed so need to clean it up.
      close();
      throw ioe;
    } catch (KijiNotInstalledException kie) {
      // Some clients handle this unchecked Exception so do the same here.
      close();
      throw kie;
    }

    // TODO(SCHEMA-491) Share ZooKeeperClient instances when possible.
    if (mSystemVersion.compareTo(Versions.MIN_SYS_VER_FOR_LAYOUT_VALIDATION) >= 0) {
      // system-2.0 clients must connect to ZooKeeper:
      //  - to register themselves as table users;
      //  - to receive table layout updates.
      mZKClient = HBaseFactory.Provider.get().getZooKeeperClient(mURI);
      try {
        mMonitor = new ZooKeeperMonitor(mZKClient);
        mMonitor.registerInstanceUser(mURI, mKijiClientId, mSystemVersion.toString());
      } catch (KeeperException ke) {
        // Unrecoverable KeeperException:
        throw new IOException(ke);
      }
    } else {
      // system-1.x clients do not need a ZooKeeper connection.
      mZKClient = null;
      mMonitor = null;
    }
  }
Пример #30
0
 /** {@inheritDoc} */
 @Override
 public int hashCode() {
   return mURI.hashCode();
 }