@Test public void testRetrieveAll() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji()); manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen()); manager.storePolicy("user", "info:visits", TestProducer.class, new NeverFreshen()); LOG.info(KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString()); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(), "--do=retrieve-all")); assertEquals( "Freshness policy attached to column: info:visits\n" + " Freshness policy class: org.kiji.scoring.lib.NeverFreshen\n" + " Freshness policy state: \n" + " Producer class: org.kiji.scoring.tools.TestFreshTool$TestProducer\n" + "Freshness policy attached to column: info:name\n" + " Freshness policy class: org.kiji.scoring.lib.NeverFreshen\n" + " Freshness policy state: \n" + " Producer class: org.kiji.scoring.tools.TestFreshTool$TestProducer\n", mToolOutputStr); }
@Test public void testListIntances() throws Exception { final Kiji kiji = getKiji(); final KijiURI hbaseURI = KijiURI.newBuilder(kiji.getURI()).withInstanceName(null).build(); final LsTool ls = new LsTool(); assertEquals(BaseTool.SUCCESS, runTool(ls, "--kiji=" + hbaseURI)); assertEquals(1, mToolOutputLines.length); assertEquals(kiji.getURI(), KijiURI.newBuilder(mToolOutputLines[0]).build()); }
/** * Gets a set of input splits for a MapReduce job running over a Kiji table. One split is created * per region in the input Kiji table. * * @param configuration of the job using the splits. The configuration should specify the input * Kiji table being used, through the configuration variable {@link * KijiConfKeys#KIJI_INPUT_TABLE_URI}. * @param numSplits desired for the job. This framework hint is ignored by this method. * @return an array of input splits to be operated on in the MapReduce job. * @throws IOException if an I/O error occurs while communicating with HBase to determine the * regions in the Kiji table. */ @Override public InputSplit[] getSplits(JobConf configuration, int numSplits) throws IOException { final String uriString = Preconditions.checkNotNull(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI)); final KijiURI inputTableURI = KijiURI.newBuilder(uriString).build(); final Kiji kiji = Kiji.Factory.open(inputTableURI, configuration); try { final KijiTable table = kiji.openTable(inputTableURI.getTable()); try { final HTableInterface htable = HBaseKijiTable.downcast(table).getHTable(); final List<InputSplit> splits = Lists.newArrayList(); for (KijiRegion region : table.getRegions()) { final byte[] startKey = region.getStartKey(); // TODO(KIJIMR-65): For now pick the first available location (ie. region server), if any. final String location = region.getLocations().isEmpty() ? null : region.getLocations().iterator().next(); final TableSplit tableSplit = new TableSplit(htable.getTableName(), startKey, region.getEndKey(), location); splits.add(new KijiTableSplit(tableSplit)); } return splits.toArray(new InputSplit[0]); } finally { table.release(); } } finally { kiji.release(); } }
/** * {@inheritDoc} * * @throws IOException when instance in configuration can not be opened and closed. */ @Override public final void run(final KijiRESTConfiguration configuration, final Environment environment) throws IOException { final KijiURI clusterURI = KijiURI.newBuilder(configuration.getClusterURI()).build(); // Load specified instances and health checks for each. final Set<KijiURI> instances = InstanceUtil.getInstances( clusterURI, new InstancesMapTo<KijiURI>() { public KijiURI apply(KijiURI instanceURI) throws IOException { InstanceUtil.openAndCloseInstance(instanceURI); LOG.info("Loading instance {} upon startup.", instanceURI.toOrderedString()); environment.addHealthCheck(new InstanceHealthCheck(instanceURI)); return instanceURI; } }); final ManagedKijiClient managedKijiClient = new ManagedKijiClient(instances); environment.manage(managedKijiClient); // Remove all built-in Dropwizard ExceptionHandler. // Always depend on custom ones. // Inspired by Jeremy Whitlock's suggestion on thoughtspark.org. Set<Object> jerseyResources = environment.getJerseyResourceConfig().getSingletons(); Iterator<Object> jerseyResourcesIterator = jerseyResources.iterator(); while (jerseyResourcesIterator.hasNext()) { Object jerseyResource = jerseyResourcesIterator.next(); if (jerseyResource instanceof ExceptionMapper && jerseyResource.getClass().getName().startsWith("com.yammer.dropwizard.jersey")) { jerseyResourcesIterator.remove(); } } // Update instances periodically. final RefreshInstances instanceRefresher = new RefreshInstances(clusterURI, managedKijiClient); ScheduledExecutorService scheduler = environment.managedScheduledExecutorService("instance_refresh_scheduler", 1); scheduler.scheduleAtFixedRate( instanceRefresher, INSTANCE_REFRESH_PERIOD_MINUTES, // Start a period from now. INSTANCE_REFRESH_PERIOD_MINUTES, MINUTES); // Load admin task to manually update instances. environment.addTask(new RefreshInstancesTask(instanceRefresher)); // Load resources. for (KijiRestPlugin plugin : Lookups.get(KijiRestPlugin.class)) { LOG.info("Loading plugin {}", plugin.getClass()); plugin.install(managedKijiClient, configuration, environment); } // Allow global CORS filter. CORS off by default. if (configuration.getCORS()) { environment.addFilter( CrossOriginFilter.class, configuration.getHttpConfiguration().getRootPath()); LOG.info("Global cross-origin resource sharing is allowed."); } }
@Test public void testValidate() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji()); manager.storePolicy("user", "info:name", TestProducer.class, new AlwaysFreshen()); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(), "--do=validate-all")); KijiFreshnessPolicyRecord record = KijiFreshnessPolicyRecord.newBuilder() .setRecordVersion(ProtocolVersion.parse("policyrecord-0.1").toCanonicalString()) .setProducerClass(TestProducer.class.getName()) .setFreshnessPolicyClass(AlwaysFreshen.class.getName()) .setFreshnessPolicyState("") .build(); final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); final EncoderFactory encoderFactory = EncoderFactory.get(); Encoder encoder = encoderFactory.directBinaryEncoder(outputStream, null); final DatumWriter<KijiFreshnessPolicyRecord> recordWriter = new SpecificDatumWriter<KijiFreshnessPolicyRecord>(KijiFreshnessPolicyRecord.SCHEMA$); recordWriter.write(record, encoder); getKiji() .getMetaTable() .putValue("user", "kiji.scoring.fresh.columnName", outputStream.toByteArray()); assertEquals( BaseTool.FAILURE, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(), "--do=validate-all")); assertEquals( "Freshness policy attached to column: columnName is not valid.", mToolOutputLines[1]); assertEquals( "NO_FAMILY_IN_TABLE: java.lang.IllegalArgumentException: Table: user does not " + "contain family: columnName", mToolOutputLines[2]); }
@Test public void testListInstances() throws Exception { final Kiji kiji = getKiji(); final KijiURI hbaseURI = KijiURI.newBuilder(kiji.getURI()).withInstanceName(null).build(); final LsTool ls = new LsTool(); assertEquals(BaseTool.SUCCESS, runTool(ls, hbaseURI.toString())); final Set<String> instances = Sets.newHashSet(mToolOutputLines); assertTrue(instances.contains(kiji.getURI().toString())); }
@Test public void testRetrieveAllEmpty() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(), "--do=retrieve-all")); assertEquals( "There are no freshness policies attached to columns in table: user", mToolOutputStr); }
@Test public void testIllegalRegister() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()) .withTableName("user") .withColumnNames(Lists.newArrayList("info:name")) .build() .toString(), "--do=register", "--policy-class=org.kiji..badname", "--policy-state={\"shelfLife\":10}", "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer", "--interactive=false"); assertEquals( "BAD_POLICY_NAME: java.lang.IllegalArgumentException: Policy class name: " + "org.kiji..badname is not a valid Java class identifier.", mToolOutputLines[0]); runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()) .withTableName("user") .withColumnNames(Lists.newArrayList("info:name")) .build() .toString(), "--do=register", "--policy-class=org.kiji.class", "--policy-state={\"shelfLife\":10}", "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer.", "--interactive=false"); assertEquals( "BAD_PRODUCER_NAME: java.lang.IllegalArgumentException: Producer class name: org." + "kiji.scoring.tools.TestFreshTool$TestProducer. is not a valid Java class identifier.", mToolOutputLines[0]); }
/** * Creates a Kiji table in an HBase instance, without checking for validation compatibility and * without applying permissions. * * @param tableLayout The initial layout of the table (with unassigned column ids). * @param splitKeys The initial key boundaries between regions. There will be splitKeys + 1 * regions created. Pass null to specify the default single region. * @throws IOException on I/O error. * @throws KijiAlreadyExistsException if the table already exists. */ private void createTableUnchecked(TableLayoutDesc tableLayout, byte[][] splitKeys) throws IOException { final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build(); // This will validate the layout and may throw an InvalidLayoutException. final KijiTableLayout kijiTableLayout = KijiTableLayout.newLayout(tableLayout); if (getMetaTable().tableExists(tableLayout.getName())) { throw new KijiAlreadyExistsException( String.format("Kiji table '%s' already exists.", tableURI), tableURI); } if (tableLayout.getKeysFormat() instanceof RowKeyFormat) { LOG.warn("Usage of 'RowKeyFormat' is deprecated. New tables should use 'RowKeyFormat2'."); } getMetaTable().updateTableLayout(tableLayout.getName(), tableLayout); if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) { // system-2.0 clients retrieve the table layout from ZooKeeper as a stream of notifications. // Invariant: ZooKeeper hold the most recent layout of the table. LOG.debug("Writing initial table layout in ZooKeeper for table {}.", tableURI); try { final ZooKeeperMonitor monitor = new ZooKeeperMonitor(mZKClient); try { final byte[] layoutId = Bytes.toBytes(kijiTableLayout.getDesc().getLayoutId()); monitor.notifyNewTableLayout(tableURI, layoutId, -1); } finally { monitor.close(); } } catch (KeeperException ke) { throw new IOException(ke); } } try { final HTableSchemaTranslator translator = new HTableSchemaTranslator(); final HTableDescriptor desc = translator.toHTableDescriptor(mURI.getInstance(), kijiTableLayout); LOG.debug("Creating HBase table '{}'.", desc.getNameAsString()); if (null != splitKeys) { getHBaseAdmin().createTable(desc, splitKeys); } else { getHBaseAdmin().createTable(desc); } } catch (TableExistsException tee) { throw new KijiAlreadyExistsException( String.format("Kiji table '%s' already exists.", tableURI), tableURI); } }
@Test public void testCreateTableWithInvalidSchemaClassInLayout() throws Exception { final TableLayoutDesc layout = KijiTableLayouts.getLayout(KijiTableLayouts.INVALID_SCHEMA); final File layoutFile = getTempLayoutFile(layout); final KijiURI tableURI = KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build(); assertEquals( BaseTool.FAILURE, runTool(new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile)); assertEquals(2, mToolOutputLines.length); assertTrue( mToolOutputLines[1].startsWith( "Error: Schema with type 'class' must be a valid Java identifier.")); }
@Test public void testUnregisterAll() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji()); manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen()); manager.storePolicy("user", "info:visits", TestProducer.class, new NeverFreshen()); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()).withTableName("user").build().toString(), "--do=unregister-all")); assertEquals(0, manager.retrievePolicies("user").size()); assertEquals("All freshness policies removed from table: user", mToolOutputStr); }
public GatherReduceDriverTemplate(String outputFilePath, String inputTableName) { PropertyConfigurator.configure( this.getClass().getResourceAsStream(Constants.LOG4J_PROPERTIES_FILE_PATH)); try { Configuration hBaseConfiguration = HBaseConfiguration.addHbaseResources(new Configuration(true)); hBaseConfiguration.set("mapred.textoutputformat.separator", "|"); hBaseConfiguration.setInt("hbase.client.scanner.caching", 1000); KijiURI tableUri = KijiURI.newBuilder(String.format("kiji://.env/default/%s", inputTableName)).build(); String additionalJarsPath = ""; try { additionalJarsPath = InetAddress.getLocalHost().getHostName().equals("master") ? GlobalConstants.ADDTIONAL_JARS_PATH_KIJI_CLUSTER : GlobalConstants.ADDTIONAL_JARS_PATH_BENTO; } catch (UnknownHostException unknownHostException) { logger.error(unknownHostException); unknownHostException.printStackTrace(); System.exit(-1); } this.mapReduceJob = KijiGatherJobBuilder.create() .withConf(hBaseConfiguration) .withGatherer(GathererTemplate.class) .withReducer(ReducerTemplate.class) .withInputTable(tableUri) .withOutput(new TextMapReduceJobOutput(new Path(outputFilePath), 1)) .addJarDirectory(new Path(additionalJarsPath)) .build(); } catch (IOException ioException) { logger.error("IO Exception while configuring MapReduce Job", ioException); System.exit(1); } catch (Exception unknownException) { logger.error("Unknown Exception while configuring MapReduce Job", unknownException); System.exit(1); } }
@Test public void testRetrieveEmpty() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()) .withTableName("user") .withColumnNames(Lists.newArrayList("info:name")) .build() .toString(), "--do=retrieve")); assertEquals( "There is no freshness policy attached to column: info:name in table: user", mToolOutputStr); }
/** * Initializes a new table-wide record writer. * * @param oformat KijiHFileOutputFormat this writer is built from. * @param context Context of the task. * @throws IOException on I/O error. */ public TableRecordWriter(KijiHFileOutputFormat oformat, TaskAttemptContext context) throws IOException { mContext = Preconditions.checkNotNull(context); mConf = mContext.getConfiguration(); mLatestTimestamp = mConf.getLong(CONF_LATEST_TIMESTAMP, System.currentTimeMillis()); mLatestTimestampBytes = toBytes(mLatestTimestamp); mOutputDir = oformat.getDefaultWorkFile(mContext, OUTPUT_EXTENSION); mFileSystem = mOutputDir.getFileSystem(mConf); mTableURI = KijiURI.newBuilder(mConf.get(KijiConfKeys.KIJI_OUTPUT_TABLE_URI)).build(); final Kiji kiji = Kiji.Factory.open(mTableURI, mConf); final KijiTable table = kiji.openTable(mTableURI.getTable()); mLayout = table.getLayout(); ResourceUtils.releaseOrLog(table); ResourceUtils.releaseOrLog(kiji); }
@Test public void testUnregister() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji()); manager.storePolicy("user", "info:name", TestProducer.class, new NeverFreshen()); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()) .withTableName("user") .withColumnNames(Lists.newArrayList("info:name")) .build() .toString(), "--do=unregister")); assertEquals(null, manager.retrievePolicy("user", "info:name")); assertEquals("Freshness policy removed from column: info:name in table user", mToolOutputStr); }
/** {@inheritDoc} */ @Deprecated @Override public void createTable(String tableName, KijiTableLayout tableLayout, byte[][] splitKeys) throws IOException { if (getMetaTable().tableExists(tableName)) { final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build(); throw new KijiAlreadyExistsException( String.format("Kiji table '%s' already exists.", tableURI), tableURI); } if (!tableName.equals(tableLayout.getName())) { throw new RuntimeException( String.format( "Table name from layout descriptor '%s' does match table name '%s'.", tableLayout.getName(), tableName)); } createTable(tableLayout.getDesc(), splitKeys); }
@Test public void testCreateHashedTableWithNumRegions() throws Exception { final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST); final File layoutFile = getTempLayoutFile(layout); final KijiURI tableURI = KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build(); assertEquals( BaseTool.SUCCESS, runTool( new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile, "--num-regions=" + 2, "--debug")); assertEquals(2, mToolOutputLines.length); assertTrue(mToolOutputLines[0].startsWith("Parsing table layout: ")); assertTrue(mToolOutputLines[1].startsWith("Creating Kiji table")); }
@Test public void testCreateUnhashedTableWithNumRegions() throws Exception { final KijiTableLayout layout = KijiTableLayout.newLayout(KijiTableLayouts.getFooUnhashedTestLayout()); final File layoutFile = getTempLayoutFile(layout); final KijiURI tableURI = KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build(); assertEquals( BaseTool.FAILURE, runTool( new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile, "--num-regions=4")); assertEquals(3, mToolOutputLines.length); assertTrue( mToolOutputLines[2].startsWith( "Error: May not use numRegions > 1 if row key hashing is disabled in the layout")); }
@Test public void testCreateHashedTableWithSplitKeys() throws Exception { final KijiTableLayout layout = KijiTableLayouts.getTableLayout(KijiTableLayouts.FOO_TEST); final File layoutFile = getTempLayoutFile(layout); final KijiURI tableURI = KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build(); final String splitKeyFile = getClass().getClassLoader().getResource(REGION_SPLIT_KEY_FILE).getPath(); assertEquals( BaseTool.FAILURE, runTool( new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile, "--split-key-file=file://" + splitKeyFile)); assertEquals(3, mToolOutputLines.length); assertTrue( mToolOutputLines[2].startsWith( "Error: Row key hashing is enabled for the table. Use --num-regions=N instead.")); }
@Test public void testCreateUnhashedTableWithSplitKeys() throws Exception { final KijiTableLayout layout = KijiTableLayout.newLayout(KijiTableLayouts.getFooUnhashedTestLayout()); final File layoutFile = getTempLayoutFile(layout); final KijiURI tableURI = KijiURI.newBuilder(getKiji().getURI()).withTableName(layout.getName()).build(); final String splitKeyFile = getClass().getClassLoader().getResource(REGION_SPLIT_KEY_FILE).getPath(); assertEquals( BaseTool.SUCCESS, runTool( new CreateTableTool(), "--table=" + tableURI, "--layout=" + layoutFile, "--split-key-file=file://" + splitKeyFile, "--debug")); assertEquals(2, mToolOutputLines.length); assertTrue(mToolOutputLines[0].startsWith("Parsing table layout: ")); assertTrue(mToolOutputLines[1].startsWith("Creating Kiji table")); }
/** * Creates a new record reader that scans over a subset of rows from a Kiji table. The record * reader will scan over rows in the table specified in the provided input split, subject to row * limits specified in the data request serialized into the specified configuration. * * @param split for the MapReduce task that will use this record reader. The split specifies a * subset of rows from a Kiji table. * @param configuration for the MapReduce job using this record reader. The configuration should * specify the input Kiji table through the configuration variable {@link * KijiConfKeys#KIJI_INPUT_TABLE_URI} and a serialized {@link KijiDataRequest} through the * configuration variable {@link KijiConfKeys#KIJI_INPUT_DATA_REQUEST}. * @throws IOException if there is a problem constructing the record reader and opening the * resources it requires. */ public KijiTableRecordReader(InputSplit split, Configuration configuration) throws IOException { // Get data request from the job configuration. final String dataRequestB64 = configuration.get(KijiConfKeys.KIJI_INPUT_DATA_REQUEST); Preconditions.checkNotNull(dataRequestB64, "Missing data request in job configuration."); final byte[] dataRequestBytes = Base64.decodeBase64(Bytes.toBytes(dataRequestB64)); mDataRequest = (KijiDataRequest) SerializationUtils.deserialize(dataRequestBytes); // Open connections to Kiji. assert split instanceof KijiTableSplit; mSplit = (KijiTableSplit) split; final KijiURI inputURI = KijiURI.newBuilder(configuration.get(KijiConfKeys.KIJI_INPUT_TABLE_URI)).build(); final KijiScannerOptions scannerOptions = new KijiScannerOptions() .setStartRow(new HBaseEntityId(mSplit.getStartRow())) .setStopRow(new HBaseEntityId(mSplit.getEndRow())); mKiji = Kiji.Factory.open(inputURI, configuration); mTable = mKiji.openTable(inputURI.getTable()); mReader = mTable.openTableReader(); mScanner = mReader.getScanner(mDataRequest, scannerOptions); mIterator = mScanner.iterator(); }
/** {@inheritDoc} */ @Override public void createTable(TableLayoutDesc tableLayout, byte[][] splitKeys) throws IOException { final State state = mState.get(); Preconditions.checkState( state == State.OPEN, "Cannot create table in Kiji instance %s in state %s.", this, state); final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableLayout.getName()).build(); LOG.debug("Creating Kiji table '{}'.", tableURI); ensureValidationCompatibility(tableLayout); // If security is enabled, apply the permissions to the new table. if (isSecurityEnabled()) { getSecurityManager().lock(); try { createTableUnchecked(tableLayout, splitKeys); getSecurityManager().applyPermissionsToNewTable(tableURI); } finally { getSecurityManager().unlock(); } } else { createTableUnchecked(tableLayout, splitKeys); } }
/** {@inheritDoc} */ @Override public KijiTableLayout modifyTableLayout( TableLayoutDesc update, boolean dryRun, PrintStream printStream) throws IOException { final State state = mState.get(); Preconditions.checkState( state == State.OPEN, "Cannot modify table layout in Kiji instance %s in state %s.", this, state); Preconditions.checkNotNull(update); ensureValidationCompatibility(update); if (dryRun && (null == printStream)) { printStream = System.out; } final KijiMetaTable metaTable = getMetaTable(); final String tableName = update.getName(); // Throws a KijiTableNotFoundException if there is no table. metaTable.getTableLayout(tableName); final KijiURI tableURI = KijiURI.newBuilder(mURI).withTableName(tableName).build(); LOG.debug("Applying layout update {} on table {}", update, tableURI); KijiTableLayout newLayout = null; if (dryRun) { // Process column ids and perform validation, but don't actually update the meta table. final List<KijiTableLayout> layouts = metaTable.getTableLayoutVersions(tableName, 1); final KijiTableLayout currentLayout = layouts.isEmpty() ? null : layouts.get(0); newLayout = KijiTableLayout.createUpdatedLayout(update, currentLayout); } else { // Actually set it. if (mSystemVersion.compareTo(Versions.SYSTEM_2_0) >= 0) { try { // Use ZooKeeper to inform all watchers that a new table layout is available. final HBaseTableLayoutUpdater updater = new HBaseTableLayoutUpdater(this, tableURI, update); try { updater.update(); newLayout = updater.getNewLayout(); } finally { updater.close(); } } catch (KeeperException ke) { throw new IOException(ke); } } else { // System versions before system-2.0 do not enforce table layout update consistency or // validation. newLayout = metaTable.updateTableLayout(tableName, update); } } Preconditions.checkState(newLayout != null); if (dryRun) { printStream.println("This table layout is valid."); } LOG.debug("Computing new HBase schema"); final HTableSchemaTranslator translator = new HTableSchemaTranslator(); final HTableDescriptor newTableDescriptor = translator.toHTableDescriptor(mURI.getInstance(), newLayout); LOG.debug("Reading existing HBase schema"); final KijiManagedHBaseTableName hbaseTableName = KijiManagedHBaseTableName.getKijiTableName(mURI.getInstance(), tableName); HTableDescriptor currentTableDescriptor = null; byte[] tableNameAsBytes = hbaseTableName.toBytes(); try { currentTableDescriptor = getHBaseAdmin().getTableDescriptor(tableNameAsBytes); } catch (TableNotFoundException tnfe) { if (!dryRun) { throw tnfe; // Not in dry-run mode; table needs to exist. Rethrow exception. } } if (currentTableDescriptor == null) { if (dryRun) { printStream.println("Would create new table: " + tableName); currentTableDescriptor = HTableDescriptorComparator.makeEmptyTableDescriptor(hbaseTableName); } else { throw new RuntimeException( "Table " + hbaseTableName.getKijiTableName() + " does not exist"); } } LOG.debug("Existing table descriptor: {}", currentTableDescriptor); LOG.debug("New table descriptor: {}", newTableDescriptor); LOG.debug("Checking for differences between the new HBase schema and the existing one"); final HTableDescriptorComparator comparator = new HTableDescriptorComparator(); if (0 == comparator.compare(currentTableDescriptor, newTableDescriptor)) { LOG.debug("HBase schemas are the same. No need to change HBase schema"); if (dryRun) { printStream.println("This layout does not require any physical table schema changes."); } } else { LOG.debug("HBase schema must be changed, but no columns will be deleted"); if (dryRun) { printStream.println("Changes caused by this table layout:"); } else { LOG.debug("Disabling HBase table"); getHBaseAdmin().disableTable(hbaseTableName.toString()); } for (HColumnDescriptor newColumnDescriptor : newTableDescriptor.getFamilies()) { final String columnName = Bytes.toString(newColumnDescriptor.getName()); final ColumnId columnId = ColumnId.fromString(columnName); final String lgName = newLayout.getLocalityGroupIdNameMap().get(columnId); final HColumnDescriptor currentColumnDescriptor = currentTableDescriptor.getFamily(newColumnDescriptor.getName()); if (null == currentColumnDescriptor) { if (dryRun) { printStream.println(" Creating new locality group: " + lgName); } else { LOG.debug("Creating new column " + columnName); getHBaseAdmin().addColumn(hbaseTableName.toString(), newColumnDescriptor); } } else if (!newColumnDescriptor.equals(currentColumnDescriptor)) { if (dryRun) { printStream.println(" Modifying locality group: " + lgName); } else { LOG.debug("Modifying column " + columnName); getHBaseAdmin().modifyColumn(hbaseTableName.toString(), newColumnDescriptor); } } else { LOG.debug("No changes needed for column " + columnName); } } if (dryRun) { if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) { printStream.printf( " Changing max_filesize from %d to %d: %n", currentTableDescriptor.getMaxFileSize(), newTableDescriptor.getMaxFileSize()); } if (newTableDescriptor.getMaxFileSize() != currentTableDescriptor.getMaxFileSize()) { printStream.printf( " Changing memstore_flushsize from %d to %d: %n", currentTableDescriptor.getMemStoreFlushSize(), newTableDescriptor.getMemStoreFlushSize()); } } else { LOG.debug("Modifying table descriptor"); getHBaseAdmin().modifyTable(tableNameAsBytes, newTableDescriptor); } if (!dryRun) { LOG.debug("Re-enabling HBase table"); getHBaseAdmin().enableTable(hbaseTableName.toString()); } } return newLayout; }
@Test public void testRegister() throws Exception { getKiji().createTable(KijiTableLayouts.getLayout(KijiTableLayouts.COUNTER_TEST)); assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), KijiURI.newBuilder(getKiji().getURI()) .withTableName("user") .withColumnNames(Lists.newArrayList("info:name")) .build() .toString(), "--do=register", "--policy-class=org.kiji.scoring.lib.ShelfLife", "--policy-state={\"shelfLife\":10}", "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer")); final KijiFreshnessPolicyRecord record = KijiFreshnessPolicyRecord.newBuilder() .setRecordVersion("policyrecord-0.1.0") .setProducerClass(TestProducer.class.getName()) .setFreshnessPolicyClass( Class.forName("org.kiji.scoring.lib.ShelfLife") .asSubclass(KijiFreshnessPolicy.class) .getName()) .setFreshnessPolicyState("{\"shelfLife\":10}") .build(); final KijiFreshnessManager manager = KijiFreshnessManager.create(getKiji()); assertEquals(record, manager.retrievePolicy("user", "info:name")); assertEquals( "Freshness policy: org.kiji.scoring.lib.ShelfLife with state: {\"shelfLife\":10} " + "and producer: org.kiji.scoring.tools.TestFreshTool$TestProducer\n" + "attached to column: info:name in table: user", mToolOutputStr); // Test another ordering for arguments assertEquals( BaseTool.SUCCESS, runTool( new FreshTool(), "--do=register", KijiURI.newBuilder(getKiji().getURI()) .withTableName("user") .withColumnNames(Lists.newArrayList("info:visits")) .build() .toString(), "--policy-state={\"shelfLife\":10}", "--producer-class=org.kiji.scoring.tools.TestFreshTool$TestProducer", "--policy-class=org.kiji.scoring.lib.ShelfLife")); final KijiFreshnessPolicyRecord record2 = KijiFreshnessPolicyRecord.newBuilder() .setRecordVersion("policyrecord-0.1.0") .setProducerClass(TestProducer.class.getName()) .setFreshnessPolicyClass( Class.forName("org.kiji.scoring.lib.ShelfLife") .asSubclass(KijiFreshnessPolicy.class) .getName()) .setFreshnessPolicyState("{\"shelfLife\":10}") .build(); assertEquals(record2, manager.retrievePolicy("user", "info:visits")); assertEquals( "Freshness policy: org.kiji.scoring.lib.ShelfLife with state: {\"shelfLife\":10} " + "and producer: org.kiji.scoring.tools.TestFreshTool$TestProducer\n" + "attached to column: info:visits in table: user", mToolOutputStr); }