@Test public void TestMap() throws Exception { String prefix = "0000"; final String fileName = "19691231f2cd014ea28f42788214560a21a44cef"; final String mobFilePath = prefix + fileName; ImmutableBytesWritable r = new ImmutableBytesWritable(Bytes.toBytes("r")); final KeyValue[] kvList = new KeyValue[1]; kvList[0] = new KeyValue( Bytes.toBytes("row"), Bytes.toBytes("family"), Bytes.toBytes("column"), Bytes.toBytes(mobFilePath)); Result columns = mock(Result.class); when(columns.rawCells()).thenReturn(kvList); Configuration configuration = new Configuration(TEST_UTIL.getConfiguration()); ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable()); TableName tn = TableName.valueOf("testSweepMapper"); TableName lockName = MobUtils.getTableLockName(tn); String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString()); configuration.set(SweepJob.SWEEP_JOB_ID, "1"); configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode); ServerName serverName = SweepJob.getCurrentServerName(configuration); configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString()); TableLockManager tableLockManager = TableLockManager.createTableLockManager(configuration, zkw, serverName); TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool"); lock.acquire(); try { Mapper<ImmutableBytesWritable, Result, Text, KeyValue>.Context ctx = mock(Mapper.Context.class); when(ctx.getConfiguration()).thenReturn(configuration); SweepMapper map = new SweepMapper(); doAnswer( new Answer<Void>() { @Override public Void answer(InvocationOnMock invocation) throws Throwable { Text text = (Text) invocation.getArguments()[0]; KeyValue kv = (KeyValue) invocation.getArguments()[1]; assertEquals(Bytes.toString(text.getBytes(), 0, text.getLength()), fileName); assertEquals(0, Bytes.compareTo(kv.getKey(), kvList[0].getKey())); return null; } }) .when(ctx) .write(any(Text.class), any(KeyValue.class)); map.map(r, columns, ctx); } finally { lock.release(); } }
public static void addDaughter( final CatalogTracker catalogTracker, final HRegionInfo regionInfo, final ServerName sn) throws NotAllMetaRegionsOnlineException, IOException { Put put = new Put(regionInfo.getRegionName()); addRegionInfo(put, regionInfo); if (sn != null) addLocation(put, sn); putToMetaTable(catalogTracker, put); LOG.info( "Added daughter " + regionInfo.getRegionNameAsString() + (sn == null ? ", serverName=null" : ", serverName=" + sn.toString())); }
protected void deleteRegion(Configuration conf, final Table tbl, byte[] startKey, byte[] endKey) throws IOException { LOG.info("Before delete:"); HTableDescriptor htd = tbl.getTableDescriptor(); dumpMeta(htd); List<HRegionLocation> regions; try (RegionLocator rl = connection.getRegionLocator(tbl.getName())) { regions = rl.getAllRegionLocations(); } for (HRegionLocation e : regions) { HRegionInfo hri = e.getRegionInfo(); ServerName hsa = e.getServerName(); if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 && Bytes.compareTo(hri.getEndKey(), endKey) == 0) { LOG.info("RegionName: " + hri.getRegionNameAsString()); byte[] deleteRow = hri.getRegionName(); TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true); LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString()); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()), hri.getEncodedName()); fs.delete(p, true); try (Table meta = this.connection.getTable(TableName.META_TABLE_NAME)) { Delete delete = new Delete(deleteRow); meta.delete(delete); } } LOG.info(hri.toString() + hsa.toString()); } TEST_UTIL.getMetaTableRows(htd.getTableName()); LOG.info("After delete:"); dumpMeta(htd); }
@Test public void testRun() throws Exception { TableName tn = TableName.valueOf(tableName); byte[] mobValueBytes = new byte[100]; // get the path where mob files lie in Path mobFamilyPath = MobUtils.getMobFamilyPath(TEST_UTIL.getConfiguration(), tn, family); Put put = new Put(Bytes.toBytes(row)); put.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes); Put put2 = new Put(Bytes.toBytes(row + "ignore")); put2.addColumn(Bytes.toBytes(family), Bytes.toBytes(qf), 1, mobValueBytes); table.mutate(put); table.mutate(put2); table.flush(); admin.flush(tn); FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); // check the generation of a mob file assertEquals(1, fileStatuses.length); String mobFile1 = fileStatuses[0].getPath().getName(); Configuration configuration = new Configuration(TEST_UTIL.getConfiguration()); configuration.setFloat(MobConstants.MOB_SWEEP_TOOL_COMPACTION_RATIO, 0.6f); configuration.setStrings(TableInputFormat.INPUT_TABLE, tableName); configuration.setStrings(TableInputFormat.SCAN_COLUMN_FAMILY, family); configuration.setStrings(SweepJob.WORKING_VISITED_DIR_KEY, "jobWorkingNamesDir"); configuration.setStrings(SweepJob.WORKING_FILES_DIR_KEY, "compactionFileDir"); configuration.setStrings( CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, JavaSerialization.class.getName()); configuration.set(SweepJob.WORKING_VISITED_DIR_KEY, "compactionVisitedDir"); configuration.setLong( MobConstants.MOB_SWEEP_TOOL_COMPACTION_START_DATE, System.currentTimeMillis() + 24 * 3600 * 1000); ZooKeeperWatcher zkw = new ZooKeeperWatcher(configuration, "1", new DummyMobAbortable()); TableName lockName = MobUtils.getTableLockName(tn); String znode = ZKUtil.joinZNode(zkw.tableLockZNode, lockName.getNameAsString()); configuration.set(SweepJob.SWEEP_JOB_ID, "1"); configuration.set(SweepJob.SWEEP_JOB_TABLE_NODE, znode); ServerName serverName = SweepJob.getCurrentServerName(configuration); configuration.set(SweepJob.SWEEP_JOB_SERVERNAME, serverName.toString()); TableLockManager tableLockManager = TableLockManager.createTableLockManager(configuration, zkw, serverName); TableLock lock = tableLockManager.writeLock(lockName, "Run sweep tool"); lock.acquire(); try { // use the same counter when mocking Counter counter = new GenericCounter(); Reducer<Text, KeyValue, Writable, Writable>.Context ctx = mock(Reducer.Context.class); when(ctx.getConfiguration()).thenReturn(configuration); when(ctx.getCounter(Matchers.any(SweepCounter.class))).thenReturn(counter); when(ctx.nextKey()).thenReturn(true).thenReturn(false); when(ctx.getCurrentKey()).thenReturn(new Text(mobFile1)); byte[] refBytes = Bytes.toBytes(mobFile1); long valueLength = refBytes.length; byte[] newValue = Bytes.add(Bytes.toBytes(valueLength), refBytes); KeyValue kv2 = new KeyValue( Bytes.toBytes(row), Bytes.toBytes(family), Bytes.toBytes(qf), 1, KeyValue.Type.Put, newValue); List<KeyValue> list = new ArrayList<KeyValue>(); list.add(kv2); when(ctx.getValues()).thenReturn(list); SweepReducer reducer = new SweepReducer(); reducer.run(ctx); } finally { lock.release(); } FileStatus[] filsStatuses2 = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath); String mobFile2 = filsStatuses2[0].getPath().getName(); // new mob file is generated, old one has been archived assertEquals(1, filsStatuses2.length); assertEquals(false, mobFile2.equalsIgnoreCase(mobFile1)); // test sequence file String workingPath = configuration.get(SweepJob.WORKING_VISITED_DIR_KEY); FileStatus[] statuses = TEST_UTIL.getTestFileSystem().listStatus(new Path(workingPath)); Set<String> files = new TreeSet<String>(); for (FileStatus st : statuses) { files.addAll( getKeyFromSequenceFile(TEST_UTIL.getTestFileSystem(), st.getPath(), configuration)); } assertEquals(1, files.size()); assertEquals(true, files.contains(mobFile1)); }
public void splitLog(final List<ServerName> serverNames) throws IOException { long splitTime = 0, splitLogSize = 0; List<Path> logDirs = new ArrayList<Path>(); for(ServerName serverName: serverNames){ Path logDir = new Path(this.rootdir, HLog.getHLogDirectoryName(serverName.toString())); Path splitDir = logDir.suffix(HLog.SPLITTING_EXT); // rename the directory so a rogue RS doesn't create more HLogs if (fs.exists(logDir)) { if (!this.fs.rename(logDir, splitDir)) { throw new IOException("Failed fs.rename for log split: " + logDir); } logDir = splitDir; LOG.debug("Renamed region directory: " + splitDir); } else if (!fs.exists(splitDir)) { LOG.info("Log dir for server " + serverName + " does not exist"); continue; } logDirs.add(splitDir); } if (logDirs.isEmpty()) { LOG.info("No logs to split"); return; } if (distributedLogSplitting) { splitLogManager.handleDeadWorkers(serverNames); splitTime = EnvironmentEdgeManager.currentTimeMillis(); splitLogSize = splitLogManager.splitLogDistributed(logDirs); splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime; } else { for(Path logDir: logDirs){ // splitLogLock ensures that dead region servers' logs are processed // one at a time this.splitLogLock.lock(); try { HLogSplitter splitter = HLogSplitter.createLogSplitter( conf, rootdir, logDir, oldLogDir, this.fs); try { // If FS is in safe mode, just wait till out of it. FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000)); splitter.splitLog(); } catch (OrphanHLogAfterSplitException e) { LOG.warn("Retrying splitting because of:", e); //An HLogSplitter instance can only be used once. Get new instance. splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir, oldLogDir, this.fs); splitter.splitLog(); } splitTime = splitter.getTime(); splitLogSize = splitter.getSize(); } finally { this.splitLogLock.unlock(); } } } if (this.metrics != null) { this.metrics.addSplit(splitTime, splitLogSize); } }
public static void writeRegionAndServer( HRegionInfo region, ServerName serverName, OutputStreamWriter outputStreamWriter) throws IOException { outputStreamWriter.write( String.format("%s %s\n", region.getEncodedName(), serverName.toString())); }