/** * Dump a Log File (Map or Sequence) to stdout. Will read from HDFS or local file system. * * @param args - first argument is the file to print */ public static void main(String[] args) throws IOException { Opts opts = new Opts(); opts.parseArgs(LogReader.class.getName(), args); VolumeManager fs = VolumeManagerImpl.get(); Matcher rowMatcher = null; KeyExtent ke = null; Text row = null; if (opts.files.isEmpty()) { new JCommander(opts).usage(); return; } if (opts.row != null) row = new Text(opts.row); if (opts.extent != null) { String sa[] = opts.extent.split(";"); ke = new KeyExtent(new Text(sa[0]), new Text(sa[1]), new Text(sa[2])); } if (opts.regexp != null) { Pattern pattern = Pattern.compile(opts.regexp); rowMatcher = pattern.matcher(""); } Set<Integer> tabletIds = new HashSet<Integer>(); for (String file : opts.files) { Path path = new Path(file); LogFileKey key = new LogFileKey(); LogFileValue value = new LogFileValue(); if (fs.isFile(path)) { // read log entries from a simple hdfs file DFSLoggerInputStreams streams = DfsLogger.readHeaderAndReturnStream(fs, path, SiteConfiguration.getInstance()); DataInputStream input = streams.getDecryptingInputStream(); try { while (true) { try { key.readFields(input); value.readFields(input); } catch (EOFException ex) { break; } printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations); } } finally { input.close(); } } else { // read the log entries sorted in a map file MultiReader input = new MultiReader(fs, path); while (input.next(key, value)) { printLogEvent(key, value, row, rowMatcher, ke, tabletIds, opts.maxMutations); } } } }
public static Pair<List<LogEntry>, SortedMap<FileRef, DataFileValue>> getFileAndLogEntries( Credentials credentials, KeyExtent extent) throws KeeperException, InterruptedException, IOException { ArrayList<LogEntry> result = new ArrayList<LogEntry>(); TreeMap<FileRef, DataFileValue> sizes = new TreeMap<FileRef, DataFileValue>(); VolumeManager fs = VolumeManagerImpl.get(); if (extent.isRootTablet()) { getRootLogEntries(result); Path rootDir = new Path(getRootTabletDir()); FileStatus[] files = fs.listStatus(rootDir); for (FileStatus fileStatus : files) { if (fileStatus.getPath().toString().endsWith("_tmp")) { continue; } DataFileValue dfv = new DataFileValue(0, 0); sizes.put(new FileRef(fileStatus.getPath().toString(), fileStatus.getPath()), dfv); } } else { String systemTableToCheck = extent.isMeta() ? RootTable.ID : MetadataTable.ID; Scanner scanner = new ScannerImpl( HdfsZooInstance.getInstance(), credentials, systemTableToCheck, Authorizations.EMPTY); scanner.fetchColumnFamily(LogColumnFamily.NAME); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); scanner.setRange(extent.toMetadataRange()); for (Entry<Key, Value> entry : scanner) { if (!entry.getKey().getRow().equals(extent.getMetadataEntry())) { throw new RuntimeException( "Unexpected row " + entry.getKey().getRow() + " expected " + extent.getMetadataEntry()); } if (entry.getKey().getColumnFamily().equals(LogColumnFamily.NAME)) { result.add(LogEntry.fromKeyValue(entry.getKey(), entry.getValue())); } else if (entry.getKey().getColumnFamily().equals(DataFileColumnFamily.NAME)) { DataFileValue dfv = new DataFileValue(entry.getValue().get()); sizes.put(new FileRef(fs, entry.getKey()), dfv); } else { throw new RuntimeException("Unexpected col fam " + entry.getKey().getColumnFamily()); } } } return new Pair<List<LogEntry>, SortedMap<FileRef, DataFileValue>>(result, sizes); }
private void addBlocks( VolumeManager fs, String host, ArrayList<String> files, Map<String, Long> totalBlocks, Map<String, Long> localBlocks) throws Exception { long allBlocks = 0; long matchingBlocks = 0; if (!totalBlocks.containsKey(host)) { totalBlocks.put(host, 0L); localBlocks.put(host, 0L); } for (String file : files) { Path filePath = new Path(file); FileSystem ns = fs.getFileSystemByPath(filePath); FileStatus fileStatus = ns.getFileStatus(filePath); BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); for (BlockLocation blockLocation : fileBlockLocations) { allBlocks++; for (String location : blockLocation.getHosts()) { HostAndPort hap = HostAndPort.fromParts(location, 0); if (hap.getHostText().equals(host)) { matchingBlocks++; break; } } } } totalBlocks.put(host, allBlocks + totalBlocks.get(host)); localBlocks.put(host, matchingBlocks + localBlocks.get(host)); }
@Test public void testMoveToTrash_UsingTrash_VolMgrFailure() throws Exception { Path path = createMock(Path.class); expect(volMgr.moveToTrash(path)).andThrow(new FileNotFoundException()); replay(volMgr); assertFalse(gc.archiveOrMoveToTrash(path)); verify(volMgr); }
@Test public void testMoveToTrash_UsingTrash() throws Exception { Path path = createMock(Path.class); expect(volMgr.moveToTrash(path)).andReturn(true); replay(volMgr); assertTrue(gc.archiveOrMoveToTrash(path)); verify(volMgr); }
public int run(String[] args) throws Exception { ClientOpts opts = new ClientOpts(); opts.parseArgs(LocalityCheck.class.getName(), args); VolumeManager fs = VolumeManagerImpl.get(); Connector connector = opts.getConnector(); Scanner scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY); scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME); scanner.fetchColumnFamily(DataFileColumnFamily.NAME); scanner.setRange(MetadataSchema.TabletsSection.getRange()); Map<String, Long> totalBlocks = new HashMap<String, Long>(); Map<String, Long> localBlocks = new HashMap<String, Long>(); ArrayList<String> files = new ArrayList<String>(); for (Entry<Key, Value> entry : scanner) { Key key = entry.getKey(); if (key.compareColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME) == 0) { String location = entry.getValue().toString(); String[] parts = location.split(":"); String host = parts[0]; addBlocks(fs, host, files, totalBlocks, localBlocks); files.clear(); } else if (key.compareColumnFamily(DataFileColumnFamily.NAME) == 0) { files.add(fs.getFullPath(key).toString()); } } System.out.println(" Server %local total blocks"); for (String host : totalBlocks.keySet()) { System.out.println( String.format( "%15s %5.1f %8d", host, (localBlocks.get(host) * 100.) / totalBlocks.get(host), totalBlocks.get(host))); } return 0; }
public static Map<TreeSet<String>, Long> getDiskUsage( AccumuloConfiguration acuConf, Set<String> tableIds, VolumeManager fs, Connector conn) throws IOException { TableDiskUsage tdu = new TableDiskUsage(); // Add each tableID for (String tableId : tableIds) tdu.addTable(tableId); HashSet<String> tablesReferenced = new HashSet<String>(tableIds); HashSet<String> emptyTableIds = new HashSet<String>(); HashSet<String> nameSpacesReferenced = new HashSet<String>(); // For each table ID for (String tableId : tableIds) { Scanner mdScanner = null; try { mdScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); } catch (TableNotFoundException e) { throw new RuntimeException(e); } mdScanner.fetchColumnFamily(DataFileColumnFamily.NAME); mdScanner.setRange(new KeyExtent(tableId, null, null).toMetadataRange()); if (!mdScanner.iterator().hasNext()) { emptyTableIds.add(tableId); } // Read each file referenced by that table for (Entry<Key, Value> entry : mdScanner) { String file = entry.getKey().getColumnQualifier().toString(); String parts[] = file.split("/"); // the filename String uniqueName = parts[parts.length - 1]; if (file.contains(":") || file.startsWith("../")) { String ref = parts[parts.length - 3]; // Track any tables which are referenced externally by the current table if (!ref.equals(tableId)) { tablesReferenced.add(ref); } if (file.contains(":") && parts.length > 3) { List<String> base = Arrays.asList(Arrays.copyOf(parts, parts.length - 3)); nameSpacesReferenced.add(Joiner.on("/").join(base)); } } // add this file to this table tdu.linkFileAndTable(tableId, uniqueName); } } // Each table seen (provided by user, or reference by table the user provided) for (String tableId : tablesReferenced) { for (String tableDir : nameSpacesReferenced) { // Find each file and add its size FileStatus[] files = fs.globStatus(new Path(tableDir + "/" + tableId + "/*/*")); if (files != null) { for (FileStatus fileStatus : files) { // Assumes that all filenames are unique String name = fileStatus.getPath().getName(); tdu.addFileSize(name, fileStatus.getLen()); } } } } // Invert tableId->tableName HashMap<String, String> reverseTableIdMap = new HashMap<String, String>(); for (Entry<String, String> entry : conn.tableOperations().tableIdMap().entrySet()) reverseTableIdMap.put(entry.getValue(), entry.getKey()); TreeMap<TreeSet<String>, Long> usage = new TreeMap<TreeSet<String>, Long>( new Comparator<TreeSet<String>>() { @Override public int compare(TreeSet<String> o1, TreeSet<String> o2) { int len1 = o1.size(); int len2 = o2.size(); int min = Math.min(len1, len2); Iterator<String> iter1 = o1.iterator(); Iterator<String> iter2 = o2.iterator(); int count = 0; while (count < min) { String s1 = iter1.next(); String s2 = iter2.next(); int cmp = s1.compareTo(s2); if (cmp != 0) return cmp; count++; } return len1 - len2; } }); for (Entry<List<String>, Long> entry : tdu.calculateUsage().entrySet()) { TreeSet<String> tableNames = new TreeSet<String>(); // Convert size shared by each table id into size shared by each table name for (String tableId : entry.getKey()) tableNames.add(reverseTableIdMap.get(tableId)); // Make table names to shared file size usage.put(tableNames, entry.getValue()); } if (!emptyTableIds.isEmpty()) { TreeSet<String> emptyTables = new TreeSet<String>(); for (String tableId : emptyTableIds) { emptyTables.add(reverseTableIdMap.get(tableId)); } usage.put(emptyTables, 0L); } return usage; }
public static void cloneTable( Instance instance, String srcTableId, String tableId, VolumeManager volumeManager) throws Exception { Connector conn = instance.getConnector( SystemCredentials.get().getPrincipal(), SystemCredentials.get().getToken()); BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig()); while (true) { try { initializeClone(srcTableId, tableId, conn, bw); // the following loop looks changes in the file that occurred during the copy.. if files // were dereferenced then they could have been GCed while (true) { int rewrites = checkClone(srcTableId, tableId, conn, bw); if (rewrites == 0) break; } bw.flush(); break; } catch (TabletIterator.TabletDeletedException tde) { // tablets were merged in the src table bw.flush(); // delete what we have cloned and try again deleteTable(tableId, false, SystemCredentials.get(), null); log.debug( "Tablets merged in table " + srcTableId + " while attempting to clone, trying again"); UtilWaitThread.sleep(100); } } // delete the clone markers and create directory entries Scanner mscanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY); mscanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange()); mscanner.fetchColumnFamily(ClonedColumnFamily.NAME); int dirCount = 0; for (Entry<Key, Value> entry : mscanner) { Key k = entry.getKey(); Mutation m = new Mutation(k.getRow()); m.putDelete(k.getColumnFamily(), k.getColumnQualifier()); String dir = volumeManager.choose(ServerConstants.getTablesDirs()) + "/" + tableId + new String(FastFormat.toZeroPaddedString(dirCount++, 8, 16, "/c-".getBytes())); TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.put(m, new Value(dir.getBytes())); bw.addMutation(m); } bw.close(); }