private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) { try { String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/')); File outputDest = File.createTempFile(fname, ".hdfs"); if (outputDest.exists()) { outputDest.delete(); } HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath); FileSystem fileSystem = hdfsInfo.getFileSystem(); FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf()); try { FileUtil.copyMerge( fileSystem, // src new Path(hdfsPath), FileSystem.getLocal(new Configuration()), // dest new Path(outputDest.toURI()), false, fileSystem.getConf(), null); } catch (IOException e) { return outputDest; } return new File(outputDest, fname); } catch (IOException ex) { throw new RuntimeCamelException(ex); } }
@BeforeClass public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException { SupportsBlocks = true; CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); cluster = new MiniDFSCluster.Builder(CONF) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .numDataNodes(2) .build(); cluster.waitClusterUp(); fHdfs = cluster.getFileSystem(0); fHdfs2 = cluster.getFileSystem(1); fHdfs .getConf() .set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString()); fHdfs2 .getConf() .set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString()); defaultWorkingDirectory = fHdfs.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); defaultWorkingDirectory2 = fHdfs2.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fHdfs.mkdirs(defaultWorkingDirectory); fHdfs2.mkdirs(defaultWorkingDirectory2); }
@Test public void testNonDefaultFS() throws IOException { FileSystem fs = cluster.getFileSystem(); Configuration conf = fs.getConf(); conf.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, fs.getUri().toString()); TestTrash.trashNonDefaultFS(conf); }
@SuppressWarnings("unchecked") public static <V extends Writable> List<V> readValues(Path path, FileSystem fs, int max) { List<V> list = new ArrayList<V>(); try { int k = 0; SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf()); Writable key = (Writable) reader.getKeyClass().newInstance(); V value = (V) reader.getValueClass().newInstance(); while (reader.next(key, value)) { k++; list.add(value); if (k >= max) { break; } value = (V) reader.getValueClass().newInstance(); } reader.close(); } catch (Exception e) { throw new RuntimeException("Error reading SequenceFile " + path); } return list; }
private void writeBuffer( String destPath, ArrayList<Pair<LogFileKey, LogFileValue>> buffer, int part) throws IOException { Path path = new Path(destPath, String.format("part-r-%05d", part++)); FileSystem ns = fs.getFileSystemByPath(path); @SuppressWarnings("deprecation") MapFile.Writer output = new MapFile.Writer( ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class); try { Collections.sort( buffer, new Comparator<Pair<LogFileKey, LogFileValue>>() { @Override public int compare( Pair<LogFileKey, LogFileValue> o1, Pair<LogFileKey, LogFileValue> o2) { return o1.getFirst().compareTo(o2.getFirst()); } }); for (Pair<LogFileKey, LogFileValue> entry : buffer) { output.append(entry.getFirst(), entry.getSecond()); } } finally { output.close(); } }
/** * Reads key-value pairs from a SequenceFile, up to a maximum number. * * @param path path to file * @param max maximum of key-value pairs to read * @return list of key-value pairs */ @SuppressWarnings("unchecked") public static <K extends Writable, V extends Writable> List<PairOfWritables<K, V>> readFile( Path path, FileSystem fs, int max) throws IOException { List<PairOfWritables<K, V>> list = new ArrayList<PairOfWritables<K, V>>(); try { int k = 0; SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf()); K key; V value; key = (K) reader.getKeyClass().newInstance(); value = (V) reader.getValueClass().newInstance(); while (reader.next(key, value)) { k++; list.add(new PairOfWritables<K, V>(key, value)); if (k >= max) { break; } // Create new objects, because the key, value gets reused key = (K) reader.getKeyClass().newInstance(); value = (V) reader.getValueClass().newInstance(); } reader.close(); } catch (IllegalAccessException e) { throw new RuntimeException("Error reading SequenceFile: " + e); } catch (InstantiationException e) { throw new RuntimeException("Error reading SequenceFile: " + e); } return list; }
/** Constructor */ public DFSAdminCommand(FileSystem fs) { super(fs.getConf()); if (!(fs instanceof DistributedFileSystem)) { throw new IllegalArgumentException( "FileSystem " + fs.getUri() + " is not a distributed file system"); } this.dfs = (DistributedFileSystem) fs; }
// creates a file but does not close it static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl) throws IOException { System.out.println("createFile: Created " + name + " with " + repl + " replica."); FSDataOutputStream stm = fileSys.create( name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, (long) blockSize); return stm; }
// creates a file and returns a descriptor for writing to it private static FSDataOutputStream writeFile( FileSystem fileSys, Path name, short repl, long blockSize) throws IOException { FSDataOutputStream stm = fileSys.create( name, true, fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), repl, blockSize); return stm; }
private void writeFile(FileSystem fileSys, Path name, int repl) throws IOException { FSDataOutputStream stm = fileSys.create( name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, (long) blockSize); byte[] buffer = new byte[fileSize]; Random rand = new Random(seed); rand.nextBytes(buffer); stm.write(buffer); stm.close(); }
private FSDataOutputStream writeFile(FileSystem fileSys, Path name, int repl) throws IOException { FSDataOutputStream stm = fileSys.create( name, true, fileSys.getConf().getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096), (short) repl, blockSize); byte[] buffer = new byte[fileSize]; Random rand = new Random(seed); rand.nextBytes(buffer); stm.write(buffer); return stm; }
private FileSystemWriter(FileSystem fs, Path path, DatasetDescriptor descriptor) { Preconditions.checkNotNull(fs, "File system is not defined"); Preconditions.checkNotNull(path, "Destination directory is not defined"); Preconditions.checkNotNull(descriptor, "Descriptor is not defined"); this.fs = fs; this.directory = path; this.descriptor = descriptor; this.conf = new Configuration(fs.getConf()); this.state = ReaderWriterState.NEW; // copy file format settings from custom properties to the Configuration for (String prop : descriptor.listProperties()) { conf.set(prop, descriptor.getProperty(prop)); } }
private Collection<TestDataModel> collectContent(FileSystem fs, FileStatus status) throws IOException { Collection<TestDataModel> results = new ArrayList<TestDataModel>(); ModelInput<TestDataModel> input = TemporaryStorage.openInput(fs.getConf(), TestDataModel.class, status.getPath()); try { TestDataModel model = new TestDataModel(); while (input.readTo(model)) { results.add(model.copy()); } } finally { input.close(); } return results; }
// creates a file using DistributedFileSystem.createNonRecursive() static FSDataOutputStream createNonRecursive( FileSystem fs, Path name, int repl, boolean overwrite) throws IOException { System.out.println("createNonRecursive: Created " + name + " with " + repl + " replica."); FSDataOutputStream stm = ((DistributedFileSystem) fs) .createNonRecursive( name, FsPermission.getDefault(), overwrite, fs.getConf().getInt("io.file.buffer.size", 4096), (short) repl, (long) blockSize, null); return stm; }
// // creates a file and populate it with data. // private void createOldFile(FileSystem fileSys, Path name, int repl, int numBlocks, long blocksize) throws IOException { FSDataOutputStream stm = fileSys.create( name, true, fileSys.getConf().getInt("io.file.buffer.size", 4096), (short) repl, blocksize); // fill data into file final byte[] b = new byte[(int) blocksize]; for (int i = 0; i < numBlocks; i++) { stm.write(b); } stm.close(); }
@Test public void testDirectoryFilterUnfixableFiles() throws Exception { conf = new Configuration(); dfsCluster = new MiniDFSCluster(conf, NUM_DATANODES, true, null); dfsCluster.waitActive(); FileSystem fs = dfsCluster.getFileSystem(); Utils.loadTestCodecs(conf, 3, 5, 1, 3, "/destraid", "/destraidrs", false, true); try { Configuration testConf = fs.getConf(); BlockIntegrityMonitor blockFixer = new LocalBlockIntegrityMonitor(testConf, false); String p1 = "/user/foo/f1"; String p2 = "/user/foo/f2"; String p3 = "/user1/foo/bar/f1"; String p4 = "/a/b"; String p5 = "/c"; String p6 = "/destraidrs/user"; String p7 = "/destraid/user1/foo"; fs.mkdirs(new Path(p6)); List<String> fileList = new ArrayList<String>(); fileList.add(p1); fileList.add(p2); fileList.add(p3); fileList.add(p4); fileList.add(p5); blockFixer.filterUnreconstructableSourceFiles(fs, fileList.iterator()); // p3 and p5 should be filtered out. assertEquals(3, fileList.size()); Set<String> filtered = new HashSet<String>(); for (String p : fileList) filtered.add(p); assertFalse("File not filtered", filtered.contains(p3)); assertFalse("File not filtered", filtered.contains(p5)); fileList.add(p3); fs.mkdirs(new Path(p7)); blockFixer.filterUnreconstructableSourceFiles(fs, fileList.iterator()); // Nothing is filtered. assertEquals(4, fileList.size()); } finally { dfsCluster.shutdown(); } }
private static int readSequenceFile(Path path, FileSystem fs, int max) throws IOException { SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf()); System.out.println("Reading " + path + "...\n"); try { System.out.println("Key type: " + reader.getKeyClass().toString()); System.out.println("Value type: " + reader.getValueClass().toString() + "\n"); } catch (Exception e) { throw new RuntimeException("Error: loading key/value class"); } Writable key = null, value; int n = 0; try { if (Tuple.class.isAssignableFrom(reader.getKeyClass())) { key = TUPLE_FACTORY.newTuple(); } else { key = (Writable) reader.getKeyClass().newInstance(); } if (Tuple.class.isAssignableFrom(reader.getValueClass())) { value = TUPLE_FACTORY.newTuple(); } else { value = (Writable) reader.getValueClass().newInstance(); } while (reader.next(key, value)) { System.out.println("Record " + n); System.out.println("Key: " + key + "\nValue: " + value); System.out.println("----------------------------------------"); n++; if (n >= max) break; } reader.close(); System.out.println(n + " records read.\n"); } catch (Exception e) { e.printStackTrace(); } return n; }
private void cloneOutput() throws IOException { List<FileStatus> listStatus = getOutputMappings(); /* * Initialize to empty list, in which case swap() will be a no-op. The reference is then replaced with a real list, which is * used in the subsequent iterations. */ List<Path> crushInput = emptyList(); Text srcFile = new Text(); Text crushOut = new Text(); Text prevCrushOut = new Text(); for (FileStatus partFile : listStatus) { Path path = partFile.getPath(); Reader reader = new Reader(fs, path, fs.getConf()); try { while (reader.next(srcFile, crushOut)) { if (!crushOut.equals(prevCrushOut)) { swap(crushInput, prevCrushOut.toString()); prevCrushOut.set(crushOut); crushInput = new LinkedList<Path>(); } crushInput.add(new Path(srcFile.toString())); } } finally { try { reader.close(); } catch (IOException e) { LOG.warn("Trapped exception when closing " + path, e); } } swap(crushInput, prevCrushOut.toString()); } }
public static RawSequenceFileWriter createWriter( FileSystem fs, Configuration conf, Path name, Class keyClass, Class valClass, CompressionType compressionType) throws IOException { return new RawSequenceFileWriter( fs, conf, name, keyClass, valClass, fs.getConf().getInt("io.file.buffer.size", 65536), fs.getDefaultReplication(), fs.getDefaultBlockSize(), null, new Metadata()); }
/** * Get token from the token sequence file. * * @param authPath * @param proxyUserName * @return Token for proxyUserName if it exists. * @throws IOException */ private Optional<Token> getTokenFromSeqFile(String authPath, String proxyUserName) throws IOException { Closer closer = Closer.create(); try { FileSystem localFs = FileSystem.getLocal(new Configuration()); SequenceFile.Reader tokenReader = closer.register(new SequenceFile.Reader(localFs, new Path(authPath), localFs.getConf())); Text key = new Text(); Token value = new Token(); while (tokenReader.next(key, value)) { LOG.info("Found token for " + key); if (key.toString().equals(proxyUserName)) { return Optional.of(value); } } } catch (Throwable t) { throw closer.rethrow(t); } finally { closer.close(); } return Optional.absent(); }
public static HiveServer create( Map<String, String> properties, File baseDir, File confDir, File logDir, FileSystem fileSystem) throws Exception { if (!properties.containsKey(WAREHOUSE_DIR)) { LOGGER.info("fileSystem " + fileSystem.getClass().getSimpleName()); if (fileSystem instanceof DistributedFileSystem) { String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString(); LOGGER.info("dfsUri " + dfsUri); properties.put(WAREHOUSE_DIR, dfsUri + "/data"); fileSystem.mkdirs(new Path("/data/"), new FsPermission((short) 0777)); } else { properties.put(WAREHOUSE_DIR, new File(baseDir, "warehouse").getPath()); fileSystem.mkdirs(new Path("/", "warehouse"), new FsPermission((short) 0777)); } } LOGGER.info("Setting an readable path to hive.exec.scratchdir"); properties.put("hive.exec.scratchdir", new File(baseDir, "scratchdir").getPath()); if (!properties.containsKey(METASTORE_CONNECTION_URL)) { properties.put( METASTORE_CONNECTION_URL, String.format( "jdbc:derby:;databaseName=%s;create=true", new File(baseDir, "metastore").getPath())); } if (!properties.containsKey(HS2_PORT)) { properties.put(HS2_PORT, String.valueOf(findPort())); } if (!properties.containsKey(SUPPORT_CONCURRENCY)) { properties.put(SUPPORT_CONCURRENCY, "false"); } if (!properties.containsKey(HADOOPBIN)) { properties.put(HADOOPBIN, "./target/test-classes/hadoop"); } // Modify the test resource to have executable permission java.nio.file.Path hadoopPath = FileSystems.getDefault().getPath("target/test-classes", "hadoop"); if (hadoopPath != null) { hadoopPath.toFile().setExecutable(true); } properties.put(METASTORE_SETUGI, "true"); properties.put(METASTORE_CLIENT_TIMEOUT, "100"); properties.put(ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS.varname, "true"); properties.put(ConfVars.HIVESTATSAUTOGATHER.varname, "false"); properties.put(ConfVars.HIVE_STATS_COLLECT_SCANCOLS.varname, "true"); String hadoopBinPath = properties.get(HADOOPBIN); Assert.assertNotNull(hadoopBinPath, "Hadoop Bin"); File hadoopBin = new File(hadoopBinPath); if (!hadoopBin.isFile()) { Assert.fail( "Path to hadoop bin " + hadoopBin.getPath() + " is invalid. " + "Perhaps you missed the download-hadoop profile."); } /* * This hack, setting the hiveSiteURL field removes a previous hack involving * setting of system properties for each property. Although both are hacks, * I prefer this hack because once the system properties are set they can * affect later tests unless those tests clear them. This hack allows for * a clean switch to a new set of defaults when a new HiveConf object is created. */ Reflection.staticField("hiveSiteURL").ofType(URL.class).in(HiveConf.class).set(null); HiveConf hiveConf = new HiveConf(); for (Map.Entry<String, String> entry : properties.entrySet()) { LOGGER.info(entry.getKey() + " => " + entry.getValue()); hiveConf.set(entry.getKey(), entry.getValue()); } File hiveSite = new File(confDir, "hive-site.xml"); hiveConf.set(HIVESERVER2_IMPERSONATION, "false"); OutputStream out = new FileOutputStream(hiveSite); hiveConf.writeXml(out); out.close(); Reflection.staticField("hiveSiteURL") .ofType(URL.class) .in(HiveConf.class) .set(hiveSite.toURI().toURL()); LOGGER.info("Creating InternalHiveServer"); return new InternalHiveServer(hiveConf); }
@Override public void visit(State state, Properties props) throws Exception { boolean userExists = SecurityHelper.getTabUserExists(state); Connector conn; try { conn = state .getInstance() .getConnector( SecurityHelper.getTabUserName(state), SecurityHelper.getTabUserPass(state)); } catch (AccumuloSecurityException ae) { if (ae.getErrorCode().equals(SecurityErrorCode.BAD_CREDENTIALS)) { if (userExists) throw new AccumuloException( "User didn't exist when they should (or worse- password mismatch)", ae); else return; } throw new AccumuloException("Unexpected exception!", ae); } String action = props.getProperty("action", "_random"); TablePermission tp; if ("_random".equalsIgnoreCase(action)) { Random r = new Random(); tp = TablePermission.values()[r.nextInt(TablePermission.values().length)]; } else { tp = TablePermission.valueOf(action); } boolean tableExists = SecurityHelper.getTableExists(state); boolean hasPerm = SecurityHelper.getTabPerm(state, SecurityHelper.getTabUserName(state), tp); String tableName = state.getString("secTableName"); switch (tp) { case READ: Authorizations auths = SecurityHelper.getUserAuths(state, SecurityHelper.getTabUserName(state)); boolean canRead = SecurityHelper.getTabPerm( state, SecurityHelper.getTabUserName(state), TablePermission.READ); try { Scanner scan = conn.createScanner( tableName, conn.securityOperations() .getUserAuthorizations(SecurityHelper.getTabUserName(state))); int seen = 0; Iterator<Entry<Key, Value>> iter = scan.iterator(); while (iter.hasNext()) { Entry<Key, Value> entry = iter.next(); Key k = entry.getKey(); seen++; if (!auths.contains(k.getColumnVisibilityData())) throw new AccumuloException( "Got data I should not be capable of seeing: " + k + " table " + tableName); } if (!canRead) throw new AccumuloException( "Was able to read when I shouldn't have had the perm with connection user " + conn.whoami() + " table " + tableName); for (Entry<String, Integer> entry : SecurityHelper.getAuthsMap(state).entrySet()) { if (auths.contains(entry.getKey().getBytes())) seen = seen - entry.getValue(); } if (seen != 0) throw new AccumuloException("Got mismatched amounts of data"); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException( "Accumulo and test suite out of sync: table " + tableName, tnfe); return; } catch (AccumuloSecurityException ae) { if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, ae); else return; } throw new AccumuloException("Unexpected exception!", ae); } catch (RuntimeException re) { if (re.getCause() instanceof AccumuloSecurityException && ((AccumuloSecurityException) re.getCause()) .getErrorCode() .equals(SecurityErrorCode.PERMISSION_DENIED)) { if (canRead) throw new AccumuloException( "Table read permission out of sync with Accumulo: table " + tableName, re.getCause()); else return; } throw new AccumuloException("Unexpected exception!", re); } break; case WRITE: String key = SecurityHelper.getLastKey(state) + "1"; Mutation m = new Mutation(new Text(key)); for (String s : SecurityHelper.getAuthsArray()) { m.put(new Text(), new Text(), new ColumnVisibility(s), new Value("value".getBytes())); } BatchWriter writer; try { writer = conn.createBatchWriter(tableName, 9000l, 0l, 1); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } boolean works = true; try { writer.addMutation(m); } catch (MutationsRejectedException mre) { throw new AccumuloException("Mutation exception!", mre); } if (works) for (String s : SecurityHelper.getAuthsArray()) SecurityHelper.increaseAuthMap(state, s, 1); break; case BULK_IMPORT: key = SecurityHelper.getLastKey(state) + "1"; SortedSet<Key> keys = new TreeSet<Key>(); for (String s : SecurityHelper.getAuthsArray()) { Key k = new Key(key, "", "", s); keys.add(k); } Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); Path fail = new Path(dir.toString() + "_fail"); FileSystem fs = SecurityHelper.getFs(state); FileSKVWriter f = FileOperations.getInstance() .openWriter( dir + "/securityBulk." + RFile.EXTENSION, fs, fs.getConf(), AccumuloConfiguration.getDefaultConfiguration()); f.startDefaultLocalityGroup(); fs.mkdirs(fail); for (Key k : keys) f.append(k, new Value("Value".getBytes())); f.close(); try { conn.tableOperations().importDirectory(tableName, dir.toString(), fail.toString(), true); } catch (TableNotFoundException tnfe) { if (tableExists) throw new AccumuloException("Table didn't exist when it should have: " + tableName); return; } catch (AccumuloSecurityException ae) { if (ae.getErrorCode().equals(SecurityErrorCode.PERMISSION_DENIED)) { if (hasPerm) throw new AccumuloException( "Bulk Import failed when it should have worked: " + tableName); return; } throw new AccumuloException("Unexpected exception!", ae); } for (String s : SecurityHelper.getAuthsArray()) SecurityHelper.increaseAuthMap(state, s, 1); fs.delete(dir, true); fs.delete(fail, true); if (!hasPerm) throw new AccumuloException( "Bulk Import succeeded when it should have failed: " + dir + " table " + tableName); break; case ALTER_TABLE: AlterTable.renameTable(conn, state, tableName, tableName + "plus", hasPerm, tableExists); break; case GRANT: props.setProperty("task", "grant"); props.setProperty("perm", "random"); props.setProperty("source", "table"); props.setProperty("target", "system"); AlterTablePerm.alter(state, props); break; case DROP_TABLE: props.setProperty("source", "table"); DropTable.dropTable(state, props); break; } }
/** * Log job submitted event to history. Creates a new file in history for the job. if history file * creation fails, it disables history for all other events. * * @param jobConfPath path to job conf xml file in HDFS. * @param submitTime time when job tracker received the job * @throws IOException */ public void logSubmitted(String jobConfPath, long submitTime, String jobTrackerId) throws IOException { if (disableHistory) { return; } // create output stream for logging in hadoop.job.history.location int defaultBufferSize = logDirFs.getConf().getInt("io.file.buffer.size", 4096); try { FSDataOutputStream out = null; PrintWriter writer = null; // In case the old JT is still running, but we can't connect to it, we // should ensure that it won't write to our (new JT's) job history file. if (logDirFs.exists(logFile)) { LOG.info("Remove the old history file " + logFile); logDirFs.delete(logFile, true); } out = logDirFs.create( logFile, new FsPermission(HISTORY_FILE_PERMISSION), true, defaultBufferSize, logDirFs.getDefaultReplication(), jobHistoryBlockSize, null); writer = new PrintWriter(out); fileManager.addWriter(jobId, writer); // cache it ... fileManager.setHistoryFile(jobId, logFile); writers = fileManager.getWriters(jobId); if (null != writers) { log( writers, RecordTypes.Meta, new Keys[] {Keys.VERSION}, new String[] {String.valueOf(JobHistory.VERSION)}); } String jobName = getJobName(); String user = getUserName(); // add to writer as well log( writers, RecordTypes.Job, new Keys[] { Keys.JOBID, Keys.JOBNAME, Keys.USER, Keys.SUBMIT_TIME, Keys.JOBCONF, Keys.JOBTRACKERID }, new String[] { jobId.toString(), jobName, user, String.valueOf(submitTime), jobConfPath, jobTrackerId }); } catch (IOException e) { // Disable history if we have errors other than in the user log. disableHistory = true; } /* Storing the job conf on the log dir */ Path jobFilePath = new Path(logDir, CoronaJobHistoryFilesManager.getConfFilename(jobId)); fileManager.setConfFile(jobId, jobFilePath); FSDataOutputStream jobFileOut = null; try { if (!logDirFs.exists(jobFilePath)) { jobFileOut = logDirFs.create( jobFilePath, new FsPermission(HISTORY_FILE_PERMISSION), true, defaultBufferSize, logDirFs.getDefaultReplication(), logDirFs.getDefaultBlockSize(), null); conf.writeXml(jobFileOut); jobFileOut.close(); } } catch (IOException ioe) { LOG.error("Failed to store job conf in the log dir", ioe); } finally { if (jobFileOut != null) { try { jobFileOut.close(); } catch (IOException ie) { LOG.info( "Failed to close the job configuration file " + StringUtils.stringifyException(ie)); } } } }
/** * Moves the skipped files to the output directory. Called when operation in normal (non-clone) * mode. */ private void moveOutput() throws IOException { List<FileStatus> listStatus = getOutputMappings(); Text srcFile = new Text(); Text crushOut = new Text(); Set<String> crushOutputFiles = new HashSet<String>(nBuckets); for (FileStatus partFile : listStatus) { Path path = partFile.getPath(); Reader reader = new Reader(fs, path, fs.getConf()); try { while (reader.next(srcFile, crushOut)) { crushOutputFiles.add(new Path(crushOut.toString()).toUri().getPath()); } } finally { try { reader.close(); } catch (IOException e) { LOG.warn("Trapped exception when closing " + path, e); } } } assert crushOutputFiles.size() == nBuckets; /* * The crushoutput files will appear in a subdirectory of the output directory. The subdirectory will be the full path of the * input directory that was crushed. E.g. * * Crush input: * /user/me/input/dir1/file1 * /user/me/input/dir1/file2 * /user/me/input/dir2/file3 * /user/me/input/dir2/file4 * /user/me/input/dir3/dir4/file5 * /user/me/input/dir3/dir4/file6 * * Crush output: * /user/me/output/user/me/input/dir1/crushed_file ... * /user/me/output/user/me/input/dir2/crushed_file ... * /user/me/output/user/me/input/dir2/dir3/dir4/crushed_file ... * * We need to collapse this down to: * /user/me/output/dir1/crushed_file ... * /user/me/output/dir2/crushed_file ... * /user/me/output/dir2/dir3/dir4/crushed_file ... */ String srcDirName = fs.makeQualified(srcDir).toUri().getPath(); String destName = fs.makeQualified(dest).toUri().getPath(); String partToReplace = fs.makeQualified(outDir).toUri().getPath() + "/crush" + srcDirName; print(Verbosity.INFO, "\n\nCopying crush files to " + destName); for (String crushOutputFile : crushOutputFiles) { Path srcPath = new Path(crushOutputFile); Path destPath = new Path(destName + crushOutputFile.substring(partToReplace.length())).getParent(); rename(srcPath, destPath, null); } print(Verbosity.INFO, "\n\nMoving skipped files to " + destName); /* * Don't forget to move the files that were not crushed to the output dir so that the output dir has all the data that was in * the input dir, the difference being there are fewer files in the output dir. */ for (String name : skippedFiles) { Path srcPath = new Path(name); Path destPath = new Path(destName + name.substring(srcDirName.length())).getParent(); rename(srcPath, destPath, null); } }
public FileLinkInputStream(final FileSystem fs, final FileLink fileLink) throws IOException { this(fs, fileLink, fs.getConf().getInt("io.file.buffer.size", 4096)); }
@Override public Object getConf() { return mFs.getConf(); }
static void bulkLoadLots(Logger log, State state, Environment env, Value value) throws Exception { final Path dir = new Path("/tmp", "bulk_" + UUID.randomUUID().toString()); final Path fail = new Path(dir.toString() + "_fail"); final DefaultConfiguration defaultConfiguration = AccumuloConfiguration.getDefaultConfiguration(); final Random rand = (Random) state.get("rand"); final FileSystem fs = (FileSystem) state.get("fs"); fs.mkdirs(fail); final int parts = rand.nextInt(10) + 1; TreeSet<Integer> startRows = new TreeSet<Integer>(); startRows.add(0); while (startRows.size() < parts) startRows.add(rand.nextInt(LOTS)); List<String> printRows = new ArrayList<String>(startRows.size()); for (Integer row : startRows) printRows.add(String.format(FMT, row)); String markerColumnQualifier = String.format("%07d", counter.incrementAndGet()); log.debug( "preparing bulk files with start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier); List<Integer> rows = new ArrayList<Integer>(startRows); rows.add(LOTS); for (int i = 0; i < parts; i++) { String fileName = dir + "/" + String.format("part_%d.", i) + RFile.EXTENSION; FileSKVWriter f = FileOperations.getInstance().openWriter(fileName, fs, fs.getConf(), defaultConfiguration); f.startDefaultLocalityGroup(); int start = rows.get(i); int end = rows.get(i + 1); for (int j = start; j < end; j++) { Text row = new Text(String.format(FMT, j)); for (Column col : COLNAMES) { f.append(new Key(row, col.getColumnFamily(), col.getColumnQualifier()), value); } f.append(new Key(row, MARKER_CF, new Text(markerColumnQualifier)), ONE); } f.close(); } env.getConnector() .tableOperations() .importDirectory(Setup.getTableName(), dir.toString(), fail.toString(), true); fs.delete(dir, true); FileStatus[] failures = fs.listStatus(fail); if (failures != null && failures.length > 0) { state.set("bulkImportSuccess", "false"); throw new Exception(failures.length + " failure files found importing files from " + dir); } fs.delete(fail, true); log.debug( "Finished bulk import, start rows " + printRows + " last row " + String.format(FMT, LOTS - 1) + " marker " + markerColumnQualifier); }
@SuppressWarnings("static-access") public URI getDFSUri() throws IOException { return fileSystem.getDefaultUri(fileSystem.getConf()); }
/** * Test case that stops a writer after finalizing a block but before calling completeFile, * recovers a file from another writer, starts writing from that writer, and then has the old * lease holder call completeFile */ @Test(timeout = 60000) public void testCompleteOtherLeaseHoldersFile() throws Throwable { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); try { cluster.waitActive(); NameNode preSpyNN = cluster.getNameNode(); NameNode spyNN = spy(preSpyNN); // Delay completeFile DelayAnswer delayer = new DelayAnswer(); doAnswer(delayer).when(spyNN).complete(anyString(), anyString(), (Block) anyObject()); DFSClient client = new DFSClient(null, spyNN, conf, null); file1 = new Path("/testCompleteOtherLease"); final OutputStream stm = client.create("/testCompleteOtherLease", true); // write 1/2 block AppendTestUtil.write(stm, 0, 4096); final AtomicReference<Throwable> err = new AtomicReference<Throwable>(); Thread t = new Thread() { public void run() { try { stm.close(); } catch (Throwable t) { err.set(t); } } }; t.start(); LOG.info("Waiting for close to get to latch..."); delayer.waitForCall(); // At this point, the block is finalized on the DNs, but the file // has not been completed in the NN. // Lose the leases LOG.info("Killing lease checker"); client.leasechecker.interruptAndJoin(); FileSystem fs1 = cluster.getFileSystem(); FileSystem fs2 = AppendTestUtil.createHdfsWithDifferentUsername(fs1.getConf()); LOG.info("Recovering file"); recoverFile(fs2); LOG.info("Opening file for append from new fs"); FSDataOutputStream appenderStream = fs2.append(file1); LOG.info("Writing some data from new appender"); AppendTestUtil.write(appenderStream, 0, 4096); LOG.info("Telling old close to proceed."); delayer.proceed(); LOG.info("Waiting for close to finish."); t.join(); LOG.info("Close finished."); // We expect that close will get a "Lease mismatch" // error. Throwable thrownByClose = err.get(); assertNotNull(thrownByClose); assertTrue(thrownByClose instanceof IOException); if (!thrownByClose.getMessage().contains("Lease mismatch")) throw thrownByClose; // The appender should be able to close properly appenderStream.close(); } finally { cluster.shutdown(); } }
@BeforeClass public static void setupTestStaticConfiguration() throws Exception { LOGGER.info("AbstractTestWithStaticConfiguration setupTestStaticConfiguration"); properties = Maps.newHashMap(); if (!policyOnHdfs) { policyOnHdfs = new Boolean(System.getProperty("sentry.e2etest.policyonhdfs", "false")); } if (testServerType != null) { properties.put("sentry.e2etest.hiveServer2Type", testServerType); } baseDir = Files.createTempDir(); LOGGER.info("BaseDir = " + baseDir); logDir = assertCreateDir(new File(baseDir, "log")); confDir = assertCreateDir(new File(baseDir, "etc")); dataDir = assertCreateDir(new File(baseDir, "data")); policyFileLocation = new File(confDir, HiveServerFactory.AUTHZ_PROVIDER_FILENAME); String dfsType = System.getProperty(DFSFactory.FS_TYPE); dfs = DFSFactory.create(dfsType, baseDir, testServerType); fileSystem = dfs.getFileSystem(); PolicyFile policyFile = PolicyFile.setAdminOnServer1(ADMIN1) .setUserGroupMapping(StaticUserGroup.getStaticMapping()); policyFile.write(policyFileLocation); String policyURI; if (policyOnHdfs) { String dfsUri = FileSystem.getDefaultUri(fileSystem.getConf()).toString(); LOGGER.error("dfsUri " + dfsUri); policyURI = dfsUri + System.getProperty("sentry.e2etest.hive.policy.location", "/user/hive/sentry"); policyURI += "/" + HiveServerFactory.AUTHZ_PROVIDER_FILENAME; } else { policyURI = policyFileLocation.getPath(); } boolean startSentry = new Boolean(System.getProperty(EXTERNAL_SENTRY_SERVICE, "false")); if ("true".equalsIgnoreCase(System.getProperty(ENABLE_SENTRY_HA, "false"))) { enableSentryHA = true; } if (useSentryService && (!startSentry)) { setupSentryService(); } if (enableHiveConcurrency) { properties.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "true"); properties.put( HiveConf.ConfVars.HIVE_TXN_MANAGER.varname, "org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager"); properties.put( HiveConf.ConfVars.HIVE_LOCK_MANAGER.varname, "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager"); } hiveServer = create(properties, baseDir, confDir, logDir, policyURI, fileSystem); hiveServer.start(); createContext(); // Create tmp as scratch dir if it doesn't exist Path tmpPath = new Path("/tmp"); if (!fileSystem.exists(tmpPath)) { fileSystem.mkdirs(tmpPath, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); } }