public static void baseSetup() throws Exception { MiniDFSShim dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); fs = dfs.getFileSystem(); baseDfsDir = new Path(new Path(fs.getUri()), "/base"); fs.mkdirs(baseDfsDir); warehouseDir = new Path(baseDfsDir, "warehouse"); fs.mkdirs(warehouseDir); conf.setVar(ConfVars.METASTOREWAREHOUSE, warehouseDir.toString()); // Assuming the tests are run either in C or D drive in Windows OS! dataFileDir = conf.get("test.data.files") .replace('\\', '/') .replace("c:", "") .replace("C:", "") .replace("D:", "") .replace("d:", ""); dataFilePath = new Path(dataFileDir, "kv1.txt"); // Set up scratch directory Path scratchDir = new Path(baseDfsDir, "scratchdir"); conf.setVar(HiveConf.ConfVars.SCRATCHDIR, scratchDir.toString()); // set hive conf vars conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setBoolVar(HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS, true); conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict"); int port = MetaStoreUtils.findFreePort(); MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge()); SessionState.start(new CliSessionState(conf)); driver = new Driver(conf); setupDataTable(); }
@BeforeClass public static void clusterSetupAtBegining() throws IOException, LoginException, URISyntaxException { SupportsBlocks = true; CONF.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true); cluster = new MiniDFSCluster.Builder(CONF) .nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)) .numDataNodes(2) .build(); cluster.waitClusterUp(); fHdfs = cluster.getFileSystem(0); fHdfs2 = cluster.getFileSystem(1); fHdfs .getConf() .set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString()); fHdfs2 .getConf() .set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, FsConstants.VIEWFS_URI.toString()); defaultWorkingDirectory = fHdfs.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); defaultWorkingDirectory2 = fHdfs2.makeQualified( new Path("/user/" + UserGroupInformation.getCurrentUser().getShortUserName())); fHdfs.mkdirs(defaultWorkingDirectory); fHdfs2.mkdirs(defaultWorkingDirectory2); }
@org.junit.Test public void testPermissions() throws Exception { Path myFile = new Path("filePerm.txt"); fs.create(myFile); short perm = 0777; fs.setPermission(myFile, new FsPermission(perm)); assertEquals(fs.getFileStatus(myFile).getPermission().toShort(), perm); perm = 0700; fs.setPermission(myFile, new FsPermission(perm)); assertEquals(fs.getFileStatus(myFile).getPermission().toShort(), perm); fs.delete(myFile); assertFalse(fs.exists(myFile)); /* directory permissions */ Path directory = new Path("aa/bb/cc"); perm = 0700; fs.mkdirs(directory, new FsPermission(perm)); assertEquals(fs.getFileStatus(directory).getPermission().toShort(), perm); fs.delete(new Path("aa"), true); assertFalse(fs.exists(directory)); perm = 0777; fs.mkdirs(directory, new FsPermission(perm)); assertEquals(fs.getFileStatus(directory).getPermission().toShort(), perm); fs.delete(new Path("aa"), true); assertFalse(fs.exists(directory)); }
private void testMovingFiles(boolean useAcl) throws Exception { // Create a tmp directory with wide-open permissions and sticky bit Path tmpPath = new Path("/tmp"); Path tmpPath2 = new Path("/tmp2"); hdfs.mkdirs(tmpPath); hdfs.mkdirs(tmpPath2); hdfs.setPermission(tmpPath, new FsPermission((short) 01777)); if (useAcl) { applyAcl(tmpPath); } hdfs.setPermission(tmpPath2, new FsPermission((short) 01777)); if (useAcl) { applyAcl(tmpPath2); } // Write a file to the new tmp directory as a regular user Path file = new Path(tmpPath, "foo"); writeFile(hdfsAsUser1, file); // Log onto cluster as another user and attempt to move the file try { hdfsAsUser2.rename(file, new Path(tmpPath2, "renamed")); fail("Shouldn't be able to rename someone else's file with SB on"); } catch (IOException ioe) { assertTrue(ioe instanceof AccessControlException); assertTrue(ioe.getMessage().contains("sticky bit")); } }
@Override public void start(CoprocessorEnvironment env) { this.env = (RegionCoprocessorEnvironment) env; random = new SecureRandom(); conf = env.getConfiguration(); baseStagingDir = SecureBulkLoadUtil.getBaseStagingDir(conf); this.userProvider = UserProvider.instantiate(conf); try { fs = FileSystem.get(conf); fs.mkdirs(baseStagingDir, PERM_HIDDEN); fs.setPermission(baseStagingDir, PERM_HIDDEN); // no sticky bit in hadoop-1.0, making directory nonempty so it never gets erased fs.mkdirs(new Path(baseStagingDir, "DONOTERASE"), PERM_HIDDEN); FileStatus status = fs.getFileStatus(baseStagingDir); if (status == null) { throw new IllegalStateException("Failed to create staging directory"); } if (!status.getPermission().equals(PERM_HIDDEN)) { throw new IllegalStateException( "Directory already exists but permissions aren't set to '-rwx--x--x' "); } } catch (IOException e) { throw new IllegalStateException("Failed to get FileSystem instance", e); } }
@Test public void errorRenameTest() throws Exception { // Rename /dirA to /dirA/dirB should fail { Path dirA = new Path("/dirA"); Path finalDst = new Path("/dirA/dirB"); sTFS.mkdirs(dirA); Assert.assertFalse(sTFS.rename(dirA, finalDst)); Assert.assertFalse(sTFS.exists(finalDst)); Assert.assertTrue(sTFS.exists(dirA)); cleanup(sTFS); } // Rename /fileA to /fileB should fail if /fileB exists { Path fileA = new Path("/fileA"); Path fileB = new Path("/fileB"); create(sTFS, fileA); create(sTFS, fileB); Assert.assertFalse(sTFS.rename(fileA, fileB)); Assert.assertTrue(sTFS.exists(fileA)); Assert.assertTrue(sTFS.exists(fileB)); cleanup(sTFS); } // Rename /fileA to /dirA/fileA should fail if /dirA/fileA exists { Path fileA = new Path("/fileA"); Path dirA = new Path("/dirA"); Path finalDst = new Path("/dirA/fileA"); create(sTFS, fileA); create(sTFS, finalDst); sTFS.mkdirs(dirA); Assert.assertFalse(sTFS.rename(fileA, dirA)); Assert.assertTrue(sTFS.exists(fileA)); Assert.assertTrue(sTFS.exists(dirA)); Assert.assertTrue(sTFS.exists(finalDst)); cleanup(sTFS); } // Rename /fileA to an nonexistent path should fail { Path fileA = new Path("/fileA"); Path nonexistentPath = new Path("/doesNotExist/fileA"); create(sTFS, fileA); Assert.assertFalse(sTFS.rename(fileA, nonexistentPath)); Assert.assertTrue(sTFS.exists(fileA)); cleanup(sTFS); } }
@Override protected synchronized void startInternal() throws Exception { // create filesystem only now, as part of service-start. By this time, RM is // authenticated with kerberos so we are good to create a file-system // handle. fs = fsWorkingPath.getFileSystem(getConfig()); fs.mkdirs(rmDTSecretManagerRoot); fs.mkdirs(rmAppRoot); }
@Override public void visit(State state, Properties props) throws Exception { Connector conn = state.getConnector(); Random rand = (Random) state.get("rand"); @SuppressWarnings("unchecked") List<String> tableNames = (List<String>) state.get("tables"); String tableName = tableNames.get(rand.nextInt(tableNames.size())); Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", Math.abs(rand.nextLong())); fs.mkdirs(new Path(bulkDir)); fs.mkdirs(new Path(bulkDir + "_f")); try { BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf"); try { TreeSet<Long> rows = new TreeSet<Long>(); int numRows = rand.nextInt(100000); for (int i = 0; i < numRows; i++) { rows.add(Math.abs(rand.nextLong())); } for (Long row : rows) { Mutation m = new Mutation(String.format("%016x", row)); long val = Math.abs(rand.nextLong()); for (int j = 0; j < 10; j++) { m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes())); } bw.addMutation(m); } } finally { bw.close(); } conn.tableOperations() .importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean()); log.debug("BulkImported to " + tableName); } catch (TableNotFoundException e) { log.debug("BulkImport " + tableName + " failed, doesnt exist"); } catch (TableOfflineException toe) { log.debug("BulkImport " + tableName + " failed, offline"); } finally { fs.delete(new Path(bulkDir), true); fs.delete(new Path(bulkDir + "_f"), true); } }
@Test public void testGeneralSBBehavior() throws Exception { Path baseDir = new Path("/mcgann"); hdfs.mkdirs(baseDir); // Create a tmp directory with wide-open permissions and sticky bit Path p = new Path(baseDir, "tmp"); hdfs.mkdirs(p); hdfs.setPermission(p, new FsPermission((short) 01777)); confirmCanAppend(conf, p); baseDir = new Path("/eccleston"); hdfs.mkdirs(baseDir); p = new Path(baseDir, "roguetraders"); hdfs.mkdirs(p); confirmSettingAndGetting(hdfs, p, baseDir); baseDir = new Path("/tennant"); hdfs.mkdirs(baseDir); p = new Path(baseDir, "contemporary"); hdfs.mkdirs(p); hdfs.setPermission(p, new FsPermission((short) 01777)); confirmDeletingFiles(conf, p); baseDir = new Path("/smith"); hdfs.mkdirs(baseDir); p = new Path(baseDir, "scissorsisters"); // Turn on its sticky bit hdfs.mkdirs(p, new FsPermission((short) 01666)); confirmStickyBitDoesntPropagate(hdfs, baseDir); }
public void createDirectories( Configuration conf, FileSystem fs, String inputDir, String failedDir) throws Exception { // System.out.println(fs.getHomeDirectory()); Path input = new Path(inputDir); Path failput = new Path(failedDir); fs.mkdirs(input); fs.mkdirs(failput); // Tweek permissions if needed // fs.setPermission(input, FsPermission.createImmutable((short) 0777)); // fs.setOwner(input, "accumulo", "supergroup"); // fs.setOwner(output, "accumulo", "supergroup"); }
public void setUp() throws Exception { readMessages = new ArrayList<Message>(); fs = rootDir.getFileSystem(conf); defaultConf = new JobConf(new Configuration()); dataFiles = new String[] {TestUtil.files[0], TestUtil.files[1]}; setUpCluster(); fs.delete(new Path(cluster.getRootDir()), true); Path streamDir = new Path(cluster.getDataDir(), testStream); fs.delete(streamDir, true); fs.mkdirs(streamDir); collectorDir = new Path(streamDir, collectors); fs.delete(collectorDir, true); fs.mkdirs(collectorDir); TestUtil.setUpFiles(cluster, collectors, dataFiles, null, null, 1, 0); }
/** * Assert that getSplitEditFilesSorted returns files in expected order and that it skips * moved-aside files. * * @throws IOException */ @Test public void testGetSplitEditFilesSorted() throws IOException { FileSystem fs = FileSystem.get(util.getConfiguration()); Path regiondir = util.getDataTestDir("regiondir"); fs.delete(regiondir, true); fs.mkdirs(regiondir); Path recoverededits = WALSplitter.getRegionDirRecoveredEditsDir(regiondir); String first = WALSplitter.formatRecoveredEditsFileName(-1); createFile(fs, recoverededits, first); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(0)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(1)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(11)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(2)); createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(50)); String last = WALSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE); createFile(fs, recoverededits, last); createFile( fs, recoverededits, Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis()); final Configuration walConf = new Configuration(util.getConfiguration()); FSUtils.setRootDir(walConf, regiondir); (new WALFactory(walConf, null, "dummyLogName")).getWAL(new byte[] {}, null); NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir); assertEquals(7, files.size()); assertEquals(files.pollFirst().getName(), first); assertEquals(files.pollLast().getName(), last); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(0)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(1)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(2)); assertEquals(files.pollFirst().getName(), WALSplitter.formatRecoveredEditsFileName(11)); }
@Test public void testVanishingTaskZNode() throws Exception { LOG.info("testVanishingTaskZNode"); conf.setInt("hbase.splitlog.manager.unassigned.timeout", 0); slm = new SplitLogManager(zkw, conf, stopper, "dummy-master", null); slm.finishInitialization(); FileSystem fs = TEST_UTIL.getTestFileSystem(); final Path logDir = new Path(fs.getWorkingDirectory(), UUID.randomUUID().toString()); fs.mkdirs(logDir); Path logFile = new Path(logDir, UUID.randomUUID().toString()); fs.createNewFile(logFile); new Thread() { public void run() { try { // this call will block because there are no SplitLogWorkers slm.splitLogDistributed(logDir); } catch (Exception e) { LOG.warn("splitLogDistributed failed", e); fail(); } } }.start(); waitForCounter(tot_mgr_node_create_result, 0, 1, 10000); String znode = ZKSplitLog.getEncodedNodeName(zkw, logFile.toString()); // remove the task znode ZKUtil.deleteNode(zkw, znode); waitForCounter(tot_mgr_get_data_nonode, 0, 1, 30000); waitForCounter(tot_mgr_log_split_batch_success, 0, 1, 1000); assertTrue(fs.exists(logFile)); fs.delete(logDir, true); }
public void testLease() throws Exception { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null); try { FileSystem fs = cluster.getFileSystem(); assertTrue(fs.mkdirs(dir)); Path a = new Path(dir, "a"); Path b = new Path(dir, "b"); DataOutputStream a_out = fs.create(a); a_out.writeBytes("something"); assertTrue(hasLease(cluster, a)); assertTrue(!hasLease(cluster, b)); DataOutputStream b_out = fs.create(b); b_out.writeBytes("something"); assertTrue(hasLease(cluster, a)); assertTrue(hasLease(cluster, b)); a_out.close(); b_out.close(); assertTrue(!hasLease(cluster, a)); assertTrue(!hasLease(cluster, b)); fs.delete(dir, true); } finally { if (cluster != null) { cluster.shutdown(); } } }
private void touch(FileSystem fs, String path, boolean generateFiles) throws Exception { if (generateFiles) { fs.create(new Path(path)).close(); } else { fs.mkdirs(new Path(path)); } }
/** * Generate random data, compress it, index and md5 hash the data. Then read it all back and md5 * that too, to verify that it all went ok. * * @param testWithIndex Should we index or not? * @param charsToOutput How many characters of random data should we output. * @throws IOException * @throws NoSuchAlgorithmException * @throws InterruptedException */ private void runTest(boolean testWithIndex, int charsToOutput) throws IOException, NoSuchAlgorithmException, InterruptedException { Configuration conf = new Configuration(); conf.setLong("fs.local.block.size", charsToOutput / 2); // reducing block size to force a split of the tiny file conf.set("io.compression.codecs", LzopCodec.class.getName()); Assume.assumeTrue(CoreTestUtil.okToRunLzoTests(conf)); FileSystem.getLocal(conf).close(); // remove cached filesystem (if any) FileSystem localFs = FileSystem.getLocal(conf); localFs.delete(outputDir_, true); localFs.mkdirs(outputDir_); Job job = new Job(conf); TextOutputFormat.setCompressOutput(job, true); TextOutputFormat.setOutputCompressorClass(job, LzopCodec.class); TextOutputFormat.setOutputPath(job, outputDir_); TaskAttemptContext attemptContext = new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID("123", 0, false, 1, 2)); // create some input data byte[] expectedMd5 = createTestInput(outputDir_, localFs, attemptContext, charsToOutput); if (testWithIndex) { Path lzoFile = new Path(outputDir_, lzoFileName_); LzoIndex.createIndex(localFs, lzoFile); } LzoTextInputFormat inputFormat = new LzoTextInputFormat(); TextInputFormat.setInputPaths(job, outputDir_); List<InputSplit> is = inputFormat.getSplits(job); // verify we have the right number of lzo chunks if (testWithIndex && OUTPUT_BIG == charsToOutput) { assertEquals(3, is.size()); } else { assertEquals(1, is.size()); } // let's read it all and calculate the md5 hash for (InputSplit inputSplit : is) { RecordReader<LongWritable, Text> rr = inputFormat.createRecordReader(inputSplit, attemptContext); rr.initialize(inputSplit, attemptContext); while (rr.nextKeyValue()) { Text value = rr.getCurrentValue(); md5_.update(value.getBytes(), 0, value.getLength()); } rr.close(); } localFs.close(); assertTrue(Arrays.equals(expectedMd5, md5_.digest())); }
public static boolean mkdir(final String outPathName) throws IOException { Configuration conf = new Configuration(); Path path = new Path(outPathName); FileSystem fs = path.getFileSystem(conf); fs.mkdirs(path); return false; }
@Before public void setup() throws IOException { configuration = new YarnConfiguration(); configuration.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getCanonicalName()); fs = FileSystem.get(configuration); workingPath = new Path( new File("target", this.getClass().getSimpleName() + "-remoteDir").getAbsolutePath()); configuration.set(YarnConfiguration.FS_BASED_RM_CONF_STORE, workingPath.toString()); tmpDir = new Path(new File("target", this.getClass().getSimpleName() + "-tmpDir").getAbsolutePath()); fs.delete(workingPath, true); fs.delete(tmpDir, true); fs.mkdirs(workingPath); fs.mkdirs(tmpDir); }
/* * Write some entries in the log file. * 7 different tables with name "testtb-%d" * 10 region per table with name "tableName-region-%d" * 50 entry with row key "row-%d" */ private void writeTestLog(final Path logFile) throws IOException { fs.mkdirs(logFile.getParent()); HLog.Writer writer = HLog.createWriter(fs, logFile, conf); try { for (int i = 0; i < 7; ++i) { byte[] tableName = getTableName(i); for (int j = 0; j < 10; ++j) { byte[] regionName = getRegionName(tableName, j); for (int k = 0; k < 50; ++k) { byte[] rowkey = Bytes.toBytes("row-" + k); HLogKey key = new HLogKey( regionName, tableName, (long) k, System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID); WALEdit edit = new WALEdit(); edit.add(new KeyValue(rowkey, TEST_FAMILY, TEST_QUALIFIER, rowkey)); writer.append(new HLog.Entry(key, edit)); } } } } finally { writer.close(); } }
private void createDirIfNotExist(Path path) throws IOException { if (!fs.exists(path)) { if (!fs.mkdirs(path)) { throw new IOException("Unable to create: " + path); } } }
@Before public void setUp() throws Exception { fsTarget = fHdfs; fsTarget2 = fHdfs2; targetTestRoot = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget); targetTestRoot2 = fileSystemTestHelper.getAbsoluteTestRootPath(fsTarget2); fsTarget.delete(targetTestRoot, true); fsTarget2.delete(targetTestRoot2, true); fsTarget.mkdirs(targetTestRoot); fsTarget2.mkdirs(targetTestRoot2); fsViewConf = ViewFileSystemTestSetup.createConfig(); setupMountPoints(); fsView = FileSystem.get(FsConstants.VIEWFS_URI, fsViewConf); }
/** * * Function to copy the image files from local file system to hdfs * * @param conf * @param sourcePath * @param destinationPath * @throws Exception */ public static void copyFromLocal(JobConf conf, String sourcePath, String destinationPath) throws Exception { FileSystem fs = FileSystem.get(conf); Path interPath = new Path(destinationPath); fs.mkdirs(interPath); fs.copyFromLocalFile(new Path(sourcePath), interPath); }
// Create temp directory in HDFS to store logsearch logs before sorting public void tmpDirHDFS( boolean quiet, boolean silent, FileSystem fs, Configuration conf, String tmp, boolean log) { logConsole(quiet, silent, info, "Creating new Temp Directory in HDFS: " + tmp); try { Path path = new Path(tmp); if (!(fs.exists(path))) { // Create directory fs.mkdirs(path); if (log != true) { fs.deleteOnExit(path); } } } catch (IOException e) { if (e.toString().contains("Failed to find any Kerberos")) { logConsole(true, true, error, "No/bad Kerberos ticket - please authenticate."); System.exit(1); } else if (e.toString().contains("quota") && e.toString().contains("exceeded")) { logConsole(true, true, error, "Disk quota Exceeded."); System.exit(1); } e.printStackTrace(); System.exit(1); } }
public static void addTxt( HttpServletRequest request, HttpServletResponse response, OutputStreamWriter outStream) throws Exception { SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd"); String day = fmt.format(new Date()); Map stormconf = Utils.readStormConfig(); String store = (String) stormconf.get("higo.download.offline.store") + "/" + day + "/upload_" + java.util.UUID.randomUUID().toString(); Configuration conf = getConf(stormconf); FileSystem fs = FileSystem.get(conf); if (!fs.exists(new Path(store))) { fs.mkdirs(new Path(store)); } HashMap<String, String> params = new HashMap<String, String>(); Path outpath = new Path(store, String.valueOf(System.currentTimeMillis())); FSDataOutputStream out = fs.create(outpath); Upload up = new Upload(); up.mergerTo(request, response, "utf-8", out, params); out.close(); String rtn = addTxt(params.get("tableName"), store, params.get("callback")); outStream.append(rtn); }
/** * 디렉토리가 존재하지 않는다면 생성한다. * * @param directory 디렉토리 * @param conf Hadoop Configuration * @throws java.io.IOException HDFS 작업을 실패한 경우 */ public static void makeDirectoryIfNotExists(String directory, Configuration conf) throws IOException { FileSystem fileSystem = FileSystem.get(conf); if (!isExist(conf, directory) && !isDirectory(fileSystem, directory)) { fileSystem.mkdirs(new Path(directory)); } }
@BeforeClass public static void setUpBeforeClass() throws Exception { File minidfsDir = new File("target/minidfs-" + UUID.randomUUID()).getAbsoluteFile(); minidfsDir.mkdirs(); Assert.assertTrue(minidfsDir.exists()); System.setProperty(MiniDFSCluster.PROP_TEST_BUILD_DATA, minidfsDir.getPath()); Configuration conf = new HdfsConfiguration(); conf.set("dfs.namenode.fs-limits.min-block-size", String.valueOf(32)); EditLogFileOutputStream.setShouldSkipFsyncForTesting(true); miniDFS = new MiniDFSCluster.Builder(conf).numDataNodes(3).build(); dir = new Path(miniDFS.getURI() + "/dir"); FileSystem fs = miniDFS.getFileSystem(); fs.mkdirs(dir); writeFile(fs, new Path(dir + "/forAllTests/" + "path"), 1000); dummyEtc = new File(minidfsDir, "dummy-etc"); dummyEtc.mkdirs(); Assert.assertTrue(dummyEtc.exists()); Configuration dummyConf = new Configuration(false); for (String file : new String[] {"core", "hdfs", "mapred", "yarn"}) { File siteXml = new File(dummyEtc, file + "-site.xml"); FileOutputStream out = new FileOutputStream(siteXml); dummyConf.writeXml(out); out.close(); } resourcesDir = minidfsDir.getAbsolutePath(); hadoopConfDir = dummyEtc.getName(); System.setProperty("sdc.resources.dir", resourcesDir); ; }
public void SequenceFileRenameRetryCoreTest(int numberOfRetriesRequired, boolean closeSucceed) throws Exception { String hdfsPath = "file:///tmp/flume-test." + Calendar.getInstance().getTimeInMillis() + "." + Thread.currentThread().getId(); Context context = new Context(); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path dirPath = new Path(hdfsPath); fs.delete(dirPath, true); fs.mkdirs(dirPath); context.put("hdfs.path", hdfsPath); context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired)); context.put("hdfs.rollCount", "1"); context.put("hdfs.retryInterval", "1"); context.put("hdfs.callTimeout", Long.toString(1000)); MockFileSystem mockFs = new MockFileSystem(fs, numberOfRetriesRequired, closeSucceed); BucketWriter bucketWriter = new BucketWriter( 0, 0, 1, 1, ctx, hdfsPath, hdfsPath, "singleBucket", ".tmp", null, null, null, new MockDataStream(mockFs), timedRollerPool, proxy, new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()), 0, null, null, 30000, Executors.newSingleThreadExecutor(), 1, numberOfRetriesRequired); bucketWriter.setFileSystem(mockFs); // At this point, we checked if isFileClosed is available in // this JVM, so lets make it check again. Event event = EventBuilder.withBody("test", Charsets.UTF_8); bucketWriter.append(event); // This is what triggers the close, so a 2nd append is required :/ bucketWriter.append(event); TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2); Assert.assertTrue( "Expected " + numberOfRetriesRequired + " " + "but got " + bucketWriter.renameTries.get(), bucketWriter.renameTries.get() == numberOfRetriesRequired); }
@SuppressWarnings("rawtypes") static Path createConfigurationFileInFs( FileSystem fs, String appHome, Map stormConf, YarnConfiguration yarnConf) throws IOException { // dump stringwriter's content into FS conf/storm.yaml Path confDst = new Path(fs.getHomeDirectory(), appHome + Path.SEPARATOR + STORM_CONF_PATH_STRING); Path dirDst = confDst.getParent(); fs.mkdirs(dirDst); // storm.yaml FSDataOutputStream out = fs.create(confDst); Yaml yaml = new Yaml(); OutputStreamWriter writer = new OutputStreamWriter(out); rmNulls(stormConf); yaml.dump(stormConf, writer); writer.close(); out.close(); // yarn-site.xml Path yarn_site_xml = new Path(dirDst, "yarn-site.xml"); out = fs.create(yarn_site_xml); writer = new OutputStreamWriter(out); yarnConf.writeXml(writer); writer.close(); out.close(); // logback.xml Path logback_xml = new Path(dirDst, "logback.xml"); out = fs.create(logback_xml); CreateLogbackXML(out); out.close(); return dirDst; }
public HdfsDirectory(Path hdfsDirPath, LockFactory lockFactory, Configuration configuration) throws IOException { super(lockFactory); this.hdfsDirPath = hdfsDirPath; this.configuration = configuration; fileSystem = FileSystem.get(hdfsDirPath.toUri(), configuration); fileContext = FileContext.getFileContext(hdfsDirPath.toUri(), configuration); if (fileSystem instanceof DistributedFileSystem) { // Make sure dfs is not in safe mode while (((DistributedFileSystem) fileSystem).setSafeMode(SafeModeAction.SAFEMODE_GET, true)) { LOG.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again."); try { Thread.sleep(5000); } catch (InterruptedException e) { Thread.interrupted(); // continue } } } try { if (!fileSystem.exists(hdfsDirPath)) { boolean success = fileSystem.mkdirs(hdfsDirPath); if (!success) { throw new RuntimeException("Could not create directory: " + hdfsDirPath); } } } catch (Exception e) { org.apache.solr.common.util.IOUtils.closeQuietly(fileSystem); throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e); } }
@Override public boolean initializeReader(Properties configuration) { try { hdfsConfiguration = new org.apache.hadoop.conf.Configuration(false); hdfsConfiguration.set("fs.defaultFS", configuration.getProperty("hdfs.url")); hdfsConfiguration.set( "fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); hdfsConfiguration.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName()); hdfsConfiguration.set( "hdfs.url", configuration.getProperty("hdfs.url")); // "hdfs://snf-618466.vm.okeanos.grnet.gr:8020"); // hdfsConfiguration.set("prefix", "/user/vagvaz/"); hdfsConfiguration.set("prefix", configuration.getProperty("prefix")); // fileSystem = FileSystem.get(hdfsConfiguration); fileSystem = FileSystem.get( getDefaultUri(hdfsConfiguration), hdfsConfiguration, configuration.getProperty("hdfs.user")); if (!fileSystem.exists(basePath)) { log.info("Creating base path on HDFS " + configuration.getProperty("hdfs.url")); fileSystem.mkdirs(basePath); } } catch (Exception e) { log.error( "Could not create HDFS remote FileSystem using \n" + configuration.toString() + "\n"); return false; } return true; }