static {
try {
// call newInstance() instead of using a shared instance from a cache
// to avoid accidentally having it closed by someone else
FileSystem fs = FileSystem.newInstance(FileSystem.getDefaultUri(CONF), CONF);
if (!(fs instanceof DistributedFileSystem)) {
String error =
"Cannot connect to HDFS. "
+ CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY
+ "("
+ CONF.get(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY)
+ ")"
+ " might be set incorrectly";
throw new RuntimeException(error);
}
DFS = (DistributedFileSystem) fs;
} catch (IOException e) {
throw new RuntimeException("couldn't retrieve FileSystem:\n" + e.getMessage(), e);
}
SUPPORTS_VOLUME_ID =
CONF.getBoolean(
DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED,
DFSConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT);
}
@AfterClass
public static void teardown() {
Path root = new Path(applicationPath);
try {
FileSystem fs = FileSystem.newInstance(root.toUri(), new Configuration());
fs.delete(root, true);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public HdfsDirectory(Path hdfsDirPath, Configuration configuration) throws IOException {
assert hdfsDirPath.toString().startsWith("hdfs:/") : hdfsDirPath.toString();
setLockFactory(NoLockFactory.getNoLockFactory());
this.hdfsDirPath = hdfsDirPath;
this.configuration = configuration;
fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), configuration);
try {
if (!fileSystem.exists(hdfsDirPath)) {
fileSystem.mkdirs(hdfsDirPath);
}
} catch (Exception e) {
org.apache.solr.util.IOUtils.closeQuietly(fileSystem);
throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e);
}
}
@Test
public void testPurge() throws IOException, InterruptedException {
FileSystem fileSystem = FileSystem.newInstance(new Configuration());
testTransferWindowFiles();
RemoteIterator<LocatedFileStatus> iterator =
fileSystem.listLocatedStatus(new Path(testMeta.applicationPath + "/bucket_data"));
Assert.assertTrue(iterator.hasNext());
testMeta.managedStateContext.getBucketsFileSystem().deleteTimeBucketsLessThanEqualTo(200);
iterator = fileSystem.listLocatedStatus(new Path(testMeta.applicationPath + "/bucket_data"));
if (iterator.hasNext()) {
Assert.fail("All buckets should be deleted");
}
}
public void copyInitialState(Path origAppDir) throws IOException {
// locate previous snapshot
String newAppDir = this.dag.assertAppPath();
FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
// read snapshot against new dependencies
Object snapshot = recoveryHandler.restore();
if (snapshot == null) {
throw new IllegalArgumentException("No previous application state found in " + origAppDir);
}
InputStream logIs = recoveryHandler.getLog();
// modify snapshot state to switch app id
((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf);
Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);
FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
// remove the path that was created by the storage agent during deserialization and replacement
fs.delete(checkpointPath, true);
// write snapshot to new location
recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
recoveryHandler.save(snapshot);
OutputStream logOs = recoveryHandler.rotateLog();
IOUtils.copy(logIs, logOs);
logOs.flush();
logOs.close();
logIs.close();
// copy sub directories that are not present in target
FileStatus[] lFiles = fs.listStatus(origAppDir);
for (FileStatus f : lFiles) {
if (f.isDirectory()) {
String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir);
if (!fs.exists(new Path(targetPath))) {
LOG.debug("Copying {} to {}", f.getPath(), targetPath);
FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
// FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf);
} else {
LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
// FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777));
}
}
}
}
private void createFileSystem(String uri)
throws IOException, InterruptedException, URISyntaxException {
if (this.state.getPropAsBoolean(
ConfigurationKeys.SHOULD_FS_PROXY_AS_USER,
ConfigurationKeys.DEFAULT_SHOULD_FS_PROXY_AS_USER)) {
// Initialize file system as a proxy user.
this.fs =
new ProxiedFileSystemWrapper()
.getProxiedFileSystem(
this.state,
ProxiedFileSystemWrapper.AuthType.TOKEN,
this.state.getProp(ConfigurationKeys.FS_PROXY_AS_USER_TOKEN_FILE),
uri);
} else {
// Initialize file system as the current user.
this.fs = FileSystem.newInstance(URI.create(uri), this.configuration);
}
}
public HdfsDirectory(Path hdfsDirPath, Configuration configuration) throws IOException {
setLockFactory(NoLockFactory.getNoLockFactory());
this.hdfsDirPath = hdfsDirPath;
this.configuration = configuration;
fileSystem = FileSystem.newInstance(hdfsDirPath.toUri(), configuration);
while (true) {
try {
if (!fileSystem.exists(hdfsDirPath)) {
boolean success = fileSystem.mkdirs(hdfsDirPath);
if (!success) {
throw new RuntimeException("Could not create directory: " + hdfsDirPath);
}
} else {
fileSystem.mkdirs(hdfsDirPath); // check for safe mode
}
break;
} catch (RemoteException e) {
if (e.getClassName().equals("org.apache.hadoop.hdfs.server.namenode.SafeModeException")) {
LOG.warn("The NameNode is in SafeMode - Solr will wait 5 seconds and try again.");
try {
Thread.sleep(5000);
} catch (InterruptedException e1) {
Thread.interrupted();
}
continue;
}
org.apache.solr.util.IOUtils.closeQuietly(fileSystem);
throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e);
} catch (Exception e) {
org.apache.solr.util.IOUtils.closeQuietly(fileSystem);
throw new RuntimeException("Problem creating directory: " + hdfsDirPath, e);
}
}
}
/**
* Override this method to change the FileSystem instance that is used by the operator.
*
* @return A FileSystem object.
* @throws IOException
*/
protected FileSystem getFSInstance() throws IOException {
return FileSystem.newInstance(filePath.toUri(), configuration);
}
@Override
public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
return FileSystem.newInstance(uri, conf);
}
/**
* Override this method to change the FileSystem instance that is used by the operator.
*
* @return A FileSystem object.
* @throws IOException
*/
protected FileSystem getFSInstance() throws IOException {
return FileSystem.newInstance(configuration);
}