/**
* Initialize a Har filesystem per har archive. The archive home directory is the top level
* directory in the filesystem that contains the HAR archive. Be careful with this method, you do
* not want to go on creating new Filesystem instances per call to path.getFileSystem(). the uri
* of Har is har://underlyingfsscheme-host:port/archivepath. or har:///archivepath. This assumes
* the underlying filesystem to be used in case not specified.
*/
@Override
public void initialize(URI name, Configuration conf) throws IOException {
// initialize the metadata cache, if needed
initializeMetadataCache(conf);
// decode the name
URI underLyingURI = decodeHarURI(name, conf);
// we got the right har Path- now check if this is
// truly a har filesystem
Path harPath = archivePath(new Path(name.getScheme(), name.getAuthority(), name.getPath()));
if (harPath == null) {
throw new IOException("Invalid path for the Har Filesystem. " + name.toString());
}
if (fs == null) {
fs = FileSystem.get(underLyingURI, conf);
}
uri = harPath.toUri();
archivePath = new Path(uri.getPath());
harAuth = getHarAuth(underLyingURI);
// check for the underlying fs containing
// the index file
Path masterIndexPath = new Path(archivePath, "_masterindex");
Path archiveIndexPath = new Path(archivePath, "_index");
if (!fs.exists(masterIndexPath) || !fs.exists(archiveIndexPath)) {
throw new IOException(
"Invalid path for the Har Filesystem. " + "No index file in " + harPath);
}
metadata = harMetaCache.get(uri);
if (metadata != null) {
FileStatus mStat = fs.getFileStatus(masterIndexPath);
FileStatus aStat = fs.getFileStatus(archiveIndexPath);
if (mStat.getModificationTime() != metadata.getMasterIndexTimestamp()
|| aStat.getModificationTime() != metadata.getArchiveIndexTimestamp()) {
// the archive has been overwritten since we last read it
// remove the entry from the meta data cache
metadata = null;
harMetaCache.remove(uri);
}
}
if (metadata == null) {
metadata = new HarMetaData(fs, masterIndexPath, archiveIndexPath);
metadata.parseMetaData();
harMetaCache.put(uri, metadata);
}
}
private void dirCleanup() {
Configuration conf = new Configuration();
try {
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(testPath);
if (fs.exists(dirPath)) {
fs.delete(dirPath, true);
}
} catch (IOException ex) {
LOG.warn("IO Error in test cleanup", ex);
}
}
public void copyInitialState(Path origAppDir) throws IOException {
// locate previous snapshot
String newAppDir = this.dag.assertAppPath();
FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(origAppDir.toString(), conf);
// read snapshot against new dependencies
Object snapshot = recoveryHandler.restore();
if (snapshot == null) {
throw new IllegalArgumentException("No previous application state found in " + origAppDir);
}
InputStream logIs = recoveryHandler.getLog();
// modify snapshot state to switch app id
((StreamingContainerManager.CheckpointState) snapshot).setApplicationId(this.dag, conf);
Path checkpointPath = new Path(newAppDir, LogicalPlan.SUBDIR_CHECKPOINTS);
FileSystem fs = FileSystem.newInstance(origAppDir.toUri(), conf);
// remove the path that was created by the storage agent during deserialization and replacement
fs.delete(checkpointPath, true);
// write snapshot to new location
recoveryHandler = new FSRecoveryHandler(newAppDir, conf);
recoveryHandler.save(snapshot);
OutputStream logOs = recoveryHandler.rotateLog();
IOUtils.copy(logIs, logOs);
logOs.flush();
logOs.close();
logIs.close();
// copy sub directories that are not present in target
FileStatus[] lFiles = fs.listStatus(origAppDir);
for (FileStatus f : lFiles) {
if (f.isDirectory()) {
String targetPath = f.getPath().toString().replace(origAppDir.toString(), newAppDir);
if (!fs.exists(new Path(targetPath))) {
LOG.debug("Copying {} to {}", f.getPath(), targetPath);
FileUtil.copy(fs, f.getPath(), fs, new Path(targetPath), false, conf);
// FSUtil.copy(fs, f, fs, new Path(targetPath), false, false, conf);
} else {
LOG.debug("Ignoring {} as it already exists under {}", f.getPath(), targetPath);
// FSUtil.setPermission(fs, new Path(targetPath), new FsPermission((short)0777));
}
}
}
}
static void checkRecords(
Configuration defaults, int noMaps, int noReduces, Path sortInput, Path sortOutput)
throws IOException {
JobConf jobConf = new JobConf(defaults, RecordChecker.class);
jobConf.setJobName("sortvalidate-record-checker");
jobConf.setInputFormat(SequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
jobConf.setOutputKeyClass(BytesWritable.class);
jobConf.setOutputValueClass(IntWritable.class);
jobConf.setMapperClass(Map.class);
jobConf.setReducerClass(Reduce.class);
JobClient client = new JobClient(jobConf);
ClusterStatus cluster = client.getClusterStatus();
if (noMaps == -1) {
noMaps = cluster.getTaskTrackers() * jobConf.getInt("test.sortvalidate.maps_per_host", 10);
}
if (noReduces == -1) {
noReduces = (int) (cluster.getMaxReduceTasks() * 0.9);
String sortReduces = jobConf.get("test.sortvalidate.reduces_per_host");
if (sortReduces != null) {
noReduces = cluster.getTaskTrackers() * Integer.parseInt(sortReduces);
}
}
jobConf.setNumMapTasks(noMaps);
jobConf.setNumReduceTasks(noReduces);
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
Path outputPath = new Path("/tmp/sortvalidate/recordchecker");
FileSystem fs = FileSystem.get(defaults);
if (fs.exists(outputPath)) {
fs.delete(outputPath, true);
}
FileOutputFormat.setOutputPath(jobConf, outputPath);
// Uncomment to run locally in a single process
// job_conf.set("mapred.job.tracker", "local");
Path[] inputPaths = FileInputFormat.getInputPaths(jobConf);
System.out.println(
"\nSortValidator.RecordChecker: Running on "
+ cluster.getTaskTrackers()
+ " nodes to validate sort from "
+ inputPaths[0]
+ ", "
+ inputPaths[1]
+ " into "
+ FileOutputFormat.getOutputPath(jobConf)
+ " with "
+ noReduces
+ " reduces.");
Date startTime = new Date();
System.out.println("Job started: " + startTime);
JobClient.runJob(jobConf);
Date end_time = new Date();
System.out.println("Job ended: " + end_time);
System.out.println(
"The job took " + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds.");
}
static void checkRecords(Configuration defaults, Path sortInput, Path sortOutput)
throws IOException {
FileSystem inputfs = sortInput.getFileSystem(defaults);
FileSystem outputfs = sortOutput.getFileSystem(defaults);
FileSystem defaultfs = FileSystem.get(defaults);
JobConf jobConf = new JobConf(defaults, RecordStatsChecker.class);
jobConf.setJobName("sortvalidate-recordstats-checker");
int noSortReduceTasks = outputfs.listStatus(sortOutput, sortPathsFilter).length;
jobConf.setInt("sortvalidate.sort.reduce.tasks", noSortReduceTasks);
int noSortInputpaths = inputfs.listStatus(sortInput).length;
jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
jobConf.setOutputKeyClass(IntWritable.class);
jobConf.setOutputValueClass(RecordStatsChecker.RecordStatsWritable.class);
jobConf.setMapperClass(Map.class);
jobConf.setCombinerClass(Reduce.class);
jobConf.setReducerClass(Reduce.class);
jobConf.setNumMapTasks(noSortReduceTasks);
jobConf.setNumReduceTasks(1);
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker");
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
}
FileOutputFormat.setOutputPath(jobConf, outputPath);
// Uncomment to run locally in a single process
// job_conf.set("mapred.job.tracker", "local");
Path[] inputPaths = FileInputFormat.getInputPaths(jobConf);
System.out.println(
"\nSortValidator.RecordStatsChecker: Validate sort "
+ "from "
+ inputPaths[0]
+ " ("
+ noSortInputpaths
+ " files), "
+ inputPaths[1]
+ " ("
+ noSortReduceTasks
+ " files) into "
+ FileOutputFormat.getOutputPath(jobConf)
+ " with 1 reducer.");
Date startTime = new Date();
System.out.println("Job started: " + startTime);
JobClient.runJob(jobConf);
Date end_time = new Date();
System.out.println("Job ended: " + end_time);
System.out.println(
"The job took " + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds.");
// Check to ensure that the statistics of the
// framework's sort-input and sort-output match
SequenceFile.Reader stats =
new SequenceFile.Reader(defaultfs, new Path(outputPath, "part-00000"), defaults);
IntWritable k1 = new IntWritable();
IntWritable k2 = new IntWritable();
RecordStatsWritable v1 = new RecordStatsWritable();
RecordStatsWritable v2 = new RecordStatsWritable();
if (!stats.next(k1, v1)) {
throw new IOException("Failed to read record #1 from reduce's output");
}
if (!stats.next(k2, v2)) {
throw new IOException("Failed to read record #2 from reduce's output");
}
if ((v1.getBytes() != v2.getBytes())
|| (v1.getRecords() != v2.getRecords())
|| v1.getChecksum() != v2.getChecksum()) {
throw new IOException(
"("
+ v1.getBytes()
+ ", "
+ v1.getRecords()
+ ", "
+ v1.getChecksum()
+ ") v/s ("
+ v2.getBytes()
+ ", "
+ v2.getRecords()
+ ", "
+ v2.getChecksum()
+ ")");
}
}