/**
* Parse a list of strings into longs.
*
* @param strs the list of strings to parse
* @return a list of longs that were parsed. same length as strs.
*/
private static long[] parseTimestamps(String[] strs) {
if (strs == null) {
return null;
}
long[] result = new long[strs.length];
for (int i = 0; i < strs.length; ++i) {
result[i] = Long.parseLong(strs[i]);
}
return result;
}
/** Sets up configuration based on params */
private static boolean setup(Hashtable<String, String> curConf, Configuration argConf) {
if (argConf.get("file") == null) {
logger.fatal("Missing file parameter");
System.exit(1);
}
if (argConf.get("hdfs_base_path") == null) {
logger.fatal("Missing HDFS base path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("hdfs_temp_path") == null) {
logger.fatal("Missing HDFS temp path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("local_temp_path") == null) {
logger.fatal("Missing local temp path, check gestore-conf.xml");
System.exit(1);
}
// Input paramaters
curConf.put("run_id", argConf.get("run", ""));
curConf.put("task_id", argConf.get("task", ""));
curConf.put("file_id", argConf.get("file"));
curConf.put("local_path", argConf.get("path", ""));
curConf.put("type", argConf.get("type", "l2r"));
curConf.put("timestamp_start", argConf.get("timestamp_start", "1"));
curConf.put(
"timestamp_stop", argConf.get("timestamp_stop", Integer.toString(Integer.MAX_VALUE)));
curConf.put("delimiter", argConf.get("regex", "ID=.*"));
curConf.put("taxon", argConf.get("taxon", "all"));
curConf.put("intermediate", argConf.get("full_run", "false"));
curConf.put("quick_add", argConf.get("quick_add", "false"));
Boolean full_run = curConf.get("intermediate").matches("(?i).*true.*");
curConf.put("format", argConf.get("format", "unknown"));
curConf.put("split", argConf.get("split", "1"));
curConf.put("copy", argConf.get("copy", "true"));
// Constants
curConf.put("base_path", argConf.get("hdfs_base_path"));
curConf.put("temp_path", argConf.get("hdfs_temp_path"));
curConf.put("local_temp_path", argConf.get("local_temp_path"));
curConf.put("db_name_files", argConf.get("hbase_file_table"));
curConf.put("db_name_runs", argConf.get("hbase_run_table"));
curConf.put("db_name_updates", argConf.get("hbase_db_update_table"));
// Timestamps
Date currentTime = new Date();
Date endDate = new Date(new Long(curConf.get("timestamp_stop")));
curConf.put("timestamp_real", Long.toString(currentTime.getTime()));
return true;
}
@Override
public void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
String line = value.toString();
String[] tokens = line.split("\\s");
String hour = new String();
long numRequests = 0;
if (tokens.length > 4) {
// get the project name field
hour = tokens[4];
// get the number of requests field
numRequests = Long.parseLong(tokens[2]);
// output the key, value pairs where the key is a combination of
// project name and datetime and the value is number of requests
context.write(new Text(hour), new LongWritable(numRequests));
}
}
/**
* Update the maps baseDirSize and baseDirNumberSubDir when adding cache.
*
* @param cacheStatus cache status of the cache is added
*/
private static void addCacheInfoUpdate(CacheStatus cacheStatus) {
long cacheSize = cacheStatus.size;
synchronized (baseDirSize) {
Long dirSize = baseDirSize.get(cacheStatus.getBaseDir());
if (dirSize == null) {
dirSize = Long.valueOf(cacheSize);
} else {
dirSize += cacheSize;
}
baseDirSize.put(cacheStatus.getBaseDir(), dirSize);
}
synchronized (baseDirNumberSubDir) {
Integer dirSubDir = baseDirNumberSubDir.get(cacheStatus.getBaseDir());
if (dirSubDir == null) {
dirSubDir = 1;
} else {
dirSubDir += 1;
}
baseDirNumberSubDir.put(cacheStatus.getBaseDir(), dirSubDir);
}
}
/**
* Get the locally cached file or archive; it could either be previously cached (and valid) or
* copy it from the {@link FileSystem} now.
*
* @param cache the cache to be localized, this should be specified as new
* URI(hdfs://hostname:port/absolute_path_to_file#LINKNAME). If no schema or hostname:port is
* provided the file is assumed to be in the filesystem being used in the Configuration
* @param conf The Confguration file which contains the filesystem
* @param subDir The sub cache Dir where you want to localize the files/archives
* @param fileStatus The file status on the dfs.
* @param isArchive if the cache is an archive or a file. In case it is an archive with a .zip or
* .jar or .tar or .tgz or .tar.gz extension it will be unzipped/unjarred/untarred
* automatically and the directory where the archive is unzipped/unjarred/untarred is returned
* as the Path. In case of a file, the path to the file is returned
* @param confFileStamp this is the hdfs file modification timestamp to verify that the file to be
* cached hasn't changed since the job started
* @param fileLength this is the length of the cache file
* @param currentWorkDir this is the directory where you would want to create symlinks for the
* locally cached files/archives
* @param honorSymLinkConf if this is false, then the symlinks are not created even if conf says
* so (this is required for an optimization in task launches
* @param lDirAllocator LocalDirAllocator of the tracker
* @return the path to directory where the archives are unjarred in case of archives, the path to
* the file where the file is copied locally
* @throws IOException
*/
private static Path getLocalCache(
URI cache,
Configuration conf,
Path subDir,
FileStatus fileStatus,
boolean isArchive,
long confFileStamp,
long fileLength,
Path currentWorkDir,
boolean honorSymLinkConf,
MRAsyncDiskService asyncDiskService,
LocalDirAllocator lDirAllocator)
throws IOException {
String key = getKey(cache, conf, confFileStamp);
CacheStatus lcacheStatus;
Path localizedPath;
synchronized (cachedArchives) {
lcacheStatus = cachedArchives.get(key);
if (lcacheStatus == null) {
// was never localized
Path uniqueParentDir = new Path(subDir, String.valueOf(random.nextLong()));
String cachePath = new Path(uniqueParentDir, makeRelative(cache, conf)).toString();
Path localPath = lDirAllocator.getLocalPathForWrite(cachePath, fileLength, conf);
lcacheStatus =
new CacheStatus(
new Path(localPath.toString().replace(cachePath, "")), localPath, uniqueParentDir);
cachedArchives.put(key, lcacheStatus);
}
lcacheStatus.refcount++;
}
boolean initSuccessful = false;
try {
synchronized (lcacheStatus) {
if (!lcacheStatus.isInited()) {
localizedPath = localizeCache(conf, cache, confFileStamp, lcacheStatus, isArchive);
lcacheStatus.initComplete();
} else {
if (fileStatus != null) {
localizedPath =
checkCacheStatusValidity(
conf, cache, confFileStamp, lcacheStatus, fileStatus, isArchive);
} else {
// if fileStatus is null, then the md5 must be correct
// so there is no need to check for cache validity
localizedPath = lcacheStatus.localizedLoadPath;
}
}
createSymlink(conf, cache, lcacheStatus, isArchive, currentWorkDir, honorSymLinkConf);
}
// try deleting stuff if you can
long size = 0;
int numberSubDir = 0;
synchronized (lcacheStatus) {
synchronized (baseDirSize) {
Long get = baseDirSize.get(lcacheStatus.getBaseDir());
if (get != null) {
size = get.longValue();
} else {
LOG.warn("Cannot find size of baseDir: " + lcacheStatus.getBaseDir());
}
}
synchronized (baseDirNumberSubDir) {
Integer get = baseDirNumberSubDir.get(lcacheStatus.getBaseDir());
if (get != null) {
numberSubDir = get.intValue();
} else {
LOG.warn("Cannot find subdirectories limit of baseDir: " + lcacheStatus.getBaseDir());
}
}
}
// setting the cache size to a default of 10GB
long allowedSize = conf.getLong("local.cache.size", DEFAULT_CACHE_SIZE);
long allowedNumberSubDir =
conf.getLong("local.cache.numbersubdir", DEFAULT_CACHE_SUBDIR_LIMIT);
if (allowedSize < size || allowedNumberSubDir < numberSubDir) {
// try some cache deletions
LOG.debug(
"Start deleting released cache because"
+ " [size, allowedSize, numberSubDir, allowedNumberSubDir] ="
+ " ["
+ size
+ ", "
+ allowedSize
+ ", "
+ numberSubDir
+ ", "
+ allowedNumberSubDir
+ "]");
deleteCache(conf, asyncDiskService);
}
initSuccessful = true;
return localizedPath;
} finally {
if (!initSuccessful) {
synchronized (cachedArchives) {
lcacheStatus.refcount--;
}
}
}
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
// Information needed to get a single file:
// BASE_PATH, FILE_ID, TIMESTAMP_START, TIMESTAMP_STOP, SOURCE, FILESYSTEM
private static Vector<Path> getFile(
FileSystem fs, Hashtable<String, String> config, dbutil db_util) throws Exception {
Long latestVersion = latestVersion(config, db_util);
try {
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
} catch (Exception E) {
logger.error("Tryign to get file that is impossible to generate: " + getFullPath(config));
return null;
}
if (Integer.parseInt(config.get("timestamp_start"))
> Integer.parseInt(config.get("timestamp_stop"))) {
return null;
}
logger.debug(
"Getting DB for timestamp "
+ config.get("timestamp_start")
+ " to "
+ config.get("timestamp_stop"));
String final_result = getFullPath(config);
String temp_path_base =
config.get("local_temp_path")
+ "_"
+ config.get("task_id")
+ "_"
+ config.get("run_id")
+ "/";
Path newPath = new Path(final_result + "*");
Vector<Path> ret_path = new Vector<Path>();
String lockName = lock(final_result.replaceAll("/", "_"));
if (fs.globStatus(newPath).length != 0) {
ret_path.add(newPath);
unlock(lockName);
config.put("full_file_name", final_result);
return ret_path;
} else {
if (!config.get("source").equals("local")) {
config.put("temp_path_base", temp_path_base);
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
Class<?> sourceClass =
Class.forName("org.gestore.plugin.source." + config.get("source") + "Source");
Method process_data = sourceClass.getMethod("process", Hashtable.class, FileSystem.class);
Object processor = sourceClass.newInstance();
Object retVal;
try {
retVal = process_data.invoke(processor, config, fs);
} catch (InvocationTargetException E) {
Throwable exception = E.getTargetException();
logger.error("Unable to call method in child class: " + exception.toString());
exception.printStackTrace(System.out);
unlock(lockName);
return null;
}
FileStatus[] files = (FileStatus[]) retVal;
if (files == null) {
logger.error("Error getting files, no files returned");
return null;
}
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path = new Path(temp_path_base + config.get("file_id"));
String suffix = getSuffix(config.get("file_id"), cur_file.getName());
cur_local_path = cur_local_path.suffix(suffix);
Path res_path = new Path(new String(final_result + suffix));
logger.debug("Moving file" + cur_file.toString() + " to " + res_path.toString());
if (config.get("copy").equals("true")) {
fs.moveFromLocalFile(cur_file, res_path);
} else {
fs.rename(cur_file, res_path);
}
}
config.put("full_file_name", final_result);
}
}
unlock(lockName);
return ret_path;
}
@Override
public final void run() {
try {
// before preparing the job localize
// all the archives
TaskAttemptID taskid = t.getTaskID();
LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");
File jobCacheDir = null;
if (conf.getJar() != null) {
jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
}
File workDir =
new File(
lDirAlloc
.getLocalPathToRead(
TaskTracker.getJobCacheSubdir()
+ Path.SEPARATOR
+ t.getJobID()
+ Path.SEPARATOR
+ t.getTaskID()
+ Path.SEPARATOR
+ MRConstants.WORKDIR,
conf)
.toString());
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
FileStatus fileStatus;
FileSystem fileSystem;
Path localPath;
String baseDir;
if ((archives != null) || (files != null)) {
if (archives != null) {
String[] archivesTimestamps = DistributedCache.getArchiveTimestamps(conf);
Path[] p = new Path[archives.length];
for (int i = 0; i < archives.length; i++) {
fileSystem = FileSystem.get(archives[i], conf);
fileStatus = fileSystem.getFileStatus(new Path(archives[i].getPath()));
String cacheId = DistributedCache.makeRelative(archives[i], conf);
String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
if (lDirAlloc.ifExists(cachePath, conf)) {
localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
} else {
localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
}
baseDir = localPath.toString().replace(cacheId, "");
p[i] =
DistributedCache.getLocalCache(
archives[i],
conf,
new Path(baseDir),
fileStatus,
true,
Long.parseLong(archivesTimestamps[i]),
new Path(workDir.getAbsolutePath()),
false);
}
DistributedCache.setLocalArchives(conf, stringifyPathArray(p));
}
if ((files != null)) {
String[] fileTimestamps = DistributedCache.getFileTimestamps(conf);
Path[] p = new Path[files.length];
for (int i = 0; i < files.length; i++) {
fileSystem = FileSystem.get(files[i], conf);
fileStatus = fileSystem.getFileStatus(new Path(files[i].getPath()));
String cacheId = DistributedCache.makeRelative(files[i], conf);
String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId;
if (lDirAlloc.ifExists(cachePath, conf)) {
localPath = lDirAlloc.getLocalPathToRead(cachePath, conf);
} else {
localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
}
baseDir = localPath.toString().replace(cacheId, "");
p[i] =
DistributedCache.getLocalCache(
files[i],
conf,
new Path(baseDir),
fileStatus,
false,
Long.parseLong(fileTimestamps[i]),
new Path(workDir.getAbsolutePath()),
false);
}
DistributedCache.setLocalFiles(conf, stringifyPathArray(p));
}
Path localTaskFile = new Path(t.getJobFile());
FileSystem localFs = FileSystem.getLocal(conf);
localFs.delete(localTaskFile, true);
OutputStream out = localFs.create(localTaskFile);
try {
conf.writeXml(out);
} finally {
out.close();
}
}
if (!prepare()) {
return;
}
String sep = System.getProperty("path.separator");
StringBuffer classPath = new StringBuffer();
// start with same classpath as parent process
classPath.append(System.getProperty("java.class.path"));
classPath.append(sep);
if (!workDir.mkdirs()) {
if (!workDir.isDirectory()) {
LOG.fatal("Mkdirs failed to create " + workDir.toString());
}
}
String jar = conf.getJar();
if (jar != null) {
// if jar exists, it into workDir
File[] libs = new File(jobCacheDir, "lib").listFiles();
if (libs != null) {
for (int i = 0; i < libs.length; i++) {
classPath.append(sep); // add libs from jar to classpath
classPath.append(libs[i]);
}
}
classPath.append(sep);
classPath.append(new File(jobCacheDir, "classes"));
classPath.append(sep);
classPath.append(jobCacheDir);
}
// include the user specified classpath
// archive paths
Path[] archiveClasspaths = DistributedCache.getArchiveClassPaths(conf);
if (archiveClasspaths != null && archives != null) {
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
if (localArchives != null) {
for (int i = 0; i < archives.length; i++) {
for (int j = 0; j < archiveClasspaths.length; j++) {
if (archives[i].getPath().equals(archiveClasspaths[j].toString())) {
classPath.append(sep);
classPath.append(localArchives[i].toString());
}
}
}
}
}
// file paths
Path[] fileClasspaths = DistributedCache.getFileClassPaths(conf);
if (fileClasspaths != null && files != null) {
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
if (localFiles != null) {
for (int i = 0; i < files.length; i++) {
for (int j = 0; j < fileClasspaths.length; j++) {
if (files[i].getPath().equals(fileClasspaths[j].toString())) {
classPath.append(sep);
classPath.append(localFiles[i].toString());
}
}
}
}
}
classPath.append(sep);
classPath.append(workDir);
// Build exec child jmv args.
Vector<String> vargs = new Vector<String>(8);
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
vargs.add(jvm.toString());
// Add child (task) java-vm options.
//
// The following symbols if present in mapred.child.java.opts value are
// replaced:
// + @taskid@ is interpolated with value of TaskID.
// Other occurrences of @ will not be altered.
//
// Example with multiple arguments and substitutions, showing
// jvm GC logging, and start of a passwordless JVM JMX agent so can
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
// <property>
// <name>mapred.child.java.opts</name>
// <value>-verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
String javaOpts = conf.get("mapred.child.java.opts", "-Xmx200m");
javaOpts = javaOpts.replace("@taskid@", taskid.toString());
String[] javaOptsSplit = javaOpts.split(" ");
// Add java.library.path; necessary for loading native libraries.
//
// 1. To support native-hadoop library i.e. libhadoop.so, we add the
// parent processes' java.library.path to the child.
// 2. We also add the 'cwd' of the task to it's java.library.path to help
// users distribute native libraries via the DistributedCache.
// 3. The user can also specify extra paths to be added to the
// java.library.path via mapred.child.java.opts.
//
String libraryPath = System.getProperty("java.library.path");
if (libraryPath == null) {
libraryPath = workDir.getAbsolutePath();
} else {
libraryPath += sep + workDir;
}
boolean hasUserLDPath = false;
for (int i = 0; i < javaOptsSplit.length; i++) {
if (javaOptsSplit[i].startsWith("-Djava.library.path=")) {
javaOptsSplit[i] += sep + libraryPath;
hasUserLDPath = true;
break;
}
}
if (!hasUserLDPath) {
vargs.add("-Djava.library.path=" + libraryPath);
}
for (int i = 0; i < javaOptsSplit.length; i++) {
vargs.add(javaOptsSplit[i]);
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
}
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
vargs.add("-Djava.io.tmpdir=" + tmpDir.toString());
// Add classpath.
vargs.add("-classpath");
vargs.add(classPath.toString());
// Setup the log4j prop
long logSize = TaskLog.getTaskLogLength(conf);
vargs.add(
"-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
vargs.add("-Dhadoop.root.logger=INFO,TLA");
vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);
if (conf.getProfileEnabled()) {
if (conf.getProfileTaskRange(t.isMapTask()).isIncluded(t.getPartition())) {
File prof = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.PROFILE);
vargs.add(String.format(conf.getProfileParams(), prof.toString()));
}
}
// Add main class and its arguments
vargs.add(Child.class.getName()); // main of Child
// pass umbilical address
InetSocketAddress address = tracker.getTaskTrackerReportAddress();
vargs.add(address.getAddress().getHostAddress());
vargs.add(Integer.toString(address.getPort()));
vargs.add(taskid.toString()); // pass task identifier
String pidFile = null;
if (tracker.isTaskMemoryManagerEnabled()) {
pidFile =
lDirAlloc
.getLocalPathForWrite(
(TaskTracker.getPidFilesSubdir() + Path.SEPARATOR + taskid), this.conf)
.toString();
}
// set memory limit using ulimit if feasible and necessary ...
String[] ulimitCmd = Shell.getUlimitMemoryCommand(conf);
List<String> setup = null;
if (ulimitCmd != null) {
setup = new ArrayList<String>();
for (String arg : ulimitCmd) {
setup.add(arg);
}
}
// Set up the redirection of the task's stdout and stderr streams
File stdout = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDOUT);
File stderr = TaskLog.getTaskLogFile(taskid, TaskLog.LogName.STDERR);
stdout.getParentFile().mkdirs();
tracker.getTaskTrackerInstrumentation().reportTaskLaunch(taskid, stdout, stderr);
Map<String, String> env = new HashMap<String, String>();
StringBuffer ldLibraryPath = new StringBuffer();
ldLibraryPath.append(workDir.toString());
String oldLdLibraryPath = null;
oldLdLibraryPath = System.getenv("LD_LIBRARY_PATH");
if (oldLdLibraryPath != null) {
ldLibraryPath.append(sep);
ldLibraryPath.append(oldLdLibraryPath);
}
env.put("LD_LIBRARY_PATH", ldLibraryPath.toString());
jvmManager.launchJvm(
this,
jvmManager.constructJvmEnv(
setup, vargs, stdout, stderr, logSize, workDir, env, pidFile, conf));
synchronized (lock) {
while (!done) {
lock.wait();
}
}
tracker.getTaskTrackerInstrumentation().reportTaskEnd(t.getTaskID());
if (exitCodeSet) {
if (!killed && exitCode != 0) {
if (exitCode == 65) {
tracker.getTaskTrackerInstrumentation().taskFailedPing(t.getTaskID());
}
throw new IOException("Task process exit with nonzero status of " + exitCode + ".");
}
}
} catch (FSError e) {
LOG.fatal("FSError", e);
try {
tracker.fsError(t.getTaskID(), e.getMessage());
} catch (IOException ie) {
LOG.fatal(t.getTaskID() + " reporting FSError", ie);
}
} catch (Throwable throwable) {
LOG.warn(t.getTaskID() + " Child Error", throwable);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
throwable.printStackTrace(new PrintStream(baos));
try {
tracker.reportDiagnosticInfo(t.getTaskID(), baos.toString());
} catch (IOException e) {
LOG.warn(t.getTaskID() + " Reporting Diagnostics", e);
}
} finally {
try {
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
if (archives != null) {
for (int i = 0; i < archives.length; i++) {
DistributedCache.releaseCache(archives[i], conf);
}
}
if (files != null) {
for (int i = 0; i < files.length; i++) {
DistributedCache.releaseCache(files[i], conf);
}
}
} catch (IOException ie) {
LOG.warn("Error releasing caches : Cache files might not have been cleaned up");
}
tracker.reportTaskFinished(t.getTaskID(), false);
if (t.isMapTask()) {
tracker.addFreeMapSlot();
} else {
tracker.addFreeReduceSlot();
}
}
}
public void generateFileDetails(JspWriter out, HttpServletRequest req, Configuration conf)
throws IOException, InterruptedException {
int chunkSizeToView = 0;
long startOffset = 0;
int datanodePort;
String blockIdStr = null;
long currBlockId = 0;
blockIdStr = req.getParameter("blockId");
if (blockIdStr == null) {
out.print("Invalid input (blockId absent)");
return;
}
currBlockId = Long.parseLong(blockIdStr);
String datanodePortStr = req.getParameter("datanodePort");
if (datanodePortStr == null) {
out.print("Invalid input (datanodePort absent)");
return;
}
datanodePort = Integer.parseInt(datanodePortStr);
String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
int namenodeInfoPort = -1;
if (namenodeInfoPortStr != null) namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
String chunkSizeToViewStr = req.getParameter("chunkSizeToView");
if (chunkSizeToViewStr != null && Integer.parseInt(chunkSizeToViewStr) > 0) {
chunkSizeToView = Integer.parseInt(chunkSizeToViewStr);
} else {
chunkSizeToView = JspHelper.getDefaultChunkSize(conf);
}
String startOffsetStr = req.getParameter("startOffset");
if (startOffsetStr == null || Long.parseLong(startOffsetStr) < 0) startOffset = 0;
else startOffset = Long.parseLong(startOffsetStr);
String filename = HtmlQuoting.unquoteHtmlChars(req.getParameter("filename"));
if (filename == null || filename.length() == 0) {
out.print("Invalid input");
return;
}
String blockSizeStr = req.getParameter("blockSize");
long blockSize = 0;
if (blockSizeStr == null || blockSizeStr.length() == 0) {
out.print("Invalid input");
return;
}
blockSize = Long.parseLong(blockSizeStr);
String tokenString = req.getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
UserGroupInformation ugi = JspHelper.getUGI(req, conf);
DFSClient dfs = JspHelper.getDFSClient(ugi, jspHelper.nameNodeAddr, conf);
List<LocatedBlock> blocks =
dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
// Add the various links for looking at the file contents
// URL for downloading the full file
String downloadUrl =
"http://"
+ req.getServerName()
+ ":"
+ +req.getServerPort()
+ "/streamFile"
+ URLEncoder.encode(filename, "UTF-8")
+ "?"
+ JspHelper.DELEGATION_PARAMETER_NAME
+ "="
+ tokenString;
out.print("<a name=\"viewOptions\"></a>");
out.print("<a href=\"" + downloadUrl + "\">Download this file</a><br>");
DatanodeInfo chosenNode;
// URL for TAIL
LocatedBlock lastBlk = blocks.get(blocks.size() - 1);
long blockId = lastBlk.getBlock().getBlockId();
try {
chosenNode = jspHelper.bestNode(lastBlk);
} catch (IOException e) {
out.print(e.toString());
dfs.close();
return;
}
String fqdn = InetAddress.getByName(chosenNode.getHost()).getCanonicalHostName();
String tailUrl =
"http://"
+ fqdn
+ ":"
+ chosenNode.getInfoPort()
+ "/tail.jsp?filename="
+ URLEncoder.encode(filename, "UTF-8")
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ "&chunkSizeToView="
+ chunkSizeToView
+ "&referrer="
+ URLEncoder.encode(req.getRequestURL() + "?" + req.getQueryString(), "UTF-8")
+ JspHelper.getDelegationTokenUrlParam(tokenString);
out.print("<a href=\"" + tailUrl + "\">Tail this file</a><br>");
out.print("<form action=\"/browseBlock.jsp\" method=GET>");
out.print("<b>Chunk size to view (in bytes, up to file's DFS block size): </b>");
out.print("<input type=\"hidden\" name=\"blockId\" value=\"" + currBlockId + "\">");
out.print("<input type=\"hidden\" name=\"blockSize\" value=\"" + blockSize + "\">");
out.print("<input type=\"hidden\" name=\"startOffset\" value=\"" + startOffset + "\">");
out.print("<input type=\"hidden\" name=\"filename\" value=\"" + filename + "\">");
out.print("<input type=\"hidden\" name=\"datanodePort\" value=\"" + datanodePort + "\">");
out.print(
"<input type=\"hidden\" name=\"namenodeInfoPort\" value=\"" + namenodeInfoPort + "\">");
out.print(
"<input type=\"text\" name=\"chunkSizeToView\" value="
+ chunkSizeToView
+ " size=10 maxlength=10>");
out.print(" <input type=\"submit\" name=\"submit\" value=\"Refresh\">");
out.print("</form>");
out.print("<hr>");
out.print("<a name=\"blockDetails\"></a>");
out.print("<B>Total number of blocks: " + blocks.size() + "</B><br>");
// generate a table and dump the info
out.println("\n<table>");
for (LocatedBlock cur : blocks) {
out.print("<tr>");
blockId = cur.getBlock().getBlockId();
blockSize = cur.getBlock().getNumBytes();
String blk = "blk_" + Long.toString(blockId);
out.print("<td>" + Long.toString(blockId) + ":</td>");
DatanodeInfo[] locs = cur.getLocations();
for (int j = 0; j < locs.length; j++) {
String datanodeAddr = locs[j].getName();
datanodePort =
Integer.parseInt(
datanodeAddr.substring(datanodeAddr.indexOf(':') + 1, datanodeAddr.length()));
fqdn = InetAddress.getByName(locs[j].getHost()).getCanonicalHostName();
String blockUrl =
"http://"
+ fqdn
+ ":"
+ locs[j].getInfoPort()
+ "/browseBlock.jsp?blockId="
+ Long.toString(blockId)
+ "&blockSize="
+ blockSize
+ "&filename="
+ URLEncoder.encode(filename, "UTF-8")
+ "&datanodePort="
+ datanodePort
+ "&genstamp="
+ cur.getBlock().getGenerationStamp()
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ "&chunkSizeToView="
+ chunkSizeToView;
out.print(
"<td> </td>" + "<td><a href=\"" + blockUrl + "\">" + datanodeAddr + "</a></td>");
}
out.println("</tr>");
}
out.println("</table>");
out.print("<hr>");
String namenodeHost = jspHelper.nameNodeAddr.getHostName();
out.print(
"<br><a href=\"http://"
+ InetAddress.getByName(namenodeHost).getCanonicalHostName()
+ ":"
+ namenodeInfoPort
+ "/dfshealth.jsp\">Go back to DFS home</a>");
dfs.close();
}
public void generateFileChunks(JspWriter out, HttpServletRequest req, Configuration conf)
throws IOException, InterruptedException {
long startOffset = 0;
int datanodePort = 0;
int chunkSizeToView = 0;
String namenodeInfoPortStr = req.getParameter("namenodeInfoPort");
int namenodeInfoPort = -1;
if (namenodeInfoPortStr != null) namenodeInfoPort = Integer.parseInt(namenodeInfoPortStr);
String filename = HtmlQuoting.unquoteHtmlChars(req.getParameter("filename"));
if (filename == null) {
out.print("Invalid input (filename absent)");
return;
}
String blockIdStr = null;
long blockId = 0;
blockIdStr = req.getParameter("blockId");
if (blockIdStr == null) {
out.print("Invalid input (blockId absent)");
return;
}
blockId = Long.parseLong(blockIdStr);
String tokenString = req.getParameter(JspHelper.DELEGATION_PARAMETER_NAME);
UserGroupInformation ugi = JspHelper.getUGI(req, conf);
final DFSClient dfs = JspHelper.getDFSClient(ugi, jspHelper.nameNodeAddr, conf);
Token<BlockTokenIdentifier> accessToken = BlockTokenSecretManager.DUMMY_TOKEN;
if (conf.getBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, false)) {
List<LocatedBlock> blks =
dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
if (blks == null || blks.size() == 0) {
out.print("Can't locate file blocks");
dfs.close();
return;
}
for (int i = 0; i < blks.size(); i++) {
if (blks.get(i).getBlock().getBlockId() == blockId) {
accessToken = blks.get(i).getBlockToken();
break;
}
}
}
String blockGenStamp = null;
long genStamp = 0;
blockGenStamp = req.getParameter("genstamp");
if (blockGenStamp == null) {
out.print("Invalid input (genstamp absent)");
return;
}
genStamp = Long.parseLong(blockGenStamp);
String blockSizeStr;
long blockSize = 0;
blockSizeStr = req.getParameter("blockSize");
if (blockSizeStr == null) {
out.print("Invalid input (blockSize absent)");
return;
}
blockSize = Long.parseLong(blockSizeStr);
String chunkSizeToViewStr = req.getParameter("chunkSizeToView");
if (chunkSizeToViewStr != null && Integer.parseInt(chunkSizeToViewStr) > 0)
chunkSizeToView = Integer.parseInt(chunkSizeToViewStr);
else chunkSizeToView = JspHelper.getDefaultChunkSize(conf);
String startOffsetStr = req.getParameter("startOffset");
if (startOffsetStr == null || Long.parseLong(startOffsetStr) < 0) startOffset = 0;
else startOffset = Long.parseLong(startOffsetStr);
String datanodePortStr = req.getParameter("datanodePort");
if (datanodePortStr == null) {
out.print("Invalid input (datanodePort absent)");
return;
}
datanodePort = Integer.parseInt(datanodePortStr);
out.print("<h3>File: ");
JspHelper.printPathWithLinks(
HtmlQuoting.quoteHtmlChars(filename), out, namenodeInfoPort, tokenString);
out.print("</h3><hr>");
String parent = new File(filename).getParent();
JspHelper.printGotoForm(out, namenodeInfoPort, tokenString, HtmlQuoting.quoteHtmlChars(parent));
out.print("<hr>");
out.print(
"<a href=\"http://"
+ req.getServerName()
+ ":"
+ req.getServerPort()
+ "/browseDirectory.jsp?dir="
+ URLEncoder.encode(parent, "UTF-8")
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ "\"><i>Go back to dir listing</i></a><br>");
out.print("<a href=\"#viewOptions\">Advanced view/download options</a><br>");
out.print("<hr>");
// Determine the prev & next blocks
long nextStartOffset = 0;
long nextBlockSize = 0;
String nextBlockIdStr = null;
String nextGenStamp = null;
String nextHost = req.getServerName();
int nextPort = req.getServerPort();
int nextDatanodePort = datanodePort;
// determine data for the next link
if (startOffset + chunkSizeToView >= blockSize) {
// we have to go to the next block from this point onwards
List<LocatedBlock> blocks =
dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
for (int i = 0; i < blocks.size(); i++) {
if (blocks.get(i).getBlock().getBlockId() == blockId) {
if (i != blocks.size() - 1) {
LocatedBlock nextBlock = blocks.get(i + 1);
nextBlockIdStr = Long.toString(nextBlock.getBlock().getBlockId());
nextGenStamp = Long.toString(nextBlock.getBlock().getGenerationStamp());
nextStartOffset = 0;
nextBlockSize = nextBlock.getBlock().getNumBytes();
DatanodeInfo d = jspHelper.bestNode(nextBlock);
String datanodeAddr = d.getName();
nextDatanodePort =
Integer.parseInt(
datanodeAddr.substring(datanodeAddr.indexOf(':') + 1, datanodeAddr.length()));
nextHost = InetAddress.getByName(d.getHost()).getCanonicalHostName();
nextPort = d.getInfoPort();
}
}
}
} else {
// we are in the same block
nextBlockIdStr = blockIdStr;
nextStartOffset = startOffset + chunkSizeToView;
nextBlockSize = blockSize;
nextGenStamp = blockGenStamp;
}
String nextUrl = null;
if (nextBlockIdStr != null) {
nextUrl =
"http://"
+ nextHost
+ ":"
+ nextPort
+ "/browseBlock.jsp?blockId="
+ nextBlockIdStr
+ "&blockSize="
+ nextBlockSize
+ "&startOffset="
+ nextStartOffset
+ "&genstamp="
+ nextGenStamp
+ "&filename="
+ URLEncoder.encode(filename, "UTF-8")
+ "&chunkSizeToView="
+ chunkSizeToView
+ "&datanodePort="
+ nextDatanodePort
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ JspHelper.getDelegationTokenUrlParam(tokenString);
out.print("<a href=\"" + nextUrl + "\">View Next chunk</a> ");
}
// determine data for the prev link
String prevBlockIdStr = null;
String prevGenStamp = null;
long prevStartOffset = 0;
long prevBlockSize = 0;
String prevHost = req.getServerName();
int prevPort = req.getServerPort();
int prevDatanodePort = datanodePort;
if (startOffset == 0) {
List<LocatedBlock> blocks =
dfs.namenode.getBlockLocations(filename, 0, Long.MAX_VALUE).getLocatedBlocks();
for (int i = 0; i < blocks.size(); i++) {
if (blocks.get(i).getBlock().getBlockId() == blockId) {
if (i != 0) {
LocatedBlock prevBlock = blocks.get(i - 1);
prevBlockIdStr = Long.toString(prevBlock.getBlock().getBlockId());
prevGenStamp = Long.toString(prevBlock.getBlock().getGenerationStamp());
prevStartOffset = prevBlock.getBlock().getNumBytes() - chunkSizeToView;
if (prevStartOffset < 0) prevStartOffset = 0;
prevBlockSize = prevBlock.getBlock().getNumBytes();
DatanodeInfo d = jspHelper.bestNode(prevBlock);
String datanodeAddr = d.getName();
prevDatanodePort =
Integer.parseInt(
datanodeAddr.substring(datanodeAddr.indexOf(':') + 1, datanodeAddr.length()));
prevHost = InetAddress.getByName(d.getHost()).getCanonicalHostName();
prevPort = d.getInfoPort();
}
}
}
} else {
// we are in the same block
prevBlockIdStr = blockIdStr;
prevStartOffset = startOffset - chunkSizeToView;
if (prevStartOffset < 0) prevStartOffset = 0;
prevBlockSize = blockSize;
prevGenStamp = blockGenStamp;
}
String prevUrl = null;
if (prevBlockIdStr != null) {
prevUrl =
"http://"
+ prevHost
+ ":"
+ prevPort
+ "/browseBlock.jsp?blockId="
+ prevBlockIdStr
+ "&blockSize="
+ prevBlockSize
+ "&startOffset="
+ prevStartOffset
+ "&filename="
+ URLEncoder.encode(filename, "UTF-8")
+ "&chunkSizeToView="
+ chunkSizeToView
+ "&genstamp="
+ prevGenStamp
+ "&datanodePort="
+ prevDatanodePort
+ "&namenodeInfoPort="
+ namenodeInfoPort
+ JspHelper.getDelegationTokenUrlParam(tokenString);
out.print("<a href=\"" + prevUrl + "\">View Prev chunk</a> ");
}
out.print("<hr>");
out.print("<textarea cols=\"100\" rows=\"25\" wrap=\"virtual\" style=\"width:100%\" READONLY>");
try {
jspHelper.streamBlockInAscii(
new InetSocketAddress(req.getServerName(), datanodePort),
blockId,
accessToken,
genStamp,
blockSize,
startOffset,
chunkSizeToView,
out,
conf);
} catch (Exception e) {
out.print(e);
}
out.print("</textarea>");
dfs.close();
}