public void initialize(InputSplit genericSplit, TaskAttemptContext context) throws IOException {
FileSplit split = (FileSplit) genericSplit;
Configuration job = context.getConfiguration();
m_Sb.setLength(0);
m_Start = split.getStart();
m_End = m_Start + split.getLength();
final Path file = split.getPath();
compressionCodecs = new CompressionCodecFactory(job);
final CompressionCodec codec = compressionCodecs.getCodec(file);
// open the file and seek to the m_Start of the split
FileSystem fs = file.getFileSystem(job);
// getFileStatus fileStatus = fs.getFileStatus(split.getPath());
//noinspection deprecation
@SuppressWarnings(value = "deprecated")
long length = fs.getLength(file);
FSDataInputStream fileIn = fs.open(split.getPath());
if (m_Start > 0) fileIn.seek(m_Start);
if (codec != null) {
CompressionInputStream inputStream = codec.createInputStream(fileIn);
m_Input = new BufferedReader(new InputStreamReader(inputStream));
m_End = length;
} else {
m_Input = new BufferedReader(new InputStreamReader(fileIn));
}
m_Current = m_Start;
m_Key = split.getPath().getName();
}
/**
* Create a map-only Hadoop Job out of the passed in parameters. Does not set the
* Job name.
*
* @see #getCustomJobName(String, org.apache.hadoop.mapreduce.JobContext, Class, Class)
*/
@SuppressWarnings("rawtypes")
public static Job prepareJob(Path inputPath,
Path outputPath,
Class<? extends InputFormat> inputFormat,
Class<? extends Mapper> mapper,
Class<? extends Writable> mapperKey,
Class<? extends Writable> mapperValue,
Class<? extends OutputFormat> outputFormat, Configuration conf) throws IOException {
//Job job = new Job(new Configuration(conf));
Job job = Job.getInstance(conf);
Configuration jobConf = job.getConfiguration();
if (mapper.equals(Mapper.class)) {
throw new IllegalStateException("Can't figure out the user class jar file from mapper/reducer");
}
job.setJarByClass(mapper);
job.setInputFormatClass(inputFormat);
jobConf.set("mapred.input.dir", inputPath.toString());
job.setMapperClass(mapper);
job.setMapOutputKeyClass(mapperKey);
job.setMapOutputValueClass(mapperValue);
job.setOutputKeyClass(mapperKey);
job.setOutputValueClass(mapperValue);
jobConf.setBoolean("mapred.compress.map.output", true);
job.setNumReduceTasks(0);
job.setOutputFormatClass(outputFormat);
jobConf.set("mapred.output.dir", outputPath.toString());
return job;
}
/**
* Generate the list of files and make them into FileSplits. This needs to be copied to insert a
* filter on acceptable data
*/
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
long maxSize = getMaxSplitSize(job);
long desiredMappers =
job.getConfiguration().getLong("org.systemsbiology.jxtandem.DesiredXMLInputMappers", 0);
// generate splits
List<InputSplit> splits = new ArrayList<InputSplit>();
List<FileStatus> fileStatuses = listStatus(job);
boolean forceNumberMappers = fileStatuses.size() == 1;
for (FileStatus file : fileStatuses) {
Path path = file.getPath();
if (!isPathAcceptable(path)) // filter acceptable data
continue;
FileSystem fs = path.getFileSystem(job.getConfiguration());
long length = file.getLen();
BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0, length);
if ((length != 0) && isSplitable(job, path)) {
long blockSize = file.getBlockSize();
// use desired mappers to force more splits
if (forceNumberMappers && desiredMappers > 0)
maxSize = Math.min(maxSize, (length / desiredMappers));
long splitSize = computeSplitSize(blockSize, minSize, maxSize);
long bytesRemaining = length;
while (withinSlop(splitSize, bytesRemaining)) {
int blkIndex = getBlockIndex(blkLocations, length - bytesRemaining);
splits.add(
new FileSplit(
path, length - bytesRemaining, splitSize, blkLocations[blkIndex].getHosts()));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
splits.add(
new FileSplit(
path,
length - bytesRemaining,
bytesRemaining,
blkLocations[blkLocations.length - 1].getHosts()));
}
} else if (length != 0) {
splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
} else {
// Create empty hosts array for zero length files
splits.add(new FileSplit(path, 0, length, new String[0]));
}
}
System.out.println("Total # of splits: " + splits.size());
// LOG.debug("Total # of splits: " + splits.size());
return splits;
}
public static void delete(Configuration conf, Iterable<Path> paths) throws IOException {
if (conf == null) {
conf = new Configuration();
}
for (Path path : paths) {
FileSystem fs = path.getFileSystem(conf);
if (fs.exists(path)) {
log.info("Deleting {}", path);
fs.delete(path, true);
}
}
}
@Override
protected boolean isSplitable(JobContext context, Path file) {
String fname = file.getName().toLowerCase();
//noinspection SimplifiableIfStatementf,RedundantIfStatement
if (fname.endsWith(".gz")) return false;
return true;
}
public static void writeInt(int value, Path path, Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(path.toUri(), conf);
FSDataOutputStream out = fs.create(path);
try {
out.writeInt(value);
} finally {
out.close();
}
}
public static int readInt(Path path, Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(path.toUri(), conf);
//FileSystem fs = path.getFileSystem(conf);
FSDataInputStream in = fs.open(path);
try {
return in.readInt();
} finally {
in.close();
}
}
protected boolean isPathAcceptable(final Path pPath1) {
String path = pPath1.toString().toLowerCase();
if (path.startsWith("part-r-")) return true;
String extension = getExtension();
if (extension != null && path.endsWith(extension.toLowerCase())) return true;
if (extension != null && path.endsWith(extension.toLowerCase() + ".gz")) return true;
//noinspection SimplifiableIfStatement,RedundantIfStatement
if (extension == null) return true;
return false;
}
private static boolean writeChecksum(Path localPath, String checksum) {
try {
java.io.BufferedWriter writer =
Files.newBufferedWriter(
java.nio.file.Paths.get(localPath.toString() + ".crc"), Charset.forName("UTF-8"));
writer.write(checksum);
writer.flush();
writer.close();
return true;
} catch (Exception e) {
logger.warn("Unable to write CRC file!");
return false;
}
}
public static void recursePath(Configuration conf, Path path, Job job) {
try {
FileSystem fs = path.getFileSystem(conf);
FileStatus[] fstats = fs.listStatus(path);
if (fstats != null) {
for (FileStatus f : fstats) {
Path p = f.getPath();
;
if (fs.isFile(p)) {
// connection times out otherwise
System.err.println("file:" + p.toString());
FileInputFormat.addInputPath(job, p);
} else {
System.err.println("dir:" + p.toString());
recursePath(conf, p, job);
}
}
}
} catch (IOException e) {
// shouldn't be here
throw new RuntimeException(e);
}
}
private static boolean checksumLocalTest(Path localPath, String checksum) {
String crcPath = localPath.toString() + ".crc";
if (Files.isReadable(java.nio.file.Paths.get(crcPath))) {
try {
List<String> lines =
Files.readAllLines(java.nio.file.Paths.get(crcPath), Charset.forName("UTF-8"));
for (String line : lines) {
if (line.equals(checksum)) {
return true;
}
}
} catch (Exception e) {
return false;
}
}
return false;
}
public static void cacheFiles(Path fileToCache, Configuration conf) {
DistributedCache.setCacheFiles(new URI[]{fileToCache.toUri()}, conf);
}
public static InputStream openStream(Path path, Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(path.toUri(), conf);
return fs.open(path.makeQualified(fs));
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
// Information needed to get a single file:
// BASE_PATH, FILE_ID, TIMESTAMP_START, TIMESTAMP_STOP, SOURCE, FILESYSTEM
private static Vector<Path> getFile(
FileSystem fs, Hashtable<String, String> config, dbutil db_util) throws Exception {
Long latestVersion = latestVersion(config, db_util);
try {
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
} catch (Exception E) {
logger.error("Tryign to get file that is impossible to generate: " + getFullPath(config));
return null;
}
if (Integer.parseInt(config.get("timestamp_start"))
> Integer.parseInt(config.get("timestamp_stop"))) {
return null;
}
logger.debug(
"Getting DB for timestamp "
+ config.get("timestamp_start")
+ " to "
+ config.get("timestamp_stop"));
String final_result = getFullPath(config);
String temp_path_base =
config.get("local_temp_path")
+ "_"
+ config.get("task_id")
+ "_"
+ config.get("run_id")
+ "/";
Path newPath = new Path(final_result + "*");
Vector<Path> ret_path = new Vector<Path>();
String lockName = lock(final_result.replaceAll("/", "_"));
if (fs.globStatus(newPath).length != 0) {
ret_path.add(newPath);
unlock(lockName);
config.put("full_file_name", final_result);
return ret_path;
} else {
if (!config.get("source").equals("local")) {
config.put("temp_path_base", temp_path_base);
config.put("timestamp_start", config.get("timestamp_start"));
config.put("timestamp_real", latestVersion.toString());
config.put("timestamp_stop", latestVersion.toString());
Class<?> sourceClass =
Class.forName("org.gestore.plugin.source." + config.get("source") + "Source");
Method process_data = sourceClass.getMethod("process", Hashtable.class, FileSystem.class);
Object processor = sourceClass.newInstance();
Object retVal;
try {
retVal = process_data.invoke(processor, config, fs);
} catch (InvocationTargetException E) {
Throwable exception = E.getTargetException();
logger.error("Unable to call method in child class: " + exception.toString());
exception.printStackTrace(System.out);
unlock(lockName);
return null;
}
FileStatus[] files = (FileStatus[]) retVal;
if (files == null) {
logger.error("Error getting files, no files returned");
return null;
}
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path = new Path(temp_path_base + config.get("file_id"));
String suffix = getSuffix(config.get("file_id"), cur_file.getName());
cur_local_path = cur_local_path.suffix(suffix);
Path res_path = new Path(new String(final_result + suffix));
logger.debug("Moving file" + cur_file.toString() + " to " + res_path.toString());
if (config.get("copy").equals("true")) {
fs.moveFromLocalFile(cur_file, res_path);
} else {
fs.rename(cur_file, res_path);
}
}
config.put("full_file_name", final_result);
}
}
unlock(lockName);
return ret_path;
}