public int run(String[] args) throws Exception {
Configuration argConf = getConf();
// JobConf conf = new JobConf(diffdb.class);
Configuration config = HBaseConfiguration.create();
HBaseAdmin hbAdmin = new HBaseAdmin(config);
dbutil db_util = new dbutil(config);
HTable runTable = new HTable(config, "gestore_runs");
Get runGet = new Get(argConf.get("id").getBytes());
Result pipeline = runTable.get(runGet);
NavigableMap<byte[], byte[]> pipeMap = pipeline.getFamilyMap("d".getBytes());
Map.Entry<byte[], byte[]> results = pipeMap.pollFirstEntry();
HashMap<String, HashMap<String, String>> resultMap =
new HashMap<String, HashMap<String, String>>();
while (results != null) {
String resultKey = new String(results.getKey());
String resultValue = new String(results.getValue());
String field = "type";
HashMap<String, String> tempMap = new HashMap<String, String>();
String entry = resultKey;
if (resultKey.endsWith("_db_timestamp")) {
field = "db_timestamp";
entry = resultKey.substring(0, resultKey.lastIndexOf("_db_timestamp"));
} else if (resultKey.endsWith("_filename")) {
field = "filename";
entry = resultKey.substring(0, resultKey.lastIndexOf("_filename"));
} else if (resultKey.endsWith("_regex")) {
field = "regex";
entry = resultKey.substring(0, resultKey.lastIndexOf("_regex"));
}
if (resultMap.containsKey(entry)) {
tempMap = resultMap.get(entry);
}
tempMap.put(field, resultValue);
resultMap.put(entry, tempMap);
// System.out.println("Key: " + resultKey + " Value: " + resultValue);
results = pipeMap.pollFirstEntry();
}
for (String key : resultMap.keySet()) {
System.out.println("File ID: " + key);
for (String subKey : resultMap.get(key).keySet()) {
// System.out.println("\t " + subKey + "\t\t" + resultMap.get(key).get(subKey));
System.out.format(" %1$-20s %2$s\n", subKey, resultMap.get(key).get(subKey));
}
}
return 0;
}
protected void setup(Context context) {
Configuration config = context.getConfiguration();
inMovies = config.get("inParameter");
}
/** Sets up configuration based on params */
private static boolean setup(Hashtable<String, String> curConf, Configuration argConf) {
if (argConf.get("file") == null) {
logger.fatal("Missing file parameter");
System.exit(1);
}
if (argConf.get("hdfs_base_path") == null) {
logger.fatal("Missing HDFS base path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("hdfs_temp_path") == null) {
logger.fatal("Missing HDFS temp path, check gestore-conf.xml");
System.exit(1);
}
if (argConf.get("local_temp_path") == null) {
logger.fatal("Missing local temp path, check gestore-conf.xml");
System.exit(1);
}
// Input paramaters
curConf.put("run_id", argConf.get("run", ""));
curConf.put("task_id", argConf.get("task", ""));
curConf.put("file_id", argConf.get("file"));
curConf.put("local_path", argConf.get("path", ""));
curConf.put("type", argConf.get("type", "l2r"));
curConf.put("timestamp_start", argConf.get("timestamp_start", "1"));
curConf.put(
"timestamp_stop", argConf.get("timestamp_stop", Integer.toString(Integer.MAX_VALUE)));
curConf.put("delimiter", argConf.get("regex", "ID=.*"));
curConf.put("taxon", argConf.get("taxon", "all"));
curConf.put("intermediate", argConf.get("full_run", "false"));
curConf.put("quick_add", argConf.get("quick_add", "false"));
Boolean full_run = curConf.get("intermediate").matches("(?i).*true.*");
curConf.put("format", argConf.get("format", "unknown"));
curConf.put("split", argConf.get("split", "1"));
curConf.put("copy", argConf.get("copy", "true"));
// Constants
curConf.put("base_path", argConf.get("hdfs_base_path"));
curConf.put("temp_path", argConf.get("hdfs_temp_path"));
curConf.put("local_temp_path", argConf.get("local_temp_path"));
curConf.put("db_name_files", argConf.get("hbase_file_table"));
curConf.put("db_name_runs", argConf.get("hbase_run_table"));
curConf.put("db_name_updates", argConf.get("hbase_db_update_table"));
// Timestamps
Date currentTime = new Date();
Date endDate = new Date(new Long(curConf.get("timestamp_stop")));
curConf.put("timestamp_real", Long.toString(currentTime.getTime()));
return true;
}
