private static boolean hasFile(
dbutil db_util, FileSystem fs, String db_name, String file_id, String file_path)
throws Exception {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names == null) {
return false;
}
String all_files = new String(file_names.getValue());
String[] files = all_files.split("\n");
for (String line : files) {
if (line.equals(file_path)) {
if (fs.globStatus(new Path(line + "*")).length == 0) {
Put new_put = new Put(file_id.getBytes());
new_put.add(
"d".getBytes(),
"filenames".getBytes(),
all_files.replace(file_path + "\n", "").getBytes());
db_util.doPut(db_name, new_put);
return false;
}
return true;
}
}
return false;
}
private static boolean putRunEntry(
dbutil db_util,
String db_name,
String run_id,
String file_id,
String type,
String timestamp,
String timestamp_stop,
String path,
String regex)
throws Exception {
Put run_id_put = new Put(run_id.getBytes());
run_id_put.add("d".getBytes(), file_id.getBytes(), new Long(timestamp), type.getBytes());
run_id_put.add(
"d".getBytes(),
(file_id + "_db_timestamp").getBytes(),
new Long(timestamp),
timestamp_stop.getBytes());
run_id_put.add(
"d".getBytes(), (file_id + "_filename").getBytes(), new Long(timestamp), path.getBytes());
run_id_put.add(
"d".getBytes(), (file_id + "_regex").getBytes(), new Long(timestamp), regex.getBytes());
db_util.doPut(db_name, run_id_put);
return true;
}
private static boolean putFileEntry(
dbutil db_util,
FileSystem fs,
String db_name,
String file_id,
String file_path,
String source)
throws Exception {
String all_paths = file_path;
if (hasFile(db_util, fs, db_name, file_id, file_path)) {
logger.debug("File already found, putFileEntry aborting");
return false;
} else {
Get file_id_get = new Get(file_id.getBytes());
Result file_result = db_util.doGet(db_name, file_id_get);
KeyValue file_names = file_result.getColumnLatest("d".getBytes(), "filenames".getBytes());
if (file_names != null) {
String paths = new String(file_names.getValue());
all_paths = paths + "\n" + file_path;
}
}
Put file_id_put = new Put(file_id.getBytes());
file_id_put.add("d".getBytes(), "source".getBytes(), source.getBytes());
if (!source.equals("fullfile")) {
file_id_put.add("d".getBytes(), "filenames".getBytes(), all_paths.getBytes());
}
db_util.doPut(db_name, file_id_put);
return true;
}