public static void main(String[] args) throws IOException {
Path baseDir = null;
String localPath = null;
String preservePath = null;
String sIgnoreTablesFilename = null;
String sNoPreserveFilename = null;
String sDateString = null;
long size = 0;
// UNIX dates for right now
long now = new java.util.Date().getTime() / 1000;
long maxDate = now;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("--hdfs-path")) {
baseDir = new Path(args[++i]);
continue;
}
if (args[i].equals("--local-path")) {
localPath = args[++i];
continue;
}
if (args[i].equals("--preserve-path")) {
preservePath = args[++i];
continue;
}
if (args[i].equals("--no-preserve")) {
sNoPreserveFilename = args[++i];
continue;
}
if (args[i].equals("--ignore-tables")) {
sIgnoreTablesFilename = args[++i];
continue;
}
if (args[i].equals("--sleep")) {
try {
m_nSleepSeconds = Integer.parseInt(args[++i]);
} catch (Exception e) {
System.err.println("ERROR: " + e.toString() + "\n");
usage();
}
continue;
}
if (args[i].equals("--dry-run")) {
m_bDryRun = true;
continue;
}
if (args[i].equals("--date")) {
sDateString = args[++i];
continue;
}
if (args[i].equals("--max-date")) {
maxDate = Long.parseLong(args[++i]);
continue;
}
if (args[i].equals("--max-bytes")) {
size = Long.parseLong(args[++i]);
continue;
}
System.err.println("ERROR: unknown arg " + args[i]);
usage();
}
if (baseDir == null || localPath == null || preservePath == null || sDateString == null) {
usage();
}
long minDate;
if ("yesterday".equals(sDateString)) {
// figure out yesterday's dates
Calendar cal = Calendar.getInstance();
cal.roll(Calendar.DAY_OF_YEAR, -1);
// yesterday midnight
cal.set(Calendar.HOUR_OF_DAY, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MILLISECOND, 0);
minDate = cal.getTimeInMillis() / 1000;
// yesterday end of day
cal.set(Calendar.HOUR_OF_DAY, 23);
cal.set(Calendar.MINUTE, 59);
cal.set(Calendar.SECOND, 59);
cal.set(Calendar.MILLISECOND, 999);
maxDate = cal.getTimeInMillis() / 1000;
} else if ("last-week".equals(sDateString)) {
minDate = maxDate - (7 * 24 * 60 * 60);
} else if ("last-day".equals(sDateString)) {
minDate = maxDate - (24 * 60 * 60);
} else {
// UNIX date since epoch of last backup
minDate = Long.parseLong(sDateString);
}
long tmpDate = 0;
BackupHdfs bak = new BackupHdfs();
// initialize the list of tables to ignore
if (sIgnoreTablesFilename != null) {
bak.initializeTablesToIgnore(sIgnoreTablesFilename);
}
// initialize list of files to not preserve
if (sNoPreserveFilename != null) {
bak.initializeNoPreserve(sNoPreserveFilename);
}
ArrayList<Path> pathList = new ArrayList<Path>(2000);
HashMap<Path, Long> hmTimestamps = new HashMap<Path, Long>();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
// If the HDFS path is a dir continue
if (fs.getFileStatus(baseDir).isDir()) {
Calendar cal = Calendar.getInstance();
System.err.println("");
cal.setTimeInMillis(minDate * 1000);
System.err.println("min date = " + cal.getTime().toString());
cal.setTimeInMillis(maxDate * 1000);
System.err.println("max date = " + cal.getTime().toString());
System.err.println("");
System.err.println("Searching filesystem: " + baseDir.toUri().getPath());
bak.checkDir(fs, minDate, maxDate, baseDir, pathList, hmTimestamps);
System.err.println("");
System.err.println("Skipped " + m_nIgnoredTables + " files due to ignored tables");
System.err.println("");
System.err.println("Number of files to backup = " + pathList.size());
System.err.println("Total bytes to backup = " + prettyPrintBytes(m_nTotalBytes));
System.err.println("");
System.err.println("sorting list of files...");
Collections.sort(pathList, new DateComparator(hmTimestamps));
System.err.println("done");
System.err.println("");
System.err.println("starting backup...");
tmpDate = bak.backupFiles(localPath, preservePath, fs, pathList, size);
bak.closeFiles();
System.err.println("");
System.err.println("backup completed...");
}
if (tmpDate == 0) {
// If not size limit reached print out date for right now
System.out.println(maxDate);
} else {
// Print out date for last file backed up
System.err.println("Size limit reached.");
System.out.println(tmpDate);
}
System.exit(0);
}
public void configure(JobConf conf) {
numberOfCenters = Integer.valueOf(conf.get("numberOfCenters"));
centersDirectory = conf.get("centersReadDirectory");
try {
Configuration c = new Configuration();
FileSystem fs = FileSystem.get(c);
for (int index = 0; index < numberOfCenters; ++index) {
SequenceFile.Reader reader =
new SequenceFile.Reader(fs, new Path(centersDirectory + "/centers/" + index), c);
LongWritable key = new LongWritable();
Point value = new Point();
reader.next(key, value);
Point center = (Point) value;
centers.add(center);
reader.close();
}
} catch (IOException e) {
// do nothing
// I hope this doesn't happen
System.out.println("well, damn.");
e.printStackTrace();
}
}
public void inject(Path crawlDb, Path urlDir) throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long start = System.currentTimeMillis();
if (LOG.isInfoEnabled()) {
LOG.info("Injector: starting at " + sdf.format(start));
LOG.info("Injector: crawlDb: " + crawlDb);
LOG.info("Injector: urlDir: " + urlDir);
}
Path tempDir =
new Path(
getConf().get("mapred.temp.dir", ".")
+ "/inject-temp-"
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
// map text input file to a <url,CrawlDatum> file
if (LOG.isInfoEnabled()) {
LOG.info("Injector: Converting injected urls to crawl db entries.");
}
JobConf sortJob = new NutchJob(getConf());
sortJob.setJobName("inject " + urlDir);
FileInputFormat.addInputPath(sortJob, urlDir);
sortJob.setMapperClass(InjectMapper.class);
FileOutputFormat.setOutputPath(sortJob, tempDir);
sortJob.setOutputFormat(SequenceFileOutputFormat.class);
sortJob.setOutputKeyClass(Text.class);
sortJob.setOutputValueClass(CrawlDatum.class);
sortJob.setLong("injector.current.time", System.currentTimeMillis());
RunningJob mapJob = JobClient.runJob(sortJob);
long urlsInjected = mapJob.getCounters().findCounter("injector", "urls_injected").getValue();
long urlsFiltered = mapJob.getCounters().findCounter("injector", "urls_filtered").getValue();
LOG.info("Injector: total number of urls rejected by filters: " + urlsFiltered);
LOG.info(
"Injector: total number of urls injected after normalization and filtering: "
+ urlsInjected);
// merge with existing crawl db
if (LOG.isInfoEnabled()) {
LOG.info("Injector: Merging injected urls into crawl db.");
}
JobConf mergeJob = CrawlDb.createJob(getConf(), crawlDb);
FileInputFormat.addInputPath(mergeJob, tempDir);
mergeJob.setReducerClass(InjectReducer.class);
JobClient.runJob(mergeJob);
CrawlDb.install(mergeJob, crawlDb);
// clean up
FileSystem fs = FileSystem.get(getConf());
fs.delete(tempDir, true);
long end = System.currentTimeMillis();
LOG.info(
"Injector: finished at "
+ sdf.format(end)
+ ", elapsed: "
+ TimingUtil.elapsedTime(start, end));
}
private void dirCleanup() {
Configuration conf = new Configuration();
try {
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(testPath);
if (fs.exists(dirPath)) {
fs.delete(dirPath, true);
}
} catch (IOException ex) {
LOG.warn("IO Error in test cleanup", ex);
}
}
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
Path pt = new Path("/user/yao/query/query");
FileSystem fs = FileSystem.get(new Configuration());
BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(pt)));
String line = br.readLine();
String[] keywords = line.split(",");
k0 = keywords[0];
k1 = keywords[1];
k2 = keywords[2];
br.close();
}
/*
* Returns the relative path of the dir this cache will be localized in
* relative path that this cache will be localized in. For
* hdfs://hostname:port/absolute_path -- the relative path is
* hostname/absolute path -- if it is just /absolute_path -- then the
* relative path is hostname of DFS this mapred cluster is running
* on/absolute_path
*/
public static String makeRelative(URI cache, Configuration conf) throws IOException {
String host = cache.getHost();
if (host == null) {
host = cache.getScheme();
}
if (host == null) {
URI defaultUri = FileSystem.get(conf).getUri();
host = defaultUri.getHost();
if (host == null) {
host = defaultUri.getScheme();
}
}
String path = host + cache.getPath();
path = path.replace(":/", "/"); // remove windows device colon
return path;
}
/** Start the JobTracker process, listen on the indicated port */
JobTracker(Configuration conf) throws IOException {
//
// Grab some static constants
//
maxCurrentTasks = conf.getInt("mapred.tasktracker.tasks.maximum", 2);
RETIRE_JOB_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.interval", 24 * 60 * 60 * 1000);
RETIRE_JOB_CHECK_INTERVAL = conf.getLong("mapred.jobtracker.retirejob.check", 60 * 1000);
TASK_ALLOC_EPSILON = conf.getFloat("mapred.jobtracker.taskalloc.loadbalance.epsilon", 0.2f);
PAD_FRACTION = conf.getFloat("mapred.jobtracker.taskalloc.capacitypad", 0.1f);
MIN_SLOTS_FOR_PADDING = 3 * maxCurrentTasks;
// This is a directory of temporary submission files. We delete it
// on startup, and can delete any files that we're done with
this.conf = conf;
JobConf jobConf = new JobConf(conf);
this.systemDir = jobConf.getSystemDir();
this.fs = FileSystem.get(conf);
FileUtil.fullyDelete(fs, systemDir);
fs.mkdirs(systemDir);
// Same with 'localDir' except it's always on the local disk.
jobConf.deleteLocalFiles(SUBDIR);
// Set ports, start RPC servers, etc.
InetSocketAddress addr = getAddress(conf);
this.localMachine = addr.getHostName();
this.port = addr.getPort();
this.interTrackerServer = RPC.getServer(this, addr.getPort(), 10, false, conf);
this.interTrackerServer.start();
Properties p = System.getProperties();
for (Iterator it = p.keySet().iterator(); it.hasNext(); ) {
String key = (String) it.next();
String val = (String) p.getProperty(key);
LOG.info("Property '" + key + "' is " + val);
}
this.infoPort = conf.getInt("mapred.job.tracker.info.port", 50030);
this.infoServer = new JobTrackerInfoServer(this, infoPort);
this.infoServer.start();
this.startTime = System.currentTimeMillis();
new Thread(this.expireTrackers).start();
new Thread(this.retireJobs).start();
new Thread(this.initJobs).start();
}
/**
* Initialize a Har filesystem per har archive. The archive home directory is the top level
* directory in the filesystem that contains the HAR archive. Be careful with this method, you do
* not want to go on creating new Filesystem instances per call to path.getFileSystem(). the uri
* of Har is har://underlyingfsscheme-host:port/archivepath. or har:///archivepath. This assumes
* the underlying filesystem to be used in case not specified.
*/
@Override
public void initialize(URI name, Configuration conf) throws IOException {
// initialize the metadata cache, if needed
initializeMetadataCache(conf);
// decode the name
URI underLyingURI = decodeHarURI(name, conf);
// we got the right har Path- now check if this is
// truly a har filesystem
Path harPath = archivePath(new Path(name.getScheme(), name.getAuthority(), name.getPath()));
if (harPath == null) {
throw new IOException("Invalid path for the Har Filesystem. " + name.toString());
}
if (fs == null) {
fs = FileSystem.get(underLyingURI, conf);
}
uri = harPath.toUri();
archivePath = new Path(uri.getPath());
harAuth = getHarAuth(underLyingURI);
// check for the underlying fs containing
// the index file
Path masterIndexPath = new Path(archivePath, "_masterindex");
Path archiveIndexPath = new Path(archivePath, "_index");
if (!fs.exists(masterIndexPath) || !fs.exists(archiveIndexPath)) {
throw new IOException(
"Invalid path for the Har Filesystem. " + "No index file in " + harPath);
}
metadata = harMetaCache.get(uri);
if (metadata != null) {
FileStatus mStat = fs.getFileStatus(masterIndexPath);
FileStatus aStat = fs.getFileStatus(archiveIndexPath);
if (mStat.getModificationTime() != metadata.getMasterIndexTimestamp()
|| aStat.getModificationTime() != metadata.getArchiveIndexTimestamp()) {
// the archive has been overwritten since we last read it
// remove the entry from the meta data cache
metadata = null;
harMetaCache.remove(uri);
}
}
if (metadata == null) {
metadata = new HarMetaData(fs, masterIndexPath, archiveIndexPath);
metadata.parseMetaData();
harMetaCache.put(uri, metadata);
}
}
private static Path checkCacheStatusValidity(
Configuration conf,
URI cache,
long confFileStamp,
CacheStatus cacheStatus,
FileStatus fileStatus,
boolean isArchive)
throws IOException {
FileSystem fs = FileSystem.get(cache, conf);
// Has to be
if (!ifExistsAndFresh(conf, fs, cache, confFileStamp, cacheStatus, fileStatus)) {
throw new IOException(
"Stale cache file: " + cacheStatus.localizedLoadPath + " for cache-file: " + cache);
}
LOG.info(
String.format(
"Using existing cache of %s->%s", cache.toString(), cacheStatus.localizedLoadPath));
return cacheStatus.localizedLoadPath;
}
public static void extractQueryFeatures2HDFS(String filename, Job job) throws IOException {
// Read the local image.jpg as a Mat
Mat query_mat_float =
Highgui.imread(LOCAL_USER_DIR + ID + INPUT + "/" + filename, CvType.CV_32FC3);
// Convert RGB to GRAY
Mat query_gray = new Mat();
Imgproc.cvtColor(query_mat_float, query_gray, Imgproc.COLOR_RGB2GRAY);
// Convert the float type to unsigned integer(required by SIFT)
Mat query_mat_byte = new Mat();
query_gray.convertTo(query_mat_byte, CvType.CV_8UC3);
// // Resize the image to 1/FACTOR both width and height
// Mat query_mat_byte = FeatureExtraction.resize(query_mat_byte);
// Extract the feature from the (Mat)image
Mat query_features = FeatureExtraction.extractFeature(query_mat_byte);
System.out.println(PREFIX + "Extracting the query image feature...");
System.out.println("query_mat(float,color):" + query_mat_float);
System.out.println("query_mat(float,gray):" + query_gray);
System.out.println("query_mat(byte,gray):" + query_mat_byte);
System.out.println("query_mat_features:" + query_features);
System.out.println();
// Store the feature to the hdfs in order to use it later in different map tasks
System.out.println(PREFIX + "Generating the feature file for the query image in HDFS...");
FileSystem fs = FileSystem.get(job.getConfiguration());
String featureFileName = filename.substring(0, filename.lastIndexOf(".")) + ".json";
FSDataOutputStream fsDataOutputStream =
fs.create(new Path(HDFS_HOME + USER + ID + INPUT + "/" + featureFileName));
BufferedWriter bw =
new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
bw.write(FeatureExtraction.mat2json(query_features));
bw.close();
System.out.println(PREFIX + "Query feature extraction finished...");
System.out.println();
}
static void checkRecords(Configuration defaults, Path sortInput, Path sortOutput)
throws IOException {
FileSystem inputfs = sortInput.getFileSystem(defaults);
FileSystem outputfs = sortOutput.getFileSystem(defaults);
FileSystem defaultfs = FileSystem.get(defaults);
JobConf jobConf = new JobConf(defaults, RecordStatsChecker.class);
jobConf.setJobName("sortvalidate-recordstats-checker");
int noSortReduceTasks = outputfs.listStatus(sortOutput, sortPathsFilter).length;
jobConf.setInt("sortvalidate.sort.reduce.tasks", noSortReduceTasks);
int noSortInputpaths = inputfs.listStatus(sortInput).length;
jobConf.setInputFormat(NonSplitableSequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
jobConf.setOutputKeyClass(IntWritable.class);
jobConf.setOutputValueClass(RecordStatsChecker.RecordStatsWritable.class);
jobConf.setMapperClass(Map.class);
jobConf.setCombinerClass(Reduce.class);
jobConf.setReducerClass(Reduce.class);
jobConf.setNumMapTasks(noSortReduceTasks);
jobConf.setNumReduceTasks(1);
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
Path outputPath = new Path("/tmp/sortvalidate/recordstatschecker");
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
}
FileOutputFormat.setOutputPath(jobConf, outputPath);
// Uncomment to run locally in a single process
// job_conf.set("mapred.job.tracker", "local");
Path[] inputPaths = FileInputFormat.getInputPaths(jobConf);
System.out.println(
"\nSortValidator.RecordStatsChecker: Validate sort "
+ "from "
+ inputPaths[0]
+ " ("
+ noSortInputpaths
+ " files), "
+ inputPaths[1]
+ " ("
+ noSortReduceTasks
+ " files) into "
+ FileOutputFormat.getOutputPath(jobConf)
+ " with 1 reducer.");
Date startTime = new Date();
System.out.println("Job started: " + startTime);
JobClient.runJob(jobConf);
Date end_time = new Date();
System.out.println("Job ended: " + end_time);
System.out.println(
"The job took " + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds.");
// Check to ensure that the statistics of the
// framework's sort-input and sort-output match
SequenceFile.Reader stats =
new SequenceFile.Reader(defaultfs, new Path(outputPath, "part-00000"), defaults);
IntWritable k1 = new IntWritable();
IntWritable k2 = new IntWritable();
RecordStatsWritable v1 = new RecordStatsWritable();
RecordStatsWritable v2 = new RecordStatsWritable();
if (!stats.next(k1, v1)) {
throw new IOException("Failed to read record #1 from reduce's output");
}
if (!stats.next(k2, v2)) {
throw new IOException("Failed to read record #2 from reduce's output");
}
if ((v1.getBytes() != v2.getBytes())
|| (v1.getRecords() != v2.getRecords())
|| v1.getChecksum() != v2.getChecksum()) {
throw new IOException(
"("
+ v1.getBytes()
+ ", "
+ v1.getRecords()
+ ", "
+ v1.getChecksum()
+ ") v/s ("
+ v2.getBytes()
+ ", "
+ v2.getRecords()
+ ", "
+ v2.getChecksum()
+ ")");
}
}
public void map(Text key, Text value, Context context)
throws InterruptedException, IOException {
String filename = key.toString();
String json = value.toString();
// Make sure the input is valid
if (!(filename.isEmpty() || json.isEmpty())) {
// Change the json-type feature to Mat-type feature
Mat descriptor = json2mat(json);
if (descriptor != null) {
// Read the query feature from the cache in Hadoop
Mat query_features;
String pathStr = context.getConfiguration().get("featureFilePath");
FileSystem fs = FileSystem.get(context.getConfiguration());
FSDataInputStream fsDataInputStream = fs.open(new Path(pathStr));
StringBuilder sb = new StringBuilder();
// Use a buffer to read the query_feature
int remain = fsDataInputStream.available();
while (remain > 0) {
int read;
byte[] buf = new byte[BUF_SIZE];
read = fsDataInputStream.read(buf, fsDataInputStream.available() - remain, BUF_SIZE);
sb.append(new String(buf, 0, read, StandardCharsets.UTF_8));
remain = remain - read;
System.out.println("remain:" + remain + "\tread:" + read + "\tsb.size:" + sb.length());
}
// Read the query_feature line by line
// Scanner sc = new Scanner(fsDataInputStream, "UTF-8");
// StringBuilder sb = new StringBuilder();
// while (sc.hasNextLine()) {
// sb.append(sc.nextLine());
// }
// String query_json = sb.toString();
// String query_json = new String(buf, StandardCharsets.UTF_8);
String query_json = sb.toString();
fsDataInputStream.close();
query_features = json2mat(query_json);
// Get the similarity of the current database image against the query image
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.FLANNBASED);
MatOfDMatch matches = new MatOfDMatch();
// Ensure the two features have same length of cols (the feature extracted are all 128
// cols(at least in this case))
if (query_features.cols() == descriptor.cols()) {
matcher.match(query_features, descriptor, matches);
DMatch[] dMatches = matches.toArray();
// Calculate the max/min distances
// double max_dist = Double.MAX_VALUE;
// double min_dist = Double.MIN_VALUE;
double max_dist = 0;
double min_dist = 100;
for (int i = 0; i < dMatches.length; i++) {
double dist = dMatches[i].distance;
if (min_dist > dist) min_dist = dist;
if (max_dist < dist) max_dist = dist;
}
// Only distances ≤ threshold are good matches
double threshold = max_dist * THRESHOLD_FACTOR;
// double threshold = min_dist * 2;
LinkedList<DMatch> goodMatches = new LinkedList<DMatch>();
for (int i = 0; i < dMatches.length; i++) {
if (dMatches[i].distance <= threshold) {
goodMatches.addLast(dMatches[i]);
}
}
// Get the ratio of good_matches to all_matches
double ratio = (double) goodMatches.size() / (double) dMatches.length;
System.out.println("*** current_record_filename:" + filename + " ***");
System.out.println("feature:" + descriptor + "\nquery_feature:" + query_features);
System.out.println(
"min_dist of keypoints:" + min_dist + " max_dist of keypoints:" + max_dist);
System.out.println(
"total_matches:" + dMatches.length + "\tgood_matches:" + goodMatches.size());
// System.out.println("type:" + descriptor.type() + " channels:" +
// descriptor.channels() + " rows:" + descriptor.rows() + " cols:" + descriptor.cols());
// System.out.println("qtype:" + query_features.type() + "
// qchannels:" + query_features.channels() + " qrows:" + query_features.rows() + "
// qcols:" + query_features.cols());
System.out.println();
if (ratio > PERCENTAGE_THRESHOLD) {
// Key:1 Value:filename|ratio
context.write(ONE, new Text(filename + "|" + ratio));
// context.write(ONE, new Text(filename + "|" +
// String.valueOf(goodMatches.size())));
}
} else {
System.out.println("The size of the features are not equal");
}
} else {
// a null pointer, do nothing
System.out.println("A broken/null feature:" + filename);
System.out.println();
}
}
}
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
if (args.length != 2) {
System.out.println("Usage: FeatureMatching ID <inputName.jpeg/inputName.jpg>");
System.exit(1);
}
SimpleDateFormat sdf = new SimpleDateFormat("", Locale.US);
sdf.applyPattern("yyyy-MM-dd_HH-mm-ss");
String time = sdf.format(new Date());
Job job = Job.getInstance();
ID = "/" + args[0];
String filename = args[1];
filename = filename.toLowerCase();
System.out.println("current filename:" + filename);
// Detect illegal username (if the username dir doesn't exist)
File userPath = new File(LOCAL_USER_DIR + ID);
if (!userPath.exists()) {
System.out.println("Unauthorized username!!!\nExiting......");
System.exit(1);
}
// Preprocess the input image.jpg from local dir: /local.../user/ID/input/image.jpg
// Save the features to local dir: hdfs://.../user/ID/input/image.jpg
extractQueryFeatures2HDFS(filename, job);
// Add the feature file to the hdfs cache
String featureFileName = filename.substring(0, filename.lastIndexOf(".")) + ".json";
// job.addCacheFile(new Path(HDFS_HOME + USER + ID + INPUT + "/" +
// featureFileName).toUri());
job.getConfiguration()
.set("featureFilePath", HDFS_HOME + USER + ID + INPUT + "/" + featureFileName);
// Check the file type. Only support jpeg/jpg type images
String type = filename.substring(args[1].lastIndexOf("."));
if (!(type.equals(".jpg") || type.equals(".jpeg"))) {
System.out.println("Image type not supported!!!\nExiting");
System.exit(1);
}
// Input: hdfs://.../features/
// The feature dir is a location of all features extracted from the database
String inputPathStr = HDFS_HOME + FEATURES;
// Output: hdfs://.../user/ID/output/
String outputPathStr = HDFS_HOME + USER + ID + OUTPUT + "/" + time;
job.setInputFormatClass(KeyValueTextInputFormat.class);
// job.setOutputFormatClass(TextOutputFormat.class);
// Get the lists of all feature files: /.../features/data/part-*
FileSystem fs = FileSystem.get(job.getConfiguration());
FileStatus[] statuses = fs.listStatus(new Path(inputPathStr));
StringBuffer sb = new StringBuffer();
for (FileStatus fileStatus : statuses) {
sb.append(fileStatus.getPath() + ",");
}
sb.deleteCharAt(sb.lastIndexOf(","));
job.setJarByClass(FeatureMatching.class);
job.setMapperClass(FeatureMatchMapper.class);
job.setReducerClass(FeatureMatchReducer.class);
// only need one reducer to collect the result
job.setNumReduceTasks(1);
job.setMapOutputKeyClass(IntWritable.class);
job.setMapOutputValueClass(Text.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(DoubleWritable.class);
// Input a directory, so need the recursive input
FileInputFormat.setInputDirRecursive(job, true);
// Set the PathFilter, to omit _SUCCESS files
// (This is not working correctly, as the PathFilter class is an interface rather than a class.
// But the 2nd arg asks me to extend the PathFilter)
// FileInputFormat.setInputPathFilter(job, MyPathFilter.class);
//
// FileInputFormat.setInputPaths(job, new Path(inputPathStr));
FileInputFormat.setInputPaths(job, sb.toString());
FileOutputFormat.setOutputPath(job, new Path(outputPathStr));
boolean success = job.waitForCompletion(true);
return success ? 0 : 1;
}
public void doTestTextBatchAppend() throws Exception {
LOG.debug("Starting...");
final long rollCount = 10;
final long batchSize = 2;
final String fileName = "PageView";
String newPath = testPath + "/singleTextBucket";
int totalEvents = 0;
int i = 1, j = 1;
// clear the test directory
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(newPath);
fs.delete(dirPath, true);
fs.mkdirs(dirPath);
Context context = new Context();
context.put("hdfs.path", newPath);
context.put("hdfs.rollCount", String.valueOf(rollCount));
context.put("hdfs.batchSize", String.valueOf(batchSize));
context.put("hdfs.filePrefix", "pageview");
Channel channel = new MemoryChannel();
Configurables.configure(channel, context);
sink.setChannel(channel);
sink.start();
Calendar eventDate = Calendar.getInstance();
Date currentDate = new Date();
Map<String, String> header = new HashMap<String, String>();
header.put("topic", "PageView");
List<String> bodies = Lists.newArrayList();
// 将测试的事件推入到通道中
for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
Transaction txn = channel.getTransaction();
txn.begin();
for (j = 1; j <= batchSize; j++) {
header.put("timestamp", String.valueOf(currentDate.getTime()));
Event event = new SimpleEvent();
eventDate.clear();
eventDate.set(2014, i, i, i, 0);
String body = "Test." + i + "." + j;
event.setHeaders(header);
event.setBody(body.getBytes());
bodies.add(body);
channel.put(event);
totalEvents++;
}
txn.commit();
txn.close();
// execute sink to process the events
sink.process();
}
sink.stop();
FileStatus[] dirStat = fs.listStatus(dirPath);
Path fList[] = FileUtil.stat2Paths(dirStat);
long expectedFiles = totalEvents / rollCount;
if (totalEvents % rollCount > 0) {
expectedFiles++;
}
Assert.assertEquals(
"num files wrong, found: " + Lists.newArrayList(fList), expectedFiles, fList.length);
// 检查所有写入文件的内容
verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
}
/**
* Test how IPC cache map works.
*
* @throws Exception If failed.
*/
@SuppressWarnings("unchecked")
public void testIpcCache() throws Exception {
Field cacheField = GridGgfsHadoopIpcIo.class.getDeclaredField("ipcCache");
cacheField.setAccessible(true);
Field activeCntField = GridGgfsHadoopIpcIo.class.getDeclaredField("activeCnt");
activeCntField.setAccessible(true);
Map<String, GridGgfsHadoopIpcIo> cache =
(Map<String, GridGgfsHadoopIpcIo>) cacheField.get(null);
String name = "ggfs:" + getTestGridName(0) + "@";
Configuration cfg = new Configuration();
cfg.addResource(U.resolveGridGainUrl(HADOOP_FS_CFG));
cfg.setBoolean("fs.ggfs.impl.disable.cache", true);
cfg.setBoolean(String.format(GridGgfsHadoopUtils.PARAM_GGFS_ENDPOINT_NO_EMBED, name), true);
// Ensure that existing IO is reused.
FileSystem fs1 = FileSystem.get(new URI("ggfs://" + name + "/"), cfg);
assertEquals(1, cache.size());
GridGgfsHadoopIpcIo io = null;
System.out.println("CACHE: " + cache);
for (String key : cache.keySet()) {
if (key.contains("10500")) {
io = cache.get(key);
break;
}
}
assert io != null;
assertEquals(1, ((AtomicInteger) activeCntField.get(io)).get());
// Ensure that when IO is used by multiple file systems and one of them is closed, IO is not
// stopped.
FileSystem fs2 = FileSystem.get(new URI("ggfs://" + name + "/abc"), cfg);
assertEquals(1, cache.size());
assertEquals(2, ((AtomicInteger) activeCntField.get(io)).get());
fs2.close();
assertEquals(1, cache.size());
assertEquals(1, ((AtomicInteger) activeCntField.get(io)).get());
Field stopField = GridGgfsHadoopIpcIo.class.getDeclaredField("stopping");
stopField.setAccessible(true);
assert !(Boolean) stopField.get(io);
// Ensure that IO is stopped when nobody else is need it.
fs1.close();
assert cache.isEmpty();
assert (Boolean) stopField.get(io);
}
public int run(String[] args) throws Exception {
// printUsage();
/*
* SETUP
*/
Configuration argConf = getConf();
Hashtable<String, String> confArg = new Hashtable<String, String>();
setup(confArg, argConf);
Date currentTime = new Date();
Date endDate = new Date(new Long(confArg.get("timestamp_stop")));
Boolean full_run = confArg.get("intermediate").matches("(?i).*true.*");
Boolean quick_add = confArg.get("quick_add").matches("(?i).*true.*");
logger.info("Running GeStore");
// ZooKeeper setup
Configuration config = HBaseConfiguration.create();
zkWatcher = new ZooKeeperWatcher(config, "Testing", new HBaseAdmin(config));
zkInstance =
new ZooKeeper(
ZKConfig.getZKQuorumServersString(config),
config.getInt("zookeeper.session.timeout", -1),
zkWatcher);
if (!confArg.get("task_id").isEmpty()) {
confArg.put("temp_path", confArg.get("temp_path") + confArg.get("task_id"));
}
String lockRequest = confArg.get("file_id");
if (!confArg.get("run_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("run_id") + "_";
if (!confArg.get("task_id").isEmpty())
lockRequest = lockRequest + "_" + confArg.get("task_id") + "_";
// Get type of movement
toFrom type_move = checkArgs(confArg);
if (type_move == toFrom.LOCAL2REMOTE && !confArg.get("format").equals("unknown")) {
List<String> arguments = new ArrayList<String>();
arguments.add("-Dinput=" + confArg.get("local_path"));
arguments.add("-Dtable=" + confArg.get("file_id"));
arguments.add("-Dtimestamp=" + confArg.get("timestamp_stop"));
arguments.add("-Dtype=" + confArg.get("format"));
arguments.add("-Dtarget_dir=" + confArg.get("base_path") + "_" + confArg.get("file_id"));
arguments.add("-Dtemp_hdfs_path=" + confArg.get("temp_path"));
arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("run_id").isEmpty()) arguments.add("-Drun_id=" + confArg.get("run_id"));
if (!confArg.get("task_id").isEmpty()) arguments.add("-Dtask_id=" + confArg.get("task_id"));
if (quick_add) arguments.add("-Dquick_add=" + confArg.get("quick_add"));
String lockName = lock(lockRequest);
String[] argumentString = arguments.toArray(new String[arguments.size()]);
adddb.main(argumentString);
unlock(lockName);
System.exit(0);
}
// Database registration
dbutil db_util = new dbutil(config);
db_util.register_database(confArg.get("db_name_files"), true);
db_util.register_database(confArg.get("db_name_runs"), true);
db_util.register_database(confArg.get("db_name_updates"), true);
FileSystem hdfs = FileSystem.get(config);
FileSystem localFS = FileSystem.getLocal(config);
// Get source type
confArg.put("source", getSource(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
confArg.put(
"database", isDatabase(db_util, confArg.get("db_name_files"), confArg.get("file_id")));
if (!confArg.get("source").equals("local")
&& type_move == toFrom.REMOTE2LOCAL
&& !confArg.get("timestamp_stop").equals(Integer.toString(Integer.MAX_VALUE))) {
confArg.put("timestamp_stop", Long.toString(latestVersion(confArg, db_util)));
}
/*
* Get previous timestamp
*/
Get run_id_get = new Get(confArg.get("run_id").getBytes());
Result run_get = db_util.doGet(confArg.get("db_name_runs"), run_id_get);
KeyValue run_file_prev =
run_get.getColumnLatest(
"d".getBytes(), (confArg.get("file_id") + "_db_timestamp").getBytes());
String last_timestamp = new String("0");
if (null != run_file_prev && !confArg.get("source").equals("local")) {
long last_timestamp_real = run_file_prev.getTimestamp();
Long current_timestamp = new Long(confArg.get("timestamp_real"));
if ((current_timestamp - last_timestamp_real) > 36000) {
last_timestamp = new String(run_file_prev.getValue());
Integer lastTimestamp = new Integer(last_timestamp);
lastTimestamp += 1;
last_timestamp = lastTimestamp.toString();
logger.info("Last timestamp: " + last_timestamp + " End data: " + endDate);
Date last_run = new Date(run_file_prev.getTimestamp());
if (last_run.before(endDate) && !full_run) {
confArg.put("timestamp_start", last_timestamp);
}
}
}
Integer tse = new Integer(confArg.get("timestamp_stop"));
Integer tss = new Integer(confArg.get("timestamp_start"));
if (tss > tse) {
logger.info("No new version of requested file.");
return 0;
}
/*
* Generate file
*/
String lockName = lock(lockRequest);
Get file_id_get = new Get(confArg.get("file_id").getBytes());
Result file_get = db_util.doGet(confArg.get("db_name_files"), file_id_get);
if (!file_get.isEmpty()) {
boolean found =
hasFile(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg));
if (confArg.get("source").equals("fullfile")) {
found = false;
}
String filenames_put =
getFileNames(
db_util, confArg.get("db_name_files"), confArg.get("file_id"), getFullPath(confArg));
// Filename not found in file database
if (!found && type_move == toFrom.REMOTE2LOCAL) {
if (!confArg.get("source").equals("local")) {
// Generate intermediate file
if (getFile(hdfs, confArg, db_util) == null) {
unlock(lockName);
return 1;
}
// Put generated file into file database
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
confArg.get("full_file_name"),
confArg.get("source"));
}
} else {
logger.warn("Remote file not found, and cannot be generated! File: " + confArg);
unlock(lockName);
return 1;
}
}
} else {
if (type_move == toFrom.REMOTE2LOCAL) {
logger.warn("Remote file not found, and cannot be generated.");
unlock(lockName);
return 1;
}
}
/*
* Copy file
* Update tables
*/
if (type_move == toFrom.LOCAL2REMOTE) {
if (!confArg.get("format").equals("fullfile")) {
putFileEntry(
db_util,
hdfs,
confArg.get("db_name_files"),
confArg.get("file_id"),
getFullPath(confArg),
confArg.get("source"));
}
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
hdfs.copyFromLocalFile(new Path(confArg.get("local_path")), new Path(getFullPath(confArg)));
} else if (type_move == toFrom.REMOTE2LOCAL) {
FileStatus[] files = hdfs.globStatus(new Path(getFullPath(confArg) + "*"));
putRunEntry(
db_util,
confArg.get("db_name_runs"),
confArg.get("run_id"),
confArg.get("file_id"),
confArg.get("type"),
confArg.get("timestamp_real"),
confArg.get("timestamp_stop"),
getFullPath(confArg),
confArg.get("delimiter"));
unlock(lockName);
for (FileStatus file : files) {
Path cur_file = file.getPath();
Path cur_local_path =
new Path(new String(confArg.get("local_path") + confArg.get("file_id")));
String suffix = getSuffix(getFileName(confArg), cur_file.getName());
if (suffix.length() > 0) {
cur_local_path = cur_local_path.suffix(new String("." + suffix));
}
if (confArg.get("copy").equals("true")) {
String crc = hdfs.getFileChecksum(cur_file).toString();
if (checksumLocalTest(cur_local_path, crc)) {
continue;
} else {
hdfs.copyToLocalFile(cur_file, cur_local_path);
writeChecksum(cur_local_path, crc);
}
} else {
System.out.println(cur_local_path + "\t" + cur_file);
}
}
}
unlock(lockName);
return 0;
}
private static FileSystem getFileSystem(URI cache, Configuration conf) throws IOException {
String fileSysName = getFileSysName(cache);
if (fileSysName != null) return FileSystem.getNamed(fileSysName, conf);
else return FileSystem.get(conf);
}
/**
* Returns the status of a given cache file on hdfs.
*
* @param conf configuration
* @param cache cache file
* @return FileStatus object of the file
* @throws IOException
*/
public static FileStatus getFileStatus(Configuration conf, URI cache) throws IOException {
FileSystem fileSystem = FileSystem.get(cache, conf);
Path filePath = new Path(cache.getPath());
return fileSystem.getFileStatus(filePath);
}
/**
* Returns mtime of a given cache file on hdfs.
*
* @param conf configuration
* @param cache cache file
* @return mtime of a given cache file on hdfs
* @throws IOException
*/
public static long getTimestamp(Configuration conf, URI cache) throws IOException {
FileSystem fileSystem = FileSystem.get(cache, conf);
Path filePath = new Path(cache.getPath());
return fileSystem.getFileStatus(filePath).getModificationTime();
}
static void checkRecords(
Configuration defaults, int noMaps, int noReduces, Path sortInput, Path sortOutput)
throws IOException {
JobConf jobConf = new JobConf(defaults, RecordChecker.class);
jobConf.setJobName("sortvalidate-record-checker");
jobConf.setInputFormat(SequenceFileInputFormat.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
jobConf.setOutputKeyClass(BytesWritable.class);
jobConf.setOutputValueClass(IntWritable.class);
jobConf.setMapperClass(Map.class);
jobConf.setReducerClass(Reduce.class);
JobClient client = new JobClient(jobConf);
ClusterStatus cluster = client.getClusterStatus();
if (noMaps == -1) {
noMaps = cluster.getTaskTrackers() * jobConf.getInt("test.sortvalidate.maps_per_host", 10);
}
if (noReduces == -1) {
noReduces = (int) (cluster.getMaxReduceTasks() * 0.9);
String sortReduces = jobConf.get("test.sortvalidate.reduces_per_host");
if (sortReduces != null) {
noReduces = cluster.getTaskTrackers() * Integer.parseInt(sortReduces);
}
}
jobConf.setNumMapTasks(noMaps);
jobConf.setNumReduceTasks(noReduces);
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
Path outputPath = new Path("/tmp/sortvalidate/recordchecker");
FileSystem fs = FileSystem.get(defaults);
if (fs.exists(outputPath)) {
fs.delete(outputPath, true);
}
FileOutputFormat.setOutputPath(jobConf, outputPath);
// Uncomment to run locally in a single process
// job_conf.set("mapred.job.tracker", "local");
Path[] inputPaths = FileInputFormat.getInputPaths(jobConf);
System.out.println(
"\nSortValidator.RecordChecker: Running on "
+ cluster.getTaskTrackers()
+ " nodes to validate sort from "
+ inputPaths[0]
+ ", "
+ inputPaths[1]
+ " into "
+ FileOutputFormat.getOutputPath(jobConf)
+ " with "
+ noReduces
+ " reduces.");
Date startTime = new Date();
System.out.println("Job started: " + startTime);
JobClient.runJob(jobConf);
Date end_time = new Date();
System.out.println("Job ended: " + end_time);
System.out.println(
"The job took " + (end_time.getTime() - startTime.getTime()) / 1000 + " seconds.");
}
