public int run(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: Injector <crawldb> <url_dir>");
return -1;
}
try {
inject(new Path(args[0]), new Path(args[1]));
return 0;
} catch (Exception e) {
LOG.error("Injector: " + StringUtils.stringifyException(e));
return -1;
}
}
private String runResolveCommand(List<String> args) {
int loopCount = 0;
if (args.size() == 0) {
return null;
}
StringBuffer allOutput = new StringBuffer();
int numProcessed = 0;
if (maxArgs < MIN_ALLOWABLE_ARGS) {
LOG.warn(
"Invalid value "
+ Integer.toString(maxArgs)
+ " for "
+ SCRIPT_ARG_COUNT_KEY
+ "; must be >= "
+ Integer.toString(MIN_ALLOWABLE_ARGS));
return null;
}
while (numProcessed != args.size()) {
int start = maxArgs * loopCount;
List<String> cmdList = new ArrayList<String>();
cmdList.add(scriptName);
for (numProcessed = start;
numProcessed < (start + maxArgs) && numProcessed < args.size();
numProcessed++) {
cmdList.add(args.get(numProcessed));
}
File dir = null;
String userDir;
if ((userDir = System.getProperty("user.dir")) != null) {
dir = new File(userDir);
}
ShellCommandExecutor s = new ShellCommandExecutor(cmdList.toArray(new String[0]), dir);
try {
s.execute();
allOutput.append(s.getOutput() + " ");
} catch (Exception e) {
LOG.warn(StringUtils.stringifyException(e));
return null;
}
loopCount++;
}
return allOutput.toString();
}
// TODO: Can also collect #procs, time taken by each thread to measure fairness.
@Override
public Integer call() throws IOException {
while (true) {
if (workersFailed.get()) {
return EXIT_FAILURE;
}
long procId = procIds.getAndIncrement();
if (procId >= numProcs) {
break;
}
if (procId != 0 && procId % 10000 == 0) {
long ms = System.currentTimeMillis() - start;
System.out.println("Wrote " + procId + " procedures in " + StringUtils.humanTimeDiff(ms));
}
try {
if (procId > 0 && procId % numProcsPerWal == 0) {
store.rollWriterForTesting();
System.out.println(
"Starting new log : "
+ store.getActiveLogs().get(store.getActiveLogs().size() - 1));
}
} catch (IOException ioe) {
// Ask other threads to quit too.
workersFailed.set(true);
System.err.println("Exception when rolling log file. Current procId = " + procId);
ioe.printStackTrace();
return EXIT_FAILURE;
}
ProcedureTestingUtility.TestProcedure proc =
new ProcedureTestingUtility.TestProcedure(procId);
proc.setData(serializedState);
store.insert(proc, null);
store.update(proc);
}
return EXIT_SUCCESS;
}
/**
* Return the path array of the localized files
*
* @param conf Configuration that contains the localized files
* @return A path array of localized files
* @throws IOException
*/
public static Path[] getLocalSharedCacheFiles(Configuration conf) throws IOException {
return StringUtils.stringToPath(conf.getStrings("mapred.cache.shared.localFiles"));
}
/**
* Return the path array of the localized caches
*
* @param conf Configuration that contains the localized archives
* @return A path array of localized caches
* @throws IOException
*/
public static Path[] getLocalCacheArchives(Configuration conf) throws IOException {
return StringUtils.stringToPath(conf.getStrings("mapred.cache.localArchives"));
}
/**
* Get cache files set in the Configuration
*
* @param conf The configuration which contains the files
* @return A URI array of the files set in the Configuration
* @throws IOException
*/
public static URI[] getSharedCacheFiles(Configuration conf) throws IOException {
return StringUtils.stringToURI(conf.getStrings("mapred.cache.shared.files"));
}
/**
* Get cache archives set in the Configuration
*
* @param conf The configuration which contains the archives
* @return A URI array of the caches set in the Configuration
* @throws IOException
*/
public static URI[] getCacheArchives(Configuration conf) throws IOException {
return StringUtils.stringToURI(conf.getStrings("mapred.cache.archives"));
}
/**
* Set the configuration with the given set of files
*
* @param files The list of files that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheFiles(URI[] files, Configuration conf) {
String sfiles = StringUtils.uriToString(files);
conf.set("mapred.cache.files", sfiles);
}
/**
* Set the configuration with the given set of archives
*
* @param archives The list of archives that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheArchives(URI[] archives, Configuration conf) {
String sarchives = StringUtils.uriToString(archives);
conf.set("mapred.cache.archives", sarchives);
}
/**
* Return the path array of the localized files. Intended to be used by user code.
*
* @param conf Configuration that contains the localized files
* @return A path array of localized files
* @throws IOException
*/
public static Path[] getLocalCacheFiles(Configuration conf) throws IOException {
return StringUtils.stringToPath(conf.getStrings(CACHE_LOCALFILES));
}
/**
* Get cache files set in the Configuration. Used by internal DistributedCache and MapReduce code.
*
* @param conf The configuration which contains the files
* @return Am array of the files set in the Configuration
* @throws IOException
*/
public static URI[] getCacheFiles(Configuration conf) throws IOException {
return StringUtils.stringToURI(conf.getStrings(CACHE_FILES));
}
/**
* Get cache archives set in the Configuration. Used by internal DistributedCache and MapReduce
* code.
*
* @param conf The configuration which contains the archives
* @return An array of the caches set in the Configuration
* @throws IOException
*/
public static URI[] getCacheArchives(Configuration conf) throws IOException {
return StringUtils.stringToURI(conf.getStrings(CACHE_ARCHIVES));
}
/**
* Set the configuration with the given set of files. Intended to be used by user code.
*
* @param files The list of files that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheFiles(URI[] files, Configuration conf) {
String sfiles = StringUtils.uriToString(files);
conf.set(CACHE_FILES, sfiles);
}
/**
* Set the configuration with the given set of archives. Intended to be used by user code.
*
* @param archives The list of archives that need to be localized
* @param conf Configuration which will be changed
*/
public static void setCacheArchives(URI[] archives, Configuration conf) {
String sarchives = StringUtils.uriToString(archives);
conf.set(CACHE_ARCHIVES, sarchives);
}
