/**
* This is the main driver for recursively copying directories across file systems. It takes at
* least two cmdline parameters. A source URL and a destination URL. It then essentially does an
* "ls -lR" on the source URL, and writes the output in a round-robin manner to all the map input
* files. The mapper actually copies the files allotted to it. The reduce is empty.
*/
public int run(String[] args) {
try {
copy(conf, Arguments.valueOf(args, conf));
return 0;
} catch (IllegalArgumentException e) {
System.err.println(StringUtils.stringifyException(e) + "\n" + usage);
ToolRunner.printGenericCommandUsage(System.err);
return -1;
} catch (DuplicationException e) {
System.err.println(StringUtils.stringifyException(e));
return DuplicationException.ERROR_CODE;
} catch (RemoteException e) {
final IOException unwrapped =
e.unwrapRemoteException(
FileNotFoundException.class,
AccessControlException.class,
QuotaExceededException.class);
System.err.println(StringUtils.stringifyException(unwrapped));
return -3;
} catch (Exception e) {
System.err.println(
"With failures, global counters are inaccurate; " + "consider running with -i");
System.err.println("Copy failed: " + StringUtils.stringifyException(e));
return -999;
}
}
Upgradeable instantiate() throws IOException {
try {
return (Upgradeable) Class.forName(getClassName()).newInstance();
} catch (ClassNotFoundException e) {
throw new IOException(StringUtils.stringifyException(e));
} catch (InstantiationException e) {
throw new IOException(StringUtils.stringifyException(e));
} catch (IllegalAccessException e) {
throw new IOException(StringUtils.stringifyException(e));
}
}
public void run() {
while (running) {
try {
LOG.info("LocalBlockFixer continuing to run...");
doFix();
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
} catch (Error err) {
LOG.error("Exiting after encountering " + StringUtils.stringifyException(err));
throw err;
}
}
}
public void run(
RecordReader<Text, ArcFileItem> input,
OutputCollector<Text, CrawlURLMetadata> output,
Reporter reporter)
throws IOException {
int lastValidPos = 0;
try {
// allocate key & value instances that are re-used for all entries
Text key = input.createKey();
ArcFileItem value = input.createValue();
while (input.next(key, value)) {
lastValidPos = value.getArcFilePos();
// map pair to output
map(key, value, output, reporter);
}
} catch (IOException e) {
String errorMessage =
"Exception processing Split:"
+ _splitDetails
+ " Exception:"
+ StringUtils.stringifyException(e);
LOG.error(errorMessage);
if (_attemptID.getId() == 0
|| (lastValidPos == 0 && _attemptID.getId() != _maxAttemptsPerTask - 1)) {
throw new IOException(errorMessage);
}
// and just ignore the message
} catch (Throwable e) {
String errorMessage =
"Unknown Exception processing Split:"
+ _splitDetails
+ " Exception:"
+ StringUtils.stringifyException(e);
LOG.error(errorMessage);
// if attempt number is not max attempt number configured...
if (_attemptID.getId() != _maxAttemptsPerTask - 1) {
// then bubble up exception
throw new IOException(errorMessage);
}
} finally {
close();
}
}
/**
* checks if jobs have completed and updates job and file index returns a list of failed files for
* restarting
*/
void checkJobs() throws IOException {
Iterator<Job> jobIter = jobIndex.keySet().iterator();
while (jobIter.hasNext()) {
Job job = jobIter.next();
try {
if (job.isComplete()) {
long slotSeconds =
job.getCounters().findCounter(JobInProgress.Counter.SLOTS_MILLIS_MAPS).getValue()
/ 1000;
RaidNodeMetrics.getInstance().blockFixSlotSeconds.inc(slotSeconds);
long filesSucceeded =
job.getCounters().findCounter(Counter.FILES_SUCCEEDED) != null
? job.getCounters().findCounter(Counter.FILES_SUCCEEDED).getValue()
: 0;
long filesFailed =
job.getCounters().findCounter(Counter.FILES_FAILED) != null
? job.getCounters().findCounter(Counter.FILES_FAILED).getValue()
: 0;
long filesNoAction =
job.getCounters().findCounter(Counter.FILES_NOACTION) != null
? job.getCounters().findCounter(Counter.FILES_NOACTION).getValue()
: 0;
int files = jobIndex.get(job).size();
if (job.isSuccessful()
&& (filesSucceeded + filesFailed + filesNoAction == ((long) files))) {
// job has processed all files
succeedJob(job, filesSucceeded, filesFailed);
} else {
failJob(job);
}
jobIter.remove();
} else {
LOG.info("Job " + job.getID() + "(" + job.getJobName() + " still running");
}
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
failJob(job);
try {
job.killJob();
} catch (Exception ee) {
LOG.error(StringUtils.stringifyException(ee));
}
jobIter.remove();
}
}
purgeFileIndex();
}
// Mostly for setting up the symlinks. Note that when we setup the distributed
// cache, we didn't create the symlinks. This is done on a per task basis
// by the currently executing task.
public static void setupWorkDir(JobConf conf) throws IOException {
File workDir = new File(".").getAbsoluteFile();
FileUtil.fullyDelete(workDir);
if (DistributedCache.getSymlink(conf)) {
URI[] archives = DistributedCache.getCacheArchives(conf);
URI[] files = DistributedCache.getCacheFiles(conf);
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
if (archives != null) {
for (int i = 0; i < archives.length; i++) {
String link = archives[i].getFragment();
if (link != null) {
link = workDir.toString() + Path.SEPARATOR + link;
File flink = new File(link);
if (!flink.exists()) {
FileUtil.symLink(localArchives[i].toString(), link);
}
}
}
}
if (files != null) {
for (int i = 0; i < files.length; i++) {
String link = files[i].getFragment();
if (link != null) {
link = workDir.toString() + Path.SEPARATOR + link;
File flink = new File(link);
if (!flink.exists()) {
FileUtil.symLink(localFiles[i].toString(), link);
}
}
}
}
}
File jobCacheDir = null;
if (conf.getJar() != null) {
jobCacheDir = new File(new Path(conf.getJar()).getParent().toString());
}
// create symlinks for all the files in job cache dir in current
// workingdir for streaming
try {
DistributedCache.createAllSymlink(conf, jobCacheDir, workDir);
} catch (IOException ie) {
// Do not exit even if symlinks have not been created.
LOG.warn(StringUtils.stringifyException(ie));
}
// add java.io.tmpdir given by mapred.child.tmp
String tmp = conf.get("mapred.child.tmp", "./tmp");
Path tmpDir = new Path(tmp);
// if temp directory path is not absolute
// prepend it with workDir.
if (!tmpDir.isAbsolute()) {
tmpDir = new Path(workDir.toString(), tmp);
FileSystem localFs = FileSystem.getLocal(conf);
if (!localFs.mkdirs(tmpDir) && !localFs.getFileStatus(tmpDir).isDir()) {
throw new IOException("Mkdirs failed to create " + tmpDir.toString());
}
}
}
public void run() {
LOG.info(
"Starting expired delegation token remover thread, "
+ "tokenRemoverScanInterval="
+ tokenRemoverScanInterval / (60 * 1000)
+ " min(s)");
try {
while (running) {
long now = System.currentTimeMillis();
if (lastMasterKeyUpdate + keyUpdateInterval < now) {
try {
rollMasterKey();
lastMasterKeyUpdate = now;
} catch (IOException e) {
LOG.error("Master key updating failed. " + StringUtils.stringifyException(e));
}
}
if (lastTokenCacheCleanup + tokenRemoverScanInterval < now) {
removeExpiredToken();
lastTokenCacheCleanup = now;
}
try {
Thread.sleep(5000); // 5 seconds
} catch (InterruptedException ie) {
LOG.error("InterruptedExcpetion recieved for ExpiredTokenRemover thread " + ie);
}
}
} catch (Throwable t) {
LOG.error("ExpiredTokenRemover thread received unexpected exception. " + t);
Runtime.getRuntime().exit(-1);
}
}
/** Connects to the server and instructs it to shutdown. */
public void shutdown() {
// Send the SHUTDOWN command to the server via SQL.
SqoopOptions options = new SqoopOptions(conf);
options.setConnectString("jdbc:hsqldb:hsql://localhost:" + port + "/sqoop");
options.setUsername("SA");
options.setPassword("");
HsqldbManager manager = new HsqldbManager(options);
Statement s = null;
try {
Connection c = manager.getConnection();
s = c.createStatement();
s.execute("SHUTDOWN");
} catch (SQLException sqlE) {
LOG.warn("Exception shutting down database: " + StringUtils.stringifyException(sqlE));
} finally {
if (null != s) {
try {
s.close();
} catch (SQLException sqlE) {
LOG.warn("Error closing statement: " + sqlE);
}
}
try {
manager.close();
} catch (SQLException sqlE) {
LOG.warn("Error closing manager: " + sqlE);
}
}
}
@Override
public void initialize(
HiveConf conf, QueryPlan queryPlan, DriverContext ctx, CompilationOpContext opContext) {
super.initialize(conf, queryPlan, ctx, opContext);
work.initializeForFetch(opContext);
try {
// Create a file system handle
JobConf job = new JobConf(conf);
Operator<?> source = work.getSource();
if (source instanceof TableScanOperator) {
TableScanOperator ts = (TableScanOperator) source;
// push down projections
ColumnProjectionUtils.appendReadColumns(
job, ts.getNeededColumnIDs(), ts.getNeededColumns());
// push down filters
HiveInputFormat.pushFilters(job, ts);
}
sink = work.getSink();
fetch = new FetchOperator(work, job, source, getVirtualColumns(source));
source.initialize(conf, new ObjectInspector[] {fetch.getOutputObjectInspector()});
totalRows = 0;
ExecMapper.setDone(false);
} catch (Exception e) {
// Bail out ungracefully - we should never hit
// this here - but would have hit it in SemanticAnalyzer
LOG.error(StringUtils.stringifyException(e));
throw new RuntimeException(e);
}
}
@Override
public int execute(DriverContext driverContext) {
PrintStream out = null;
try {
Path resFile = new Path(work.getResFile());
OutputStream outS = resFile.getFileSystem(conf).create(resFile);
out = new PrintStream(outS);
QB qb = work.getQb();
TokenRewriteStream stream = work.getCtx().getTokenRewriteStream();
String program = "sq rewrite";
ASTNode ast = work.getAst();
try {
addRewrites(stream, qb, program, out);
out.println(
"\nRewritten Query:\n"
+ stream.toString(program, ast.getTokenStartIndex(), ast.getTokenStopIndex()));
} finally {
stream.deleteProgram(program);
}
out.close();
out = null;
return (0);
} catch (Exception e) {
console.printError(
"Failed with exception " + e.getMessage(), "\n" + StringUtils.stringifyException(e));
return (1);
} finally {
IOUtils.closeStream(out);
}
}
@Override
public void run() {
try {
Thread.sleep(delay);
containerExecutor.signalContainer(
new ContainerSignalContext.Builder()
.setContainer(container)
.setUser(user)
.setPid(pid)
.setSignal(signal)
.build());
} catch (InterruptedException e) {
return;
} catch (IOException e) {
String message =
"Exception when user "
+ user
+ " killing task "
+ pid
+ " in DelayedProcessKiller: "
+ StringUtils.stringifyException(e);
LOG.warn(message);
container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), message));
}
}
/** Run a FileOperation */
public void map(
Text key,
PolicyInfo policy,
OutputCollector<WritableComparable, Text> out,
Reporter reporter)
throws IOException {
this.reporter = reporter;
try {
LOG.info("Raiding file=" + key.toString() + " policy=" + policy);
Path p = new Path(key.toString());
FileStatus fs = p.getFileSystem(jobconf).getFileStatus(p);
st.clear();
RaidNode.doRaid(jobconf, policy, fs, st, reporter);
++succeedcount;
reporter.incrCounter(Counter.PROCESSED_BLOCKS, st.numProcessedBlocks);
reporter.incrCounter(Counter.PROCESSED_SIZE, st.processedSize);
reporter.incrCounter(Counter.META_BLOCKS, st.numMetaBlocks);
reporter.incrCounter(Counter.META_SIZE, st.metaSize);
reporter.incrCounter(Counter.FILES_SUCCEEDED, 1);
} catch (IOException e) {
++failcount;
reporter.incrCounter(Counter.FILES_FAILED, 1);
String s = "FAIL: " + policy + ", " + key + " " + StringUtils.stringifyException(e);
out.collect(null, new Text(s));
LOG.info(s);
} finally {
reporter.setStatus(getCountString());
}
}
@Override
public int run(String[] args) throws Exception {
if (args.length < 1) {
System.err.println("Usage: InjectorJob <url_dir> [-crawlId <id>]");
return -1;
}
for (int i = 1; i < args.length; i++) {
if ("-crawlId".equals(args[i])) {
getConf().set(Nutch.CRAWL_ID_KEY, args[i + 1]);
i++;
} else {
System.err.println("Unrecognized arg " + args[i]);
return -1;
}
}
try {
inject(new Path(args[0]));
LOG.info("InjectorJob: finished");
return -0;
} catch (Exception e) {
LOG.error("InjectorJob: " + StringUtils.stringifyException(e));
return -1;
}
}
private void shutdown() {
try {
this.connection.commit();
this.connection.close();
} catch (Throwable ex) {
LOG.warn(
"Exception occurred while closing connection :" + StringUtils.stringifyException(ex));
} finally {
try {
if (this.server != null) this.server.shutdown();
} catch (Throwable ex) {
LOG.warn(
"Exception occurred while shutting down HSQLDB :" + StringUtils.stringifyException(ex));
}
}
}
/** fix a stripe */
@Override
public void map(LongWritable key, Text fileText, Context context)
throws IOException, InterruptedException {
BlockFixerHelper helper = new BlockFixerHelper(context.getConfiguration());
String fileStr = fileText.toString();
LOG.info("fixing " + fileStr);
Path file = new Path(fileStr);
try {
boolean fixed = helper.fixFile(file, context);
if (fixed) {
context.getCounter(Counter.FILES_SUCCEEDED).increment(1L);
} else {
context.getCounter(Counter.FILES_NOACTION).increment(1L);
}
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
// report file as failed
context.getCounter(Counter.FILES_FAILED).increment(1L);
String outkey = fileStr;
String outval = "failed";
context.write(new Text(outkey), new Text(outval));
}
context.progress();
}
@Override
public void run() {
while (!isStopped()) {
try {
NavigableMap<HRegionInfo, ServerName> regions =
MetaScanner.allTableRegions(connection, TABLENAME);
LOG.info("-------");
byte[] lastEndKey = HConstants.EMPTY_START_ROW;
for (HRegionInfo hri : regions.navigableKeySet()) {
long startKey = 0, endKey = Long.MAX_VALUE;
if (!Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())) {
startKey = Bytes.toLong(hri.getStartKey());
}
if (!Bytes.equals(HConstants.EMPTY_END_ROW, hri.getEndKey())) {
endKey = Bytes.toLong(hri.getEndKey());
}
LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri);
Assert.assertTrue(
"lastEndKey="
+ Bytes.toString(lastEndKey)
+ ", startKey="
+ Bytes.toString(hri.getStartKey()),
Bytes.equals(lastEndKey, hri.getStartKey()));
lastEndKey = hri.getEndKey();
}
Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW));
LOG.info("-------");
Threads.sleep(10 + random.nextInt(50));
} catch (Throwable e) {
ex = e;
Assert.fail(StringUtils.stringifyException(e));
}
}
}
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
throws IOException {
super(file, fs.getFileStatus(file).getReplication());
this.datas = fs.getRawFileSystem().open(file, bufferSize);
this.fs = fs;
Path sumFile = fs.getChecksumFile(file);
try {
int sumBufferSize = fs.getSumBufferSize(fs.getBytesPerSum(), bufferSize);
sums = fs.getRawFileSystem().open(sumFile, sumBufferSize);
byte[] version = new byte[CHECKSUM_VERSION.length];
sums.readFully(version);
if (!Arrays.equals(version, CHECKSUM_VERSION))
throw new IOException("Not a checksum file: " + sumFile);
this.bytesPerSum = sums.readInt();
set(fs.verifyChecksum, new CRC32(), bytesPerSum, 4);
} catch (FileNotFoundException e) { // quietly ignore
set(fs.verifyChecksum, null, 1, 0);
} catch (IOException e) { // loudly ignore
LOG.warn(
"Problem opening checksum file: "
+ file
+ ". Ignoring exception: "
+ StringUtils.stringifyException(e));
set(fs.verifyChecksum, null, 1, 0);
}
}
/**
* This test tests 1, merging region not online; 2, merging same two regions; 3, merging unknown
* regions. They are in one test case so that we don't have to create many tables, and these tests
* are simple.
*/
@Test
public void testMerge() throws Exception {
LOG.info("Starting testMerge");
final TableName tableName = TableName.valueOf("testMerge");
try {
// Create table and load data.
Table table = createTableAndLoadData(master, tableName);
RegionStates regionStates = master.getAssignmentManager().getRegionStates();
List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
// Fake offline one region
HRegionInfo a = regions.get(0);
HRegionInfo b = regions.get(1);
regionStates.regionOffline(a);
try {
// Merge offline region. Region a is offline here
admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
fail("Offline regions should not be able to merge");
} catch (IOException ie) {
System.out.println(ie);
assertTrue(
"Exception should mention regions not online",
StringUtils.stringifyException(ie).contains("regions not online")
&& ie instanceof MergeRegionException);
}
try {
// Merge the same region: b and b.
admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
fail("A region should not be able to merge with itself, even forcifully");
} catch (IOException ie) {
assertTrue(
"Exception should mention regions not online",
StringUtils.stringifyException(ie).contains("region to itself")
&& ie instanceof MergeRegionException);
}
try {
// Merge unknown regions
admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
fail("Unknown region could not be merged");
} catch (IOException ie) {
assertTrue("UnknownRegionException should be thrown", ie instanceof UnknownRegionException);
}
table.close();
} finally {
TEST_UTIL.deleteTable(tableName);
}
}
private DataInput getNextStream() {
try {
if (resDir != null
&& resDirFilesNum < resDirPaths.length
&& (resDirPaths[resDirFilesNum] != null)) {
return resFs.open(resDirPaths[resDirFilesNum++]);
}
} catch (FileNotFoundException e) {
LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
return null;
} catch (IOException e) {
LOG.info("getNextStream error: " + StringUtils.stringifyException(e));
return null;
}
return null;
}
/**
* Main method of the benchmark.
*
* @param args command line parameters
*/
public static void runBenchmark(Configuration conf, List<String> args) throws Exception {
if (args.size() < 2 || !args.get(0).startsWith("-op")) printUsage();
String type = args.get(1);
boolean runAll = OperationStatsBase.OP_ALL_NAME.equals(type);
NNThroughputBenchmark bench = null;
List<OperationStatsBase> ops = new ArrayList<OperationStatsBase>();
OperationStatsBase opStat = null;
try {
bench = new NNThroughputBenchmark(conf);
if (runAll || CreateFileStats.OP_CREATE_NAME.equals(type)) {
opStat = bench.new CreateFileStats(args);
ops.add(opStat);
}
if (runAll || OpenFileStats.OP_OPEN_NAME.equals(type)) {
opStat = bench.new OpenFileStats(args);
ops.add(opStat);
}
if (runAll || DeleteFileStats.OP_DELETE_NAME.equals(type)) {
opStat = bench.new DeleteFileStats(args);
ops.add(opStat);
}
if (runAll || RenameFileStats.OP_RENAME_NAME.equals(type)) {
opStat = bench.new RenameFileStats(args);
ops.add(opStat);
}
if (runAll || BlockReportStats.OP_BLOCK_REPORT_NAME.equals(type)) {
opStat = bench.new BlockReportStats(args);
ops.add(opStat);
}
if (runAll || ReplicationStats.OP_REPLICATION_NAME.equals(type)) {
opStat = bench.new ReplicationStats(args);
ops.add(opStat);
}
if (runAll || CleanAllStats.OP_CLEAN_NAME.equals(type)) {
opStat = bench.new CleanAllStats(args);
ops.add(opStat);
}
if (ops.size() == 0) printUsage();
// run each benchmark
for (OperationStatsBase op : ops) {
LOG.info("Starting benchmark: " + op.getOpName());
op.benchmark();
op.cleanUp();
}
// print statistics
for (OperationStatsBase op : ops) {
LOG.info("");
op.printResults();
}
} catch (Exception e) {
LOG.error(StringUtils.stringifyException(e));
throw e;
} finally {
if (bench != null) bench.close();
}
}
/**
* Stores an exception encountered during RPC invocation so it can be passed back through to the
* client.
*
* @param controller the controller instance provided by the client when calling the service
* @param ioe the exception encountered
*/
public static void setControllerException(RpcController controller, IOException ioe) {
if (controller != null) {
if (controller instanceof ServerRpcController) {
((ServerRpcController) controller).setFailedOn(ioe);
} else {
controller.setFailed(StringUtils.stringifyException(ioe));
}
}
}
public static void main(final String[] args) {
try {
MatMulMap2 mmm = new MatMulMap2();
mmm.run(args);
} catch (Exception e) {
System.out.println("Exception: " + StringUtils.stringifyException(e));
System.exit(-1);
}
}
private static int[] parseJobSetup(Path jobFile) {
int[] result = new int[2];
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try {
DocumentBuilder db = dbf.newDocumentBuilder();
Document doc = db.parse(jobFile.toString());
Element configElement = doc.getDocumentElement();
NodeList nodes = configElement.getElementsByTagName("property");
if (nodes != null && nodes.getLength() > 0) {
for (int i = 0; i < nodes.getLength(); i++) {
Element property = (Element) nodes.item(i);
String elName = xmlGetSingleValue(property, "name");
if (elName == "example.count") {
result[0] = Integer.parseInt(xmlGetSingleValue(property, "value"));
} else if (elName == "batch.size") {
result[1] = Integer.parseInt(xmlGetSingleValue(property, "value"));
}
}
}
} catch (ParserConfigurationException pce) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(pce));
return null;
} catch (SAXException se) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(se));
return null;
} catch (IOException ioe) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(ioe));
return null;
}
return result;
}
/**
* This method gets called everytime before any read/write to make sure that any change to
* localDirs is reflected immediately.
*/
private synchronized void confChanged(Configuration conf) throws IOException {
String newLocalDirs = conf.get(contextCfgItemName);
if (!newLocalDirs.equals(savedLocalDirs)) {
String[] localDirs = conf.getStrings(contextCfgItemName);
localFS = FileSystem.getLocal(conf);
int numDirs = localDirs.length;
ArrayList<String> dirs = new ArrayList<String>(numDirs);
ArrayList<DF> dfList = new ArrayList<DF>(numDirs);
for (int i = 0; i < numDirs; i++) {
try {
// filter problematic directories
Path tmpDir = new Path(localDirs[i]);
if (localFS.mkdirs(tmpDir) || localFS.exists(tmpDir)) {
try {
DiskChecker.checkDir(new File(localDirs[i]));
dirs.add(localDirs[i]);
dfList.add(new DF(new File(localDirs[i]), 30000));
} catch (DiskErrorException de) {
LOG.warn(localDirs[i] + "is not writable\n" + StringUtils.stringifyException(de));
}
} else {
LOG.warn("Failed to create " + localDirs[i]);
}
} catch (IOException ie) {
LOG.warn(
"Failed to create "
+ localDirs[i]
+ ": "
+ ie.getMessage()
+ "\n"
+ StringUtils.stringifyException(ie));
} // ignore
}
localDirsPath = new Path[dirs.size()];
for (int i = 0; i < localDirsPath.length; i++) {
localDirsPath[i] = new Path(dirs.get(i));
}
dirDF = dfList.toArray(new DF[dirs.size()]);
savedLocalDirs = newLocalDirs;
// randomize the first disk picked in the round-robin selection
dirNumLastAccessed = dirIndexRandomizer.nextInt(dirs.size());
}
}
public int run(String[] args) throws Exception {
if (args.length <= 0) {
System.out.println(
"Usage: GeneratorJob [-topN N] [-crawlId id] [-noFilter] [-noNorm] [-adddays numDays]");
System.out.println(
" -topN <N> - number of top URLs to be selected, default is Long.MAX_VALUE ");
System.out.println(
" -crawlId <id> - the id to prefix the schemas to operate on, \n \t \t (default: storage.crawl.id)\");");
System.out.println(
" -noFilter - do not activate the filter plugin to filter the url, default is true ");
System.out.println(
" -noNorm - do not activate the normalizer plugin to normalize the url, default is true ");
System.out.println(
" -adddays - Adds numDays to the current time to facilitate crawling urls already");
System.out.println(
" fetched sooner then db.fetch.interval.default. Default value is 0.");
System.out.println(" -batchId - the batch id ");
System.out.println("----------------------");
System.out.println("Please set the params.");
return -1;
}
long curTime = System.currentTimeMillis(), topN = Long.MAX_VALUE;
boolean filter = true, norm = true;
// generate batchId
int randomSeed = Math.abs(new Random().nextInt());
String batchId = (curTime / 1000) + "-" + randomSeed;
getConf().set(BATCH_ID, batchId);
for (int i = 0; i < args.length; i++) {
if ("-topN".equals(args[i])) {
topN = Long.parseLong(args[++i]);
} else if ("-noFilter".equals(args[i])) {
filter = false;
} else if ("-noNorm".equals(args[i])) {
norm = false;
} else if ("-crawlId".equals(args[i])) {
getConf().set(Nutch.CRAWL_ID_KEY, args[++i]);
} else if ("-adddays".equals(args[i])) {
long numDays = Integer.parseInt(args[++i]);
curTime += numDays * 1000L * 60 * 60 * 24;
} else if ("-batchId".equals(args[i])) getConf().set(BATCH_ID, args[++i]);
else {
System.err.println("Unrecognized arg " + args[i]);
return -1;
}
}
try {
return (generate(topN, curTime, filter, norm) != null) ? 0 : 1;
} catch (Exception e) {
LOG.error("GeneratorJob: " + StringUtils.stringifyException(e));
return -1;
}
}
private String toErrorMessage(Writable value, Object row, ObjectInspector inspector) {
try {
if (row != null) {
return SerDeUtils.getJSONString(row, inspector);
}
return String.valueOf(value);
} catch (Exception e) {
return "[Error getting row data with exception " + StringUtils.stringifyException(e) + " ]";
}
}
public void shutdownServletContainer() {
if (server != null)
try {
server.stop();
server = null;
RESTServlet.stop();
} catch (Exception e) {
LOG.warn(StringUtils.stringifyException(e));
}
}
/** Get the type name of the Java class. */
public static String getTypeNameFromJavaClass(Type t) {
try {
ObjectInspector oi =
ObjectInspectorFactory.getReflectionObjectInspector(t, ObjectInspectorOptions.JAVA);
return oi.getTypeName();
} catch (Throwable e) {
LOG.info(StringUtils.stringifyException(e));
return "unknown";
}
}
/**
* Push the metric to the mr. The metric is pushed only if it was updated since last push
*
* <p>Note this does NOT push to JMX (JMX gets the info via {@link #get()}
*
* @param mr
*/
public synchronized void pushMetric(final MetricsRecord mr) {
if (changed) {
try {
mr.setMetric(getName(), value);
} catch (Exception e) {
LOG.info("pushMetric failed for " + getName() + "\n" + StringUtils.stringifyException(e));
}
}
changed = false;
}
private void parseJobSetup(Path jobFile) {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
try {
DocumentBuilder db = dbf.newDocumentBuilder();
Document doc = db.parse(jobFile.toString());
Element configElement = doc.getDocumentElement();
NodeList nodes = configElement.getElementsByTagName("property");
if (nodes != null && nodes.getLength() > 0) {
for (int i = 0; i < nodes.getLength(); i++) {
Element property = (Element) nodes.item(i);
String elName = xmlGetSingleValue(property, "name");
if (elName.compareToIgnoreCase("gibbs.steps") == 0) {
this.gibbsSteps = Integer.parseInt(xmlGetSingleValue(property, "value"));
} else if (elName.compareToIgnoreCase("learning.rate") == 0) {
this.learningRate = Double.parseDouble(xmlGetSingleValue(property, "value"));
} else if (elName.compareToIgnoreCase("weight.cost") == 0) {
this.weightCost = Double.parseDouble(xmlGetSingleValue(property, "value"));
} else if (elName.compareToIgnoreCase("class.count") == 0) {
this.classCount = Integer.parseInt(xmlGetSingleValue(property, "value"));
}
}
}
} catch (ParserConfigurationException pce) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(pce));
} catch (SAXException se) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(se));
} catch (IOException ioe) {
System.err.println(
"Caught exception while parsing the cached file '"
+ jobFile
+ "' : "
+ StringUtils.stringifyException(ioe));
}
}
