private void setPageRankLinksOptions(JobConf job) throws URISyntaxException {
job.setLong("pages", options.getNumPages());
job.setLong("slotpages", options.getNumSlotPages());
job.set("delimiter", cdelim);
Utils.shareLinkZipfCore(options, job);
}
/**
* This is the main routine for launching a distributed random write job. It runs 10 maps/node and
* each node writes 1 gig of data to a DFS file. The reduce doesn't do anything.
*
* @throws IOException
*/
public int run(String[] args) throws Exception {
if (args.length == 0) {
System.out.println("Usage: writer <out-dir>");
ToolRunner.printGenericCommandUsage(System.out);
return -1;
}
Path outDir = new Path(args[0]);
JobConf job = new JobConf(getConf());
job.setJarByClass(RandomWriter.class);
job.setJobName("random-writer");
FileOutputFormat.setOutputPath(job, outDir);
job.setOutputKeyClass(BytesWritable.class);
job.setOutputValueClass(BytesWritable.class);
job.setInputFormat(RandomInputFormat.class);
job.setMapperClass(Map.class);
job.setReducerClass(IdentityReducer.class);
job.setOutputFormat(SequenceFileOutputFormat.class);
JobClient client = new JobClient(job);
ClusterStatus cluster = client.getClusterStatus();
/** 如果属性不存在 则返回默认的值 * */
int numMapsPerHost = job.getInt("test.randomwriter.maps_per_host", 10);
long numBytesToWritePerMap =
job.getLong("test.randomwrite.bytes_per_map", 1 * 1024 * 1024 * 1024);
if (numBytesToWritePerMap == 0) {
System.err.println("Cannot have test.randomwrite.bytes_per_map set to 0");
return -2;
}
long totalBytesToWrite =
job.getLong(
"test.randomwrite.total_bytes",
numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
if (numMaps == 0 && totalBytesToWrite > 0) {
numMaps = 1;
job.setLong("test.randomwrite.bytes_per_map", totalBytesToWrite);
}
job.setNumMapTasks(numMaps);
/** 建议型的 * */
System.out.println("Running " + numMaps + " maps.");
// reducer NONE
job.setNumReduceTasks(0);
Date startTime = new Date();
System.out.println("Job started: " + startTime);
JobClient.runJob(job);
Date endTime = new Date();
System.out.println("Job ended: " + endTime);
System.out.println(
"The job took " + (endTime.getTime() - startTime.getTime()) / 1000 + " seconds.");
return 0;
}
public void inject(Path crawlDb, Path urlDir) throws IOException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
long start = System.currentTimeMillis();
if (LOG.isInfoEnabled()) {
LOG.info("Injector: starting at " + sdf.format(start));
LOG.info("Injector: crawlDb: " + crawlDb);
LOG.info("Injector: urlDir: " + urlDir);
}
Path tempDir =
new Path(
getConf().get("mapred.temp.dir", ".")
+ "/inject-temp-"
+ Integer.toString(new Random().nextInt(Integer.MAX_VALUE)));
// map text input file to a <url,CrawlDatum> file
if (LOG.isInfoEnabled()) {
LOG.info("Injector: Converting injected urls to crawl db entries.");
}
JobConf sortJob = new NutchJob(getConf());
sortJob.setJobName("inject " + urlDir);
FileInputFormat.addInputPath(sortJob, urlDir);
sortJob.setMapperClass(InjectMapper.class);
FileOutputFormat.setOutputPath(sortJob, tempDir);
sortJob.setOutputFormat(SequenceFileOutputFormat.class);
sortJob.setOutputKeyClass(Text.class);
sortJob.setOutputValueClass(CrawlDatum.class);
sortJob.setLong("injector.current.time", System.currentTimeMillis());
RunningJob mapJob = JobClient.runJob(sortJob);
long urlsInjected = mapJob.getCounters().findCounter("injector", "urls_injected").getValue();
long urlsFiltered = mapJob.getCounters().findCounter("injector", "urls_filtered").getValue();
LOG.info("Injector: total number of urls rejected by filters: " + urlsFiltered);
LOG.info(
"Injector: total number of urls injected after normalization and filtering: "
+ urlsInjected);
// merge with existing crawl db
if (LOG.isInfoEnabled()) {
LOG.info("Injector: Merging injected urls into crawl db.");
}
JobConf mergeJob = CrawlDb.createJob(getConf(), crawlDb);
FileInputFormat.addInputPath(mergeJob, tempDir);
mergeJob.setReducerClass(InjectReducer.class);
JobClient.runJob(mergeJob);
CrawlDb.install(mergeJob, crawlDb);
// clean up
FileSystem fs = FileSystem.get(getConf());
fs.delete(tempDir, true);
long end = System.currentTimeMillis();
LOG.info(
"Injector: finished at "
+ sdf.format(end)
+ ", elapsed: "
+ TimingUtil.elapsedTime(start, end));
}
private void writeThenReadByRecordReader(
int intervalRecordCount,
int writeCount,
int splitNumber,
long minSplitSize,
CompressionCodec codec)
throws IOException {
Path testDir =
new Path(System.getProperty("test.data.dir", ".") + "/mapred/testsmallfirstsplit");
Path testFile = new Path(testDir, "test_rcfile");
fs.delete(testFile, true);
Configuration cloneConf = new Configuration(conf);
RCFileOutputFormat.setColumnNumber(cloneConf, bytesArray.length);
cloneConf.setInt(RCFile.RECORD_INTERVAL_CONF_STR, intervalRecordCount);
RCFile.Writer writer = new RCFile.Writer(fs, cloneConf, testFile, null, codec);
BytesRefArrayWritable bytes = new BytesRefArrayWritable(bytesArray.length);
for (int i = 0; i < bytesArray.length; i++) {
BytesRefWritable cu = null;
cu = new BytesRefWritable(bytesArray[i], 0, bytesArray[i].length);
bytes.set(i, cu);
}
for (int i = 0; i < writeCount; i++) {
if (i == intervalRecordCount) {
System.out.println("write position:" + writer.getLength());
}
writer.append(bytes);
}
writer.close();
RCFileInputFormat inputFormat = new RCFileInputFormat();
JobConf jonconf = new JobConf(cloneConf);
jonconf.set("mapred.input.dir", testDir.toString());
jonconf.setLong("mapred.min.split.size", minSplitSize);
InputSplit[] splits = inputFormat.getSplits(jonconf, splitNumber);
assertEquals("splits length should be " + splitNumber, splits.length, splitNumber);
int readCount = 0;
for (int i = 0; i < splits.length; i++) {
int previousReadCount = readCount;
RecordReader rr = inputFormat.getRecordReader(splits[i], jonconf, Reporter.NULL);
Object key = rr.createKey();
Object value = rr.createValue();
while (rr.next(key, value)) {
readCount++;
}
System.out.println("The " + i + "th split read " + (readCount - previousReadCount));
}
assertEquals("readCount should be equal to writeCount", readCount, writeCount);
}
public static Path putData() throws IOException {
CloudataConf nconf = new CloudataConf();
JobConf jobConf = new JobConf(ManyTableJob.class);
jobConf.set("user.name", nconf.getUserId());
String libDir = CloudataMapReduceUtil.initMapReduce(jobConf);
jobConf.setJobName("ManyTableJob_Put" + "(" + new Date() + ")");
jobConf.setLong("mapred.task.timeout", 30 * 60 * 1000);
Path outputPath = new Path("ManyTableJob_KEY_" + System.currentTimeMillis());
FileOutputFormat.setOutputPath(jobConf, outputPath);
// <MAP>
jobConf.setMapperClass(ManyTablePutMap.class);
jobConf.setInputFormat(SimpleInputFormat.class);
jobConf.setNumMapTasks(numOfTables);
jobConf.setMapSpeculativeExecution(false);
jobConf.setMaxMapAttempts(0);
// </MAP>
// <REDUCE>
jobConf.setNumReduceTasks(0);
// </REDUCE>
try {
// Run Job
JobClient.runJob(jobConf);
return outputPath;
} finally {
// delete temp output path
FileSystem fs = FileSystem.get(jobConf);
CloudataMapReduceUtil.clearMapReduce(libDir);
}
}
public DAG createDAG(
FileSystem remoteFs,
Configuration conf,
Path remoteStagingDir,
int numMapper,
int numReducer,
int iReduceStagesCount,
int numIReducer,
long mapSleepTime,
int mapSleepCount,
long reduceSleepTime,
int reduceSleepCount,
long iReduceSleepTime,
int iReduceSleepCount,
boolean writeSplitsToDFS,
boolean generateSplitsInAM)
throws IOException, YarnException {
Configuration mapStageConf = new JobConf(conf);
mapStageConf.setInt(MRJobConfig.NUM_MAPS, numMapper);
mapStageConf.setLong(MAP_SLEEP_TIME, mapSleepTime);
mapStageConf.setLong(REDUCE_SLEEP_TIME, reduceSleepTime);
mapStageConf.setLong(IREDUCE_SLEEP_TIME, iReduceSleepTime);
mapStageConf.setInt(MAP_SLEEP_COUNT, mapSleepCount);
mapStageConf.setInt(REDUCE_SLEEP_COUNT, reduceSleepCount);
mapStageConf.setInt(IREDUCE_SLEEP_COUNT, iReduceSleepCount);
mapStageConf.setInt(IREDUCE_STAGES_COUNT, iReduceStagesCount);
mapStageConf.setInt(IREDUCE_TASKS_COUNT, numIReducer);
mapStageConf.set(MRJobConfig.MAP_CLASS_ATTR, SleepMapper.class.getName());
mapStageConf.set(MRJobConfig.INPUT_FORMAT_CLASS_ATTR, SleepInputFormat.class.getName());
if (numIReducer == 0 && numReducer == 0) {
mapStageConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, NullOutputFormat.class.getName());
}
MRHelpers.translateVertexConfToTez(mapStageConf);
Configuration[] intermediateReduceStageConfs = null;
if (iReduceStagesCount > 0 && numIReducer > 0) {
intermediateReduceStageConfs = new JobConf[iReduceStagesCount];
for (int i = 1; i <= iReduceStagesCount; ++i) {
JobConf iReduceStageConf = new JobConf(conf);
iReduceStageConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, iReduceSleepTime);
iReduceStageConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, iReduceSleepCount);
iReduceStageConf.setInt(MRJobConfig.NUM_REDUCES, numIReducer);
iReduceStageConf.set(MRJobConfig.REDUCE_CLASS_ATTR, ISleepReducer.class.getName());
iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
iReduceStageConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
iReduceStageConf.set(
MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
MRHelpers.translateVertexConfToTez(iReduceStageConf);
intermediateReduceStageConfs[i - 1] = iReduceStageConf;
}
}
Configuration finalReduceConf = null;
if (numReducer > 0) {
finalReduceConf = new JobConf(conf);
finalReduceConf.setLong(MRRSleepJob.REDUCE_SLEEP_TIME, reduceSleepTime);
finalReduceConf.setInt(MRRSleepJob.REDUCE_SLEEP_COUNT, reduceSleepCount);
finalReduceConf.setInt(MRJobConfig.NUM_REDUCES, numReducer);
finalReduceConf.set(MRJobConfig.REDUCE_CLASS_ATTR, SleepReducer.class.getName());
finalReduceConf.set(MRJobConfig.MAP_OUTPUT_KEY_CLASS, IntWritable.class.getName());
finalReduceConf.set(MRJobConfig.MAP_OUTPUT_VALUE_CLASS, IntWritable.class.getName());
finalReduceConf.set(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR, NullOutputFormat.class.getName());
MRHelpers.translateVertexConfToTez(finalReduceConf);
}
MRHelpers.doJobClientMagic(mapStageConf);
if (iReduceStagesCount > 0 && numIReducer > 0) {
for (int i = 0; i < iReduceStagesCount; ++i) {
MRHelpers.doJobClientMagic(intermediateReduceStageConfs[i]);
}
}
if (numReducer > 0) {
MRHelpers.doJobClientMagic(finalReduceConf);
}
InputSplitInfo inputSplitInfo = null;
if (!generateSplitsInAM) {
if (writeSplitsToDFS) {
LOG.info("Writing splits to DFS");
try {
inputSplitInfo = MRHelpers.generateInputSplits(mapStageConf, remoteStagingDir);
} catch (InterruptedException e) {
throw new TezUncheckedException("Could not generate input splits", e);
} catch (ClassNotFoundException e) {
throw new TezUncheckedException("Failed to generate input splits", e);
}
} else {
try {
LOG.info("Creating in-mem splits");
inputSplitInfo = MRHelpers.generateInputSplitsToMem(mapStageConf);
} catch (ClassNotFoundException e) {
throw new TezUncheckedException("Could not generate input splits", e);
} catch (InterruptedException e) {
throw new TezUncheckedException("Could not generate input splits", e);
}
}
if (inputSplitInfo.getCredentials() != null) {
this.credentials.addAll(inputSplitInfo.getCredentials());
}
}
DAG dag = new DAG("MRRSleepJob");
String jarPath = ClassUtil.findContainingJar(getClass());
if (jarPath == null) {
throw new TezUncheckedException(
"Could not find any jar containing" + " MRRSleepJob.class in the classpath");
}
Path remoteJarPath = remoteFs.makeQualified(new Path(remoteStagingDir, "dag_job.jar"));
remoteFs.copyFromLocalFile(new Path(jarPath), remoteJarPath);
FileStatus jarFileStatus = remoteFs.getFileStatus(remoteJarPath);
TokenCache.obtainTokensForNamenodes(this.credentials, new Path[] {remoteJarPath}, mapStageConf);
Map<String, LocalResource> commonLocalResources = new HashMap<String, LocalResource>();
LocalResource dagJarLocalRsrc =
LocalResource.newInstance(
ConverterUtils.getYarnUrlFromPath(remoteJarPath),
LocalResourceType.FILE,
LocalResourceVisibility.APPLICATION,
jarFileStatus.getLen(),
jarFileStatus.getModificationTime());
commonLocalResources.put("dag_job.jar", dagJarLocalRsrc);
List<Vertex> vertices = new ArrayList<Vertex>();
byte[] mapInputPayload = null;
byte[] mapUserPayload = MRHelpers.createUserPayloadFromConf(mapStageConf);
if (writeSplitsToDFS || generateSplitsInAM) {
mapInputPayload = MRHelpers.createMRInputPayload(mapUserPayload, null);
} else {
mapInputPayload =
MRHelpers.createMRInputPayload(mapUserPayload, inputSplitInfo.getSplitsProto());
}
int numTasks = generateSplitsInAM ? -1 : numMapper;
Vertex mapVertex =
new Vertex(
"map",
new ProcessorDescriptor(MapProcessor.class.getName()).setUserPayload(mapUserPayload),
numTasks,
MRHelpers.getMapResource(mapStageConf));
if (!generateSplitsInAM) {
mapVertex.setTaskLocationsHint(inputSplitInfo.getTaskLocationHints());
}
if (writeSplitsToDFS) {
Map<String, LocalResource> mapLocalResources = new HashMap<String, LocalResource>();
mapLocalResources.putAll(commonLocalResources);
MRHelpers.updateLocalResourcesForInputSplits(remoteFs, inputSplitInfo, mapLocalResources);
mapVertex.setTaskLocalFiles(mapLocalResources);
} else {
mapVertex.setTaskLocalFiles(commonLocalResources);
}
if (generateSplitsInAM) {
MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputAMSplitGenerator.class);
} else {
if (writeSplitsToDFS) {
MRHelpers.addMRInput(mapVertex, mapInputPayload, null);
} else {
MRHelpers.addMRInput(mapVertex, mapInputPayload, MRInputSplitDistributor.class);
}
}
vertices.add(mapVertex);
if (iReduceStagesCount > 0 && numIReducer > 0) {
for (int i = 0; i < iReduceStagesCount; ++i) {
Configuration iconf = intermediateReduceStageConfs[i];
byte[] iReduceUserPayload = MRHelpers.createUserPayloadFromConf(iconf);
Vertex ivertex =
new Vertex(
"ireduce" + (i + 1),
new ProcessorDescriptor(ReduceProcessor.class.getName())
.setUserPayload(iReduceUserPayload),
numIReducer,
MRHelpers.getReduceResource(iconf));
ivertex.setTaskLocalFiles(commonLocalResources);
vertices.add(ivertex);
}
}
Vertex finalReduceVertex = null;
if (numReducer > 0) {
byte[] reducePayload = MRHelpers.createUserPayloadFromConf(finalReduceConf);
finalReduceVertex =
new Vertex(
"reduce",
new ProcessorDescriptor(ReduceProcessor.class.getName())
.setUserPayload(reducePayload),
numReducer,
MRHelpers.getReduceResource(finalReduceConf));
finalReduceVertex.setTaskLocalFiles(commonLocalResources);
MRHelpers.addMROutputLegacy(finalReduceVertex, reducePayload);
vertices.add(finalReduceVertex);
} else {
// Map only job
MRHelpers.addMROutputLegacy(mapVertex, mapUserPayload);
}
Configuration partitionerConf = new Configuration(false);
partitionerConf.set(MRJobConfig.PARTITIONER_CLASS_ATTR, MRRSleepJobPartitioner.class.getName());
OrderedPartitionedKVEdgeConfigurer edgeConf =
OrderedPartitionedKVEdgeConfigurer.newBuilder(
IntWritable.class.getName(),
IntWritable.class.getName(),
MRPartitioner.class.getName(),
partitionerConf)
.configureInput()
.useLegacyInput()
.done()
.build();
for (int i = 0; i < vertices.size(); ++i) {
dag.addVertex(vertices.get(i));
if (i != 0) {
dag.addEdge(
new Edge(vertices.get(i - 1), vertices.get(i), edgeConf.createDefaultEdgeProperty()));
}
}
return dag;
}
/**
* Run a compactor job.
*
* @param conf Hive configuration file
* @param jobName name to run this job with
* @param t metastore table
* @param sd metastore storage descriptor
* @param txns list of valid transactions
* @param isMajor is this a major compaction?
* @throws java.io.IOException if the job fails
*/
void run(
HiveConf conf,
String jobName,
Table t,
StorageDescriptor sd,
ValidTxnList txns,
boolean isMajor,
Worker.StatsUpdater su)
throws IOException {
JobConf job = new JobConf(conf);
job.setJobName(jobName);
job.setOutputKeyClass(NullWritable.class);
job.setOutputValueClass(NullWritable.class);
job.setJarByClass(CompactorMR.class);
LOG.debug("User jar set to " + job.getJar());
job.setMapperClass(CompactorMap.class);
job.setNumReduceTasks(0);
job.setInputFormat(CompactorInputFormat.class);
job.setOutputFormat(NullOutputFormat.class);
job.setOutputCommitter(CompactorOutputCommitter.class);
String queueName = conf.getVar(HiveConf.ConfVars.COMPACTOR_JOB_QUEUE);
if (queueName != null && queueName.length() > 0) {
job.setQueueName(queueName);
}
job.set(FINAL_LOCATION, sd.getLocation());
job.set(TMP_LOCATION, sd.getLocation() + "/" + TMPDIR + "_" + UUID.randomUUID().toString());
job.set(INPUT_FORMAT_CLASS_NAME, sd.getInputFormat());
job.set(OUTPUT_FORMAT_CLASS_NAME, sd.getOutputFormat());
job.setBoolean(IS_MAJOR, isMajor);
job.setBoolean(IS_COMPRESSED, sd.isCompressed());
job.set(TABLE_PROPS, new StringableMap(t.getParameters()).toString());
job.setInt(NUM_BUCKETS, sd.getNumBuckets());
job.set(ValidTxnList.VALID_TXNS_KEY, txns.toString());
setColumnTypes(job, sd.getCols());
// Figure out and encode what files we need to read. We do this here (rather than in
// getSplits below) because as part of this we discover our minimum and maximum transactions,
// and discovering that in getSplits is too late as we then have no way to pass it to our
// mapper.
AcidUtils.Directory dir = AcidUtils.getAcidState(new Path(sd.getLocation()), conf, txns, false);
StringableList dirsToSearch = new StringableList();
Path baseDir = null;
if (isMajor) {
// There may not be a base dir if the partition was empty before inserts or if this
// partition is just now being converted to ACID.
baseDir = dir.getBaseDirectory();
if (baseDir == null) {
List<HdfsFileStatusWithId> originalFiles = dir.getOriginalFiles();
if (!(originalFiles == null) && !(originalFiles.size() == 0)) {
// There are original format files
for (HdfsFileStatusWithId stat : originalFiles) {
Path path = stat.getFileStatus().getPath();
dirsToSearch.add(path);
LOG.debug("Adding original file " + path + " to dirs to search");
}
// Set base to the location so that the input format reads the original files.
baseDir = new Path(sd.getLocation());
}
} else {
// add our base to the list of directories to search for files in.
LOG.debug("Adding base directory " + baseDir + " to dirs to search");
dirsToSearch.add(baseDir);
}
}
List<AcidUtils.ParsedDelta> parsedDeltas = dir.getCurrentDirectories();
if (parsedDeltas == null || parsedDeltas.size() == 0) {
// Seriously, no deltas? Can't compact that.
LOG.error("No delta files found to compact in " + sd.getLocation());
return;
}
StringableList deltaDirs = new StringableList();
long minTxn = Long.MAX_VALUE;
long maxTxn = Long.MIN_VALUE;
for (AcidUtils.ParsedDelta delta : parsedDeltas) {
LOG.debug("Adding delta " + delta.getPath() + " to directories to search");
dirsToSearch.add(delta.getPath());
deltaDirs.add(delta.getPath());
minTxn = Math.min(minTxn, delta.getMinTransaction());
maxTxn = Math.max(maxTxn, delta.getMaxTransaction());
}
if (baseDir != null) job.set(BASE_DIR, baseDir.toString());
job.set(DELTA_DIRS, deltaDirs.toString());
job.set(DIRS_TO_SEARCH, dirsToSearch.toString());
job.setLong(MIN_TXN, minTxn);
job.setLong(MAX_TXN, maxTxn);
LOG.debug("Setting minimum transaction to " + minTxn);
LOG.debug("Setting maximume transaction to " + maxTxn);
RunningJob rj = JobClient.runJob(job);
LOG.info(
"Submitted "
+ (isMajor ? CompactionType.MAJOR : CompactionType.MINOR)
+ " compaction job '"
+ jobName
+ "' with jobID="
+ rj.getID()
+ " to "
+ job.getQueueName()
+ " queue. "
+ "(current delta dirs count="
+ dir.getCurrentDirectories().size()
+ ", obsolete delta dirs count="
+ dir.getObsolete());
rj.waitForCompletion();
su.gatherStats();
}
/**
* Initialize DFSCopyFileMapper specific job-configuration.
*
* @param conf : The dfs/mapred configuration.
* @param jobConf : The handle to the jobConf object to be initialized.
* @param args Arguments
*/
private static void setup(Configuration conf, JobConf jobConf, final Arguments args)
throws IOException {
jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
// set boolean values
final boolean update = args.flags.contains(Options.UPDATE);
final boolean overwrite = !update && args.flags.contains(Options.OVERWRITE);
jobConf.setBoolean(Options.UPDATE.propertyname, update);
jobConf.setBoolean(Options.OVERWRITE.propertyname, overwrite);
jobConf.setBoolean(
Options.IGNORE_READ_FAILURES.propertyname,
args.flags.contains(Options.IGNORE_READ_FAILURES));
jobConf.setBoolean(
Options.PRESERVE_STATUS.propertyname, args.flags.contains(Options.PRESERVE_STATUS));
final String randomId = getRandomId();
JobClient jClient = new JobClient(jobConf);
Path jobDirectory = new Path(jClient.getSystemDir(), NAME + "_" + randomId);
jobConf.set(JOB_DIR_LABEL, jobDirectory.toString());
FileSystem dstfs = args.dst.getFileSystem(conf);
boolean dstExists = dstfs.exists(args.dst);
boolean dstIsDir = false;
if (dstExists) {
dstIsDir = dstfs.getFileStatus(args.dst).isDir();
}
// default logPath
Path logPath = args.log;
if (logPath == null) {
String filename = "_distcp_logs_" + randomId;
if (!dstExists || !dstIsDir) {
Path parent = args.dst.getParent();
if (!dstfs.exists(parent)) {
dstfs.mkdirs(parent);
}
logPath = new Path(parent, filename);
} else {
logPath = new Path(args.dst, filename);
}
}
FileOutputFormat.setOutputPath(jobConf, logPath);
// create src list, dst list
FileSystem jobfs = jobDirectory.getFileSystem(jobConf);
Path srcfilelist = new Path(jobDirectory, "_distcp_src_files");
jobConf.set(SRC_LIST_LABEL, srcfilelist.toString());
SequenceFile.Writer src_writer =
SequenceFile.createWriter(
jobfs,
jobConf,
srcfilelist,
LongWritable.class,
FilePair.class,
SequenceFile.CompressionType.NONE);
Path dstfilelist = new Path(jobDirectory, "_distcp_dst_files");
SequenceFile.Writer dst_writer =
SequenceFile.createWriter(
jobfs, jobConf, dstfilelist, Text.class, Text.class, SequenceFile.CompressionType.NONE);
Path dstdirlist = new Path(jobDirectory, "_distcp_dst_dirs");
jobConf.set(DST_DIR_LIST_LABEL, dstdirlist.toString());
SequenceFile.Writer dir_writer =
SequenceFile.createWriter(
jobfs,
jobConf,
dstdirlist,
Text.class,
FilePair.class,
SequenceFile.CompressionType.NONE);
// handle the case where the destination directory doesn't exist
// and we've only a single src directory OR we're updating/overwriting
// the contents of the destination directory.
final boolean special = (args.srcs.size() == 1 && !dstExists) || update || overwrite;
int srcCount = 0, cnsyncf = 0, dirsyn = 0;
long fileCount = 0L, byteCount = 0L, cbsyncs = 0L;
try {
for (Iterator<Path> srcItr = args.srcs.iterator(); srcItr.hasNext(); ) {
final Path src = srcItr.next();
FileSystem srcfs = src.getFileSystem(conf);
FileStatus srcfilestat = srcfs.getFileStatus(src);
Path root = special && srcfilestat.isDir() ? src : src.getParent();
if (srcfilestat.isDir()) {
++srcCount;
}
Stack<FileStatus> pathstack = new Stack<FileStatus>();
for (pathstack.push(srcfilestat); !pathstack.empty(); ) {
FileStatus cur = pathstack.pop();
FileStatus[] children = srcfs.listStatus(cur.getPath());
for (int i = 0; i < children.length; i++) {
boolean skipfile = false;
final FileStatus child = children[i];
final String dst = makeRelative(root, child.getPath());
++srcCount;
if (child.isDir()) {
pathstack.push(child);
} else {
// skip file if the src and the dst files are the same.
skipfile = update && sameFile(srcfs, child, dstfs, new Path(args.dst, dst));
// skip file if it exceed file limit or size limit
skipfile |=
fileCount == args.filelimit || byteCount + child.getLen() > args.sizelimit;
if (!skipfile) {
++fileCount;
byteCount += child.getLen();
if (LOG.isTraceEnabled()) {
LOG.trace("adding file " + child.getPath());
}
++cnsyncf;
cbsyncs += child.getLen();
if (cnsyncf > SYNC_FILE_MAX || cbsyncs > BYTES_PER_MAP) {
src_writer.sync();
dst_writer.sync();
cnsyncf = 0;
cbsyncs = 0L;
}
}
}
if (!skipfile) {
src_writer.append(
new LongWritable(child.isDir() ? 0 : child.getLen()), new FilePair(child, dst));
}
dst_writer.append(new Text(dst), new Text(child.getPath().toString()));
}
if (cur.isDir()) {
String dst = makeRelative(root, cur.getPath());
dir_writer.append(new Text(dst), new FilePair(cur, dst));
if (++dirsyn > SYNC_FILE_MAX) {
dirsyn = 0;
dir_writer.sync();
}
}
}
}
} finally {
checkAndClose(src_writer);
checkAndClose(dst_writer);
checkAndClose(dir_writer);
}
FileStatus dststatus = null;
try {
dststatus = dstfs.getFileStatus(args.dst);
} catch (FileNotFoundException fnfe) {
LOG.info(args.dst + " does not exist.");
}
// create dest path dir if copying > 1 file
if (dststatus == null) {
if (srcCount > 1 && !dstfs.mkdirs(args.dst)) {
throw new IOException("Failed to create" + args.dst);
}
}
final Path sorted = new Path(jobDirectory, "_distcp_sorted");
checkDuplication(jobfs, dstfilelist, sorted, conf);
if (dststatus != null && args.flags.contains(Options.DELETE)) {
deleteNonexisting(dstfs, dststatus, sorted, jobfs, jobDirectory, jobConf, conf);
}
Path tmpDir =
new Path(
(dstExists && !dstIsDir) || (!dstExists && srcCount == 1)
? args.dst.getParent()
: args.dst,
"_distcp_tmp_" + randomId);
jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
LOG.info("srcCount=" + srcCount);
jobConf.setInt(SRC_COUNT_LABEL, srcCount);
jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
setMapCount(byteCount, jobConf);
}
private void setPageRankNodesOptions(JobConf job) {
job.setLong("pages", options.getNumPages());
job.setLong("slotpages", options.getNumSlotPages());
}
public void merge(Path out, Path[] segs, boolean filter, long slice) throws Exception {
String segmentName = Generator.generateSegmentName();
if (LOG.isInfoEnabled()) {
LOG.info("Merging " + segs.length + " segments to " + out + "/" + segmentName);
}
JobConf job = new NutchJob(getConf());
job.setJobName("mergesegs " + out + "/" + segmentName);
job.setBoolean("segment.merger.filter", filter);
job.setLong("segment.merger.slice", slice);
job.set("segment.merger.segmentName", segmentName);
FileSystem fs = FileSystem.get(getConf());
// prepare the minimal common set of input dirs
boolean g = true;
boolean f = true;
boolean p = true;
boolean c = true;
boolean pd = true;
boolean pt = true;
for (int i = 0; i < segs.length; i++) {
if (!fs.exists(segs[i])) {
if (LOG.isWarnEnabled()) {
LOG.warn("Input dir " + segs[i] + " doesn't exist, skipping.");
}
segs[i] = null;
continue;
}
if (LOG.isInfoEnabled()) {
LOG.info("SegmentMerger: adding " + segs[i]);
}
Path cDir = new Path(segs[i], Content.DIR_NAME);
Path gDir = new Path(segs[i], CrawlDatum.GENERATE_DIR_NAME);
Path fDir = new Path(segs[i], CrawlDatum.FETCH_DIR_NAME);
Path pDir = new Path(segs[i], CrawlDatum.PARSE_DIR_NAME);
Path pdDir = new Path(segs[i], ParseData.DIR_NAME);
Path ptDir = new Path(segs[i], ParseText.DIR_NAME);
c = c && fs.exists(cDir);
g = g && fs.exists(gDir);
f = f && fs.exists(fDir);
p = p && fs.exists(pDir);
pd = pd && fs.exists(pdDir);
pt = pt && fs.exists(ptDir);
}
StringBuffer sb = new StringBuffer();
if (c) sb.append(" " + Content.DIR_NAME);
if (g) sb.append(" " + CrawlDatum.GENERATE_DIR_NAME);
if (f) sb.append(" " + CrawlDatum.FETCH_DIR_NAME);
if (p) sb.append(" " + CrawlDatum.PARSE_DIR_NAME);
if (pd) sb.append(" " + ParseData.DIR_NAME);
if (pt) sb.append(" " + ParseText.DIR_NAME);
if (LOG.isInfoEnabled()) {
LOG.info("SegmentMerger: using segment data from:" + sb.toString());
}
for (int i = 0; i < segs.length; i++) {
if (segs[i] == null) continue;
if (g) {
Path gDir = new Path(segs[i], CrawlDatum.GENERATE_DIR_NAME);
job.addInputPath(gDir);
}
if (c) {
Path cDir = new Path(segs[i], Content.DIR_NAME);
job.addInputPath(cDir);
}
if (f) {
Path fDir = new Path(segs[i], CrawlDatum.FETCH_DIR_NAME);
job.addInputPath(fDir);
}
if (p) {
Path pDir = new Path(segs[i], CrawlDatum.PARSE_DIR_NAME);
job.addInputPath(pDir);
}
if (pd) {
Path pdDir = new Path(segs[i], ParseData.DIR_NAME);
job.addInputPath(pdDir);
}
if (pt) {
Path ptDir = new Path(segs[i], ParseText.DIR_NAME);
job.addInputPath(ptDir);
}
}
job.setInputFormat(ObjectInputFormat.class);
job.setMapperClass(SegmentMerger.class);
job.setReducerClass(SegmentMerger.class);
job.setOutputPath(out);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(MetaWrapper.class);
job.setOutputFormat(SegmentOutputFormat.class);
setConf(job);
JobClient.runJob(job);
}
public JobBuilder set(String key, long value) throws IOException {
_jobConf.setLong(key, value);
return this;
}
public JobBuilder minSplitSize(long minSplitSize) throws IOException {
_jobConf.setLong("mapred.min.split.size", minSplitSize);
return this;
}