@SuppressWarnings("unchecked")
@Before
public void setUp() throws Exception {
utils = mock(DagUtils.class);
fs = mock(FileSystem.class);
path = mock(Path.class);
when(path.getFileSystem(any(Configuration.class))).thenReturn(fs);
when(utils.getTezDir(any(Path.class))).thenReturn(path);
when(utils.createVertex(
any(JobConf.class),
any(BaseWork.class),
any(Path.class),
any(LocalResource.class),
any(List.class),
any(FileSystem.class),
any(Context.class),
anyBoolean(),
any(TezWork.class),
any(VertexType.class)))
.thenAnswer(
new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
return Vertex.create(
((BaseWork) args[1]).getName(),
mock(ProcessorDescriptor.class),
0,
mock(Resource.class));
}
});
when(utils.createEdge(
any(JobConf.class),
any(Vertex.class),
any(Vertex.class),
any(TezEdgeProperty.class),
any(VertexType.class)))
.thenAnswer(
new Answer<Edge>() {
@Override
public Edge answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
return Edge.create((Vertex) args[1], (Vertex) args[2], mock(EdgeProperty.class));
}
});
work = new TezWork("");
mws = new MapWork[] {new MapWork(), new MapWork()};
rws = new ReduceWork[] {new ReduceWork(), new ReduceWork()};
work.addAll(mws);
work.addAll(rws);
int i = 0;
for (BaseWork w : work.getAllWork()) {
w.setName("Work " + (++i));
}
op = mock(Operator.class);
LinkedHashMap<String, Operator<? extends OperatorDesc>> map =
new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
map.put("foo", op);
mws[0].setAliasToWork(map);
mws[1].setAliasToWork(map);
LinkedHashMap<String, ArrayList<String>> pathMap =
new LinkedHashMap<String, ArrayList<String>>();
ArrayList<String> aliasList = new ArrayList<String>();
aliasList.add("foo");
pathMap.put("foo", aliasList);
mws[0].setPathToAliases(pathMap);
mws[1].setPathToAliases(pathMap);
rws[0].setReducer(op);
rws[1].setReducer(op);
TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE);
work.connect(mws[0], rws[0], edgeProp);
work.connect(mws[1], rws[0], edgeProp);
work.connect(rws[0], rws[1], edgeProp);
task = new TezTask(utils);
task.setWork(work);
task.setConsole(mock(LogHelper.class));
conf = new JobConf();
appLr = mock(LocalResource.class);
SessionState.start(new HiveConf());
session = mock(TezClient.class);
sessionState = mock(TezSessionState.class);
when(sessionState.getSession()).thenReturn(session);
when(session.submitDAG(any(DAG.class)))
.thenThrow(new SessionNotRunning(""))
.thenReturn(mock(DAGClient.class));
}
public int run(String[] args) throws Exception {
if (args.length < 1) {
System.err.println(
"MRRSleepJob [-m numMapper] [-r numReducer]"
+ " [-ir numIntermediateReducer]"
+ " [-irs numIntermediateReducerStages]"
+ " [-mt mapSleepTime (msec)] [-rt reduceSleepTime (msec)]"
+ " [-irt intermediateReduceSleepTime]"
+ " [-recordt recordSleepTime (msec)]"
+ " [-generateSplitsInAM (false)/true]"
+ " [-writeSplitsToDfs (false)/true]");
ToolRunner.printGenericCommandUsage(System.err);
return 2;
}
int numMapper = 1, numReducer = 1, numIReducer = 1;
long mapSleepTime = 100, reduceSleepTime = 100, recSleepTime = 100, iReduceSleepTime = 1;
int mapSleepCount = 1, reduceSleepCount = 1, iReduceSleepCount = 1;
int iReduceStagesCount = 1;
boolean writeSplitsToDfs = false;
boolean generateSplitsInAM = false;
boolean splitsOptionFound = false;
for (int i = 0; i < args.length; i++) {
if (args[i].equals("-m")) {
numMapper = Integer.parseInt(args[++i]);
} else if (args[i].equals("-r")) {
numReducer = Integer.parseInt(args[++i]);
} else if (args[i].equals("-ir")) {
numIReducer = Integer.parseInt(args[++i]);
} else if (args[i].equals("-mt")) {
mapSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-rt")) {
reduceSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-irt")) {
iReduceSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-irs")) {
iReduceStagesCount = Integer.parseInt(args[++i]);
} else if (args[i].equals("-recordt")) {
recSleepTime = Long.parseLong(args[++i]);
} else if (args[i].equals("-generateSplitsInAM")) {
if (splitsOptionFound) {
throw new RuntimeException(
"Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
}
splitsOptionFound = true;
generateSplitsInAM = Boolean.parseBoolean(args[++i]);
} else if (args[i].equals("-writeSplitsToDfs")) {
if (splitsOptionFound) {
throw new RuntimeException(
"Cannot use both -generateSplitsInAm and -writeSplitsToDfs together");
}
splitsOptionFound = true;
writeSplitsToDfs = Boolean.parseBoolean(args[++i]);
}
}
if (numIReducer > 0 && numReducer <= 0) {
throw new RuntimeException("Cannot have intermediate reduces without" + " a final reduce");
}
// sleep for *SleepTime duration in Task by recSleepTime per record
mapSleepCount = (int) Math.ceil(mapSleepTime / ((double) recSleepTime));
reduceSleepCount = (int) Math.ceil(reduceSleepTime / ((double) recSleepTime));
iReduceSleepCount = (int) Math.ceil(iReduceSleepTime / ((double) recSleepTime));
TezConfiguration conf = new TezConfiguration(getConf());
FileSystem remoteFs = FileSystem.get(conf);
conf.set(
TezConfiguration.TEZ_AM_STAGING_DIR,
conf.get(TezConfiguration.TEZ_AM_STAGING_DIR, TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
Path remoteStagingDir =
remoteFs.makeQualified(
new Path(
conf.get(
TezConfiguration.TEZ_AM_STAGING_DIR,
TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT),
Long.toString(System.currentTimeMillis())));
TezClientUtils.ensureStagingDirExists(conf, remoteStagingDir);
DAG dag =
createDAG(
remoteFs,
conf,
remoteStagingDir,
numMapper,
numReducer,
iReduceStagesCount,
numIReducer,
mapSleepTime,
mapSleepCount,
reduceSleepTime,
reduceSleepCount,
iReduceSleepTime,
iReduceSleepCount,
writeSplitsToDfs,
generateSplitsInAM);
TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
tezSession.start();
DAGClient dagClient = tezSession.submitDAG(dag);
while (true) {
DAGStatus status = dagClient.getDAGStatus(null);
LOG.info("DAG Status: " + status);
if (status.isCompleted()) {
break;
}
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// do nothing
}
}
tezSession.stop();
return dagClient.getDAGStatus(null).getState().equals(DAGStatus.State.SUCCEEDED) ? 0 : 1;
}