@Test
public void testGetExtraLocalResources() throws Exception {
final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"};
LocalResource res = mock(LocalResource.class);
final List<LocalResource> resources = Collections.singletonList(res);
final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
resMap.put("foo.jar", res);
when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
when(utils.getBaseName(res)).thenReturn("foo.jar");
assertEquals(resMap, task.getExtraLocalResources(conf, path, inputOutputJars));
}
@Test
public void testExistingSessionGetsStorageHandlerResources() throws Exception {
final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"};
LocalResource res = mock(LocalResource.class);
final List<LocalResource> resources = Collections.singletonList(res);
final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
resMap.put("foo.jar", res);
when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
when(utils.getBaseName(res)).thenReturn("foo.jar");
when(sessionState.isOpen()).thenReturn(true);
when(sessionState.isOpening()).thenReturn(false);
when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
task.updateSession(sessionState, conf, path, inputOutputJars, resMap);
verify(session).addAppMasterLocalFiles(resMap);
}
@Test
public void testExtraResourcesAddedToDag() throws Exception {
final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"};
LocalResource res = mock(LocalResource.class);
final List<LocalResource> resources = Collections.singletonList(res);
final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
resMap.put("foo.jar", res);
DAG dag = mock(DAG.class);
when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
when(utils.getBaseName(res)).thenReturn("foo.jar");
when(sessionState.isOpen()).thenReturn(true);
when(sessionState.isOpening()).thenReturn(false);
when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
task.addExtraResourcesToDag(sessionState, dag, inputOutputJars, resMap);
verify(dag).addTaskLocalFiles(resMap);
}
@Test
public void testSubmit() throws Exception {
DAG dag = DAG.create("test");
task.submit(
conf,
dag,
path,
appLr,
sessionState,
Collections.<LocalResource>emptyList(),
new String[0],
Collections.<String, LocalResource>emptyMap());
// validate close/reopen
verify(sessionState, times(1)).open(any(HiveConf.class), any(String[].class));
verify(sessionState, times(1)).close(eq(true)); // now uses pool after HIVE-7043
verify(session, times(2)).submitDAG(any(DAG.class));
}
@Test
public void testBuildDag() throws IllegalArgumentException, IOException, Exception {
DAG dag = task.build(conf, work, path, appLr, null, new Context(conf));
for (BaseWork w : work.getAllWork()) {
Vertex v = dag.getVertex(w.getName());
assertNotNull(v);
List<Vertex> outs = v.getOutputVertices();
for (BaseWork x : work.getChildren(w)) {
boolean found = false;
for (Vertex u : outs) {
if (u.getName().equals(x.getName())) {
found = true;
break;
}
}
assertTrue(found);
}
}
}
@SuppressWarnings("unchecked")
@Before
public void setUp() throws Exception {
utils = mock(DagUtils.class);
fs = mock(FileSystem.class);
path = mock(Path.class);
when(path.getFileSystem(any(Configuration.class))).thenReturn(fs);
when(utils.getTezDir(any(Path.class))).thenReturn(path);
when(utils.createVertex(
any(JobConf.class),
any(BaseWork.class),
any(Path.class),
any(LocalResource.class),
any(List.class),
any(FileSystem.class),
any(Context.class),
anyBoolean(),
any(TezWork.class),
any(VertexType.class)))
.thenAnswer(
new Answer<Vertex>() {
@Override
public Vertex answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
return Vertex.create(
((BaseWork) args[1]).getName(),
mock(ProcessorDescriptor.class),
0,
mock(Resource.class));
}
});
when(utils.createEdge(
any(JobConf.class),
any(Vertex.class),
any(Vertex.class),
any(TezEdgeProperty.class),
any(VertexType.class)))
.thenAnswer(
new Answer<Edge>() {
@Override
public Edge answer(InvocationOnMock invocation) throws Throwable {
Object[] args = invocation.getArguments();
return Edge.create((Vertex) args[1], (Vertex) args[2], mock(EdgeProperty.class));
}
});
work = new TezWork("");
mws = new MapWork[] {new MapWork(), new MapWork()};
rws = new ReduceWork[] {new ReduceWork(), new ReduceWork()};
work.addAll(mws);
work.addAll(rws);
int i = 0;
for (BaseWork w : work.getAllWork()) {
w.setName("Work " + (++i));
}
op = mock(Operator.class);
LinkedHashMap<String, Operator<? extends OperatorDesc>> map =
new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
map.put("foo", op);
mws[0].setAliasToWork(map);
mws[1].setAliasToWork(map);
LinkedHashMap<String, ArrayList<String>> pathMap =
new LinkedHashMap<String, ArrayList<String>>();
ArrayList<String> aliasList = new ArrayList<String>();
aliasList.add("foo");
pathMap.put("foo", aliasList);
mws[0].setPathToAliases(pathMap);
mws[1].setPathToAliases(pathMap);
rws[0].setReducer(op);
rws[1].setReducer(op);
TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE);
work.connect(mws[0], rws[0], edgeProp);
work.connect(mws[1], rws[0], edgeProp);
work.connect(rws[0], rws[1], edgeProp);
task = new TezTask(utils);
task.setWork(work);
task.setConsole(mock(LogHelper.class));
conf = new JobConf();
appLr = mock(LocalResource.class);
SessionState.start(new HiveConf());
session = mock(TezClient.class);
sessionState = mock(TezSessionState.class);
when(sessionState.getSession()).thenReturn(session);
when(session.submitDAG(any(DAG.class)))
.thenThrow(new SessionNotRunning(""))
.thenReturn(mock(DAGClient.class));
}
@Test
public void testClose() throws HiveException {
task.close(work, 0);
verify(op, times(4)).jobClose(any(Configuration.class), eq(true));
}
@Test
public void testEmptyWork() throws IllegalArgumentException, IOException, Exception {
DAG dag = task.build(conf, new TezWork(""), path, appLr, null, new Context(conf));
assertEquals(dag.getVertices().size(), 0);
}