Exemplo n.º 1
0
  @Test
  public void testGetExtraLocalResources() throws Exception {
    final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"};
    LocalResource res = mock(LocalResource.class);
    final List<LocalResource> resources = Collections.singletonList(res);
    final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
    resMap.put("foo.jar", res);

    when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
    when(utils.getBaseName(res)).thenReturn("foo.jar");

    assertEquals(resMap, task.getExtraLocalResources(conf, path, inputOutputJars));
  }
Exemplo n.º 2
0
  @Test
  public void testExistingSessionGetsStorageHandlerResources() throws Exception {
    final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"};
    LocalResource res = mock(LocalResource.class);
    final List<LocalResource> resources = Collections.singletonList(res);
    final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
    resMap.put("foo.jar", res);

    when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
    when(utils.getBaseName(res)).thenReturn("foo.jar");
    when(sessionState.isOpen()).thenReturn(true);
    when(sessionState.isOpening()).thenReturn(false);
    when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
    task.updateSession(sessionState, conf, path, inputOutputJars, resMap);
    verify(session).addAppMasterLocalFiles(resMap);
  }
Exemplo n.º 3
0
  @Test
  public void testExtraResourcesAddedToDag() throws Exception {
    final String[] inputOutputJars = new String[] {"file:///tmp/foo.jar"};
    LocalResource res = mock(LocalResource.class);
    final List<LocalResource> resources = Collections.singletonList(res);
    final Map<String, LocalResource> resMap = new HashMap<String, LocalResource>();
    resMap.put("foo.jar", res);
    DAG dag = mock(DAG.class);

    when(utils.localizeTempFiles(path.toString(), conf, inputOutputJars)).thenReturn(resources);
    when(utils.getBaseName(res)).thenReturn("foo.jar");
    when(sessionState.isOpen()).thenReturn(true);
    when(sessionState.isOpening()).thenReturn(false);
    when(sessionState.hasResources(inputOutputJars)).thenReturn(false);
    task.addExtraResourcesToDag(sessionState, dag, inputOutputJars, resMap);
    verify(dag).addTaskLocalFiles(resMap);
  }
Exemplo n.º 4
0
  @SuppressWarnings("unchecked")
  @Before
  public void setUp() throws Exception {
    utils = mock(DagUtils.class);
    fs = mock(FileSystem.class);
    path = mock(Path.class);
    when(path.getFileSystem(any(Configuration.class))).thenReturn(fs);
    when(utils.getTezDir(any(Path.class))).thenReturn(path);
    when(utils.createVertex(
            any(JobConf.class),
            any(BaseWork.class),
            any(Path.class),
            any(LocalResource.class),
            any(List.class),
            any(FileSystem.class),
            any(Context.class),
            anyBoolean(),
            any(TezWork.class),
            any(VertexType.class)))
        .thenAnswer(
            new Answer<Vertex>() {

              @Override
              public Vertex answer(InvocationOnMock invocation) throws Throwable {
                Object[] args = invocation.getArguments();
                return Vertex.create(
                    ((BaseWork) args[1]).getName(),
                    mock(ProcessorDescriptor.class),
                    0,
                    mock(Resource.class));
              }
            });

    when(utils.createEdge(
            any(JobConf.class),
            any(Vertex.class),
            any(Vertex.class),
            any(TezEdgeProperty.class),
            any(VertexType.class)))
        .thenAnswer(
            new Answer<Edge>() {

              @Override
              public Edge answer(InvocationOnMock invocation) throws Throwable {
                Object[] args = invocation.getArguments();
                return Edge.create((Vertex) args[1], (Vertex) args[2], mock(EdgeProperty.class));
              }
            });

    work = new TezWork("");

    mws = new MapWork[] {new MapWork(), new MapWork()};
    rws = new ReduceWork[] {new ReduceWork(), new ReduceWork()};

    work.addAll(mws);
    work.addAll(rws);

    int i = 0;
    for (BaseWork w : work.getAllWork()) {
      w.setName("Work " + (++i));
    }

    op = mock(Operator.class);

    LinkedHashMap<String, Operator<? extends OperatorDesc>> map =
        new LinkedHashMap<String, Operator<? extends OperatorDesc>>();
    map.put("foo", op);
    mws[0].setAliasToWork(map);
    mws[1].setAliasToWork(map);

    LinkedHashMap<String, ArrayList<String>> pathMap =
        new LinkedHashMap<String, ArrayList<String>>();
    ArrayList<String> aliasList = new ArrayList<String>();
    aliasList.add("foo");
    pathMap.put("foo", aliasList);

    mws[0].setPathToAliases(pathMap);
    mws[1].setPathToAliases(pathMap);

    rws[0].setReducer(op);
    rws[1].setReducer(op);

    TezEdgeProperty edgeProp = new TezEdgeProperty(EdgeType.SIMPLE_EDGE);
    work.connect(mws[0], rws[0], edgeProp);
    work.connect(mws[1], rws[0], edgeProp);
    work.connect(rws[0], rws[1], edgeProp);

    task = new TezTask(utils);
    task.setWork(work);
    task.setConsole(mock(LogHelper.class));

    conf = new JobConf();
    appLr = mock(LocalResource.class);

    SessionState.start(new HiveConf());
    session = mock(TezClient.class);
    sessionState = mock(TezSessionState.class);
    when(sessionState.getSession()).thenReturn(session);
    when(session.submitDAG(any(DAG.class)))
        .thenThrow(new SessionNotRunning(""))
        .thenReturn(mock(DAGClient.class));
  }