コード例 #1
0
  @Test
  public void testDoFilterForClientBackwardsCompatibility() throws Exception {
    Filter filter = new FalconAuthenticationFilter();

    final String userName = System.getProperty("user.name");
    final String httpAuthType =
        StartupProperties.get().getProperty("falcon.http.authentication.type", "simple");
    try {
      System.setProperty("user.name", "");
      StartupProperties.get()
          .setProperty(
              "falcon.http.authentication.type",
              "org.apache.falcon.security.RemoteUserInHeaderBasedAuthenticationHandler");

      synchronized (StartupProperties.get()) {
        filter.init(mockConfig);
      }

      Mockito.when(mockRequest.getMethod()).thenReturn("POST");
      Mockito.when(mockRequest.getQueryString()).thenReturn("");
      Mockito.when(mockRequest.getRemoteUser()).thenReturn(null);
      Mockito.when(mockRequest.getHeader("Remote-User")).thenReturn("remote-user");

      filter.doFilter(mockRequest, mockResponse, mockChain);

      Assert.assertEquals(CurrentUser.getUser(), "remote-user");

    } finally {
      System.setProperty("user.name", userName);
      StartupProperties.get().setProperty("falcon.http.authentication.type", httpAuthType);
    }
  }
コード例 #2
0
  @Test
  public void testGetEntityListFilterBy() throws Exception {

    Entity process2 =
        buildProcess("processAuthUserFilterBy", System.getProperty("user.name"), "", "USER-DATA");
    configStore.publish(EntityType.PROCESS, process2);

    EntityList entityList =
        this.getEntityList("", "", "", "process", "", "PIPELINES:USER-DATA", "", "asc", 0, 10, "");
    Assert.assertNotNull(entityList.getElements());
    Assert.assertEquals(entityList.getElements().length, 1);
    Assert.assertNotNull(entityList.getElements()[0].pipeline);
    Assert.assertEquals(entityList.getElements()[0].pipeline.get(0), "USER-DATA");

    /*
     * Both entities should be returned when the user is SuperUser.
     */
    StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
    CurrentUser.authenticate(System.getProperty("user.name"));
    entityList =
        this.getEntityList("", "", "", "process", "", "PIPELINES:USER-DATA", "", "desc", 0, 10, "");
    Assert.assertNotNull(entityList.getElements());
    Assert.assertEquals(entityList.getElements().length, 1);
    Assert.assertNotNull(entityList.getElements()[0].pipeline);
    Assert.assertEquals(entityList.getElements()[0].pipeline.get(0), "USER-DATA");

    // reset values
    StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false");
    CurrentUser.authenticate(System.getProperty("user.name"));
  }
コード例 #3
0
  @Override
  public void init() throws FalconException {
    graph = initializeGraphDB();
    createIndicesForVertexKeys();
    // todo - create Edge Cardinality Constraints
    LOG.info("Initialized graph db: {}", graph);

    vertexIndexedKeys = getIndexableGraph().getIndexedKeys(Vertex.class);
    LOG.info("Init vertex property keys: {}", vertexIndexedKeys);

    edgeIndexedKeys = getIndexableGraph().getIndexedKeys(Edge.class);
    LOG.info("Init edge property keys: {}", edgeIndexedKeys);

    boolean preserveHistory =
        Boolean.valueOf(
            StartupProperties.get().getProperty("falcon.graph.preserve.history", "false"));
    entityGraphBuilder = new EntityRelationshipGraphBuilder(graph, preserveHistory);
    instanceGraphBuilder = new InstanceRelationshipGraphBuilder(graph, preserveHistory);

    ConfigurationStore.get().registerListener(this);
    Services.get()
        .<WorkflowJobEndNotificationService>getService(
            WorkflowJobEndNotificationService.SERVICE_NAME)
        .registerListener(this);
    try {
      transactionRetries =
          Integer.parseInt(
              StartupProperties.get().getProperty("falcon.graph.transaction.retry.count", "3"));
      transactionRetryDelayInMillis =
          Long.parseLong(
              StartupProperties.get().getProperty("falcon.graph.transaction.retry.delay", "5"));
    } catch (NumberFormatException e) {
      throw new FalconException("Invalid values for graph transaction retry delay/count " + e);
    }
  }
コード例 #4
0
  @AfterClass
  public void tearDown() throws Exception {
    GraphUtils.dump(service.getGraph(), System.out);

    cleanUp();
    StartupProperties.get().setProperty("falcon.graph.preserve.history", "false");
  }
コード例 #5
0
  @Test
  public void testGetEntityListBadUser() throws Exception {
    CurrentUser.authenticate("fakeUser");
    try {
      Entity process1 = buildProcess("processFakeUser", "fakeUser", "", "");
      configStore.publish(EntityType.PROCESS, process1);
      Assert.fail();
    } catch (Throwable ignore) {
      // do nothing
    }

    /*
     * Only one entity should be returned when the auth is enabled.
     */
    try {
      getEntityList("", "", "", "process", "", "", "", "", 0, 10, "");
      Assert.fail();
    } catch (Throwable ignore) {
      // do nothing
    }

    // reset values
    StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false");
    CurrentUser.authenticate(System.getProperty("user.name"));
  }
コード例 #6
0
 @BeforeClass
 public void setup() throws Exception {
   StartupProperties.get()
       .setProperty("falcon.state.store.impl", "org.apache.falcon.state.store.InMemoryStateStore");
   super.setup();
   this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
   this.conf = dfsCluster.getConf();
 }
コード例 #7
0
 private void copyLibsToHDFS(Cluster cluster) throws IOException {
   // set up kahadb to be sent as part of workflows
   StartupProperties.get().setProperty("libext.paths", "./target/libext");
   String libext =
       ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath() + "/libext";
   String targetStorageUrl = ClusterHelper.getStorageUrl(cluster);
   FSUtils.copyOozieShareLibsToHDFS("./target/libext", targetStorageUrl + libext);
 }
コード例 #8
0
 public int getThreadCount() {
   try {
     return Integer.parseInt(
         StartupProperties.get().getProperty("falcon.logMoveService.threadCount", "200"));
   } catch (NumberFormatException e) {
     LOG.error("Exception in LogMoverService", e);
     return 50;
   }
 }
コード例 #9
0
  @BeforeClass
  public void setUp() throws Exception {
    CurrentUser.authenticate(FALCON_USER);

    configStore = ConfigurationStore.get();

    Services.get().register(new WorkflowJobEndNotificationService());
    StartupProperties.get()
        .setProperty(
            "falcon.graph.storage.directory", "target/graphdb-" + System.currentTimeMillis());
    StartupProperties.get().setProperty("falcon.graph.preserve.history", "true");
    service = new MetadataMappingService();
    service.init();

    Set<String> vertexPropertyKeys = service.getVertexIndexedKeys();
    System.out.println("Got vertex property keys: " + vertexPropertyKeys);

    Set<String> edgePropertyKeys = service.getEdgeIndexedKeys();
    System.out.println("Got edge property keys: " + edgePropertyKeys);
  }
コード例 #10
0
  protected void verifyBrokerProperties(Cluster cluster, HashMap<String, String> props) {
    Assert.assertEquals(
        props.get(WorkflowExecutionArgs.USER_BRKR_URL.getName()),
        ClusterHelper.getMessageBrokerUrl(cluster));
    Assert.assertEquals(
        props.get(WorkflowExecutionArgs.USER_BRKR_IMPL_CLASS.getName()),
        ClusterHelper.getMessageBrokerImplClass(cluster));

    String falconBrokerUrl =
        StartupProperties.get().getProperty("broker.url", "tcp://localhost:61616?daemon=true");
    Assert.assertEquals(props.get(WorkflowExecutionArgs.BRKR_URL.getName()), falconBrokerUrl);

    String falconBrokerImplClass =
        StartupProperties.get()
            .getProperty("broker.impl.class", ClusterHelper.DEFAULT_BROKER_IMPL_CLASS);
    Assert.assertEquals(
        props.get(WorkflowExecutionArgs.BRKR_IMPL_CLASS.getName()), falconBrokerImplClass);

    String jmsMessageTTL =
        StartupProperties.get().getProperty("broker.ttlInMins", String.valueOf(3 * 24 * 60L));
    Assert.assertEquals(props.get(WorkflowExecutionArgs.BRKR_TTL.getName()), jmsMessageTTL);
  }
コード例 #11
0
  @Test
  public void testGetKerberosPrincipalWithSubstitutedHostNonSecure() throws Exception {
    String principal =
        StartupProperties.get().getProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL);
    Configuration conf = new Configuration(false);
    conf.set("hadoop.security.authentication", "simple");
    UserGroupInformation.setConfiguration(conf);
    Assert.assertFalse(UserGroupInformation.isSecurityEnabled());

    FalconAuthenticationFilter filter = new FalconAuthenticationFilter();
    Properties properties = filter.getConfiguration(FalconAuthenticationFilter.FALCON_PREFIX, null);
    Assert.assertEquals(properties.get(KerberosAuthenticationHandler.PRINCIPAL), principal);
  }
コード例 #12
0
 protected void copySharedLibs(Cluster cluster, Path libPath) throws FalconException {
   try {
     FileSystem fs =
         HadoopClientFactory.get()
             .createProxiedFileSystem(libPath.toUri(), ClusterHelper.getConfiguration(cluster));
     SharedLibraryHostingService.pushLibsToHDFS(
         fs,
         StartupProperties.get().getProperty("system.lib.location"),
         libPath,
         FALCON_JAR_FILTER);
   } catch (IOException e) {
     throw new FalconException("Failed to copy shared libs on cluster " + cluster.getName(), e);
   }
 }
コード例 #13
0
  public static Configuration getConfiguration() {
    Configuration graphConfig = new BaseConfiguration();

    Properties configProperties = StartupProperties.get();
    for (Map.Entry entry : configProperties.entrySet()) {
      String name = (String) entry.getKey();
      if (name.startsWith(FALCON_PREFIX)) {
        String value = (String) entry.getValue();
        name = name.substring(FALCON_PREFIX.length());
        graphConfig.setProperty(name, value);
      }
    }

    return graphConfig;
  }
コード例 #14
0
 private void registerServices() throws FalconException {
   mockTimeService = Mockito.mock(AlarmService.class);
   Mockito.when(mockTimeService.getName()).thenReturn("AlarmService");
   Mockito.when(
           mockTimeService.createRequestBuilder(
               Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
       .thenCallRealMethod();
   mockDataService = Mockito.mock(DataAvailabilityService.class);
   Mockito.when(mockDataService.getName()).thenReturn("DataAvailabilityService");
   Mockito.when(
           mockDataService.createRequestBuilder(
               Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
       .thenCallRealMethod();
   dagEngine = Mockito.mock(OozieDAGEngine.class);
   Mockito.doNothing().when(dagEngine).resume(Mockito.any(ExecutionInstance.class));
   mockSchedulerService = Mockito.mock(SchedulerService.class);
   Mockito.when(mockSchedulerService.getName()).thenReturn("JobSchedulerService");
   StartupProperties.get().setProperty("dag.engine.impl", MockDAGEngine.class.getName());
   StartupProperties.get()
       .setProperty("execution.service.impl", FalconExecutionService.class.getName());
   dagEngine = Mockito.spy(DAGEngineFactory.getDAGEngine("testCluster"));
   Mockito.when(
           mockSchedulerService.createRequestBuilder(
               Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
       .thenCallRealMethod();
   mockCompletionService = Mockito.mock(JobCompletionService.class);
   Mockito.when(mockCompletionService.getName()).thenReturn("JobCompletionService");
   Mockito.when(
           mockCompletionService.createRequestBuilder(
               Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
       .thenCallRealMethod();
   Services.get().register(mockTimeService);
   Services.get().register(mockDataService);
   Services.get().register(mockSchedulerService);
   Services.get().register(mockCompletionService);
 }
コード例 #15
0
  @Test
  public void testGetKerberosPrincipalWithSubstitutedHostSecure() throws Exception {
    String principal =
        StartupProperties.get().getProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL);

    String expectedPrincipal =
        "falcon/" + SecurityUtil.getLocalHostName().toLowerCase() + "@Example.com";
    try {
      Configuration conf = new Configuration(false);
      conf.set("hadoop.security.authentication", "kerberos");
      UserGroupInformation.setConfiguration(conf);
      Assert.assertTrue(UserGroupInformation.isSecurityEnabled());

      StartupProperties.get()
          .setProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL, "falcon/[email protected]");
      FalconAuthenticationFilter filter = new FalconAuthenticationFilter();
      Properties properties =
          filter.getConfiguration(FalconAuthenticationFilter.FALCON_PREFIX, null);
      Assert.assertEquals(
          properties.get(KerberosAuthenticationHandler.PRINCIPAL), expectedPrincipal);
    } finally {
      StartupProperties.get().setProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL, principal);
    }
  }
コード例 #16
0
  @Test
  public void testAnonymous() throws Exception {
    Filter filter = new FalconAuthenticationFilter();

    synchronized (StartupProperties.get()) {
      filter.init(mockConfig);
    }

    CurrentUser.authenticate("nouser");
    Assert.assertEquals(CurrentUser.getUser(), "nouser");

    CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getRemoteUser()).thenReturn(FalconTestUtil.TEST_USER_2);
    filter.doFilter(mockRequest, mockResponse, mockChain);
    Assert.assertEquals(CurrentUser.getUser(), FalconTestUtil.TEST_USER_2);
  }
コード例 #17
0
  @Test
  public void testDoFilterWithEmptyDoAsUser() throws Exception {
    Filter filter = new FalconAuthenticationFilter();
    synchronized (StartupProperties.get()) {
      filter.init(mockConfig);
    }

    CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getMethod()).thenReturn("POST");
    Mockito.when(mockRequest.getQueryString())
        .thenReturn("user.name=" + FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getRemoteUser()).thenReturn(FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getParameter(FalconAuthenticationFilter.DO_AS_PARAM)).thenReturn("");
    filter.doFilter(mockRequest, mockResponse, mockChain);
    Assert.assertEquals(CurrentUser.getUser(), FalconTestUtil.TEST_USER_2);
  }
コード例 #18
0
  @Override
  public void init() throws FalconException {
    String listenerClassNames = StartupProperties.get().getProperty("workflow.execution.listeners");
    if (StringUtils.isEmpty(listenerClassNames)) {
      return;
    }

    for (String listenerClassName : listenerClassNames.split(",")) {
      listenerClassName = listenerClassName.trim();
      if (listenerClassName.isEmpty()) {
        continue;
      }
      WorkflowExecutionListener listener =
          ReflectionUtils.getInstanceByClassName(listenerClassName);
      registerListener(listener);
    }
  }
コード例 #19
0
  @Test
  public void testDoFilterWithDoAsUser() throws Exception {
    Filter filter = new FalconAuthenticationFilter();
    HostnameFilter.HOSTNAME_TL.set("localhost");
    synchronized (StartupProperties.get()) {
      filter.init(mockConfig);
    }

    CurrentUser.authenticate("foo");
    Mockito.when(mockRequest.getMethod()).thenReturn("POST");
    Mockito.when(mockRequest.getQueryString()).thenReturn("user.name=foo");
    Mockito.when(mockRequest.getRemoteUser()).thenReturn("foo");
    Mockito.when(mockRequest.getParameter(FalconAuthenticationFilter.DO_AS_PARAM))
        .thenReturn("doAsProxyUser");
    Mockito.when(mockRequest.getMethod()).thenReturn("POST");
    filter.doFilter(mockRequest, mockResponse, mockChain);
    Assert.assertEquals(CurrentUser.getUser(), "doAsProxyUser");
  }
コード例 #20
0
  @Test(
      expectedExceptions = AccessControlException.class,
      expectedExceptionsMessageRegExp = "User .* not defined as proxyuser.*")
  public void testDoFilterWithInvalidProxyUser() throws Exception {
    Filter filter = new FalconAuthenticationFilter();
    HostnameFilter.HOSTNAME_TL.set("localhost");
    synchronized (StartupProperties.get()) {
      filter.init(mockConfig);
    }

    CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getMethod()).thenReturn("POST");
    Mockito.when(mockRequest.getQueryString())
        .thenReturn("user.name=" + FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getRemoteUser()).thenReturn(FalconTestUtil.TEST_USER_2);
    Mockito.when(mockRequest.getParameter(FalconAuthenticationFilter.DO_AS_PARAM))
        .thenReturn("doAsProxyUser");
    Mockito.when(mockRequest.getMethod()).thenReturn("POST");
    filter.doFilter(mockRequest, mockResponse, mockChain);
    Assert.assertEquals(CurrentUser.getUser(), "doAsProxyUser");
  }
コード例 #21
0
  @Test
  public void testEmptyUser() throws Exception {
    Filter filter = new FalconAuthenticationFilter();

    synchronized (StartupProperties.get()) {
      filter.init(mockConfig);
    }

    final String userName = System.getProperty("user.name");
    try {
      System.setProperty("user.name", "");

      Mockito.when(mockRequest.getMethod()).thenReturn("POST");
      Mockito.when(mockRequest.getQueryString()).thenReturn("");
      Mockito.when(mockRequest.getRemoteUser()).thenReturn(null);

      HttpServletResponse errorResponse = Mockito.mock(HttpServletResponse.class);
      filter.doFilter(mockRequest, errorResponse, mockChain);
    } finally {
      System.setProperty("user.name", userName);
    }
  }
コード例 #22
0
/** Moves Falcon logs. */
public class LogMoverService implements WorkflowExecutionListener {

  private static final Logger LOG = LoggerFactory.getLogger(LogMoverService.class);

  public static final String ENABLE_POSTPROCESSING =
      StartupProperties.get().getProperty("falcon.postprocessing.enable");

  private BlockingQueue<Runnable> blockingQueue =
      new ArrayBlockingQueue<>(
          Integer.parseInt(
              StartupProperties.get()
                  .getProperty("falcon.logMoveService.blockingQueue.length", "50")));
  private ExecutorService executorService =
      new ThreadPoolExecutor(20, getThreadCount(), 120, TimeUnit.SECONDS, blockingQueue);

  public int getThreadCount() {
    try {
      return Integer.parseInt(
          StartupProperties.get().getProperty("falcon.logMoveService.threadCount", "200"));
    } catch (NumberFormatException e) {
      LOG.error("Exception in LogMoverService", e);
      return 50;
    }
  }

  @Override
  public void onSuccess(WorkflowExecutionContext context) throws FalconException {
    onEnd(context);
  }

  @Override
  public void onFailure(WorkflowExecutionContext context) throws FalconException {
    onEnd(context);
  }

  @Override
  public void onStart(WorkflowExecutionContext context) throws FalconException {
    // Do Nothing
  }

  @Override
  public void onSuspend(WorkflowExecutionContext context) throws FalconException {
    // DO Nothing
  }

  @Override
  public void onWait(WorkflowExecutionContext context) throws FalconException {
    // DO Nothing
  }

  private void onEnd(WorkflowExecutionContext context) {
    if (Boolean.parseBoolean(ENABLE_POSTPROCESSING)) {
      return;
    }
    while (0 < blockingQueue.remainingCapacity()) {
      try {
        LOG.trace("Sleeping, no capacity in threadpool....");
        TimeUnit.MILLISECONDS.sleep(500);
      } catch (InterruptedException e) {
        LOG.error("Exception in LogMoverService", e);
      }
    }
    executorService.execute(new LogMover(context));
  }

  private static class LogMover implements Runnable {
    private WorkflowExecutionContext context;

    public LogMover(@Nonnull WorkflowExecutionContext context) {
      this.context = context;
    }

    @Override
    public void run() {
      new JobLogMover().moveLog(context);
    }
  }
}
コード例 #23
0
 private String getPropertyValue(JMSProps prop) {
   return StartupProperties.get().getProperty(prop.propName, prop.defaultPropValue);
 }