@Test
public void testDoFilterForClientBackwardsCompatibility() throws Exception {
Filter filter = new FalconAuthenticationFilter();
final String userName = System.getProperty("user.name");
final String httpAuthType =
StartupProperties.get().getProperty("falcon.http.authentication.type", "simple");
try {
System.setProperty("user.name", "");
StartupProperties.get()
.setProperty(
"falcon.http.authentication.type",
"org.apache.falcon.security.RemoteUserInHeaderBasedAuthenticationHandler");
synchronized (StartupProperties.get()) {
filter.init(mockConfig);
}
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
Mockito.when(mockRequest.getQueryString()).thenReturn("");
Mockito.when(mockRequest.getRemoteUser()).thenReturn(null);
Mockito.when(mockRequest.getHeader("Remote-User")).thenReturn("remote-user");
filter.doFilter(mockRequest, mockResponse, mockChain);
Assert.assertEquals(CurrentUser.getUser(), "remote-user");
} finally {
System.setProperty("user.name", userName);
StartupProperties.get().setProperty("falcon.http.authentication.type", httpAuthType);
}
}
@Test
public void testGetEntityListFilterBy() throws Exception {
Entity process2 =
buildProcess("processAuthUserFilterBy", System.getProperty("user.name"), "", "USER-DATA");
configStore.publish(EntityType.PROCESS, process2);
EntityList entityList =
this.getEntityList("", "", "", "process", "", "PIPELINES:USER-DATA", "", "asc", 0, 10, "");
Assert.assertNotNull(entityList.getElements());
Assert.assertEquals(entityList.getElements().length, 1);
Assert.assertNotNull(entityList.getElements()[0].pipeline);
Assert.assertEquals(entityList.getElements()[0].pipeline.get(0), "USER-DATA");
/*
* Both entities should be returned when the user is SuperUser.
*/
StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
CurrentUser.authenticate(System.getProperty("user.name"));
entityList =
this.getEntityList("", "", "", "process", "", "PIPELINES:USER-DATA", "", "desc", 0, 10, "");
Assert.assertNotNull(entityList.getElements());
Assert.assertEquals(entityList.getElements().length, 1);
Assert.assertNotNull(entityList.getElements()[0].pipeline);
Assert.assertEquals(entityList.getElements()[0].pipeline.get(0), "USER-DATA");
// reset values
StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false");
CurrentUser.authenticate(System.getProperty("user.name"));
}
@Override
public void init() throws FalconException {
graph = initializeGraphDB();
createIndicesForVertexKeys();
// todo - create Edge Cardinality Constraints
LOG.info("Initialized graph db: {}", graph);
vertexIndexedKeys = getIndexableGraph().getIndexedKeys(Vertex.class);
LOG.info("Init vertex property keys: {}", vertexIndexedKeys);
edgeIndexedKeys = getIndexableGraph().getIndexedKeys(Edge.class);
LOG.info("Init edge property keys: {}", edgeIndexedKeys);
boolean preserveHistory =
Boolean.valueOf(
StartupProperties.get().getProperty("falcon.graph.preserve.history", "false"));
entityGraphBuilder = new EntityRelationshipGraphBuilder(graph, preserveHistory);
instanceGraphBuilder = new InstanceRelationshipGraphBuilder(graph, preserveHistory);
ConfigurationStore.get().registerListener(this);
Services.get()
.<WorkflowJobEndNotificationService>getService(
WorkflowJobEndNotificationService.SERVICE_NAME)
.registerListener(this);
try {
transactionRetries =
Integer.parseInt(
StartupProperties.get().getProperty("falcon.graph.transaction.retry.count", "3"));
transactionRetryDelayInMillis =
Long.parseLong(
StartupProperties.get().getProperty("falcon.graph.transaction.retry.delay", "5"));
} catch (NumberFormatException e) {
throw new FalconException("Invalid values for graph transaction retry delay/count " + e);
}
}
@AfterClass
public void tearDown() throws Exception {
GraphUtils.dump(service.getGraph(), System.out);
cleanUp();
StartupProperties.get().setProperty("falcon.graph.preserve.history", "false");
}
@Test
public void testGetEntityListBadUser() throws Exception {
CurrentUser.authenticate("fakeUser");
try {
Entity process1 = buildProcess("processFakeUser", "fakeUser", "", "");
configStore.publish(EntityType.PROCESS, process1);
Assert.fail();
} catch (Throwable ignore) {
// do nothing
}
/*
* Only one entity should be returned when the auth is enabled.
*/
try {
getEntityList("", "", "", "process", "", "", "", "", 0, 10, "");
Assert.fail();
} catch (Throwable ignore) {
// do nothing
}
// reset values
StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false");
CurrentUser.authenticate(System.getProperty("user.name"));
}
@BeforeClass
public void setup() throws Exception {
StartupProperties.get()
.setProperty("falcon.state.store.impl", "org.apache.falcon.state.store.InMemoryStateStore");
super.setup();
this.dfsCluster = EmbeddedCluster.newCluster("testCluster");
this.conf = dfsCluster.getConf();
}
private void copyLibsToHDFS(Cluster cluster) throws IOException {
// set up kahadb to be sent as part of workflows
StartupProperties.get().setProperty("libext.paths", "./target/libext");
String libext =
ClusterHelper.getLocation(cluster, ClusterLocationType.WORKING).getPath() + "/libext";
String targetStorageUrl = ClusterHelper.getStorageUrl(cluster);
FSUtils.copyOozieShareLibsToHDFS("./target/libext", targetStorageUrl + libext);
}
public int getThreadCount() {
try {
return Integer.parseInt(
StartupProperties.get().getProperty("falcon.logMoveService.threadCount", "200"));
} catch (NumberFormatException e) {
LOG.error("Exception in LogMoverService", e);
return 50;
}
}
@BeforeClass
public void setUp() throws Exception {
CurrentUser.authenticate(FALCON_USER);
configStore = ConfigurationStore.get();
Services.get().register(new WorkflowJobEndNotificationService());
StartupProperties.get()
.setProperty(
"falcon.graph.storage.directory", "target/graphdb-" + System.currentTimeMillis());
StartupProperties.get().setProperty("falcon.graph.preserve.history", "true");
service = new MetadataMappingService();
service.init();
Set<String> vertexPropertyKeys = service.getVertexIndexedKeys();
System.out.println("Got vertex property keys: " + vertexPropertyKeys);
Set<String> edgePropertyKeys = service.getEdgeIndexedKeys();
System.out.println("Got edge property keys: " + edgePropertyKeys);
}
protected void verifyBrokerProperties(Cluster cluster, HashMap<String, String> props) {
Assert.assertEquals(
props.get(WorkflowExecutionArgs.USER_BRKR_URL.getName()),
ClusterHelper.getMessageBrokerUrl(cluster));
Assert.assertEquals(
props.get(WorkflowExecutionArgs.USER_BRKR_IMPL_CLASS.getName()),
ClusterHelper.getMessageBrokerImplClass(cluster));
String falconBrokerUrl =
StartupProperties.get().getProperty("broker.url", "tcp://localhost:61616?daemon=true");
Assert.assertEquals(props.get(WorkflowExecutionArgs.BRKR_URL.getName()), falconBrokerUrl);
String falconBrokerImplClass =
StartupProperties.get()
.getProperty("broker.impl.class", ClusterHelper.DEFAULT_BROKER_IMPL_CLASS);
Assert.assertEquals(
props.get(WorkflowExecutionArgs.BRKR_IMPL_CLASS.getName()), falconBrokerImplClass);
String jmsMessageTTL =
StartupProperties.get().getProperty("broker.ttlInMins", String.valueOf(3 * 24 * 60L));
Assert.assertEquals(props.get(WorkflowExecutionArgs.BRKR_TTL.getName()), jmsMessageTTL);
}
@Test
public void testGetKerberosPrincipalWithSubstitutedHostNonSecure() throws Exception {
String principal =
StartupProperties.get().getProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL);
Configuration conf = new Configuration(false);
conf.set("hadoop.security.authentication", "simple");
UserGroupInformation.setConfiguration(conf);
Assert.assertFalse(UserGroupInformation.isSecurityEnabled());
FalconAuthenticationFilter filter = new FalconAuthenticationFilter();
Properties properties = filter.getConfiguration(FalconAuthenticationFilter.FALCON_PREFIX, null);
Assert.assertEquals(properties.get(KerberosAuthenticationHandler.PRINCIPAL), principal);
}
protected void copySharedLibs(Cluster cluster, Path libPath) throws FalconException {
try {
FileSystem fs =
HadoopClientFactory.get()
.createProxiedFileSystem(libPath.toUri(), ClusterHelper.getConfiguration(cluster));
SharedLibraryHostingService.pushLibsToHDFS(
fs,
StartupProperties.get().getProperty("system.lib.location"),
libPath,
FALCON_JAR_FILTER);
} catch (IOException e) {
throw new FalconException("Failed to copy shared libs on cluster " + cluster.getName(), e);
}
}
public static Configuration getConfiguration() {
Configuration graphConfig = new BaseConfiguration();
Properties configProperties = StartupProperties.get();
for (Map.Entry entry : configProperties.entrySet()) {
String name = (String) entry.getKey();
if (name.startsWith(FALCON_PREFIX)) {
String value = (String) entry.getValue();
name = name.substring(FALCON_PREFIX.length());
graphConfig.setProperty(name, value);
}
}
return graphConfig;
}
private void registerServices() throws FalconException {
mockTimeService = Mockito.mock(AlarmService.class);
Mockito.when(mockTimeService.getName()).thenReturn("AlarmService");
Mockito.when(
mockTimeService.createRequestBuilder(
Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
.thenCallRealMethod();
mockDataService = Mockito.mock(DataAvailabilityService.class);
Mockito.when(mockDataService.getName()).thenReturn("DataAvailabilityService");
Mockito.when(
mockDataService.createRequestBuilder(
Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
.thenCallRealMethod();
dagEngine = Mockito.mock(OozieDAGEngine.class);
Mockito.doNothing().when(dagEngine).resume(Mockito.any(ExecutionInstance.class));
mockSchedulerService = Mockito.mock(SchedulerService.class);
Mockito.when(mockSchedulerService.getName()).thenReturn("JobSchedulerService");
StartupProperties.get().setProperty("dag.engine.impl", MockDAGEngine.class.getName());
StartupProperties.get()
.setProperty("execution.service.impl", FalconExecutionService.class.getName());
dagEngine = Mockito.spy(DAGEngineFactory.getDAGEngine("testCluster"));
Mockito.when(
mockSchedulerService.createRequestBuilder(
Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
.thenCallRealMethod();
mockCompletionService = Mockito.mock(JobCompletionService.class);
Mockito.when(mockCompletionService.getName()).thenReturn("JobCompletionService");
Mockito.when(
mockCompletionService.createRequestBuilder(
Mockito.any(NotificationHandler.class), Mockito.any(ID.class)))
.thenCallRealMethod();
Services.get().register(mockTimeService);
Services.get().register(mockDataService);
Services.get().register(mockSchedulerService);
Services.get().register(mockCompletionService);
}
@Test
public void testGetKerberosPrincipalWithSubstitutedHostSecure() throws Exception {
String principal =
StartupProperties.get().getProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL);
String expectedPrincipal =
"falcon/" + SecurityUtil.getLocalHostName().toLowerCase() + "@Example.com";
try {
Configuration conf = new Configuration(false);
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
Assert.assertTrue(UserGroupInformation.isSecurityEnabled());
StartupProperties.get()
.setProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL, "falcon/[email protected]");
FalconAuthenticationFilter filter = new FalconAuthenticationFilter();
Properties properties =
filter.getConfiguration(FalconAuthenticationFilter.FALCON_PREFIX, null);
Assert.assertEquals(
properties.get(KerberosAuthenticationHandler.PRINCIPAL), expectedPrincipal);
} finally {
StartupProperties.get().setProperty(FalconAuthenticationFilter.KERBEROS_PRINCIPAL, principal);
}
}
@Test
public void testAnonymous() throws Exception {
Filter filter = new FalconAuthenticationFilter();
synchronized (StartupProperties.get()) {
filter.init(mockConfig);
}
CurrentUser.authenticate("nouser");
Assert.assertEquals(CurrentUser.getUser(), "nouser");
CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getRemoteUser()).thenReturn(FalconTestUtil.TEST_USER_2);
filter.doFilter(mockRequest, mockResponse, mockChain);
Assert.assertEquals(CurrentUser.getUser(), FalconTestUtil.TEST_USER_2);
}
@Test
public void testDoFilterWithEmptyDoAsUser() throws Exception {
Filter filter = new FalconAuthenticationFilter();
synchronized (StartupProperties.get()) {
filter.init(mockConfig);
}
CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
Mockito.when(mockRequest.getQueryString())
.thenReturn("user.name=" + FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getRemoteUser()).thenReturn(FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getParameter(FalconAuthenticationFilter.DO_AS_PARAM)).thenReturn("");
filter.doFilter(mockRequest, mockResponse, mockChain);
Assert.assertEquals(CurrentUser.getUser(), FalconTestUtil.TEST_USER_2);
}
@Override
public void init() throws FalconException {
String listenerClassNames = StartupProperties.get().getProperty("workflow.execution.listeners");
if (StringUtils.isEmpty(listenerClassNames)) {
return;
}
for (String listenerClassName : listenerClassNames.split(",")) {
listenerClassName = listenerClassName.trim();
if (listenerClassName.isEmpty()) {
continue;
}
WorkflowExecutionListener listener =
ReflectionUtils.getInstanceByClassName(listenerClassName);
registerListener(listener);
}
}
@Test
public void testDoFilterWithDoAsUser() throws Exception {
Filter filter = new FalconAuthenticationFilter();
HostnameFilter.HOSTNAME_TL.set("localhost");
synchronized (StartupProperties.get()) {
filter.init(mockConfig);
}
CurrentUser.authenticate("foo");
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
Mockito.when(mockRequest.getQueryString()).thenReturn("user.name=foo");
Mockito.when(mockRequest.getRemoteUser()).thenReturn("foo");
Mockito.when(mockRequest.getParameter(FalconAuthenticationFilter.DO_AS_PARAM))
.thenReturn("doAsProxyUser");
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
filter.doFilter(mockRequest, mockResponse, mockChain);
Assert.assertEquals(CurrentUser.getUser(), "doAsProxyUser");
}
@Test(
expectedExceptions = AccessControlException.class,
expectedExceptionsMessageRegExp = "User .* not defined as proxyuser.*")
public void testDoFilterWithInvalidProxyUser() throws Exception {
Filter filter = new FalconAuthenticationFilter();
HostnameFilter.HOSTNAME_TL.set("localhost");
synchronized (StartupProperties.get()) {
filter.init(mockConfig);
}
CurrentUser.authenticate(FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
Mockito.when(mockRequest.getQueryString())
.thenReturn("user.name=" + FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getRemoteUser()).thenReturn(FalconTestUtil.TEST_USER_2);
Mockito.when(mockRequest.getParameter(FalconAuthenticationFilter.DO_AS_PARAM))
.thenReturn("doAsProxyUser");
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
filter.doFilter(mockRequest, mockResponse, mockChain);
Assert.assertEquals(CurrentUser.getUser(), "doAsProxyUser");
}
@Test
public void testEmptyUser() throws Exception {
Filter filter = new FalconAuthenticationFilter();
synchronized (StartupProperties.get()) {
filter.init(mockConfig);
}
final String userName = System.getProperty("user.name");
try {
System.setProperty("user.name", "");
Mockito.when(mockRequest.getMethod()).thenReturn("POST");
Mockito.when(mockRequest.getQueryString()).thenReturn("");
Mockito.when(mockRequest.getRemoteUser()).thenReturn(null);
HttpServletResponse errorResponse = Mockito.mock(HttpServletResponse.class);
filter.doFilter(mockRequest, errorResponse, mockChain);
} finally {
System.setProperty("user.name", userName);
}
}
/** Moves Falcon logs. */
public class LogMoverService implements WorkflowExecutionListener {
private static final Logger LOG = LoggerFactory.getLogger(LogMoverService.class);
public static final String ENABLE_POSTPROCESSING =
StartupProperties.get().getProperty("falcon.postprocessing.enable");
private BlockingQueue<Runnable> blockingQueue =
new ArrayBlockingQueue<>(
Integer.parseInt(
StartupProperties.get()
.getProperty("falcon.logMoveService.blockingQueue.length", "50")));
private ExecutorService executorService =
new ThreadPoolExecutor(20, getThreadCount(), 120, TimeUnit.SECONDS, blockingQueue);
public int getThreadCount() {
try {
return Integer.parseInt(
StartupProperties.get().getProperty("falcon.logMoveService.threadCount", "200"));
} catch (NumberFormatException e) {
LOG.error("Exception in LogMoverService", e);
return 50;
}
}
@Override
public void onSuccess(WorkflowExecutionContext context) throws FalconException {
onEnd(context);
}
@Override
public void onFailure(WorkflowExecutionContext context) throws FalconException {
onEnd(context);
}
@Override
public void onStart(WorkflowExecutionContext context) throws FalconException {
// Do Nothing
}
@Override
public void onSuspend(WorkflowExecutionContext context) throws FalconException {
// DO Nothing
}
@Override
public void onWait(WorkflowExecutionContext context) throws FalconException {
// DO Nothing
}
private void onEnd(WorkflowExecutionContext context) {
if (Boolean.parseBoolean(ENABLE_POSTPROCESSING)) {
return;
}
while (0 < blockingQueue.remainingCapacity()) {
try {
LOG.trace("Sleeping, no capacity in threadpool....");
TimeUnit.MILLISECONDS.sleep(500);
} catch (InterruptedException e) {
LOG.error("Exception in LogMoverService", e);
}
}
executorService.execute(new LogMover(context));
}
private static class LogMover implements Runnable {
private WorkflowExecutionContext context;
public LogMover(@Nonnull WorkflowExecutionContext context) {
this.context = context;
}
@Override
public void run() {
new JobLogMover().moveLog(context);
}
}
}
private String getPropertyValue(JMSProps prop) {
return StartupProperties.get().getProperty(prop.propName, prop.defaultPropValue);
}