public void testScenario() throws Throwable {
final AnalysisJob job;
try (DatastoreConnection connection = datastore.openConnection(); ) {
final DataContext dataContext = connection.getDataContext();
final Table table = dataContext.getTableByQualifiedLabel("PUBLIC.CUSTOMERS");
final Row row =
MetaModelHelper.executeSingleRowQuery(
dataContext, dataContext.query().from(table).selectCount().toQuery());
assertEquals(recordsInTable, ((Number) row.getValue(0)).intValue());
try (AnalysisJobBuilder jobBuilder = new AnalysisJobBuilder(configuration)) {
jobBuilder.setDatastore(datastore);
jobBuilder.addSourceColumns("CUSTOMERS.CONTACTFIRSTNAME");
jobBuilder.addSourceColumns("CUSTOMERS.CONTACTLASTNAME");
// although not semantically correct, we pretend that EVEN is
// the
// success-state in our cleansing street and that ODD is the
// reject-state.
final Category valid = org.datacleaner.test.mock.EvenOddFilter.Category.EVEN;
final Category invalid = org.datacleaner.test.mock.EvenOddFilter.Category.ODD;
final TransformerComponentBuilder<MockTransformer> trans1 =
jobBuilder.addTransformer(MockTransformer.class);
trans1.setName("trans1");
trans1.addInputColumn(jobBuilder.getSourceColumns().get(0));
final FilterComponentBuilder<
EvenOddFilter, org.datacleaner.test.mock.EvenOddFilter.Category>
filter1 = jobBuilder.addFilter(EvenOddFilter.class);
filter1.setName("filter1");
filter1.addInputColumn(trans1.getOutputColumns().get(0));
final TransformerComponentBuilder<MockTransformer> trans2 =
jobBuilder.addTransformer(MockTransformer.class);
trans2.setName("trans2");
trans2.addInputColumn(jobBuilder.getSourceColumns().get(1));
trans2.setRequirement(filter1, valid);
final FilterComponentBuilder<
EvenOddFilter, org.datacleaner.test.mock.EvenOddFilter.Category>
filter2 = jobBuilder.addFilter(EvenOddFilter.class);
filter2.setName("filter2");
filter2.addInputColumn(trans2.getOutputColumns().get(0));
final AnalyzerComponentBuilder<MockAnalyzer> analyzer1 =
jobBuilder.addAnalyzer(MockAnalyzer.class);
analyzer1.setName("success");
analyzer1.addInputColumn(jobBuilder.getSourceColumns().get(0));
analyzer1.addInputColumn(jobBuilder.getSourceColumns().get(1));
analyzer1.addInputColumn(trans1.getOutputColumns().get(0));
analyzer1.addInputColumn(trans2.getOutputColumns().get(0));
analyzer1.setRequirement(filter2, valid);
final FilterOutcome invalid1 = filter1.getFilterOutcome(invalid);
final FilterOutcome invalid2 = filter2.getFilterOutcome(invalid);
final AnalyzerComponentBuilder<MockAnalyzer> analyzer2 =
jobBuilder.addAnalyzer(MockAnalyzer.class);
analyzer2.setName("rejects");
analyzer2.addInputColumn(jobBuilder.getSourceColumns().get(0));
analyzer2.addInputColumn(jobBuilder.getSourceColumns().get(1));
analyzer2.setComponentRequirement(new CompoundComponentRequirement(invalid1, invalid2));
job = jobBuilder.toAnalysisJob();
}
}
final AnalysisRunner runner = new AnalysisRunnerImpl(configuration);
final AnalysisResultFuture resultFuture = runner.run(job);
resultFuture.await();
if (resultFuture.isErrornous()) {
throw resultFuture.getErrors().get(0);
}
int recordsInResults = 0;
final Map<ComponentJob, AnalyzerResult> map = resultFuture.getResultMap();
for (Entry<ComponentJob, AnalyzerResult> entry : map.entrySet()) {
final ComponentJob componentJob = entry.getKey();
@SuppressWarnings("unchecked")
final ListResult<InputRow> result = (ListResult<InputRow>) entry.getValue();
final List<InputRow> values = result.getValues();
final int recordsInResult = values.size();
recordsInResults += recordsInResult;
switch (componentJob.getName()) {
case "success":
case "rejects":
// expected states
assertTrue(
"Expected records in all buckets of the cleansing street, but did not find any in: "
+ componentJob,
recordsInResult > 0);
assertTrue(
"Expected records to be distributed across buckets, but found all in: "
+ componentJob,
recordsInResult != recordsInTable);
break;
default:
fail("Unexpected component in result map: " + componentJob);
}
}
assertEquals(recordsInTable, recordsInResults);
}
public void runScenario() {
MultiThreadedTaskRunner taskRunner = TestEnvironment.getMultiThreadedTaskRunner();
ThreadPoolExecutor executorService = (ThreadPoolExecutor) taskRunner.getExecutorService();
assertEquals(TestEnvironment.THREAD_COUNT, executorService.getMaximumPoolSize());
assertEquals(0, executorService.getActiveCount());
DataCleanerConfiguration configuration =
new DataCleanerConfigurationImpl()
.withEnvironment(new DataCleanerEnvironmentImpl().withTaskRunner(taskRunner));
AnalysisRunner runner = new AnalysisRunnerImpl(configuration);
Datastore ds = TestHelper.createSampleDatabaseDatastore("foobar");
try (DatastoreConnection con = ds.openConnection()) {
AnalysisJob job;
try (AnalysisJobBuilder analysisJobBuilder = new AnalysisJobBuilder(configuration)) {
analysisJobBuilder.setDatastore(ds);
Table table = con.getDataContext().getDefaultSchema().getTableByName("ORDERFACT");
assertNotNull(table);
Column statusColumn = table.getColumnByName("STATUS");
Column commentsColumn = table.getColumnByName("COMMENTS");
analysisJobBuilder.addSourceColumns(statusColumn, commentsColumn);
analysisJobBuilder
.addAnalyzer(AnalyzerMock.class)
.addInputColumns(analysisJobBuilder.getSourceColumns());
job = analysisJobBuilder.toAnalysisJob();
}
AnalysisResultFuture resultFuture = runner.run(job);
try {
Thread.sleep(550);
} catch (InterruptedException e) {
e.printStackTrace();
fail("Interrupted! " + e.getMessage());
}
resultFuture.cancel();
assertFalse(resultFuture.isSuccessful());
assertTrue(resultFuture.isCancelled());
assertTrue(resultFuture.isErrornous());
try {
Thread.sleep(400);
} catch (InterruptedException e) {
e.printStackTrace();
fail("Interrupted! " + e.getMessage());
}
assertEquals(TestEnvironment.THREAD_COUNT, executorService.getMaximumPoolSize());
long completedTaskCount = executorService.getCompletedTaskCount();
assertTrue("completedTaskCount was: " + completedTaskCount, completedTaskCount > 3);
int largestPoolSize = executorService.getLargestPoolSize();
assertTrue("largestPoolSize was: " + largestPoolSize, largestPoolSize > 5);
assertEquals(0, executorService.getActiveCount());
}
taskRunner.shutdown();
}