public Data get(Number640 key) {
long startTime = System.currentTimeMillis();
FutureGet futureGet =
peerDHT
.get(key.locationKey())
.contentKey(key.contentKey())
.domainKey(key.domainKey())
.versionKey(key.versionKey())
.routingConfiguration(routingConfig)
.requestP2PConfiguration(requestConfig)
.start()
.awaitUninterruptibly();
putStats.report(System.currentTimeMillis() - startTime, futureGet.isSuccess());
LOG.debug("Get is success {}. Reason: {}", futureGet.isSuccess(), futureGet.failedReason());
if (futureGet.data() != null) {
return futureGet.data();
} else {
return null;
}
}
@Test
public void testEverythingFromSameExecutor() throws Exception {
int nrOfFiles = 10;
int nrOfWordsPerFile = 10;
addDHTData(dhtCon, job, nrOfFiles, nrOfWordsPerFile);
initialInputDomain =
JobProcedureDomain.create(
job.id(), job.submissionCount(), "S1", DomainProvider.INITIAL_PROCEDURE, -1, 0)
.expectedNrOfFiles(nrOfFiles);
JobProcedureDomain executorDomain =
JobProcedureDomain.create(
job.id(),
job.submissionCount(),
JobCalculationExecutor.classId,
job.currentProcedure().executable().getClass().getSimpleName(),
job.currentProcedure().procedureIndex(),
job.currentProcedure().currentExecutionNumber())
.resultHash(Number160.ZERO);
List<ExecutorTaskDomain> initialETDs = new ArrayList<>();
initialETDs.addAll(initialETDsAndTaskValues.keySet());
for (int i = 0; i < initialETDsAndTaskValues.keySet().size(); ++i) {
msgConsumer.handleCompletedTask(job, initialETDs.subList(i, (i + 1)), initialInputDomain);
Thread.sleep(50); // Wait for data to be transferred from etd to jpd
// The only thing that should happen here is that the data is transferred from etd to this
// executors jpd. Thus, tasks should also not be retrieved or executed
FutureGet getJPD =
dhtCon
.getAll(DomainProvider.PROCEDURE_OUTPUT_RESULT_KEYS, executorDomain.toString())
.awaitUninterruptibly();
if (getJPD.isSuccess()) {
assertEquals((i + 1), getJPD.dataMap().size());
for (Number640 n : getJPD.dataMap().keySet()) {
String taskKey = (String) getJPD.dataMap().get(n).object();
boolean contains = false;
for (ExecutorTaskDomain e : initialETDs) {
if (e.taskId().equals(taskKey)) {
contains = true;
}
}
assertEquals(true, contains);
FutureGet getValuesForKeys =
dhtCon.getAll(taskKey, executorDomain.toString()).awaitUninterruptibly();
if (getValuesForKeys.isSuccess()) {
assertEquals(1, getValuesForKeys.dataMap().size());
for (Number640 n2 : getValuesForKeys.dataMap().keySet()) {
String taskValues =
(String) ((Value) getValuesForKeys.dataMap().get(n2).object()).value();
assertEquals(initialETDsAndTaskValues.get(getTaskKey(taskKey)), taskValues);
}
} else {
fail();
}
}
} else {
fail();
}
System.err.println(job.currentProcedure().tasksSize());
// Additionally, there should be no tasks retrieved from the dht, thus procedure should not
// have more tasks
assertEquals((i + 1), job.currentProcedure().tasksSize());
}
msgConsumer.handleCompletedProcedure(
job, executorDomain.expectedNrOfFiles(10), initialInputDomain);
assertEquals(
WordCountMapper.class.getSimpleName(),
job.currentProcedure().executable().getClass().getSimpleName());
// Adding msgs from external completed tasks
JobProcedureDomain other =
JobProcedureDomain.create(
job.id(),
job.submissionCount(),
"E2",
StartProcedure.class.getSimpleName(),
job.currentProcedure().procedureIndex(),
job.currentProcedure().currentExecutionNumber())
.expectedNrOfFiles(nrOfWordsPerFile);
List<ExecutorTaskDomain> etds = new ArrayList<>();
etds.add(ExecutorTaskDomain.create("task_1", "E2", 0, other).resultHash(Number160.ZERO));
etds.add(ExecutorTaskDomain.create("task_2", "E2", 0, other).resultHash(Number160.ZERO));
etds.add(ExecutorTaskDomain.create("task_3", "E2", 0, other).resultHash(Number160.ZERO));
etds.add(ExecutorTaskDomain.create("task_4", "E2", 0, other).resultHash(Number160.ZERO));
msgConsumer.handleCompletedTask(
job, etds.subList(0, 1), job.currentProcedure().dataInputDomain());
msgConsumer.handleCompletedTask(
job, etds.subList(1, 2), job.currentProcedure().dataInputDomain());
msgConsumer.handleCompletedTask(
job, etds.subList(2, 3), job.currentProcedure().dataInputDomain());
msgConsumer.handleCompletedTask(
job, etds.subList(3, 4), job.currentProcedure().dataInputDomain());
msgConsumer.handleCompletedProcedure(
job,
JobProcedureDomain.create(
job.id(),
job.submissionCount(),
"E2",
job.currentProcedure().executable().getClass().getSimpleName(),
job.currentProcedure().procedureIndex(),
0)
.resultHash(Number160.ZERO),
job.currentProcedure().dataInputDomain());
Thread.sleep(10000);
}