@Override
public boolean addJobToCurrent(Job j) throws Exception {
IAtomicReference<Job> r = h.getAtomicReference("job-" + j.getWorkerId());
if (r.get() != null || !r.isNull()) {
boolean sent = false;
while (!sent) {
// always update
for (String s : workers()) {
if (jobFor(s) == null) {
log.info(
"Redirecting worker "
+ j.getWorkerId()
+ " to "
+ s
+ " due to work already being allocated");
r = h.getAtomicReference("job-" + s);
j.setWorkerId(s);
sent = true;
}
}
}
}
r.set(j);
// iterate over jobs without the work/data
j.setWork(null);
jobs.add(j);
return true;
}
@Override
public boolean isDone() {
// reason being that isDone() may getFromOrigin called and throw errors
// this ensures a safe method call happens and just silently
// returns true in case hazelcast is shutdown
try {
return done.get();
} catch (Exception e) {
log.warn("Hazelcast already shutdown...returning true on isDone()");
return true;
}
}
/**
* Creates a training evaluator using the given neural network
*
* @param network the neural network to use
* @return a training evaluator based on the configuration of the state tracker and the given
* network
*/
@Override
public TrainingEvaluator create(BaseMultiLayerNetwork network) {
OutputLayerTrainingEvaluator eval =
new OutputLayerTrainingEvaluator.Builder()
.bestLoss(bestLoss())
.improvementThreshold(improvementThreshold())
.patience(patience())
.testSet(testSet())
.withNetwork(network)
.validationEpochs(validationEpochs())
.patienceIncrease(patienceIncrease.get())
.build();
return eval;
}
@Override
public int runPreTrainIterations() {
return numTimesPretrain.get();
}
@Override
public Job jobFor(String id) {
IAtomicReference<Job> j = h.getAtomicReference("job-" + id);
if (j.isNull() || isCurrentlyJob(id)) return null;
return j.get();
}
@Override
public boolean isPretrain() {
return isPretrain.get();
}
@Override
public E getCurrent() throws Exception {
E u = (E) master.get();
if (u == null) return null;
return u;
}
/**
* The input split to use. This means that each data applyTransformToDestination that is trained
* on and loaded will be this batch size or lower per worker
*
* @return the input split to use
*/
@Override
public int inputSplit() {
return (miniBatchSize.get() * numWorkers()) / numWorkers();
}
/**
* Increments the number of batches ran. This is purely a count and does not necessarily mean
* progress.
*
* @param numBatchesRan the number of batches ran to increment by
*/
@Override
public void incrementBatchesRan(int numBatchesRan) {
numBatches.set(numBatchesRan + numBatches.get());
}
/**
* The number of epochs to test on
*
* @return the number of epochs to test on
*/
@Override
public int validationEpochs() {
return validationEpochs.get();
}
/**
* The best validation loss so far
*
* @return the best validation loss so far
*/
@Override
public double bestLoss() {
return bestLoss.get();
}
/**
* Improvement threshold for early stopping, aka the minimum
*
* @return
*/
@Override
public double improvmentThreshold() {
return improvementThreshold.get();
}
/**
* Patience is what controls early stopping
*
* @return the patience for the trainer
*/
@Override
public double patience() {
return patience.get();
}
/**
* Whether the cluster has begun training
*
* @return whether the cluster has begun training
*/
@Override
public boolean hasBegun() {
return begunTraining.get();
}
/**
* Current mini batch size
*
* @return
*/
@Override
public int miniBatchSize() {
return miniBatchSize.get();
}
@Override
public int numTimesPreTrainRun() {
return numTimesPretrainRan.get();
}
/**
* Whether to validate against a held out test applyTransformToDestination and test for validation
* error.
*
* @return whether to validate against a held out test applyTransformToDestination and test for
* validation error.
*/
@Override
public boolean isEarlyStopTesting() {
return earlyStop.get();
}
/**
* Number of batches ran so far
*
* @return the number of batches ran so far
*/
@Override
public int numBatchesRan() {
return numBatches.get();
}
