@Override public boolean addJobToCurrent(Job j) throws Exception { IAtomicReference<Job> r = h.getAtomicReference("job-" + j.getWorkerId()); if (r.get() != null || !r.isNull()) { boolean sent = false; while (!sent) { // always update for (String s : workers()) { if (jobFor(s) == null) { log.info( "Redirecting worker " + j.getWorkerId() + " to " + s + " due to work already being allocated"); r = h.getAtomicReference("job-" + s); j.setWorkerId(s); sent = true; } } } } r.set(j); // iterate over jobs without the work/data j.setWork(null); jobs.add(j); return true; }
@Override public boolean isDone() { // reason being that isDone() may getFromOrigin called and throw errors // this ensures a safe method call happens and just silently // returns true in case hazelcast is shutdown try { return done.get(); } catch (Exception e) { log.warn("Hazelcast already shutdown...returning true on isDone()"); return true; } }
/** * Creates a training evaluator using the given neural network * * @param network the neural network to use * @return a training evaluator based on the configuration of the state tracker and the given * network */ @Override public TrainingEvaluator create(BaseMultiLayerNetwork network) { OutputLayerTrainingEvaluator eval = new OutputLayerTrainingEvaluator.Builder() .bestLoss(bestLoss()) .improvementThreshold(improvementThreshold()) .patience(patience()) .testSet(testSet()) .withNetwork(network) .validationEpochs(validationEpochs()) .patienceIncrease(patienceIncrease.get()) .build(); return eval; }
@Override public int runPreTrainIterations() { return numTimesPretrain.get(); }
@Override public Job jobFor(String id) { IAtomicReference<Job> j = h.getAtomicReference("job-" + id); if (j.isNull() || isCurrentlyJob(id)) return null; return j.get(); }
@Override public boolean isPretrain() { return isPretrain.get(); }
@Override public E getCurrent() throws Exception { E u = (E) master.get(); if (u == null) return null; return u; }
/** * The input split to use. This means that each data applyTransformToDestination that is trained * on and loaded will be this batch size or lower per worker * * @return the input split to use */ @Override public int inputSplit() { return (miniBatchSize.get() * numWorkers()) / numWorkers(); }
/** * Increments the number of batches ran. This is purely a count and does not necessarily mean * progress. * * @param numBatchesRan the number of batches ran to increment by */ @Override public void incrementBatchesRan(int numBatchesRan) { numBatches.set(numBatchesRan + numBatches.get()); }
/** * The number of epochs to test on * * @return the number of epochs to test on */ @Override public int validationEpochs() { return validationEpochs.get(); }
/** * The best validation loss so far * * @return the best validation loss so far */ @Override public double bestLoss() { return bestLoss.get(); }
/** * Improvement threshold for early stopping, aka the minimum * * @return */ @Override public double improvmentThreshold() { return improvementThreshold.get(); }
/** * Patience is what controls early stopping * * @return the patience for the trainer */ @Override public double patience() { return patience.get(); }
/** * Whether the cluster has begun training * * @return whether the cluster has begun training */ @Override public boolean hasBegun() { return begunTraining.get(); }
/** * Current mini batch size * * @return */ @Override public int miniBatchSize() { return miniBatchSize.get(); }
@Override public int numTimesPreTrainRun() { return numTimesPretrainRan.get(); }
/** * Whether to validate against a held out test applyTransformToDestination and test for validation * error. * * @return whether to validate against a held out test applyTransformToDestination and test for * validation error. */ @Override public boolean isEarlyStopTesting() { return earlyStop.get(); }
/** * Number of batches ran so far * * @return the number of batches ran so far */ @Override public int numBatchesRan() { return numBatches.get(); }