@Override
protected void compute() {
int i = start;
int j = end;
int swap = -1;
while (i < j) {
while (i < j && nums[i] <= nums[j]) j--;
if (i < j) {
swap = nums[i];
nums[i] = nums[j];
nums[j] = swap;
}
while (i < j && nums[i] < nums[j]) i++;
if (i < j) {
swap = nums[i];
nums[i] = nums[j];
nums[j] = swap;
}
}
List<ForkJoinTask<Void>> tasks = new LinkedList<>();
if (i - start > 1) {
tasks.add(new ForkJoinFastSort(nums, 0, i - 1).fork());
}
if (end - j > 1) {
tasks.add(new ForkJoinFastSort(nums, j + 1, end).fork());
}
for (ForkJoinTask<Void> task : tasks) {
task.join();
}
}
/**
* Start the ingest procedure.
*
* @param iterator the iterator to iterate on
* @return the pid of the root object, or null of something odd failed
*/
@Override
public String ingest(TreeIterator iterator) {
this.iterator = iterator;
ForkJoinPool forkJoinPool = new ForkJoinPool(concurrency);
ForkJoinTask<String> result;
result = forkJoinPool.submit(this);
forkJoinPool.shutdown();
try {
return result.get();
} catch (CancellationException | ExecutionException | InterruptedException e) {
log.warn("Shutting down pool {}", forkJoinPool);
result.cancel(true);
forkJoinPool.shutdownNow();
boolean shutdown;
try {
shutdown = forkJoinPool.awaitTermination(3, TimeUnit.MINUTES);
} catch (InterruptedException e1) {
shutdown = false;
}
if (!shutdown) {
log.error("Failed to shut down forkjoinpool {}", forkJoinPool);
System.exit(1);
}
log.debug("Pool shot down {}", forkJoinPool);
throw new IngesterShutdownException(e);
}
}
/** Completed submit(ForkJoinTask) returns result */
public void testSubmitForkJoinTask() throws Throwable {
ForkJoinPool p = new ForkJoinPool(1);
try (PoolCleaner cleaner = cleaner(p)) {
ForkJoinTask<Integer> f = p.submit(new FibTask(8));
assertEquals(21, (int) f.get());
}
}
public static void downloadEncrypted(List<NUSTitleInformation> output_, final Progress progress) {
ForkJoinPool pool = ForkJoinPool.commonPool();
List<ForkJoinTask<Boolean>> list = new ArrayList<>();
for (final NUSTitleInformation nus : output_) {
final long tID = nus.getTitleID();
list.add(
pool.submit(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
NUSTitle nusa =
new NUSTitle(
tID, nus.getSelectedVersion(), Util.ByteArrayToString(nus.getKey()));
Progress childProgress = new Progress();
progress.add(childProgress);
nusa.downloadEncryptedFiles(progress);
return true;
}
}));
}
for (ForkJoinTask<Boolean> task : list) {
try {
task.get();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
protected void cancelTasks(ForkJoinTask<?>... tasks) {
for (ForkJoinTask<?> task : tasks) {
if (task != null) {
task.cancel(true);
// task.completeExceptionally(null);
}
}
}
public static void main(String[] args) {
int[] nums = new int[] {5, 4, 6, 2, 8, 7, 9, 1, 3};
ForkJoinPool pool = new ForkJoinPool();
ForkJoinTask<Void> t = pool.submit(new ForkJoinFastSort(nums, 0, 8));
t.join();
for (int i : nums) {
System.err.print(i);
}
}
/** Pool maintains parallelism when using ManagedBlocker */
public void testBlockingForkJoinTask() throws Throwable {
ForkJoinPool p = new ForkJoinPool(4);
try {
ReentrantLock lock = new ReentrantLock();
ManagedLocker locker = new ManagedLocker(lock);
ForkJoinTask<Integer> f = new LockingFibTask(20, locker, lock);
p.execute(f);
assertEquals(6765, (int) f.get());
} finally {
p.shutdownNow(); // don't wait out shutdown
}
}
/** pollSubmission returns unexecuted submitted task, if present */
public void testPollSubmission() {
final CountDownLatch done = new CountDownLatch(1);
SubFJP p = new SubFJP();
try (PoolCleaner cleaner = cleaner(p)) {
ForkJoinTask a = p.submit(awaiter(done));
ForkJoinTask b = p.submit(awaiter(done));
ForkJoinTask c = p.submit(awaiter(done));
ForkJoinTask r = p.pollSubmission();
assertTrue(r == a || r == b || r == c);
assertFalse(r.isDone());
done.countDown();
}
}
/** drainTasksTo transfers unexecuted submitted tasks, if present */
public void testDrainTasksTo() {
final CountDownLatch done = new CountDownLatch(1);
SubFJP p = new SubFJP();
try (PoolCleaner cleaner = cleaner(p)) {
ForkJoinTask a = p.submit(awaiter(done));
ForkJoinTask b = p.submit(awaiter(done));
ForkJoinTask c = p.submit(awaiter(done));
ArrayList<ForkJoinTask> al = new ArrayList();
p.drainTasksTo(al);
assertTrue(al.size() > 0);
for (ForkJoinTask r : al) {
assertTrue(r == a || r == b || r == c);
assertFalse(r.isDone());
}
done.countDown();
}
}
@Override
protected T compute() {
List<ForkJoinTask<T>> passes = Lists.newArrayListWithCapacity(nSubOptimizers);
for (int i = 0; i < nSubOptimizers; i++) {
passes.add(new SingleOptimizationStep(initial, temp).fork());
}
T currentBest = initial;
double currentBestScore = scorer.score(initial);
for (ForkJoinTask<T> pass : passes) {
T better = pass.join();
double betterScore = scorer.score(better);
if (betterScore > currentBestScore) {
currentBest = better;
currentBestScore = betterScore;
}
}
return currentBest;
}
@Override
protected T compute() {
T currentBest = initial;
for (int step = 0; step < steps; step++) {
logger.log(
Level.INFO,
"On iteration step {0}, current best has score {1}",
new Object[] {step, scorer.score(currentBest)});
double temp = primaryTempFun.temperature(step, steps);
ForkJoinTask<T> task = new ParallelOptimizationStep(currentBest, temp);
stopwatch.start();
currentBest = task.invoke();
stopwatch.stop();
logger.log(
Level.INFO,
"Iteration step {0} took {1}ms wall clock time",
new Object[] {step, stopwatch.elapsedMillis()});
}
return currentBest;
}
/**
* Wait for all the childPid tasks to complete, and then create the relations
*
* @throws BackendMethodFailedException
* @throws BackendInvalidResourceException
* @throws BackendInvalidCredsException
*/
private void handleNodeEnd()
throws BackendMethodFailedException, BackendInvalidResourceException,
BackendInvalidCredsException {
ArrayList<String> childRealPids = new ArrayList<>();
for (ForkJoinTask<String> childPid : childTasks) {
childRealPids.add(childPid.join());
}
String comment =
"Added relationship from " + myPid + " hasPart to " + childRealPids.size() + " children";
AddRelationsRequest addRelationsRequest = new AddRelationsRequest();
addRelationsRequest.setPid(myPid);
addRelationsRequest.setSubject(null);
addRelationsRequest.setPredicate(hasPartRelation);
addRelationsRequest.setObjects(childRealPids);
addRelationsRequest.setComment("Modified by " + getClass().getSimpleName());
UniqueRelationsCreator uniqueRelationsCreator = new UniqueRelationsCreator(fedora);
uniqueRelationsCreator.addRelationships(addRelationsRequest);
// fedora.addRelations(myPid, null, hasPartRelation, childRealPids, false, comment);
log.debug("{}, " + comment, myPid);
}
private static void mergeRound1(
int segmentLen, int[] input, int[] output, ForkJoinPool mainPool) {
int twoSegmentLen = 2 * segmentLen;
int count = 0;
while (count * twoSegmentLen < input.length) {
int startIndex = count * twoSegmentLen;
int endIndex = (count + 1) * twoSegmentLen - 1;
int middle = count * twoSegmentLen + segmentLen - 1;
mainPool.invoke(
ForkJoinTask.adapt(new MergeTask(input, startIndex, middle, endIndex, output)));
count++;
}
}
@Test
public void testAsyncRangeRequests()
throws IOException, URISyntaxException, InterruptedException {
final URL testResourceUrl = new URL(VAULT_BASE_URI.toURL(), "asyncRangeRequestTestFile.txt");
final MultiThreadedHttpConnectionManager cm = new MultiThreadedHttpConnectionManager();
cm.getParams().setDefaultMaxConnectionsPerHost(50);
final HttpClient client = new HttpClient(cm);
// prepare 8MiB test data:
final byte[] plaintextData = new byte[2097152 * Integer.BYTES];
final ByteBuffer bbIn = ByteBuffer.wrap(plaintextData);
for (int i = 0; i < 2097152; i++) {
bbIn.putInt(i);
}
// put request:
final EntityEnclosingMethod putMethod = new PutMethod(testResourceUrl.toString());
putMethod.setRequestEntity(new ByteArrayRequestEntity(plaintextData));
final int putResponse = client.executeMethod(putMethod);
putMethod.releaseConnection();
Assert.assertEquals(201, putResponse);
// multiple async range requests:
final List<ForkJoinTask<?>> tasks = new ArrayList<>();
final Random generator = new Random(System.currentTimeMillis());
final AtomicBoolean success = new AtomicBoolean(true);
// 10 full interrupted requests:
for (int i = 0; i < 10; i++) {
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 200) {
LOG.error("Invalid status code for interrupted full request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 crappy interrupted range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length);
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final int statusCode = client.executeMethod(getMethod);
if (statusCode != 206) {
LOG.error("Invalid status code for interrupted range request");
success.set(false);
}
getMethod.getResponseBodyAsStream().read();
getMethod.getResponseBodyAsStream().close();
getMethod.releaseConnection();
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 50 normal open range requests:
for (int i = 0; i < 50; i++) {
final int lower = generator.nextInt(plaintextData.length - 512);
final int upper = plaintextData.length - 1;
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-");
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 10];
final int bytesRead =
IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for open range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for open range request");
success.set(false);
} else if (!Arrays.equals(
expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for open range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
// 200 normal closed range requests:
for (int i = 0; i < 200; i++) {
final int pos1 = generator.nextInt(plaintextData.length - 512);
final int pos2 = pos1 + 512;
final ForkJoinTask<?> task =
ForkJoinTask.adapt(
() -> {
try {
final int lower = Math.min(pos1, pos2);
final int upper = Math.max(pos1, pos2);
final HttpMethod getMethod = new GetMethod(testResourceUrl.toString());
getMethod.addRequestHeader("Range", "bytes=" + lower + "-" + upper);
final byte[] expected = Arrays.copyOfRange(plaintextData, lower, upper + 1);
final int statusCode = client.executeMethod(getMethod);
final byte[] responseBody = new byte[upper - lower + 1];
final int bytesRead =
IOUtils.read(getMethod.getResponseBodyAsStream(), responseBody);
getMethod.releaseConnection();
if (statusCode != 206) {
LOG.error("Invalid status code for closed range request");
success.set(false);
} else if (upper - lower + 1 != bytesRead) {
LOG.error("Invalid response length for closed range request");
success.set(false);
} else if (!Arrays.equals(
expected, Arrays.copyOfRange(responseBody, 0, bytesRead))) {
LOG.error("Invalid response body for closed range request");
success.set(false);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
});
tasks.add(task);
}
Collections.shuffle(tasks, generator);
final ForkJoinPool pool = new ForkJoinPool(4);
for (ForkJoinTask<?> task : tasks) {
pool.execute(task);
}
for (ForkJoinTask<?> task : tasks) {
task.join();
}
pool.shutdown();
cm.shutdown();
Assert.assertTrue(success.get());
}
public static void downloadEncryptedAllVersions(
List<NUSTitleInformation> output_, final Progress progress) {
ForkJoinPool pool = new ForkJoinPool(25);
List<ForkJoinTask<Boolean>> list = new ArrayList<>();
final int outputsize = output_.size();
for (final NUSTitleInformation nus : output_) {
final long tID = nus.getTitleID();
list.add(
pool.submit(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
int count = 1;
for (Integer i : nus.getAllVersions()) {
NUSTitle nusa = new NUSTitle(tID, i, Util.ByteArrayToString(nus.getKey()));
Progress childProgress = new Progress();
progress.add(childProgress);
nusa.downloadEncryptedFiles(progress);
System.out.println(
"Update download progress "
+ "("
+ nus.getLongnameEN()
+ ") version "
+ i
+ " complete! This was "
+ count
+ " of "
+ nus.getAllVersions().size()
+ "!");
count++;
}
System.out.println(
"Update download complete "
+ "("
+ nus.getLongnameEN()
+ ")"
+ "! Loaded updates for "
+ nus.getAllVersions().size()
+ " version. Now are "
+ finished.incrementAndGet()
+ " of "
+ outputsize
+ " done! ");
return true;
}
}));
}
for (ForkJoinTask<Boolean> task : list) {
try {
task.get();
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ExecutionException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
/**
* Sorts all the elements of the given array using the ForkJoin framework
*
* @param array the array to sort
*/
public void sort(int[] array) {
ForkJoinTask<Void> job = pool.submit(new MergeSortTask(array, 0, array.length));
job.join();
}
@Override
protected T compute() {
long start = System.currentTimeMillis();
long dur = duration.getMillis();
long lastUpdate = start;
int step;
T currentBest = initial;
Csv.Builder builder = Csv.newBuilder();
long totalTime = 0;
for (step = 0; System.currentTimeMillis() - start < dur; step++) {
logger.log(
Level.INFO,
"On iteration step {0}, current best has score {1}",
new Object[] {step, scorer.score(currentBest)});
double temp =
primaryTempFun.temperature((int) (System.currentTimeMillis() - start), (int) dur);
ForkJoinTask<T> task = new ParallelOptimizationStep(currentBest, temp);
stopwatch.start();
currentBest = task.invoke();
stopwatch.stop();
if (step % 20 == 0) {
builder.add(
Csv.newRowBuilder()
.add("%d", System.currentTimeMillis() - start)
.add("%8.3f", scorer.score(currentBest))
.build());
}
logger.log(
Level.FINE,
"Iteration {0} took {1} of wall clock time",
new Object[] {
step,
Converters.PERIOD_FORMATTER.print(Period.millis((int) stopwatch.elapsedMillis()))
});
totalTime += stopwatch.elapsedMillis();
logger.log(
Level.FINER,
"Average step time: {0}",
new Object[] {
Converters.PERIOD_FORMATTER.print(Period.millis((int) (totalTime / (step + 1))))
});
stopwatch.reset();
if ((System.currentTimeMillis() - lastUpdate) > noProgressCancel.getMillis()) {
logger.log(Level.INFO, "Cutting off optimization for lack of progress");
break;
}
}
if (step != 0) {
logger.log(
Level.INFO,
"Average optimizer iteration took {0}",
Duration.millis((long) (System.currentTimeMillis() - start) / step)
.toPeriod()
.toString(Converters.PERIOD_FORMATTER));
}
try {
Files.write(builder.build().toString(), new File("optimization-log.csv"), Charsets.UTF_8);
} catch (IOException e) {
logger.throwing("ConcurrentOptimizer", "log", e);
}
return currentBest;
}
