private PoolSummary compilePoolSummary(Pool pool) {
PoolSummary.Builder poolSummaryBuilder = aPoolSummary().withPoolName(pool.getName());
for (Device device : pool.getDevices()) {
compileResultsForDevice(pool, poolSummaryBuilder, device);
}
Device watchdog = getPoolWatchdog(pool.getName());
compileResultsForDevice(pool, poolSummaryBuilder, watchdog);
return poolSummaryBuilder.build();
}
public static void createClientCache(String host, Integer port1, Integer port2) throws Exception {
PORT1 = port1.intValue();
PORT2 = port2.intValue();
Properties props = new Properties();
props.setProperty(DistributionConfig.MCAST_PORT_NAME, "0");
props.setProperty(DistributionConfig.LOCATORS_NAME, "");
new DestroyEntryPropagationDUnitTest("temp").createCache(props);
CacheServerTestUtil.disableShufflingOfEndpoints();
Pool p;
try {
p =
PoolManager.createFactory()
.addServer(host, PORT1)
.addServer(host, PORT2)
.setSubscriptionEnabled(true)
.setSubscriptionRedundancy(-1)
.setReadTimeout(2000)
.setSocketBufferSize(1000)
.setMinConnections(4)
// .setRetryAttempts(2)
// .setRetryInterval(250)
.create("EntryPropagationDUnitTestPool");
} finally {
CacheServerTestUtil.enableShufflingOfEndpoints();
}
AttributesFactory factory = new AttributesFactory();
factory.setScope(Scope.DISTRIBUTED_ACK);
factory.setPoolName(p.getName());
factory.setCacheListener(new CertifiableTestCacheListener(getLogWriter()));
RegionAttributes attrs = factory.create();
cache.createRegion(REGION_NAME, attrs);
}
/** Get all pool names that have been seen either in the allocation file or in a MapReduce job. */
public synchronized Collection<String> getPoolNames() {
List<String> list = new ArrayList<String>();
for (Pool pool : getPools()) {
list.add(pool.getName());
}
Collections.sort(list);
return list;
}
private void updateMinSlots() {
// Clear old minSlots
for (JobInfo info : infos.values()) {
info.minMaps = 0;
info.minReduces = 0;
}
// For each pool, distribute its task allocation among jobs in it that need
// slots. This is a little tricky since some jobs in the pool might not be
// able to use all the slots, e.g. they might have only a few tasks left.
// To deal with this, we repeatedly split up the available task slots
// between the jobs left, give each job min(its alloc, # of slots it needs),
// and redistribute any slots that are left over between jobs that still
// need slots on the next pass. If, in total, the jobs in our pool don't
// need all its allocation, we leave the leftover slots for general use.
PoolManager poolMgr = getPoolManager();
for (Pool pool : poolMgr.getPools()) {
for (final TaskType type : TaskType.values()) {
Set<JobInProgress> jobs = new HashSet<JobInProgress>(pool.getJobs());
int slotsLeft = poolMgr.getAllocation(pool.getName(), type);
// Keep assigning slots until none are left
while (slotsLeft > 0) {
// Figure out total weight of jobs that still need slots
double totalWeight = 0;
for (Iterator<JobInProgress> it = jobs.iterator(); it.hasNext(); ) {
JobInProgress job = it.next();
if (isRunnable(job) && runnableTasks(job, type) > minTasks(job, type)) {
totalWeight += weight(job, type);
} else {
it.remove();
}
}
if (totalWeight == 0) // No jobs that can use more slots are left
break;
// Assign slots to jobs, using the floor of their weight divided by
// total weight. This ensures that all jobs get some chance to take
// a slot. Then, if no slots were assigned this way, we do another
// pass where we use ceil, in case some slots were still left over.
int oldSlots = slotsLeft; // Copy slotsLeft so we can modify it
for (JobInProgress job : jobs) {
double weight = weight(job, type);
int share = (int) Math.floor(oldSlots * weight / totalWeight);
slotsLeft = giveMinSlots(job, type, slotsLeft, share);
}
if (slotsLeft == oldSlots) {
// No tasks were assigned; do another pass using ceil, giving the
// extra slots to jobs in order of weight then deficit
List<JobInProgress> sortedJobs = new ArrayList<JobInProgress>(jobs);
Collections.sort(
sortedJobs,
new Comparator<JobInProgress>() {
public int compare(JobInProgress j1, JobInProgress j2) {
double dif = weight(j2, type) - weight(j1, type);
if (dif == 0) // Weights are equal, compare by deficit
dif = deficit(j2, type) - deficit(j1, type);
return (int) Math.signum(dif);
}
});
for (JobInProgress job : sortedJobs) {
double weight = weight(job, type);
int share = (int) Math.ceil(oldSlots * weight / totalWeight);
slotsLeft = giveMinSlots(job, type, slotsLeft, share);
}
}
}
}
}
}
