@Override
public void run() {
long start = System.currentTimeMillis();
TUserGroupInfo userInfo = new TUserGroupInfo(user.user, "*", user.priority);
try {
client.submitJob(APPLICATION_ID, request, userInfo);
LOG.debug("Submitted job: " + request + " for user " + userInfo);
} catch (TException e) {
LOG.error("Scheduling request failed!", e);
}
long end = System.currentTimeMillis();
LOG.debug("Scheduling request duration " + (end - start));
}
public void run(String[] args) {
try {
OptionParser parser = new OptionParser();
parser.accepts("c", "configuration file").withRequiredArg().ofType(String.class);
parser.accepts("help", "print help statement");
OptionSet options = parser.parse(args);
if (options.has("help")) {
parser.printHelpOn(System.out);
System.exit(-1);
}
// Logger configuration: log to the console
BasicConfigurator.configure();
LOG.setLevel(Level.DEBUG);
Configuration conf = new PropertiesConfiguration();
if (options.has("c")) {
String configFile = (String) options.valueOf("c");
conf = new PropertiesConfiguration(configFile);
}
double warmup_lambda =
conf.getDouble("warmup_job_arrival_rate_s", DEFAULT_WARMUP_JOB_ARRIVAL_RATE_S);
int warmup_duration_s = conf.getInt("warmup_s", DEFAULT_WARMUP_S);
int post_warmup_s = conf.getInt("post_warmup_s", DEFAULT_POST_WARMUP_S);
// We use this to represent the the rate to fully load the cluster. This is a hack.
double lambda = conf.getDouble("job_arrival_rate_s", DEFAULT_JOB_ARRIVAL_RATE_S);
int experiment_duration_s = conf.getInt("experiment_s", DEFAULT_EXPERIMENT_S);
LOG.debug(
"Using arrival rate of "
+ lambda
+ " tasks per second and running experiment for "
+ experiment_duration_s
+ " seconds.");
int tasksPerJob = conf.getInt("tasks_per_job", DEFAULT_TASKS_PER_JOB);
int numPreferredNodes = conf.getInt("num_preferred_nodes", DEFAULT_NUM_PREFERRED_NODES);
LOG.debug("Using " + numPreferredNodes + " preferred nodes for each task.");
int benchmarkIterations = conf.getInt("benchmark.iterations", DEFAULT_BENCHMARK_ITERATIONS);
int benchmarkId = conf.getInt("benchmark.id", DEFAULT_TASK_BENCHMARK);
List<String> backends = new ArrayList<String>();
if (numPreferredNodes > 0) {
/* Attempt to parse the list of slaves, which we'll need to (randomly) select preferred
* nodes. */
if (!conf.containsKey(BACKENDS)) {
LOG.fatal(
"Missing configuration backend list, which is needed to randomly select "
+ "preferred nodes (num_preferred_nodes set to "
+ numPreferredNodes
+ ")");
}
for (String node : conf.getStringArray(BACKENDS)) {
backends.add(node);
}
if (backends.size() < numPreferredNodes) {
LOG.fatal("Number of backends smaller than number of preferred nodes!");
}
}
List<SubExperiment> experiments = new ArrayList<SubExperiment>();
double fullyUtilizedArrivalRate = lambda;
// For the first twenty seconds, the first user submits at a rate to fully utilize the
// cluster.
List<UserInfo> onlyUser0 = new ArrayList<UserInfo>();
onlyUser0.add(new UserInfo("user0", 1, 0));
experiments.add(new SubExperiment(onlyUser0, 20, fullyUtilizedArrivalRate));
// For the next 10 seconds, user1 increases her rate to 25% of the cluster.
List<UserInfo> user1QuarterDemand = new ArrayList<UserInfo>();
user1QuarterDemand.add(new UserInfo("user0", 4, 0));
user1QuarterDemand.add(new UserInfo("user1", 5, 0));
experiments.add(new SubExperiment(user1QuarterDemand, 10, 1.25 * fullyUtilizedArrivalRate));
// For the next 10 seconds, user 1 increases her rate to 50% of the cluster (using exactly
// her share, but no more).
List<UserInfo> user1HalfDemand = new ArrayList<UserInfo>();
user1HalfDemand.add(new UserInfo("user0", 2, 0));
user1HalfDemand.add(new UserInfo("user1", 3, 0));
experiments.add(new SubExperiment(user1HalfDemand, 10, 1.5 * fullyUtilizedArrivalRate));
// Next user 1 goes back down to 25%.
experiments.add(new SubExperiment(user1QuarterDemand, 10, 1.25 * fullyUtilizedArrivalRate));
// Finally user 1 goes back to 0.
experiments.add(new SubExperiment(onlyUser0, 20, fullyUtilizedArrivalRate));
SparrowFrontendClient client = new SparrowFrontendClient();
int schedulerPort =
conf.getInt("scheduler_port", SchedulerThrift.DEFAULT_SCHEDULER_THRIFT_PORT);
client.initialize(new InetSocketAddress("localhost", schedulerPort), APPLICATION_ID, this);
if (warmup_duration_s > 0) {
List<SubExperiment> warmupExperiment = new ArrayList<SubExperiment>();
List<UserInfo> warmupUsers = new ArrayList<UserInfo>();
warmupUsers.add(new UserInfo("warmupUser", 1, 0));
warmupExperiment.add(new SubExperiment(warmupUsers, warmup_duration_s, warmup_lambda));
LOG.debug(
"Warming up for "
+ warmup_duration_s
+ " seconds at arrival rate of "
+ warmup_lambda
+ " jobs per second");
launchTasks(
warmupExperiment,
tasksPerJob,
numPreferredNodes,
benchmarkIterations,
benchmarkId,
backends,
client);
LOG.debug(
"Waiting for queues to drain after warmup (waiting " + post_warmup_s + " seconds)");
Thread.sleep(post_warmup_s * 1000);
}
LOG.debug("Launching experiment for " + experiment_duration_s + " seconds");
launchTasks(
experiments,
tasksPerJob,
numPreferredNodes,
benchmarkIterations,
benchmarkId,
backends,
client);
} catch (Exception e) {
LOG.error("Fatal exception", e);
}
}