/**
* Runs the command line interface given some arguments.
*
* @param arguments the command line arguments
* @return whether or not the command successfully executed
* @throws Exception if something goes wrong
*/
public boolean run(String... arguments) throws Exception {
try {
if (isFlag(HELP, arguments)) {
parser.printHelp(stdOut);
} else if (isFlag(VERSION, arguments)) {
parser.printVersion(stdOut);
} else {
final Namespace namespace = parser.parseArgs(arguments);
if (namespace.get("is-help") == null) {
final Command command = commands.get(namespace.getString(COMMAND_NAME_ATTR));
command.run(bootstrap, namespace);
}
}
return true;
} catch (ArgumentParserException e) {
// TODO: 5/25/13 <coda> -- make ArgumentParser#handleError not depend on System.err
stdErr.println(e.getMessage());
e.getParser().printHelp(stdErr);
return false;
} catch (ConfigurationException e) {
// TODO: 7/26/13 <ntelford> -- as above, this probably shouldn't depend on System.err
stdErr.println(e.getMessage());
return false;
}
}
@Override
protected void run(final Namespace namespace, final Flyway flyway) throws Exception {
flyway.setOutOfOrder(namespace.getBoolean(OUT_OF_ORDER));
flyway.setCleanOnValidationError(namespace.getBoolean(CLEAN_ON_VALIDATION_ERROR));
flyway.validate();
}
public LoggingConfig getLoggingConfig() {
return new LoggingConfig(
options.getInt(verboseArg.getDest()),
options.getBoolean(syslogArg.getDest()),
(File) options.get(logconfigArg.getDest()),
options.getBoolean(noLogSetupArg.getDest()));
}
private static void loadEnvironment(String[] args) {
final String defaultConfDir = getProperty("dash.application.conf.dir");
final String defaultConfFile =
(defaultConfDir != null ? defaultConfDir + File.separatorChar : "") + "conf.yml";
ArgumentParser parser =
ArgumentParsers.newArgumentParser("Dash", true)
.description("Runs a performance test mix.")
.version("${prog} " + Version.id())
.epilog("Dash is a free software under Apache License Version 2.0");
parser
.addArgument("-c", "--conf")
.help("the config file containing the test specification to run (default: ../conf/conf.yml")
.required(false)
.setDefault(new File(defaultConfFile))
.type(File.class);
parser
.addArgument("-t", "--test")
.help("the name of the test to run")
.required(true)
.type(String.class);
parser
.addArgument("-v", "--version")
.help("print the version number")
.action(Arguments.version());
try {
YamlEnv yamlEnv = new YamlEnv();
Namespace namespace = parser.parseArgs(args);
@SuppressWarnings("unchecked")
HashMap<String, Object> env =
(HashMap<String, Object>) yamlEnv.loadProperties((File) namespace.get("conf"));
String test = namespace.getString("test");
@SuppressWarnings("unchecked")
HashMap<String, Object> testSpec = (HashMap<String, Object>) env.get(test);
if (testSpec == null) {
System.err.println("Test spec (" + test + ") does not exist in the config file.");
throw new Error(); // todo: message or log or exit
}
for (Map.Entry<String, Object> entry : testSpec.entrySet()) {
if (entry.getValue() != null) {
System.setProperty(entry.getKey(), entry.getValue().toString());
}
}
} catch (ArgumentParserException e) {
parser.handleError(e);
} catch (IOException e) {
e.printStackTrace();
}
}
protected FastForwardConfig ffwdConfig(final Namespace options) {
if (!options.getBoolean(ffwdEnabled.getDest())) {
return null;
}
return new FastForwardConfig(
Optional.ofNullable(options.getString(ffwdAddress.getDest())),
options.getInt(ffwdInterval.getDest()),
options.getString(ffwdMetricKey.getDest()));
}
@Override
protected int runWithJobId(
final Namespace options,
final HeliosClient client,
final PrintStream out,
final boolean json,
final JobId jobId)
throws ExecutionException, InterruptedException, IOException {
final List<String> hosts = options.getList(hostsArg.getDest());
final Deployment deployment =
new Deployment.Builder().setGoal(Goal.STOP).setJobId(jobId).build();
out.printf("Stopping %s on %s%n", jobId, hosts);
int code = 0;
for (final String host : hosts) {
out.printf("%s: ", host);
final SetGoalResponse result = client.setGoal(deployment, host).get();
if (result.getStatus() == SetGoalResponse.Status.OK) {
out.printf("done%n");
} else {
out.printf("failed: %s%n", result);
code = 1;
}
}
return code;
}
public static Namespace handleArgumentString(String[] args, ArgumentParser parser) {
Namespace parsedArgs = null;
try {
parsedArgs = parser.parseArgs(args);
String selectedModule = parsedArgs.getString("Module");
try {
String propPath = parsedArgs.getString("propertyFile");
if (propPath != null) {
parsedArgs = loadProperties(propPath);
// 선택된 모듈이 무엇이었는지 다시 기억
parsedArgs.getAttrs().put("Module", selectedModule);
}
} catch (IOException e) {
e.printStackTrace();
System.exit(1);
}
} catch (ArgumentParserException e) {
parser.handleError(e);
System.exit(1);
}
// 만약 output 디렉토리가 존재하지 않으면 새로 생성
Path outputPath = null;
try {
String outputPathString = parsedArgs.getString("outputDirPath");
if (outputPathString != null) {
outputPath = Paths.get(outputPathString);
if (!Files.exists(outputPath)) {
Files.createDirectories(outputPath);
}
}
} catch (NullPointerException e) {
// System.err.println();
} catch (InvalidPathException e) {
} catch (IOException e) {
e.printStackTrace();
}
return parsedArgs;
}
Symbol getPrintChannel() {
String name = options.get("print_channel");
if (name == null) {
return null;
} else {
return Symbol.of(name);
}
}
@Test
public void testDeploymentGroupInspectCommandNotFound() throws Exception {
when(options.getString("name")).thenReturn(NON_EXISTENT_NAME);
final int ret = command.run(options, client, out, false, null);
assertEquals(1, ret);
final String output = baos.toString();
assertThat(output, containsString("Unknown deployment group: " + NON_EXISTENT_NAME));
}
@Test
public void testDeploymentGroupInspectCommandJson() throws Exception {
when(options.getString("name")).thenReturn(NAME);
final int ret = command.run(options, client, out, true, null);
assertEquals(0, ret);
final DeploymentGroup output = Json.read(baos.toString(), DeploymentGroup.class);
assertEquals(DEPLOYMENT_GROUP, output);
}
@Override
int run(Namespace options, HeliosClient client, PrintStream out, final boolean json)
throws ExecutionException, InterruptedException {
final String host = options.getString(hostArg.getDest());
final String id = options.getString(idArg.getDest());
out.printf("Registering host %s with id %s%n", host, id);
int code = 0;
out.printf("%s: ", host);
final int result = client.registerHost(host, id).get();
if (result == 200) {
out.printf("done%n");
} else {
out.printf("failed: %s%n", result);
code = 1;
}
return code;
}
@Test
public void testDeploymentGroupInspectCommandNotFoundJson() throws Exception {
when(options.getString("name")).thenReturn(NON_EXISTENT_NAME);
final int ret = command.run(options, client, out, true, null);
assertEquals(1, ret);
final Map<String, Object> output =
Json.read(baos.toString(), new TypeReference<Map<String, Object>>() {});
assertEquals("DEPLOYMENT_GROUP_NOT_FOUND", output.get("status"));
}
public void run(String[] args, Namespace parsedArgs) {
String corpusDirPath = parsedArgs.getString("corpusDirPath");
String outputDirPath = parsedArgs.getString("outputDirPath");
String inputFilePath = parsedArgs.getString("inputFilePath");
String domainFilePath = parsedArgs.getString("domainFilePath");
String pmiThreshold = parsedArgs.getString("pmiThreshold");
String coThreshold = parsedArgs.getString("coThreshold");
try {
target(
corpusDirPath,
inputFilePath,
domainFilePath,
outputDirPath,
Double.parseDouble(pmiThreshold),
Double.parseDouble(coThreshold));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Test
public void testDeploymentGroupInspectCommand() throws Exception {
when(options.getString("name")).thenReturn(NAME);
final int ret = command.run(options, client, out, false, null);
assertEquals(0, ret);
final String output = baos.toString();
assertThat(output, containsString("Name: " + NAME));
assertThat(output, containsString("Host selectors:"));
assertThat(output, containsString(" foo = bar"));
assertThat(output, containsString(" baz = qux"));
assertThat(output, containsString("Job: " + JOB.toString()));
}
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void execute() throws IOException, RecommenderBuildException {
LenskitRecommenderEngine engine = loadEngine();
long user = options.getLong("user");
List<Long> items = options.get("items");
LenskitRecommender rec = engine.createRecommender();
RatingPredictor pred = rec.getRatingPredictor();
if (pred == null) {
logger.error("recommender has no rating predictor");
throw new UnsupportedOperationException("no rating predictor");
}
logger.info("predicting {} items", items.size());
Symbol pchan = getPrintChannel();
Stopwatch timer = Stopwatch.createStarted();
SparseVector preds = pred.predict(user, items);
Long2ObjectMap channel = null;
if (pchan != null) {
for (TypedSymbol sym : preds.getChannelSymbols()) {
if (sym.getRawSymbol().equals(pchan)) {
channel = preds.getChannel(sym);
}
}
}
for (VectorEntry e : preds) {
System.out.format(" %d: %.3f", e.getKey(), e.getValue());
if (channel != null) {
System.out.format(" (%s)", channel.get(e.getKey()));
}
System.out.println();
}
timer.stop();
logger.info("predicted for {} items in {}", items.size(), timer);
}
@Override
@SuppressWarnings("unchecked")
public final void run(Bootstrap<?> bootstrap, Namespace namespace) throws Exception {
final T configuration =
parseConfiguration(
bootstrap.getConfigurationProvider(),
namespace.getString("file"),
getConfigurationClass(),
bootstrap.getObjectMapper());
if (configuration != null) {
configuration
.getLoggingFactory()
.configure(bootstrap.getMetricRegistry(), bootstrap.getApplication().getName());
}
run((Bootstrap<T>) bootstrap, namespace, configuration);
}
private LenskitRecommenderEngine loadEngine() throws RecommenderBuildException, IOException {
File modelFile = options.get("model_file");
if (modelFile == null) {
logger.info("creating fresh recommender");
LenskitRecommenderEngineBuilder builder = LenskitRecommenderEngine.newBuilder();
for (LenskitConfiguration config : environment.loadConfigurations(getConfigFiles())) {
builder.addConfiguration(config);
}
builder.addConfiguration(input.getConfiguration());
Stopwatch timer = Stopwatch.createStarted();
LenskitRecommenderEngine engine = builder.build();
timer.stop();
logger.info("built recommender in {}", timer);
return engine;
} else {
logger.info("loading recommender from {}", modelFile);
LenskitRecommenderEngineLoader loader = LenskitRecommenderEngine.newLoader();
for (LenskitConfiguration config : environment.loadConfigurations(getConfigFiles())) {
loader.addConfiguration(config);
}
loader.addConfiguration(input.getConfiguration());
Stopwatch timer = Stopwatch.createStarted();
LenskitRecommenderEngine engine;
CompressionMode comp = CompressionMode.autodetect(modelFile);
InputStream input = new FileInputStream(modelFile);
try {
input = comp.wrapInput(input);
engine = loader.load(input);
} finally {
input.close();
}
timer.stop();
logger.info("loaded recommender in {}", timer);
return engine;
}
}
public String getStatsdHostPort() {
return options.getString(statsdHostPortArg.getDest());
}
public int getZooKeeperSessionTimeoutMillis() {
return options.getInt(zooKeeperSessiontimeoutArg.getDest());
}
public String getZooKeeperClusterId() {
return options.getString(zooKeeperClusterId.getDest());
}
@Override
protected int runWithJobId(
final Namespace options,
final HeliosClient client,
final PrintStream out,
final boolean json,
final JobId jobId,
final BufferedReader stdin)
throws ExecutionException, InterruptedException, IOException {
final String name = options.getString(nameArg.getDest());
final long timeout = options.getLong(timeoutArg.getDest());
final int parallelism = options.getInt(parallelismArg.getDest());
final boolean async = options.getBoolean(asyncArg.getDest());
final long rolloutTimeout = options.getLong(rolloutTimeoutArg.getDest());
final boolean migrate = options.getBoolean(migrateArg.getDest());
final boolean overlap = options.getBoolean(overlapArg.getDest());
final String token = options.getString(tokenArg.getDest());
checkArgument(timeout > 0, "Timeout must be greater than 0");
checkArgument(parallelism > 0, "Parallelism must be greater than 0");
checkArgument(rolloutTimeout > 0, "Rollout timeout must be greater than 0");
final long startTime = timeSupplier.get();
final RolloutOptions rolloutOptions =
RolloutOptions.newBuilder()
.setTimeout(timeout)
.setParallelism(parallelism)
.setMigrate(migrate)
.setOverlap(overlap)
.setToken(token)
.build();
final RollingUpdateResponse response = client.rollingUpdate(name, jobId, rolloutOptions).get();
if (response.getStatus() != RollingUpdateResponse.Status.OK) {
if (!json) {
out.println("Failed: " + response);
} else {
out.println(response.toJsonString());
}
return 1;
}
if (!json) {
out.println(
format(
"Rolling update%s started: %s -> %s "
+ "(parallelism=%d, timeout=%d, overlap=%b, token=%s)%s",
async ? " (async)" : "",
name,
jobId.toShortString(),
parallelism,
timeout,
overlap,
token,
async ? "" : "\n"));
}
final Map<String, Object> jsonOutput = Maps.newHashMap();
jsonOutput.put("parallelism", parallelism);
jsonOutput.put("timeout", timeout);
jsonOutput.put("overlap", overlap);
jsonOutput.put("token", token);
if (async) {
if (json) {
jsonOutput.put("status", response.getStatus());
out.println(Json.asStringUnchecked(jsonOutput));
}
return 0;
}
String error = "";
boolean failed = false;
boolean timedOut = false;
final Set<String> reported = Sets.newHashSet();
while (true) {
final DeploymentGroupStatusResponse status = client.deploymentGroupStatus(name).get();
if (status == null) {
failed = true;
error = "Failed to fetch deployment-group status";
break;
}
if (!jobId.equals(status.getDeploymentGroup().getJobId())) {
// Another rolling-update was started, overriding this one -- exit
failed = true;
error = "Deployment-group job id changed during rolling-update";
break;
}
if (!json) {
for (DeploymentGroupStatusResponse.HostStatus hostStatus : status.getHostStatuses()) {
final JobId hostJobId = hostStatus.getJobId();
final String host = hostStatus.getHost();
final TaskStatus.State state = hostStatus.getState();
final boolean done =
hostJobId != null && hostJobId.equals(jobId) && state == TaskStatus.State.RUNNING;
if (done && reported.add(host)) {
out.println(
format(
"%s -> %s (%d/%d)",
host, state, reported.size(), status.getHostStatuses().size()));
}
}
}
if (status.getStatus() != DeploymentGroupStatusResponse.Status.ROLLING_OUT) {
if (status.getStatus() == DeploymentGroupStatusResponse.Status.FAILED) {
failed = true;
error = status.getError();
}
break;
}
if (timeSupplier.get() - startTime > TimeUnit.MINUTES.toMillis(rolloutTimeout)) {
// Rollout timed out
timedOut = true;
break;
}
sleepFunction.sleep(POLL_INTERVAL_MILLIS);
}
final double duration = (timeSupplier.get() - startTime) / 1000.0;
if (json) {
if (failed) {
jsonOutput.put("status", "FAILED");
jsonOutput.put("error", error);
} else if (timedOut) {
jsonOutput.put("status", "TIMEOUT");
} else {
jsonOutput.put("status", "DONE");
}
jsonOutput.put("duration", duration);
out.println(Json.asStringUnchecked(jsonOutput));
} else {
out.println();
if (failed) {
out.println(format("Failed: %s", error));
} else if (timedOut) {
out.println("Timed out! (rolling-update still in progress)");
} else {
out.println("Done.");
}
out.println(format("Duration: %.2f s", duration));
}
return (failed || timedOut) ? 1 : 0;
}
public Boolean getNoZooKeeperRegistration() {
return fromNullable(options.getBoolean(noZooKeeperRegistrationArg.getDest())).or(false);
}
public boolean getZooKeeperEnableAcls() {
return options.getBoolean(zooKeeperEnableAcls.getDest());
}
public Path getStateDirectory() {
return Paths.get(options.getString(stateDirArg.getDest()));
}
public List<File> getConfigFiles() {
return options.get("config");
}
public String getZooKeeperAclAgentUser() {
return options.getString(zooKeeperAclAgentUser.getDest());
}
public String getRiemannHostPort() {
return options.getString(riemannHostPortArg.getDest());
}
List<File> getConfigFiles() {
return options.getList("config_file");
}
public File getOutputFile() {
return options.get("output_file");
}
public List<String> getKafkaBrokers() {
final List<String> kafkaBrokers = options.getList(kafkaArg.getDest());
return kafkaBrokers.isEmpty() ? null : kafkaBrokers;
}