private void processExamples(String group, List<Example> examples) {
Evaluation evaluation = new Evaluation();
if (examples.isEmpty()) return;
final String prefix = "iter=0." + group;
Execution.putOutput("group", group);
LogInfo.begin_track_printAll("Processing %s: %s examples", prefix, examples.size());
LogInfo.begin_track("Dumping metadata");
dumpMetadata(group, examples);
LogInfo.end_track();
LogInfo.begin_track("Examples");
for (int e = 0; e < examples.size(); e++) {
Example ex = examples.get(e);
LogInfo.begin_track_printAll("%s: example %s/%s: %s", prefix, e, examples.size(), ex.id);
ex.log();
Execution.putOutput("example", e);
StopWatchSet.begin("Parser.parse");
ParserState state = builder.parser.parse(params, ex, false);
StopWatchSet.end();
out.printf("########## Example %s ##########\n", ex.id);
dumpExample(exampleToLispTree(state));
LogInfo.logs("Current: %s", ex.evaluation.summary());
evaluation.add(ex.evaluation);
LogInfo.logs("Cumulative(%s): %s", prefix, evaluation.summary());
LogInfo.end_track();
ex.predDerivations.clear(); // To save memory
}
LogInfo.end_track();
LogInfo.logs("Stats for %s: %s", prefix, evaluation.summary());
evaluation.logStats(prefix);
evaluation.putOutput(prefix);
LogInfo.end_track();
}
public double getExecutionTime() {
long executionTime = 0;
for (Execution execution : executions) {
executionTime += execution.getExecutionTime();
}
return (double) executionTime / 1000000000.0;
}
public double getNumberOfFailures() {
double numberOfFailures = 0;
for (Execution execution : executions) {
if (execution.getResult().equals(Result.Failed)) {
numberOfFailures++;
}
}
return numberOfFailures;
}
public static void raiseException(Throwable t) {
error(t + ":\n" + StrUtils.join(t.getStackTrace(), "\n"));
t = t.getCause();
if (t != null) error("Caused by " + t + ":\n" + StrUtils.join(t.getStackTrace(), "\n"));
Execution.putOutput("exec.status", "exception");
exitCode = 1;
}
/**
* Informs the user whether the user has an account, login or create a new account with given
* information.
*
* @throws IOException
*/
private static void createUserID() throws IOException {
InputStreamReader isr = new java.io.InputStreamReader(System.in);
BufferedReader br = new BufferedReader(isr);
// log in screen prompted at every start of program
while (true) {
// will prompt user for input until correct input is given, and then
// exits the loop
System.out.println("New User?(Enter 'yes', or 'no', or 'quit')");
String newuser = br.readLine();
if (newuser.equals("yes") || newuser.equals("no")) {
System.out.println("Username: "******"Password: "******"Email: ");
String useremail = br.readLine();
boolean suc = false;
if (newuser.equals("yes")) {
// initializes main library where all player methods are
// run through. Created based on user input
suc = Execution.createUser(username, useremail, userpass);
} else if (newuser.equals("no")) {
suc = Execution.varifyUser(useremail, userpass);
}
if (suc) {
library = MusicLibrary.getMusicLibrary(userpass, useremail);
Execution.readPlaylistXml(library);
break;
}
} else if (newuser.equals("quit")) {
Execution.quit(player);
System.exit(0);
}
}
}
// This should be all we need to put in a main function.
// args are the commandline arguments
// First object is the Runnable object to call run on.
// All of them are objects whose options args is to supposed to populate.
public static void run(String[] args, Object... objects) {
init(args, objects);
if (makeThunk) {
setExecStatus("thunk", true);
printOutputMap(Execution.getFile("output.map"));
List<String> cmd = new ArrayList();
cmd.add("java");
if (thunkJavaOpts != null) cmd.add(thunkJavaOpts);
cmd.add("-cp " + StrUtils.join(jarFiles, ":") + ":" + System.getProperty("java.class.path"));
cmd.addAll(
ListUtils.newList(
objects[0].getClass().getName(),
"++" + virtualExecDir, // Load these options
// Next time when we run, just run in the same path that we used to create the thunk
"-execDir",
virtualExecDir,
"-overwriteExecDir"));
IOUtils.printLinesHard(
Execution.getFile("job.map"),
ListUtils.newList(
"workingDir\t" + SysInfoUtils.getcwd(), // Run from current directory
"command\t" + StrUtils.join(cmd, "\t"),
"priority\t" + thunkPriority));
} else {
if (dontCatchExceptions) {
((Runnable) objects[0]).run();
} else {
try {
((Runnable) objects[0]).run();
} catch (Throwable t) {
raiseException(t);
}
}
}
finish();
}
@Override
public void run() {
builder = new Builder();
builder.build();
dataset = new Dataset();
dataset.read();
params = new Params();
for (String group : dataset.groups()) {
String filename = Execution.getFile("dumped-" + group + ".gz");
out = IOUtils.openOutHard(filename);
processExamples(group, dataset.examples(group));
out.close();
LogInfo.logs("Finished dumping to %s", filename);
StopWatchSet.logStats();
}
}
void processCommand(String cmd) {
cmd = cmd.trim();
if (cmd.equals("")) {
// Print status
Execution.getInfo().print(stderr);
Execution.printOutputMapToStderr();
StopWatchSet.getStats().print(stderr);
stderr.println(Execution.getVirtualExecDir());
} else if (cmd.equals("kill")) {
stderr.println("MonitorThread: KILLING");
Execution.setExecStatus("killed", true);
Execution.printOutputMap(Execution.getFile("output.map"));
throw new RuntimeException("Killed by input command");
} else if (cmd.equals("bail")) {
// Up to program to look at this flag and actually gracefully stop
stderr.println("MonitorThread: BAILING OUT");
Execution.shouldBail = true;
} else stderr.println("Invalid command: '" + cmd + "'");
}
public void run() {
try {
while (!stop) {
if (LogInfo.writeToStdout) readAndProcessCommand();
// Input commands
Execution.inputMap.readEasy(Execution.getFile("input.map"));
boolean killed = Execution.create && new File(Execution.getFile("kill")).exists();
if (killed) Execution.setExecStatus("killed", true);
// Output status
Execution.putOutput("log.note", LogInfo.note);
Execution.putOutput("exec.memory", SysInfoUtils.getUsedMemoryStr());
Execution.putOutput(
"exec.time", new StopWatch(LogInfo.getWatch().getCurrTimeLong()).toString());
Execution.putOutput("exec.errors", "" + LogInfo.getNumErrors());
Execution.putOutput("exec.warnings", "" + LogInfo.getNumWarnings());
Execution.setExecStatus("running", false);
Execution.printOutputMap(Execution.getFile("output.map"));
if (killed) throw new RuntimeException("Killed by 'kill' file");
Utils.sleep(timeInterval);
}
} catch (Exception e) {
e.printStackTrace();
System.exit(1); // Die completely
}
}
class Eval {
// Predictions text file: context, actual, predicted, rank
private static final PrintWriter predOut = IOUtils.openOutEasy(Execution.getFile("predictions"));
// Machine-readable entropies: file, location, isIdent, entropy, reciprocal
// rank
private static final PrintWriter entOut = IOUtils.openOutEasy(Execution.getFile("entropies"));
public static Evaluation eval(InferState state) {
// Print out information about how well we're doing.
Evaluation evaluation = new Evaluation();
Candidate trueCandidate = state.getTrueCandidate();
Candidate predCandidate = state.getCandidates().get(0);
PredictionContext context = state.getContext();
NgramContext ngramContext = NgramContext.get(context);
Statistics statistics = state.getStatistics();
Corpus corpus = statistics.getProjectLangCorpus(context.getPath());
DataSummary summary = statistics.getStatistic(NgramKNCounts.class, corpus).getSummary();
Params params = state.getParams();
boolean oracle = state.isOracle();
int rank = state.getRank();
double entropy = state.getEntropy();
double reciprocalRank = state.getReciprocalRank();
boolean isIdent = state.isIdent();
boolean correct = state.isCorrect();
String path = context.getPath();
String trueTokenStr = trueCandidate.token;
String predToken = predCandidate.token;
evaluation.add("accuracy", correct);
evaluation.add("oracle", oracle);
evaluation.add("rank", rank);
evaluation.add("reciprocalRank", reciprocalRank);
if (oracle) {
evaluation.add("entropy", entropy);
}
if (isIdent) {
evaluation.add("identAccuracy", correct);
evaluation.add("identOracle", oracle);
if (oracle) {
evaluation.add("identEntropy", entropy);
evaluation.add("identReciprocalRank", reciprocalRank);
for (int i = 0; i < Main.clusters; i++) {
evaluation.add("identEntropy" + i, -Math.log(trueCandidate.clusterProbs[i]));
}
}
}
String contextStr = ngramContext.contextStr();
if (Main.verbose >= 2) {
String entropyStr = oracle ? Fmt.D(entropy) : "N/A";
begin_track(
"Example %s [%s]: %s (%d candidates, rank %s, entropy %s)",
path,
correct ? "CORRECT" : "WRONG",
contextStr,
state.getCandidates().size(),
rank,
entropyStr);
logs("True (prob= %s): [%s]", Fmt.D(trueCandidate.prob), trueTokenStr);
logs("Pred (prob= %s): [%s]", Fmt.D(predCandidate.prob), predToken);
if (oracle) {
KneserNey.logKNIs(true);
KneserNey.computeProb(CandidateNgram.get(context, trueCandidate), summary);
KneserNey.logKNIs(false);
}
// begin_track("True");
FeatureVector.logFeatureWeights("True", trueCandidate.features.toMap(), params);
// for (int i = 0; i < Main.clusters; i++) {
// logs("cluster=%d, score %s, prob %s", i, Fmt.D(trueCandidate.clusterScores[i]),
// Fmt.D(trueCandidate.clusterProbs[i]));
// FeatureVector.logFeatureWeights("cluster=" + i,
// trueCandidate.clusterFeatures.toMap(),
// params, Main.clusterDecorators[i]);
// }
// end_track();
KneserNey.logKNIs(true);
KneserNey.computeProb(CandidateNgram.get(context, predCandidate), summary);
KneserNey.logKNIs(false);
FeatureVector.logFeatureWeights("Pred", predCandidate.features.toMap(), params);
// for (Candidate candidate : candidates) {
// begin_track("Candidate " + candidate.token);
// for (int i = 0; i < Main.clusters; i++) {
// logs("cluster=%d, score %s, prob %s", i, Fmt.D(candidate.clusterScores[i]),
// Fmt.D(candidate.clusterProbs[i]));
// FeatureVector.logFeatureWeights("cluster=" + i,
// candidate.clusterFeatures.toMap(),
// params, Main.clusterDecorators[i]);
// }
// end_track();
// }
FeatureVector.logFeatureDiff(
"True - Pred", trueCandidate.features, predCandidate.features, params);
end_track();
}
// Longest context that has been seen
int context_max_n = ngramContext.getMax_n() - 1;
while (context_max_n > 0
&& !summary.counts[context_max_n].containsKey(ngramContext.subContext(context_max_n + 1)))
context_max_n--;
evaluation.add("context_max_n", context_max_n);
predOut.println(
"path="
+ path
+ "\tident="
+ (isIdent ? 1 : 0)
+ "\tcontext="
+ contextStr
+ "\tcontext_max_n="
+ context_max_n
+ "\ttrue="
+ trueTokenStr
+ "\tpred="
+ predToken
+ "\trank="
+ rank
+ "\tentropy="
+ entropy);
predOut.flush();
entOut.println(
path
+ "\t"
+ state.getTrueToken().loc()
+ "\t"
+ (isIdent ? 1 : 0)
+ "\t"
+ (oracle ? entropy : (state.isOov() ? "oov" : "offBeam"))
+ "\t"
+ reciprocalRank);
entOut.flush();
return evaluation;
}
}
/**
* Prompt the user for input and interpret the command recieved.
*
* @throws MidiUnavailableException, InvalidMidiDataException
*/
public static void executeUserCommand()
throws MidiUnavailableException, InvalidMidiDataException {
while (true) {
System.out.println();
System.out.print("Enter User Command. (Type 'quit' to quit) ");
System.out.println();
InputStreamReader isr = new java.io.InputStreamReader(System.in);
BufferedReader br = new BufferedReader(isr);
try {
String userInput = br.readLine();
// the userInput is evaluated to different functions encased in
// the following if blocks.
// some functions take in user input and evaluates what the user
// wants by separating it into different sections
if (userInput.equals("quit")) {
Execution.quit(player);
Execution.writePlaylistsToFile(library);
break;
} else if (userInput.startsWith("create playlist") && userInput.endsWith(")")) {
library = Execution.createPlaylist(userInput, library);
Execution.writePlaylistsToFile(library);
} else if (userInput.startsWith("play playlist")) {
String[] userInputSplited = userInput.split(" ");
String playlist = userInputSplited[2];
if (userInputSplited.length == 4 & userInput.matches(".*\\d.*")) {
// When input length is 4, then play specified song that
// user inputs
int songIndex;
try {
songIndex = Integer.parseInt(userInputSplited[3]) - 1;
} catch (NumberFormatException nfe) {
System.out.println(
"Please enter a valid index that is in this playlist. Please try again.");
continue;
}
try {
player = Execution.playPlaylistSong(library, player, playlist, songIndex);
} catch (IndexOutOfBoundsException iobe) {
System.out.println(
"Please enter a valid index that is in this playlist. Please try again.");
continue;
} catch (NullPointerException e) {
System.out.println("Playlist does not exist. Please try again.");
continue;
}
} else if (userInputSplited.length == 3 | (!userInput.matches(".*\\d.*"))) {
// if command does not contain integer, play entire
// playlist
Playlist pl;
pl = library.findPlaylist(playlist);
// checks if playlist exists
if (pl == null) {
System.out.println("Playlist does not exist. Please try again.");
continue;
}
for (int i = 0; i < pl.numberOfSongs(); i++) {
// plays the playlist songs one by one
try {
player = Execution.playPlaylistSong(library, player, pl, i);
} catch (IOException e) {
System.out.println("Midi file not available!");
}
player.checkIfRunning();
}
}
} else if (userInput.startsWith("delete playlist")) {
library = Execution.deletePlaylist(library, userInput);
} else if (userInput.startsWith("find songs by")) {
library = Execution.findSongByArtist(library, userInput);
} else if (userInput.startsWith("play song")) {
String[] userInputSplited = userInput.split(" ");
int songIndex;
try {
songIndex = Integer.parseInt(userInputSplited[2]) - 1;
} catch (NumberFormatException nfe) {
// the user input can only be an integer
System.out.println("Please enter a song index!");
continue;
}
try {
player = Execution.playSong(library, player, songIndex);
} catch (IndexOutOfBoundsException iobe) {
System.out.println("Song does not exist, please try again.");
continue;
}
} else if (userInput.startsWith("delete song")) {
String[] userInputSplited = userInput.split(" ");
int songIndex;
System.out.println(
"Are you sure you want to permanently remove this song?(yes to confirm, type anything to return to menu)");
String confirm = br.readLine().trim();
// additional layer of confirmation, in case of accidental
// deletion
if (confirm.equals("yes")) {
try {
songIndex = Integer.parseInt(userInputSplited[2]) - 1;
} catch (NumberFormatException nfe) {
System.out.println("Please enter a song index!");
continue;
}
try {
library.delSong(songIndex);
library = Execution.readPlaylistXml(library);
library.displayAllLibrarySongs();
library.displayAllLibraryPlaylists();
} catch (IndexOutOfBoundsException iobe) {
System.out.println("Song does not exist, please try again.");
continue;
}
}
} else if (userInput.startsWith("play random")) {
// User may enter any of the three commands that begin with
// play random
// each is encased in an if block
List<Integer> indexes;
if (userInput.equals("play random for all")) {
int numberOfSongs = library.numberOfSongs();
indexes = new ArrayList<Integer>();
for (int i = 0; i < numberOfSongs; i++) {
indexes.add(i);
}
player = Execution.playRandom(library, player, indexes);
} else if (userInput.startsWith("play random") & userInput.endsWith(")")) {
// When user wants to play specified songs randomly
String[] userInputSplited = userInput.split(" ");
userInputSplited =
userInputSplited[2].substring(1, userInputSplited[2].length() - 1).split(",");
ArrayList<String> tempList = new ArrayList<String>(Arrays.asList(userInputSplited));
indexes = new ArrayList<Integer>(tempList.size());
for (String i : tempList) {
if (Integer.valueOf(i) > library.numberOfSongs()) {
System.out.println(
"the song of index "
+ Integer.valueOf(i)
+ " does not exist in your music libaray.");
} else {
indexes.add(Integer.valueOf(i) - 1);
}
}
player = Execution.playRandom(library, player, indexes);
} else if (userInput.startsWith("play random playlist")) {
String[] userInputSplited = userInput.split(" ");
// The 4th element in list is the playlist that user
// wants to play randomly
String playlist = userInputSplited[3];
Playlist pl = library.findPlaylist(playlist);
try {
indexes = new ArrayList<Integer>(pl.numberOfSongs());
} catch (NullPointerException npe) {
System.out.println("Playlist " + playlist + " does not exist. Please try again.");
continue;
}
for (int n = 0; n < pl.numberOfSongs(); n++) {
indexes.add(n);
}
player = Execution.playRandom(library, player, pl, indexes);
}
} else if (userInput.startsWith("download")) {
String title = userInput.split("download ")[1];
library.downloadSong(title);
} else if (userInput.startsWith("display jcloud songs")) {
library.displayJcloudSongs();
} else if (userInput.startsWith("get downloaded songs")) {
library.downloadPreviousSongs();
library.displayAllLibrarySongs();
} else if (userInput.equals("display songs")) {
library.displayAllLibrarySongs();
} else if (userInput.equals("display library")) {
library.displayAllLibrarySongs();
library.displayAllLibraryPlaylists();
} else if (userInput.equals("display playlists")) {
library.displayAllLibraryPlaylists();
// Shows nothing when no playlist have been created
} else if (userInput.equals("help")) {
displayUserManual();
} else if (userInput.contains("pause")) {
try {
player.pause();
} catch (NullPointerException e) {
continue;
}
} else if (userInput.contains("resume")) {
try {
player.resume();
} catch (NullPointerException e) {
continue;
}
} else {
System.out.println(
"Invalid command. Please pay attention to input format and try again!");
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
public static void main(String[] args) {
Execution.run(args, "SerializedDumperMain", new SerializedDumper(), Master.getOptionsParser());
}
public static void init(String[] args, Object... objects) {
// Parse options
parser = new OptionsParser();
parser.doRegister("log", LogInfo.class);
parser.doRegister("exec", Execution.class);
parser.doRegisterAll(objects);
// These options are specific to the execution, so we don't want to overwrite them
// with a previous execution's.
parser.setDefaultDirFileName("options.map");
parser.setIgnoreOptsFromFileName(
"options.map",
ListUtils.newList(
"log.file",
"exec.execDir",
"exec.execPoolDir",
"exec.actualPoolDir",
"exec.makeThunk"));
if (!parser.doParse(args)) System.exit(1);
// Load classes
if (jarFiles.size() > 0) {
List<String> names = new ArrayList();
for (String jarFile : jarFiles) names.add(new File(jarFile).getName());
stderr.println("Loading JAR files: " + StrUtils.join(names));
for (String jarFile : jarFiles) // Load classes
ClassInitializer.initializeJar(jarFile);
}
// Set character encoding
if (charEncoding != null) CharEncUtils.setCharEncoding(charEncoding);
if (printOptionsAndExit) { // Just print options and exit
parser.doGetOptionPairs().print(stdout);
System.exit(0);
}
// Create a new directory
if (create) {
createVirtualExecDir();
stderr.println(virtualExecDir);
if (!makeThunk) LogInfo.file = getFile("log");
// Copy the Jar files for reference
if (!makeThunk) {
for (String jarFile : jarFiles)
Utils.systemHard(String.format("cp %s %s", jarFile, virtualExecDir));
}
} else {
LogInfo.file = "";
}
if (!makeThunk) {
LogInfo.init();
if (startMainTrack) track("main()", true);
}
// Output options
if (!makeThunk && virtualExecDir != null) logs("Execution directory: " + virtualExecDir);
if (!makeThunk) getInfo().printEasy(getFile("info.map"));
printOptions();
if (create && addToView.size() > 0)
IOUtils.printLinesHard(Execution.getFile("addToView"), addToView);
// Start monitoring
if (!makeThunk && monitor) {
monitorThread = new MonitorThread();
monitorThread.start();
}
if (!makeThunk) Record.init(Execution.getFile("record"));
}