public void run() {
int baseKey = (index * keyCount) % keys.size();
for (int i = 0; i < requests; i++) {
int keyIndex = (baseKey + (i % keyCount)) % keys.size();
String key = keys.get(keyIndex);
Transaction txn = fac.create();
long start = System.currentTimeMillis();
try {
txn.begin();
byte[] value = txn.read(key);
txn.commit();
timeElapsed += System.currentTimeMillis() - start;
if (!silent.get()) {
System.out.println("[Thread-" + index + "] " + key + ": " + new String(value));
}
success++;
} catch (Exception e) {
timeElapsed += System.currentTimeMillis() - start;
if (!silent.get()) {
System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
}
failure++;
}
}
}
public static void toLibSVM(List<TrainingSample<BxZoneLabel>> trainingElements, String filePath)
throws IOException {
BufferedWriter svmDataFile = null;
try {
FileWriter fstream = new FileWriter(filePath);
svmDataFile = new BufferedWriter(fstream);
for (TrainingSample<BxZoneLabel> elem : trainingElements) {
if (elem.getLabel() == null) {
continue;
}
svmDataFile.write(String.valueOf(elem.getLabel().ordinal()));
svmDataFile.write(" ");
Integer featureCounter = 1;
for (Double value : elem.getFeatureVector().getValues()) {
StringBuilder sb = new StringBuilder();
Formatter formatter = new Formatter(sb, Locale.US);
formatter.format("%d:%.5f", featureCounter++, value);
svmDataFile.write(sb.toString());
svmDataFile.write(" ");
}
svmDataFile.write("\n");
}
svmDataFile.close();
} catch (Exception e) {
System.err.println("Error: " + e.getMessage());
return;
} finally {
if (svmDataFile != null) {
svmDataFile.close();
}
}
System.out.println("Done.");
}
private void initConfigDecorator() {
InputStream in =
Q2.class
.getClassLoader()
.getResourceAsStream("META-INF/org/jpos/config/Q2-decorator.properties");
try {
if (in != null) {
PropertyResourceBundle bundle = new PropertyResourceBundle(in);
String ccdClass = bundle.getString("config-decorator-class");
if (log != null) log.info("Initializing config decoration provider: " + ccdClass);
decorator =
(ConfigDecorationProvider) Q2.class.getClassLoader().loadClass(ccdClass).newInstance();
decorator.initialize(getDeployDir());
}
} catch (IOException e) {
} catch (Exception e) {
if (log != null) log.error(e);
else {
e.printStackTrace();
}
} finally {
if (in != null) {
try {
in.close();
} catch (IOException e) {
}
}
}
}
public static void main(String args[]) throws IOException {
LoaderOptions options = LoaderOptions.parseArgs(args);
try {
SSTableLoader loader =
new SSTableLoader(options.directory, new ExternalClient(options), options);
SSTableLoader.LoaderFuture future = loader.stream(options.ignores);
if (options.noProgress) {
future.get();
} else {
ProgressIndicator indicator = new ProgressIndicator(future.getPendingFiles());
indicator.start();
System.out.println("");
while (!future.isDone()) {
if (indicator.printProgress()) {
// We're done with streaming
System.out.println("\nWaiting for targets to rebuild indexes ...");
future.get();
assert future.isDone();
} else {
try {
Thread.sleep(1000L);
} catch (Exception e) {
}
}
}
}
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
private Hashtable<String, String> getJarManifestAttributes(String path) {
Hashtable<String, String> h = new Hashtable<String, String>();
JarInputStream jis = null;
try {
cp.appendln(Color.black, "Looking for " + path);
InputStream is = getClass().getResourceAsStream(path);
if (is == null) {
if (!path.endsWith("/MIRC.jar")) {
cp.appendln(Color.red, "...could not find it.");
} else {
cp.appendln(
Color.black,
"...could not find it. [OK, this is a " + programName + " installation]");
}
return null;
}
jis = new JarInputStream(is);
Manifest manifest = jis.getManifest();
h = getManifestAttributes(manifest);
} catch (Exception ex) {
ex.printStackTrace();
}
if (jis != null) {
try {
jis.close();
} catch (Exception ignore) {
}
}
return h;
}
/** runs the GB features */
public void runGB() {
MTOutputProcessor mtop = null;
if (gbMode == 1) gbXML = initialiseGBResources();
String nbestSentPath =
resourceManager.getString("input") + File.separator + targetLang + File.separator + "temp";
String ngramExecPath = resourceManager.getString("tools.ngram.path");
mtop = new MTOutputProcessor(gbXML, nbestSentPath, ngramExecPath, ngramSize);
// MorphAnalysisProcessor map = new MorphAnalysisProcessor(madaFile);
File f = new File(sourceFile);
String sourceFileName = f.getName();
f = new File(targetFile);
String targetFileName = f.getName();
String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";
String out = resourceManager.getString("output") + File.separator + getMod() + outputFileName;
System.out.println("Output will be: " + out);
String lineTarget;
try {
BufferedReader brSource = new BufferedReader(new FileReader(sourceFile));
BufferedReader brTarget = new BufferedReader(new FileReader(targetFile));
BufferedWriter output = new BufferedWriter(new FileWriter(out));
ResourceManager.printResources();
Sentence targetSent;
Sentence sourceSent;
int sentCount = 0;
String lineSource;
while (((lineSource = brSource.readLine()) != null)
&& ((lineTarget = brTarget.readLine()) != null)) {
lineSource = lineSource.trim().substring(lineSource.indexOf(" "));
sourceSent = new Sentence(lineSource, sentCount);
targetSent = new Sentence(lineTarget, sentCount);
// map.processNextSentence(sourceSent);
mtop.processNextSentence(sourceSent);
++sentCount;
output.write(featureManager.runFeatures(sourceSent, targetSent));
output.write("\r\n");
}
brSource.close();
brTarget.close();
output.close();
featureManager.printFeatureIndeces();
Logger.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Load the Configuration file with the values from the command line and config files, and place
* stuff in the DistrubutedCacheService as needed
*
* @param conf The Configuration to populate
* @param args The command line arguments
* @throws Exception
*/
public static void initalizeConf(Configuration conf, String[] args) throws Exception {
// Parse the arguments and make sure the required args are there
final CommandLine commandLine;
final Options options = constructGnuOptions();
try {
commandLine = new GnuParser().parse(options, args);
} catch (MissingOptionException ex) {
HelpFormatter help = new HelpFormatter();
help.printHelp("hadoop jar <jarFile> " + FrameworkDriver.class.getCanonicalName(), options);
return;
} catch (Exception ex) {
ex.printStackTrace();
return;
}
final String userConfFilePath = commandLine.getOptionValue("amino_config_file_path", "");
final String aminoDefaultConfigPath = commandLine.getOptionValue("amino_default_config_path");
final String baseDir = commandLine.getOptionValue("base_dir");
stopOnFirstPhase = commandLine.hasOption("stop");
// Set the base dir config value if it was provided.
if (StringUtils.isNotEmpty(baseDir)) {
conf.set(AminoConfiguration.BASE_DIR, baseDir);
}
conf.set(AminoConfiguration.DEFAULT_CONFIGURATION_PATH_KEY, aminoDefaultConfigPath);
// create a single DistributedCacheService so that multiple cache entries are deduped.
// Cache files are added after each config is loaded in case the the property value changes.
final DistributedCacheService distributedCacheService = new DistributedCacheService();
// 1. load AminoDefaults
AminoConfiguration.loadAndMergeWithDefault(conf, true);
distributedCacheService.addFilesToDistributedCache(conf);
// 2. load user config files, allowing them to overwrite
if (!StringUtils.isEmpty(userConfFilePath)) {
for (String path : getUserConfigFiles(userConfFilePath)) {
Configuration userConf = new Configuration(false);
logger.info("Grabbing configuration information from: " + path);
userConf.addResource(new FileInputStream(path));
HadoopConfigurationUtils.mergeConfs(conf, userConf);
}
}
distributedCacheService.addFilesToDistributedCache(conf);
// 3. load command line arguments as properties, allowing them to overwrite
final Properties propertyOverrides = commandLine.getOptionProperties("property_override");
for (Object key : propertyOverrides.keySet()) {
conf.set((String) key, (String) propertyOverrides.get(key));
}
distributedCacheService.addFilesToDistributedCache(conf);
}
/**
* converts a string into a valid CBR
*
* @param string the string to parse
* @return the CBR to use
* @throws Exception
*/
private CBR parseCBR(String string) throws Exception {
try {
return (string == null) ? (null) : (new CBR(this)).parse(string);
} catch (Exception e) {
System.err.println("error parsing CBR, exiting.");
e.printStackTrace();
System.exit(5);
return null;
}
}
// Take a tree of files starting in a directory in a zip file
// and copy them to a disk directory, recreating the tree.
private int unpackZipFile(
File inZipFile, String directory, String parent, boolean suppressFirstPathElement) {
int count = 0;
if (!inZipFile.exists()) return count;
parent = parent.trim();
if (!parent.endsWith(File.separator)) parent += File.separator;
if (!directory.endsWith(File.separator)) directory += File.separator;
File outFile = null;
try {
ZipFile zipFile = new ZipFile(inZipFile);
Enumeration zipEntries = zipFile.entries();
while (zipEntries.hasMoreElements()) {
ZipEntry entry = (ZipEntry) zipEntries.nextElement();
String name = entry.getName().replace('/', File.separatorChar);
if (name.startsWith(directory)) {
if (suppressFirstPathElement) name = name.substring(directory.length());
outFile = new File(parent + name);
// Create the directory, just in case
if (name.indexOf(File.separatorChar) >= 0) {
String p = name.substring(0, name.lastIndexOf(File.separatorChar) + 1);
File dirFile = new File(parent + p);
dirFile.mkdirs();
}
if (!entry.isDirectory()) {
System.out.println("Installing " + outFile);
// Copy the file
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFile));
BufferedInputStream in = new BufferedInputStream(zipFile.getInputStream(entry));
int size = 1024;
int n = 0;
byte[] b = new byte[size];
while ((n = in.read(b, 0, size)) != -1) out.write(b, 0, n);
in.close();
out.flush();
out.close();
// Count the file
count++;
}
}
}
zipFile.close();
} catch (Exception e) {
System.err.println("...an error occured while installing " + outFile);
e.printStackTrace();
System.err.println("Error copying " + outFile.getName() + "\n" + e.getMessage());
return -count;
}
System.out.println(count + " files were installed.");
return count;
}
private boolean startLauncher(File dir) {
try {
Runtime rt = Runtime.getRuntime();
ArrayList<String> command = new ArrayList<String>();
command.add("java");
command.add("-jar");
command.add("Launcher.jar");
String[] cmdarray = command.toArray(new String[command.size()]);
Process proc = rt.exec(cmdarray, null, dir);
return true;
} catch (Exception ex) {
System.err.println("Unable to start the Launcher program.\n" + ex.getMessage());
return false;
}
}
private void updateLinuxServiceInstaller() {
try {
File dir = new File(directory, "CTP");
if (suppressFirstPathElement) dir = dir.getParentFile();
Properties props = new Properties();
String ctpHome = dir.getAbsolutePath();
cp.appendln(Color.black, "...CTP_HOME: " + ctpHome);
ctpHome = ctpHome.replaceAll("\\\\", "\\\\\\\\");
props.put("CTP_HOME", ctpHome);
File javaHome = new File(System.getProperty("java.home"));
String javaBin = (new File(javaHome, "bin")).getAbsolutePath();
cp.appendln(Color.black, "...JAVA_BIN: " + javaBin);
javaBin = javaBin.replaceAll("\\\\", "\\\\\\\\");
props.put("JAVA_BIN", javaBin);
File linux = new File(dir, "linux");
File install = new File(linux, "ctpService-ubuntu.sh");
cp.appendln(Color.black, "Linux service installer:");
cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
String bat = getFileText(install);
bat = replace(bat, props); // do the substitutions
bat = bat.replace("\r", "");
setFileText(install, bat);
// If this is an ISN installation, put the script in the correct place.
String osName = System.getProperty("os.name").toLowerCase();
if (programName.equals("ISN") && !osName.contains("windows")) {
install = new File(linux, "ctpService-red.sh");
cp.appendln(Color.black, "ISN service installer:");
cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
bat = getFileText(install);
bat = replace(bat, props); // do the substitutions
bat = bat.replace("\r", "");
File initDir = new File("/etc/init.d");
File initFile = new File(initDir, "ctpService");
if (initDir.exists()) {
setOwnership(initDir, "edge", "edge");
setFileText(initFile, bat);
initFile.setReadable(true, false); // everybody can read //Java 1.6
initFile.setWritable(true); // only the owner can write //Java 1.6
initFile.setExecutable(true, false); // everybody can execute //Java 1.6
}
}
} catch (Exception ex) {
ex.printStackTrace();
System.err.println("Unable to update the Linux service ctpService.sh file");
}
}
public static String getLicensee() {
InputStream is = Q2.class.getResourceAsStream(LICENSEE);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
if (is != null) {
BufferedReader br = new BufferedReader(new InputStreamReader(is));
PrintStream p = new PrintStream(baos);
p.println();
p.println();
try {
while (br.ready()) p.println(br.readLine());
} catch (Exception e) {
e.printStackTrace(p);
}
}
return baos.toString();
}
public static void main(String[] args) {
Options options = KeyGenCLI.buildOptions();
HelpFormatter helpFormatter = new HelpFormatter();
CommandLineParser parser = new GnuParser();
try {
CommandLine commandLine = parser.parse(options, args);
String service = commandLine.getOptionValue("service");
String key = Crypto.generateAes128KeyWithSeed(service);
if (key == null) {
throw new Exception("Key was not generated!");
}
JSONObject keyObject = new JSONObject();
keyObject.put("name", service);
keyObject.put("data", key);
System.out.println(String.format("Key: `%s`\n", key));
System.out.println("Key object:");
System.out.println(keyObject.toString());
} catch (MissingOptionException e) {
System.out.println("Missing required option(s)!");
helpFormatter.printHelp(
"\nKeyGenCLI",
"This tool generates a unique key and prints the result.",
options,
"",
true);
} catch (UnrecognizedOptionException e) {
System.out.println("Unrecognized option!");
helpFormatter.printHelp(
"\nKeyGenCLI",
"This tool generates a unique key and prints the result.",
options,
"",
true);
} catch (ParseException e) {
e.printStackTrace();
} catch (JSONException e) {
e.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
}
private void parseCmdLine(String[] args) {
CommandLineParser parser = new PosixParser();
Options options = new Options();
options.addOption("v", "version", false, "Q2's version");
options.addOption("d", "deploydir", true, "Deployment directory");
options.addOption("r", "recursive", false, "Deploy subdirectories recursively");
options.addOption("h", "help", false, "Usage information");
options.addOption("C", "config", true, "Configuration bundle");
options.addOption("e", "encrypt", true, "Encrypt configuration bundle");
options.addOption("i", "cli", false, "Command Line Interface");
options.addOption("c", "command", true, "Command to execute");
try {
CommandLine line = parser.parse(options, args);
if (line.hasOption("v")) {
displayVersion();
System.exit(0);
}
if (line.hasOption("h")) {
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.printHelp("Q2", options);
System.exit(0);
}
if (line.hasOption("c")) {
cli = new CLI(this, line.getOptionValue("c"), line.hasOption("i"));
} else if (line.hasOption("i")) cli = new CLI(this, null, true);
String dir = DEFAULT_DEPLOY_DIR;
if (line.hasOption("d")) {
dir = line.getOptionValue("d");
}
recursive = line.hasOption("r");
this.deployDir = new File(dir);
if (line.hasOption("C")) deployBundle(new File(line.getOptionValue("C")), false);
if (line.hasOption("e")) deployBundle(new File(line.getOptionValue("e")), true);
} catch (MissingArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
System.exit(1);
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
}
}
@SuppressWarnings("unused")
private static void callFederatorAsWS()
throws MalformedURLException, SNAAExceptionException, AuthenticationExceptionException {
SNAA port = WisebedServiceHelper.getSNAAService("http://*****:*****@wisebed1.itm.uni-luebeck.de");
auth1.setPassword("xxx");
authTriples.add(auth1);
AuthenticationTriple auth2 = new AuthenticationTriple();
auth2.setUrnPrefix("urn:wisebed:dummy1");
auth2.setUsername("*****@*****.**");
auth2.setPassword("xxx");
authTriples.add(auth2);
AuthenticationTriple auth3 = new AuthenticationTriple();
auth3.setUrnPrefix("urn:wisebed:dummy2");
auth3.setUsername("*****@*****.**");
auth3.setPassword("xxx");
authTriples.add(auth3);
AuthenticationTriple auth4 = new AuthenticationTriple();
auth4.setUrnPrefix("urn:wisebed:shib2");
auth4.setUsername("*****@*****.**");
auth4.setPassword("xxx");
authTriples.add(auth4);
try {
List<SecretAuthenticationKey> keys = port.authenticate(authTriples);
log.info("Got authentication keys: " + keys);
Action action = new Action();
action.setAction("sth");
boolean b = port.isAuthorized(new ArrayList<SecretAuthenticationKey>(), action);
log.info("Is authorized: " + b);
} catch (Exception e) {
e.printStackTrace();
}
}
private void updateWindowsServiceInstaller() {
try {
File dir = new File(directory, "CTP");
if (suppressFirstPathElement) dir = dir.getParentFile();
File windows = new File(dir, "windows");
File install = new File(windows, "install.bat");
cp.appendln(Color.black, "Windows service installer:");
cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
String bat = getFileText(install);
Properties props = new Properties();
String home = dir.getAbsolutePath();
cp.appendln(Color.black, "...home: " + home);
home = home.replaceAll("\\\\", "\\\\\\\\");
props.put("home", home);
bat = replace(bat, props);
setFileText(install, bat);
} catch (Exception ex) {
ex.printStackTrace();
System.err.println("Unable to update the windows service install.barfile.");
}
}
/**
* runs the part of speech tagger
*
* @param file input file
* @param lang language
* @param type source or target
* @return path to the output file of the POS tagger
*/
public String runPOS(String file, String lang, String type) {
String posName = resourceManager.getString(lang + ".postagger");
String langResPath = input + File.separator + lang;
File f = new File(file);
String absoluteSourceFilePath = f.getAbsolutePath();
String fileName = f.getName();
String relativeFilePath = langResPath + File.separator + fileName + ".pos";
String absoluteOutputFilePath = (new File(relativeFilePath)).getAbsolutePath();
String posSourceTaggerPath = resourceManager.getString(lang + ".postagger.exePath");
String outPath = "";
try {
Class c = Class.forName(posName);
PosTagger tagger = (PosTagger) c.newInstance();
tagger.setParameters(
type, posName, posSourceTaggerPath, absoluteSourceFilePath, absoluteOutputFilePath);
PosTagger.ForceRun(forceRun);
outPath = tagger.run();
} catch (Exception e) {
e.printStackTrace();
}
// returns the path of the output file; this is for convenience only so
// we do't have to calculate it again
return outPath;
}
public static void main(String args[]) {
Config.setClientMode(true);
LoaderOptions options = LoaderOptions.parseArgs(args);
OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
SSTableLoader loader =
new SSTableLoader(
options.directory,
new ExternalClient(
options.hosts,
options.rpcPort,
options.user,
options.passwd,
options.transportFactory,
options.storagePort,
options.sslStoragePort,
options.serverEncOptions),
handler);
DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(options.throttle);
StreamResultFuture future = null;
try {
if (options.noProgress) future = loader.stream(options.ignores);
else future = loader.stream(options.ignores, new ProgressIndicator());
} catch (Exception e) {
System.err.println(e.getMessage());
if (e.getCause() != null) System.err.println(e.getCause());
if (options.debug) e.printStackTrace(System.err);
else System.err.println("Run with --debug to get full stack trace or --help to get help.");
System.exit(1);
}
handler.output(String.format("Streaming session ID: %s", future.planId));
try {
future.get();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println("Streaming to the following hosts failed:");
System.err.println(loader.getFailedHosts());
System.err.println(e);
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public void runAll() {
File f = new File(sourceFile);
String sourceFileName = f.getName();
f = new File(targetFile);
String targetFileName = f.getName();
String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";
String out = resourceManager.getString("output") + File.separator + outputFileName;
System.out.println("Output will be: " + out);
MTOutputProcessor mtop = null;
if (gbMode == 1) gbXML = initialiseGBResources();
String nbestSentPath =
resourceManager.getString("input") + File.separator + targetLang + File.separator + "temp";
String ngramExecPath = resourceManager.getString("tools.ngram.path");
mtop = new MTOutputProcessor(gbXML, nbestSentPath, ngramExecPath, ngramSize);
// wlv.mt.features.coherence.Coherence coh = new wlv.mt.features.coherence.Coherence(
// getTargetFile());
String pplSourcePath =
resourceManager.getString("input")
+ File.separator
+ sourceLang
+ File.separator
+ sourceFileName
+ resourceManager.getString("tools.ngram.output.ext");
String pplTargetPath =
resourceManager.getString("input")
+ File.separator
+ targetLang
+ File.separator
+ targetFileName
+ resourceManager.getString("tools.ngram.output.ext");
String pplPOSTargetPath =
resourceManager.getString("input")
+ File.separator
+ targetLang
+ File.separator
+ targetFileName
+ PosTagger.getXPOS()
+ resourceManager.getString("tools.ngram.output.ext");
runNGramPPL();
PPLProcessor pplProcSource =
new PPLProcessor(pplSourcePath, new String[] {"logprob", "ppl", "ppl1"});
PPLProcessor pplProcTarget =
new PPLProcessor(pplTargetPath, new String[] {"logprob", "ppl", "ppl1"});
FileModel fm = new FileModel(sourceFile, resourceManager.getString(sourceLang + ".corpus"));
String sourcePosOutput = runPOS(sourceFile, sourceLang, "source");
String targetPosOutput = runPOS(targetFile, targetLang, "target");
String targetPPLPos = runNGramPPLPos(targetPosOutput + PosTagger.getXPOS());
System.out.println("---------TARGET PPLPOS: " + targetPPLPos);
PPLProcessor pplPosTarget =
new PPLProcessor(targetPPLPos, new String[] {"poslogprob", "posppl", "posppl1"});
loadGiza();
processNGrams();
try {
BufferedReader brSource = new BufferedReader(new FileReader(sourceFileName));
BufferedReader brTarget = new BufferedReader(new FileReader(targetFileName));
BufferedWriter output = new BufferedWriter(new FileWriter(out));
BufferedReader posSource = null;
BufferedReader posTarget = null;
boolean posSourceExists = ResourceManager.isRegistered("sourcePosTagger");
boolean posTargetExists = ResourceManager.isRegistered("targetPosTagger");
POSProcessor posSourceProc = null;
POSProcessor posTargetProc = null;
if (posSourceExists) {
posSourceProc = new POSProcessor(sourcePosOutput);
posSource =
new BufferedReader(
new InputStreamReader(new FileInputStream(sourcePosOutput), "utf-8"));
}
if (posTargetExists) {
posTargetProc = new POSProcessor(targetPosOutput);
posTarget = new BufferedReader(new InputStreamReader(new FileInputStream(targetPosOutput)));
}
ResourceManager.printResources();
Sentence targetSent;
// HACK
Sentence sourceSent;
int sentCount = 0;
// HACK
String lineSource = brSource.readLine();
String lineTarget = brTarget.readLine();
// HACK
int result;
while ((lineSource != null) && (lineTarget != null)) {
// the MADA-tokenised files contain start each sentence with the setence ID. We put it there
// (why?) - no we've got to remove it
lineSource = lineSource.trim().substring(lineSource.indexOf(" ")).replace("+", "");
sourceSent = new Sentence(lineSource, sentCount);
targetSent = new Sentence(lineTarget, sentCount);
System.out.println("Processing sentence " + sentCount);
if (posSourceExists) {
posSourceProc.processSentence(sourceSent);
}
if (posTargetExists) {
posTargetProc.processSentence(targetSent);
}
sourceSent.computeNGrams(3);
targetSent.computeNGrams(3);
pplProcSource.processNextSentence(sourceSent);
pplProcTarget.processNextSentence(targetSent);
pplPosTarget.processNextSentence(targetSent);
// coh.processNextSentence(targetSent);
mtop.processNextSentence(sourceSent);
++sentCount;
output.write(featureManager.runFeatures(sourceSent, targetSent));
output.write("\r\n");
lineSource = brSource.readLine();
lineTarget = brTarget.readLine();
}
// featureManager.printFeatureIndeces();
if (posSource != null) {
posSource.close();
}
if (posTarget != null) {
posTarget.close();
}
brSource.close();
brTarget.close();
output.close();
Logger.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Performs some basic processing of the input source and target files For English, this consists
* of converting the input to lower case and tokenizing For Arabic, this consists of
* transliteration and tokenization. Please note that the current tools used for tokenizing Arabic
* also perform POS tagging and morphological analysis Although we could separate the tokenization
* process from the more in-depth text analysis performed by these tools, for efficiency reasons
* this is not desirable The input files are also copied to the /input folder. This is necessary
* because the MADA analyser produces its output in the same folder as the input file, which may
* cause problems if the right access rights are not available for that particular folder
*/
private static void preprocessing() {
String sourceInputFolder = input + File.separator + sourceLang;
String targetInputFolder = input + File.separator + targetLang;
File origSourceFile = new File(sourceFile);
File inputSourceFile = new File(sourceInputFolder + File.separator + origSourceFile.getName());
System.out.println("source input:" + sourceFile);
System.out.println("target input:" + targetFile);
File origTargetFile = new File(targetFile);
File inputTargetFile = new File(targetInputFolder + File.separator + origTargetFile.getName());
try {
System.out.println("copying input to " + inputSourceFile.getPath());
copyFile(origSourceFile, inputSourceFile);
System.out.println("copying input to " + inputTargetFile.getPath());
copyFile(origTargetFile, inputTargetFile);
} catch (Exception e) {
e.printStackTrace();
}
// run tokenizer for source (English)
System.out.println("running tokenizer");
String src_abbr = "";
if (sourceLang.equalsIgnoreCase("english")) src_abbr = "en";
else if (sourceLang.equalsIgnoreCase("spanish")) src_abbr = "es";
else if (sourceLang.equalsIgnoreCase("french")) src_abbr = "fr";
else if (sourceLang.equalsIgnoreCase("german")) src_abbr = "de";
else if (targetLang.equalsIgnoreCase("dutch")) src_abbr = "nl";
else if (targetLang.equalsIgnoreCase("portuguese")) src_abbr = "pt";
else if (targetLang.equalsIgnoreCase("czech")) tgt_abbr = "cs";
else System.out.println("Don't recognise the source language");
String tgt_abbr = "";
if (targetLang.equalsIgnoreCase("english")) tgt_abbr = "en";
else if (targetLang.equalsIgnoreCase("spanish")) tgt_abbr = "es";
else if (targetLang.equalsIgnoreCase("french")) tgt_abbr = "fr";
else if (targetLang.equalsIgnoreCase("german")) tgt_abbr = "de";
else if (targetLang.equalsIgnoreCase("dutch")) tgt_abbr = "nl";
else if (targetLang.equalsIgnoreCase("portuguese")) tgt_abbr = "pt";
else if (targetLang.equalsIgnoreCase("czech")) tgt_abbr = "cs";
else System.out.println("Don't recognise the target language");
String truecasePath = "";
if (null != resourceManager.getProperty(sourceLang + ".lowercase")) {
truecasePath = resourceManager.getProperty(sourceLang + ".lowercase") + " -q ";
} else {
truecasePath =
resourceManager.getString(sourceLang + ".truecase")
+ " --model "
+ resourceManager.getString(sourceLang + ".truecase.model");
}
Tokenizer enTok =
new Tokenizer(
inputSourceFile.getPath(),
inputSourceFile.getPath() + ".tok",
truecasePath,
resourceManager.getString(sourceLang + ".tokenizer"),
src_abbr,
forceRun);
// Tokenizer enTok = new Tokenizer(inputSourceFile.getPath(), inputSourceFile.getPath() +
// ".tok", resourceManager.getString("english.lowercase"),
// resourceManager.getString("english.tokenizer"), "en", forceRun);
enTok.run();
sourceFile = enTok.getTok();
System.out.println(sourceFile);
// run tokenizer for target (Spanish)
System.out.println("running tokenizer");
// Tokenizer esTok = new Tokenizer(inputTargetFile.getPath(), inputTargetFile.getPath() +
// ".tok", resourceManager.getString("spanish.lowercase"),
// resourceManager.getString("spanish.tokenizer"), "es", forceRun);
if (null != resourceManager.getProperty(targetLang + ".lowercase")) {
truecasePath = resourceManager.getProperty(targetLang + ".lowercase") + " -q ";
} else {
truecasePath =
resourceManager.getString(targetLang + ".truecase")
+ " --model "
+ resourceManager.getString(targetLang + ".truecase.model");
}
Tokenizer esTok =
new Tokenizer(
inputTargetFile.getPath(),
inputTargetFile.getPath() + ".tok",
truecasePath,
resourceManager.getString(targetLang + ".tokenizer"),
tgt_abbr,
forceRun);
esTok.run();
targetFile = esTok.getTok();
System.out.println(targetFile);
// Normalize files to avoid strange characters in UTF-8 that may break the PoS tagger
// normalize_utf8();
}
public void run() {
started = true;
Thread.currentThread().setName("Q2-" + getInstanceId().toString());
try {
/*
* The following code determines whether a MBeanServer exists
* already. If so then the first one in the list is used.
* I have not yet find a way to interrogate the server for
* information other than MBeans so to pick a specific one
* would be difficult.
*/
ArrayList mbeanServerList = MBeanServerFactory.findMBeanServer(null);
if (mbeanServerList.isEmpty()) {
server = MBeanServerFactory.createMBeanServer(JMX_NAME);
} else {
server = (MBeanServer) mbeanServerList.get(0);
}
final ObjectName loaderName = new ObjectName(Q2_CLASS_LOADER);
try {
loader =
(QClassLoader)
java.security.AccessController.doPrivileged(
new java.security.PrivilegedAction() {
public Object run() {
return new QClassLoader(server, libDir, loaderName, mainClassLoader);
}
});
server.registerMBean(loader, loaderName);
loader = loader.scan(false);
} catch (Throwable t) {
if (log != null) log.error("initial-scan", t);
else t.printStackTrace();
}
factory = new QFactory(loaderName, this);
initSystemLogger();
addShutdownHook();
q2Thread = Thread.currentThread();
q2Thread.setContextClassLoader(loader);
if (cli != null) cli.start();
initConfigDecorator();
for (int i = 1; !shutdown; i++) {
try {
boolean forceNewClassLoader = scan();
QClassLoader oldClassLoader = loader;
loader = loader.scan(forceNewClassLoader);
if (loader != oldClassLoader) {
oldClassLoader = null; // We want't this to be null so it gets GCed.
System.gc(); // force a GC
log.info(
"new classloader ["
+ Integer.toString(loader.hashCode(), 16)
+ "] has been created");
}
deploy();
checkModified();
relax(SCAN_INTERVAL);
if (i % (3600000 / SCAN_INTERVAL) == 0) logVersion();
} catch (Throwable t) {
log.error("start", t);
relax();
}
}
undeploy();
try {
server.unregisterMBean(loaderName);
} catch (InstanceNotFoundException e) {
log.error(e);
}
if (decorator != null) {
decorator.uninitialize();
}
if (exit && !shuttingDown) System.exit(0);
} catch (Exception e) {
if (log != null) log.error(e);
else e.printStackTrace();
System.exit(1);
}
}
/** loads [additional] policies from commandline (either -p or -f) */
private void loadCLPolicies() {
// we already checked to make sure we have one of the options avaliable
File pLoc = null;
PolicyObject p = null;
if (genProps.getProperty("p3pLocation", null) != null) {
pLoc = new File(genProps.getProperty("p3pLocation"));
if (!pLoc.exists()) {
System.err.println(
"no file found at p3p policy location specified by the -p3p option: "
+ genProps.getProperty("p3pLocation"));
System.err.println("current location is " + System.getProperty("user.dir"));
System.exit(1);
}
try {
p = (new P3PParser()).parse(pLoc.getAbsolutePath());
if (p.getContextDomain() == null) {
p.setContext(
new Context(
new Date(System.currentTimeMillis()),
new Date(System.currentTimeMillis()),
genProps.getProperty("p3pLocation")));
}
pdb.addPolicy(p);
} catch (Exception e) {
System.err.println("Error with parsing or database");
e.printStackTrace();
// System.exit(5);
}
}
if (genProps.getProperty("p3pDirLocation", null) != null) {
pLoc = new File(genProps.getProperty("p3pDirLocation"));
File[] pfiles = pLoc.listFiles();
// System.err.println("pfiles for p3pDirLocation: "+pfiles);
for (int i = 0; i < pfiles.length; i++) {
pLoc = (pfiles[i]);
if (!pLoc.exists()) {
System.err.println(
"no file found at p3p policy location specified by the -p3pDirLocation option, "
+ genProps.getProperty("p3pDirLocation"));
System.exit(1);
}
try {
p = (new P3PParser()).parse(pLoc.getAbsolutePath());
if (p.getContext().getDomain() == null) {
p.setContext(
new Context(
new Date(System.currentTimeMillis()),
new Date(System.currentTimeMillis()),
pfiles[i].getAbsolutePath()));
}
pdb.addPolicy(p);
} catch (Exception e) {
System.err.println("Einar needs to fix this parsing error.");
e.printStackTrace();
}
}
}
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas(false);
if (Schema.instance.getCFMetaData(options.keyspace, options.cf) == null)
throw new IllegalArgumentException(
String.format("Unknown keyspace/columnFamily %s.%s", options.keyspace, options.cf));
Keyspace keyspace = Keyspace.openWithoutSSTables(options.keyspace);
ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf);
OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
Directories.SSTableLister lister = cfs.directories.sstableLister();
if (options.snapshot != null) lister.onlyBackups(true).snapshots(options.snapshot);
else lister.includeBackups(false);
Collection<SSTableReader> readers = new ArrayList<SSTableReader>();
// Upgrade sstables
for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) {
Set<Component> components = entry.getValue();
if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX))
continue;
try {
SSTableReader sstable =
SSTableReader.openNoValidation(entry.getKey(), components, cfs.metadata);
if (sstable.descriptor.version.equals(Descriptor.Version.CURRENT)) continue;
readers.add(sstable);
} catch (Exception e) {
JVMStabilityInspector.inspectThrowable(e);
System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
continue;
}
}
int numSSTables = readers.size();
handler.output("Found " + numSSTables + " sstables that need upgrading.");
for (SSTableReader sstable : readers) {
try {
Upgrader upgrader = new Upgrader(cfs, sstable, handler);
upgrader.upgrade();
if (!options.keepSource) {
// Remove the sstable (it's been copied by upgrade)
System.out.format("Deleting table %s.%n", sstable.descriptor.baseFilename());
sstable.markObsolete();
sstable.selfRef().release();
}
} catch (Exception e) {
System.err.println(String.format("Error upgrading %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
SSTableDeletingTask.waitForDeletions();
System.exit(0);
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public static void main(String args[]) throws IOException {
Options options = Options.parseArgs(args);
try {
// load keyspace descriptions.
DatabaseDescriptor.loadSchemas();
String ksName = null;
String cfName = null;
Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
for (String filename : options.filenames) {
File file = new File(filename);
if (!file.exists()) {
System.out.println("Skipping inexisting file " + file);
continue;
}
Pair<Descriptor, Component> pair =
SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
if (pair == null) {
System.out.println("Skipping non sstable file " + file);
continue;
}
Descriptor desc = pair.left;
if (ksName == null) ksName = desc.ksname;
else if (!ksName.equals(desc.ksname))
throw new IllegalArgumentException("All sstables must be part of the same keyspace");
if (cfName == null) cfName = desc.cfname;
else if (!cfName.equals(desc.cfname))
throw new IllegalArgumentException("All sstables must be part of the same column family");
Set<Component> components =
new HashSet<Component>(
Arrays.asList(
new Component[] {
Component.DATA,
Component.PRIMARY_INDEX,
Component.FILTER,
Component.COMPRESSION_INFO,
Component.STATS
}));
Iterator<Component> iter = components.iterator();
while (iter.hasNext()) {
Component component = iter.next();
if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
}
parsedFilenames.put(desc, components);
}
if (ksName == null || cfName == null) {
System.err.println("No valid sstables to split");
System.exit(1);
}
// Do not load sstables since they might be broken
Table table = Table.openWithoutSSTables(ksName);
ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);
String snapshotName = "pre-split-" + System.currentTimeMillis();
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
try {
SSTableReader sstable =
SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
sstables.add(sstable);
if (options.snapshot) {
File snapshotDirectory =
Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
sstable.createLinks(snapshotDirectory.getPath());
}
} catch (Exception e) {
System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
if (options.snapshot)
System.out.println(
String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));
cfs.getDataTracker().markCompacting(sstables);
for (SSTableReader sstable : sstables) {
try {
new SSTableSplitter(cfs, sstable, options.sizeInMB).split();
// Remove the sstable
sstable.markCompacted();
sstable.releaseReference();
} catch (Exception e) {
System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
if (options.debug) e.printStackTrace(System.err);
}
}
SSTableDeletingTask.waitForDeletions();
System.exit(0); // We need that to stop non daemonized threads
} catch (Exception e) {
System.err.println(e.getMessage());
if (options.debug) e.printStackTrace(System.err);
System.exit(1);
}
}
public void parse() {
PerfPrediction predict = new PerfPrediction();
CommandLineParser parser = new DefaultParser();
CommandLine cmd = null;
try {
cmd = parser.parse(options, args);
if (cmd.hasOption("h")) {
help();
}
if (cmd.getOptions().length == 0) {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
if (reader.ready()) {
predict.add(reader, "");
predict.printFixedValues();
} else {
help();
}
} catch (Exception e) {
e.printStackTrace();
}
return;
}
if (cmd.hasOption("cs") && cmd.getOptionValue("cs").equals("sqlite")) {
if (cmd.hasOption("f")) {
File location = new File(cmd.getOptionValue("f"));
if (location.isDirectory()) {
setSQLitePredictionScenarioNo(location);
if (cmd.hasOption("im")) {
if (isValidMode(cmd.getOptionValue("im"))) {
if (cmd.hasOption("pm")) {
if (isValidMode(cmd.getOptionValue("pm"))) {
this.INPUT_MODE = cmd.getOptionValue("im");
this.PREDICT_MODE = cmd.getOptionValue("pm");
try {
predict.addPrediction(location, this.INPUT_MODE);
} catch (Exception e) {
e.printStackTrace();
}
predict.predict(this.PREDICT_MODE, getSQLiteDir(), location);
} else {
log.log(
Level.SEVERE, "invalid mode for option '-pm " + validModesToString() + "'");
}
} else {
log.log(
Level.SEVERE, "missing prediction mode '-pm " + validModesToString() + "'");
}
} else {
log.log(Level.SEVERE, "invalid mode for option '-im " + validModesToString() + "'");
}
} else {
log.log(Level.SEVERE, "missing input mode '-im " + validModesToString() + "'");
}
} else {
log.log(
Level.SEVERE,
"'-f <location>' does not point to a folder",
cmd.getOptionValue("f"));
}
} else {
log.log(Level.SEVERE, "Use option '-f <location>'", cmd.getOptionValue("f"));
}
}
if (cmd.hasOption("rc")) {
GenerateRandomConfig.generateConfigs(cmd.getOptionValue("rc"));
}
if (cmd.hasOption("fo")) {
try (BufferedReader reader = new BufferedReader(new FileReader(cmd.getOptionValue("fo")))) {
predict.add(reader, "");
predict.printFixedValues();
} catch (Exception e) {
e.printStackTrace();
}
}
if (cmd.hasOption("md")) {
Accumulator.enableMedian();
}
if (cmd.hasOption("acc")) {
Accumulator.start(new File(cmd.getOptionValue("acc")));
}
} catch (ParseException e) {
log.log(Level.SEVERE, "Failed to parse comand line properties", e);
help();
}
}
public static void main(String[] args) {
options = new Options();
options.addOption("h", "help", false, "display this help message");
options.addOption("v", "version", false, "display the version number");
options.addOption("i", "importer", true, "run the importer");
options.addOption("w", "webapi", true, "run the web api server");
options.addOption("s", "simulator", true, "run the simulator");
options.addOption("c", "converter", true, "run the converter");
options.addOption("u", "usage", false, "display heap usage");
options.addOption("", "host", true, "Hostname for REST server (e.g.: 'http://199.116.235.225'");
// Return value
int r = 0;
try {
CommandLineParser parser = new BasicParser();
CommandLine line = parser.parse(options, args);
//// Options that don't need any requirements should be listed first
if (line.hasOption("h")) {
// Help
print_help();
return;
}
if (line.hasOption("u")) {
// Heap Usage
print_heap_usage();
return;
}
//// Options that require arguments
// Check that only one of the importer, webapi, or simulator params are set
boolean[] proper_options = {
line.hasOption("i"), line.hasOption("w"), line.hasOption("s"), line.hasOption("c")
};
int proper_option_count = 0;
for (boolean proper_option : proper_options) {
if (proper_option) {
proper_option_count += 1;
}
}
if (proper_option_count != 1) {
print_help();
return;
}
if (line.hasOption("i")) {
// Run the importer
try {
Importer.import_from_file(line.getOptionValue("i"));
} catch (XMLStreamException e) {
e.printStackTrace();
}
} else if (line.hasOption("w")) {
// Run the webapi
String raw_port = line.getOptionValue("w");
WebApi.start_web_api(raw_port);
} else if (line.hasOption("s")) {
// Run the simulator
r = Simulator.run(line.getOptionValue('s'));
if (r != 0) {
// Kiewan complains about this, but I disagree with Kiewan.
// If the program failed, I should be able to return a non-0 exit code.
System.exit(r);
}
} else if (line.hasOption("c")) {
if (!line.hasOption("host")) {
System.err.println("Run converter with the option --host point to the REST API");
}
// Run the converter
Converter.run(line.getOptionValue('c'), line.getOptionValue("host"));
}
} catch (ParseException exp) {
System.err.println("Parsing failed. Reason: " + exp.getMessage());
} catch (Exception e) {
e.printStackTrace();
}
}
/** runs the BB features */
public void runBB() {
File f = new File(sourceFile);
String sourceFileName = f.getName();
f = new File(targetFile);
String targetFileName = f.getName();
String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";
File file = new File(resourceManager.getString("output"));
if (!file.exists()) {
System.err.println("Creating dir: " + resourceManager.getString("output"));
Logger.log("Creating dir: " + resourceManager.getString("output"));
file.mkdirs();
} else {
Logger.log("output dir exists: " + resourceManager.getString("output"));
}
String out = resourceManager.getString("output") + File.separator + outputFileName;
System.out.println("Output will be: " + out);
String pplSourcePath =
resourceManager.getString("input")
+ File.separator
+ sourceLang
+ File.separator
+ sourceFileName
+ resourceManager.getString("tools.ngram.output.ext");
String pplTargetPath =
resourceManager.getString("input")
+ File.separator
+ targetLang
+ File.separator
+ targetFileName
+ resourceManager.getString("tools.ngram.output.ext");
String pplPOSTargetPath =
resourceManager.getString("input")
+ File.separator
+ targetLang
+ File.separator
+ targetFileName
+ PosTagger.getXPOS()
+ resourceManager.getString("tools.ngram.output.ext");
runNGramPPL();
FileModel fm = new FileModel(sourceFile, resourceManager.getString(sourceLang + ".corpus"));
// FileModel fm = new FileModel(sourceFile,
// resourceManager.getString("source" + ".corpus"));
PPLProcessor pplProcSource =
new PPLProcessor(pplSourcePath, new String[] {"logprob", "ppl", "ppl1"});
PPLProcessor pplProcTarget =
new PPLProcessor(pplTargetPath, new String[] {"logprob", "ppl", "ppl1"});
String sourcePosOutput = null;
String targetPosOutput = null;
PPLProcessor pplPosTarget = null;
if (!isBaseline) {
sourcePosOutput = runPOS(sourceFile, sourceLang, "source");
targetPosOutput = runPOS(targetFile, targetLang, "target");
String targetPPLPos = runNGramPPLPos(targetPosOutput + PosTagger.getXPOS());
System.out.println("---------TARGET PPLPOS: " + targetPPLPos);
pplPosTarget =
new PPLProcessor(targetPPLPos, new String[] {"poslogprob", "posppl", "posppl1"});
}
loadGiza();
processNGrams();
boolean gl = false;
String temp0 = resourceManager.getString("GL");
if (null != temp0 && temp0.equals("1")) {
gl = true;
}
if (gl) {
loadGlobalLexicon();
}
// Preparing the indices for IR_similarity_features
Lucene sourceLuc = null;
Lucene targetLuc = null;
if (featureManager.hasFeature("1700")) {
// The indices reside under lang_resources path
String lang_resources = workDir + File.separator + "lang_resources";
// Indices are saved under: luceneIndex folder
String source_lucene_path =
lang_resources + File.separator + sourceLang + File.separator + "luceneIndex";
// The corpus to index
String source_lucene_corpus = source_lucene_path + File.separator + sourceLang + ".corpus";
// System.out.println("SOURCE: " + source_lucene_path + " ||| " + source_lucene_corpus);
try {
sourceLuc = new Lucene(source_lucene_path, source_lucene_corpus, true, true, "Source");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String target_lucene_path =
lang_resources + File.separator + targetLang + File.separator + "luceneIndex";
String target_lucene_corpus = target_lucene_path + File.separator + targetLang + ".corpus";
// System.out.println("TARGET: " + target_lucene_path + " ||| " + target_lucene_corpus);
try {
targetLuc = new Lucene(target_lucene_path, target_lucene_corpus, true, true, "Target");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// MQM kicks in
MQMManager.getInstance().initialize(resourceManager);
Context context = new Context();
context.setSourceFilePath(sourceFile);
context.setTargetFilePath(targetFile);
MQMManager.getInstance().globalProcessing(context);
try {
BufferedReader brSource = new BufferedReader(new FileReader(sourceFile));
BufferedReader brTarget = new BufferedReader(new FileReader(targetFile));
BufferedWriter output = new BufferedWriter(new FileWriter(out));
BufferedReader posSource = null;
BufferedReader posTarget = null;
boolean posSourceExists = ResourceManager.isRegistered("sourcePosTagger");
boolean posTargetExists = ResourceManager.isRegistered("targetPosTagger");
POSProcessor posSourceProc = null;
POSProcessor posTargetProc = null;
// lefterav: Berkeley parser modifications start here
// Check if user has defined the grammar files for source
// and target language
// if ( ResourceManager.isRegistered("BParser")){
boolean bp = false;
String temp = resourceManager.getString("BP");
if (null != temp && temp.equals("1")) {
bp = true;
}
BParserProcessor sourceParserProcessor = null;
BParserProcessor targetParserProcessor = null;
if (bp) {
sourceParserProcessor = new BParserProcessor();
targetParserProcessor = new BParserProcessor();
sourceParserProcessor.initialize(sourceFile, resourceManager, sourceLang);
targetParserProcessor.initialize(targetFile, resourceManager, targetLang);
}
// }
/** BEGIN: Added by Raphael Rubino for the Topic Model Features */
boolean tm = false;
String temp1 = resourceManager.getString("TM");
if (temp1 != null && temp1.equals("1")) {
tm = true;
}
TopicDistributionProcessor sourceTopicDistributionProcessor = null;
TopicDistributionProcessor targetTopicDistributionProcessor = null;
if (tm) {
String sourceTopicDistributionFile =
resourceManager.getString(sourceLang + ".topic.distribution");
String targetTopicDistributionFile =
resourceManager.getString(targetLang + ".topic.distribution");
sourceTopicDistributionProcessor =
new TopicDistributionProcessor(sourceTopicDistributionFile, "sourceTopicDistribution");
targetTopicDistributionProcessor =
new TopicDistributionProcessor(targetTopicDistributionFile, "targetTopicDistribution");
}
/* END: Added by Raphael Rubino for the Topic Model Features
*/
if (!isBaseline) {
if (posSourceExists) {
posSourceProc = new POSProcessor(sourcePosOutput);
posSource =
new BufferedReader(
new InputStreamReader(new FileInputStream(sourcePosOutput), "utf-8"));
}
if (posTargetExists) {
posTargetProc = new POSProcessor(targetPosOutput);
posTarget =
new BufferedReader(new InputStreamReader(new FileInputStream(targetPosOutput)));
}
}
ResourceManager.printResources();
Sentence sourceSent;
Sentence targetSent;
int sentCount = 0;
String lineSource = brSource.readLine();
String lineTarget = brTarget.readLine();
/** Triggers (by David Langlois) */
boolean tr = false;
String temp2 = resourceManager.getString("TR");
if (temp2 != null && temp2.equals("1")) {
tr = true;
}
Triggers itl_target = null;
TriggersProcessor itl_target_p = null;
Triggers itl_source = null;
TriggersProcessor itl_source_p = null;
// TriggersProcessor itl_source_p = null;
Triggers itl_source_target = null;
TriggersProcessor itl_source_target_p = null;
if (tr) {
itl_target =
new Triggers(
resourceManager.getString("target.intra.triggers.file"),
Integer.parseInt(resourceManager.getString("nb.max.triggers.target.intra")),
resourceManager.getString("phrase.separator"));
itl_target_p = new TriggersProcessor(itl_target);
itl_source =
new Triggers(
resourceManager.getString("source.intra.triggers.file"),
Integer.parseInt(resourceManager.getString("nb.max.triggers.source.intra")),
resourceManager.getString("phrase.separator"));
itl_source_p = new TriggersProcessor(itl_source);
itl_source_target =
new Triggers(
resourceManager.getString("source.target.inter.triggers.file"),
Integer.parseInt(resourceManager.getString("nb.max.triggers.source.target.inter")),
resourceManager.getString("phrase.separator"));
itl_source_target_p = new TriggersProcessor(itl_source_target);
}
/*
* End modification for Triggers
*/
// read in each line from the source and target files
// create a sentence from each
// process each sentence
// run the features on the sentences
while ((lineSource != null) && (lineTarget != null)) {
// lineSource = lineSource.trim().substring(lineSource.indexOf(" ")).replace("+", "");
sourceSent = new Sentence(lineSource, sentCount);
targetSent = new Sentence(lineTarget, sentCount);
// System.out.println("Processing sentence "+sentCount);
// System.out.println("SORCE: " + sourceSent.getText());
// System.out.println("TARGET: " + targetSent.getText());
if (posSourceExists) {
posSourceProc.processSentence(sourceSent);
}
if (posTargetExists) {
posTargetProc.processSentence(targetSent);
}
sourceSent.computeNGrams(3);
targetSent.computeNGrams(3);
pplProcSource.processNextSentence(sourceSent);
pplProcTarget.processNextSentence(targetSent);
if (!isBaseline) {
pplPosTarget.processNextSentence(targetSent);
}
// lefterav: Parse code here
if (bp) {
sourceParserProcessor.processNextSentence(sourceSent);
targetParserProcessor.processNextSentence(targetSent);
}
if (tm) {
sourceTopicDistributionProcessor.processNextSentence(sourceSent);
targetTopicDistributionProcessor.processNextSentence(targetSent);
}
// modified by David
if (tr) {
itl_source_p.processNextSentence(sourceSent);
itl_target_p.processNextSentence(targetSent);
itl_source_target_p.processNextParallelSentences(sourceSent, targetSent);
}
// end modification by David
// MQM kicks in
MQMManager.getInstance().processNextParallelSentences(sourceSent, targetSent);
// Ergun
if (featureManager.hasFeature("1700")) {
sourceLuc.processNextSentence(sourceSent);
targetLuc.processNextSentence(targetSent);
}
++sentCount;
output.write(featureManager.runFeatures(sourceSent, targetSent));
output.newLine();
lineSource = brSource.readLine();
lineTarget = brTarget.readLine();
}
if (posSource != null) {
posSource.close();
}
if (posTarget != null) {
posTarget.close();
}
brSource.close();
brTarget.close();
output.close();
Logger.close();
} catch (Exception e) {
e.printStackTrace();
}
}