private void fixConfigSchema() {
File configFile;
File ctpDir = new File(directory, "CTP");
if (ctpDir.exists()) configFile = new File(ctpDir, "config.xml");
else configFile = new File(directory, "config.xml");
if (configFile.exists()) {
try {
Document doc = getDocument(configFile);
Element root = doc.getDocumentElement();
Element server = getFirstNamedChild(root, "Server");
moveAttributes(server, sslAttrs, "SSL");
moveAttributes(server, proxyAttrs, "ProxyServer");
moveAttributes(server, ldapAttrs, "LDAP");
if (programName.equals("ISN")) fixRSNAROOT(server);
if (isMIRC(root)) fixFileServiceAnonymizerID(root);
setFileText(configFile, toString(doc));
} catch (Exception ex) {
cp.appendln(Color.red, "\nUnable to convert the config file schema.");
cp.appendln(Color.black, "");
}
} else {
cp.appendln(Color.red, "\nUnable to find the config file to check the schema.");
cp.appendln(Color.black, "");
}
}
private Map<String, Integer> loadFixedClasses(String file) {
FileReader input;
try {
Map<Integer, List<String>> fixedClaszzes = new HashMap<Integer, List<String>>();
Map<String, Integer> invertedFixedClasses = new HashMap<String, Integer>();
File f = new File(file);
if (!f.exists()) {
System.out.println("Extra classes file doesn't exist: " + fixedClassesFile);
return invertedFixedClasses;
}
input = new FileReader(f);
BufferedReader bufRead = new BufferedReader(input);
String line;
while ((line = bufRead.readLine()) != null) {
String parts[] = line.split(",");
int clazz = Integer.parseInt(parts[0]);
List<String> symbols = new ArrayList<String>();
symbols.addAll(Arrays.asList(parts).subList(1, parts.length));
fixedClaszzes.put(clazz, symbols);
}
for (Map.Entry<Integer, List<String>> e : fixedClaszzes.entrySet()) {
for (String s : e.getValue()) {
invertedFixedClasses.put(s, e.getKey());
}
}
return invertedFixedClasses;
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
private void adjustConfiguration(Element root, File dir) {
// If this is an ISN installation and the Edge Server
// keystore and truststore files do not exist, then set the configuration
// to use the keystore.jks and truststore.jks files instead of the ones
// in the default installation. If the Edge Server files do exist, then
// delete the keystore.jks and truststore.jks files, just to avoid
// confusion.
if (programName.equals("ISN")) {
Element server = getFirstNamedChild(root, "Server");
Element ssl = getFirstNamedChild(server, "SSL");
String rsnaroot = System.getenv("RSNA_ROOT");
rsnaroot = (rsnaroot == null) ? "/usr/local/edgeserver" : rsnaroot.trim();
String keystore = rsnaroot + "/conf/keystore.jks";
String truststore = rsnaroot + "/conf/truststore.jks";
File keystoreFile = new File(keystore);
File truststoreFile = new File(truststore);
cp.appendln(Color.black, "Looking for " + keystore);
if (keystoreFile.exists() || truststoreFile.exists()) {
cp.appendln(Color.black, "...found it [This is an EdgeServer installation]");
// Delete the default files, just to avoid confusion
File ks = new File(dir, "keystore.jks");
File ts = new File(dir, "truststore.jks");
boolean ksok = ks.delete();
boolean tsok = ts.delete();
if (ksok && tsok) cp.appendln(Color.black, "...Unused default SSL files were removed");
else {
if (!ksok) cp.appendln(Color.black, "...Unable to delete " + ks);
if (!tsok) cp.appendln(Color.black, "...Unable to delete " + ts);
}
} else {
cp.appendln(Color.black, "...not found [OK, this is a non-EdgeServer installation]");
ssl.setAttribute("keystore", "keystore.jks");
ssl.setAttribute("keystorePassword", "edge1234");
ssl.setAttribute("truststore", "truststore.jks");
ssl.setAttribute("truststorePassword", "edge1234");
cp.appendln(
Color.black, "...SSL attributes were updated for a non-EdgeServer installation");
}
}
}
private void tidyFileAway(File f, String extension) {
File rename = new File(f.getAbsolutePath() + "." + extension);
while (rename.exists()) {
rename = new File(rename.getAbsolutePath() + "." + extension);
}
getLog()
.warn(
"Tidying " + f.getAbsolutePath() + " out of the way, by adding ." + extension,
"It will be called: "
+ rename.getAbsolutePath()
+ " see log above for detail of problem.");
f.renameTo(rename);
}
private void applyLabel(String inPointsFile, String outPointsFile, List<String> symbols) {
System.out.println("Applying labels for points file: " + inPointsFile);
FileReader input;
BufferedWriter bufWriter = null;
try {
FileOutputStream fos = new FileOutputStream(outPointsFile);
bufWriter = new BufferedWriter(new OutputStreamWriter(fos));
File inFile = new File(inPointsFile);
if (!inFile.exists()) {
System.out.println("ERROR: In file doens't exist");
return;
}
input = new FileReader(inPointsFile);
BufferedReader bufRead = new BufferedReader(input);
String inputLine;
int index = 0;
while ((inputLine = bufRead.readLine()) != null && index < symbols.size()) {
Point p = Utils.readPoint(inputLine);
String symbol = symbols.get(index);
int clazz = 0;
if (this.invertedFixedClases.containsKey(symbol)) {
clazz = this.invertedFixedClases.get(symbol);
} else {
// get the corresponding symbol
// get the class for this one
String sector = invertedSectors.get(symbol);
if (sector != null) {
clazz = sectorToClazz.get(sector);
} else {
// System.out.println("No sector: " + symbol);
}
}
p.setClazz(clazz);
String s = p.serialize();
bufWriter.write(s);
bufWriter.newLine();
index++;
}
System.out.println("Read lines: " + index);
} catch (Exception e) {
throw new RuntimeException("Failed to read/write file", e);
} finally {
if (bufWriter != null) {
try {
bufWriter.close();
} catch (IOException ignore) {
}
}
}
}
// Take a tree of files starting in a directory in a zip file
// and copy them to a disk directory, recreating the tree.
private int unpackZipFile(
File inZipFile, String directory, String parent, boolean suppressFirstPathElement) {
int count = 0;
if (!inZipFile.exists()) return count;
parent = parent.trim();
if (!parent.endsWith(File.separator)) parent += File.separator;
if (!directory.endsWith(File.separator)) directory += File.separator;
File outFile = null;
try {
ZipFile zipFile = new ZipFile(inZipFile);
Enumeration zipEntries = zipFile.entries();
while (zipEntries.hasMoreElements()) {
ZipEntry entry = (ZipEntry) zipEntries.nextElement();
String name = entry.getName().replace('/', File.separatorChar);
if (name.startsWith(directory)) {
if (suppressFirstPathElement) name = name.substring(directory.length());
outFile = new File(parent + name);
// Create the directory, just in case
if (name.indexOf(File.separatorChar) >= 0) {
String p = name.substring(0, name.lastIndexOf(File.separatorChar) + 1);
File dirFile = new File(parent + p);
dirFile.mkdirs();
}
if (!entry.isDirectory()) {
System.out.println("Installing " + outFile);
// Copy the file
BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFile));
BufferedInputStream in = new BufferedInputStream(zipFile.getInputStream(entry));
int size = 1024;
int n = 0;
byte[] b = new byte[size];
while ((n = in.read(b, 0, size)) != -1) out.write(b, 0, n);
in.close();
out.flush();
out.close();
// Count the file
count++;
}
}
}
zipFile.close();
} catch (Exception e) {
System.err.println("...an error occured while installing " + outFile);
e.printStackTrace();
System.err.println("Error copying " + outFile.getName() + "\n" + e.getMessage());
return -count;
}
System.out.println(count + " files were installed.");
return count;
}
public static void deleteRecursive(File file) {
if (!file.exists()) {
return;
}
if (file.isFile()) {
file.delete();
} else if (file.isDirectory()) {
File[] files = file.listFiles();
for (int i = 0; i < files.length; i++) {
deleteRecursive(files[i]);
}
file.delete();
}
}
public static boolean deleteAll(File file) {
boolean b = true;
if ((file != null) && file.exists()) {
if (file.isDirectory()) {
try {
File[] files = file.listFiles();
for (File f : files) b &= deleteAll(f);
} catch (Exception e) {
return false;
}
}
b &= file.delete();
}
return b;
}
private void updateLinuxServiceInstaller() {
try {
File dir = new File(directory, "CTP");
if (suppressFirstPathElement) dir = dir.getParentFile();
Properties props = new Properties();
String ctpHome = dir.getAbsolutePath();
cp.appendln(Color.black, "...CTP_HOME: " + ctpHome);
ctpHome = ctpHome.replaceAll("\\\\", "\\\\\\\\");
props.put("CTP_HOME", ctpHome);
File javaHome = new File(System.getProperty("java.home"));
String javaBin = (new File(javaHome, "bin")).getAbsolutePath();
cp.appendln(Color.black, "...JAVA_BIN: " + javaBin);
javaBin = javaBin.replaceAll("\\\\", "\\\\\\\\");
props.put("JAVA_BIN", javaBin);
File linux = new File(dir, "linux");
File install = new File(linux, "ctpService-ubuntu.sh");
cp.appendln(Color.black, "Linux service installer:");
cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
String bat = getFileText(install);
bat = replace(bat, props); // do the substitutions
bat = bat.replace("\r", "");
setFileText(install, bat);
// If this is an ISN installation, put the script in the correct place.
String osName = System.getProperty("os.name").toLowerCase();
if (programName.equals("ISN") && !osName.contains("windows")) {
install = new File(linux, "ctpService-red.sh");
cp.appendln(Color.black, "ISN service installer:");
cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
bat = getFileText(install);
bat = replace(bat, props); // do the substitutions
bat = bat.replace("\r", "");
File initDir = new File("/etc/init.d");
File initFile = new File(initDir, "ctpService");
if (initDir.exists()) {
setOwnership(initDir, "edge", "edge");
setFileText(initFile, bat);
initFile.setReadable(true, false); // everybody can read //Java 1.6
initFile.setWritable(true); // only the owner can write //Java 1.6
initFile.setExecutable(true, false); // everybody can execute //Java 1.6
}
}
} catch (Exception ex) {
ex.printStackTrace();
System.err.println("Unable to update the Linux service ctpService.sh file");
}
}
public static int checkFiles(String[] filenames) {
int errors = 0;
for (String filename : filenames) {
File file = new File(filename);
if (!file.exists()) {
System.err.println("error: file not found: " + file);
++errors;
} else if (!file.canRead()) {
System.err.println("error: file not readable: " + file);
++errors;
}
}
return errors;
}
// Get the installer program file by looking in the user.dir for [programName]-installer.jar.
private File getInstallerProgramFile() {
System.out.println("Looking for the installer program file");
File programFile;
try {
programFile =
new File(this.getClass().getProtectionDomain().getCodeSource().getLocation().toURI());
} catch (Exception ex) {
String name = getProgramName();
programFile = new File(name + "-installer.jar");
}
programFile = new File(programFile.getAbsolutePath());
if (programFile.exists()) System.out.println("...found " + programFile);
else {
System.err.println("...unable to find the program file " + programFile + "\n...exiting.");
exit();
}
return programFile;
}
// If this is a new installation, ask the user for a
// port for the server; otherwise, return the negative
// of the configured port. If the user selects an illegal
// port, return zero.
private int getPort() {
// Note: directory points to the parent of the CTP directory.
File ctp = new File(directory, "CTP");
if (suppressFirstPathElement) ctp = ctp.getParentFile();
File config = new File(ctp, "config.xml");
if (!config.exists()) {
// No config file - must be a new installation.
// Figure out whether this is Windows or something else.
String os = System.getProperty("os.name").toLowerCase();
int defPort = ((os.contains("windows") && !programName.equals("ISN")) ? 80 : 1080);
int userPort = 0;
while (userPort == 0) {
String port =
JOptionPane.showInputDialog(
null,
"This is a new "
+ programName
+ " installation.\n\n"
+ "Select a port number for the web server.\n\n",
Integer.toString(defPort));
try {
userPort = Integer.parseInt(port.trim());
} catch (Exception ex) {
userPort = -1;
}
if ((userPort < 80) || (userPort > 32767)) userPort = 0;
}
return userPort;
} else {
try {
Document doc = getDocument(config);
Element root = doc.getDocumentElement();
Element server = getFirstNamedChild(root, "Server");
String port = server.getAttribute("port");
return -Integer.parseInt(port);
} catch (Exception ex) {
}
}
return 0;
}
private static void copyFile(File sourceFile, File destFile) throws IOException {
if (sourceFile.equals(destFile)) {
System.out.println("source=dest");
return;
}
if (!destFile.exists()) {
destFile.createNewFile();
}
java.nio.channels.FileChannel source = null;
java.nio.channels.FileChannel destination = null;
try {
source = new FileInputStream(sourceFile).getChannel();
destination = new FileOutputStream(destFile).getChannel();
destination.transferFrom(source, 0, source.size());
} finally {
if (source != null) {
source.close();
}
if (destination != null) {
destination.close();
}
}
}
/**
* Class constructor; creates a new Installer object, displays a JFrame introducing the program,
* allows the user to select an install directory, and copies files from the jar into the
* directory.
*/
public Installer(String args[]) {
// Inputs are --install-dir INSTALL_DIR
for (int k = 0; k < args.length; k = k + 2) {
switch (args[k]) {
case "--install-dir":
directory = new File(args[k + 1]);
System.out.println(directory);
break;
case "--port":
port = Integer.parseInt(args[k + 1]);
break;
}
}
cp = new Stream();
// Find the installer program so we can get to the files.
installer = getInstallerProgramFile();
String name = installer.getName();
programName = (name.substring(0, name.indexOf("-"))).toUpperCase();
// Get the installation information
thisJava = System.getProperty("java.version");
thisJavaBits = System.getProperty("sun.arch.data.model") + " bits";
// Find the ImageIO Tools and get the version
String javaHome = System.getProperty("java.home");
File extDir = new File(javaHome);
extDir = new File(extDir, "lib");
extDir = new File(extDir, "ext");
File clib = getFile(extDir, "clibwrapper_jiio", ".jar");
File jai = getFile(extDir, "jai_imageio", ".jar");
imageIOTools = (clib != null) && clib.exists() && (jai != null) && jai.exists();
if (imageIOTools) {
Hashtable<String, String> jaiManifest = getManifestAttributes(jai);
imageIOVersion = jaiManifest.get("Implementation-Version");
}
// Get the CTP.jar parameters
Hashtable<String, String> manifest = getJarManifestAttributes("/CTP/libraries/CTP.jar");
programDate = manifest.get("Date");
buildJava = manifest.get("Java-Version");
// Get the util.jar parameters
Hashtable<String, String> utilManifest = getJarManifestAttributes("/CTP/libraries/util.jar");
utilJava = utilManifest.get("Java-Version");
// Get the MIRC.jar parameters (if the plugin is present)
Hashtable<String, String> mircManifest = getJarManifestAttributes("/CTP/libraries/MIRC.jar");
if (mircManifest != null) {
mircJava = mircManifest.get("Java-Version");
mircDate = mircManifest.get("Date");
mircVersion = mircManifest.get("Version");
}
// Set up the installation information for display
if (imageIOVersion.equals("")) {
imageIOVersion = "<b><font color=\"red\">not installed</font></b>";
} else if (imageIOVersion.startsWith("1.0")) {
imageIOVersion = "<b><font color=\"red\">" + imageIOVersion + "</font></b>";
}
if (thisJavaBits.startsWith("64")) {
thisJavaBits = "<b><font color=\"red\">" + thisJavaBits + "</font></b>";
}
boolean javaOK = (thisJava.compareTo(buildJava) >= 0);
javaOK &= (thisJava.compareTo(utilJava) >= 0);
javaOK &= (thisJava.compareTo(mircJava) >= 0);
if (!javaOK) {
thisJava = "<b><font color=\"red\">" + thisJava + "</font></b>";
}
if (directory == null) exit();
// Point to the parent of the directory in which to install the program.
// so the copy process works correctly for directory trees.
//
// If the user has selected a directory named "CTP",
// then assume that this is the directory in which
// to install the program.
//
// If the directory is not CTP, see if it is called "RSNA" and contains
// the Launcher program, in which case we can assume that it is an
// installation that was done with Bill Weadock's all-in-one installer for Windows.
//
// If neither of those cases is true, then this is already the parent of the
// directory in which to install the program
if (directory.getName().equals("CTP")) {
directory = directory.getParentFile();
} else if (directory.getName().equals("RSNA")
&& (new File(directory, "Launcher.jar")).exists()) {
suppressFirstPathElement = true;
}
// Cleanup old releases
cleanup(directory);
// Get a port for the server.
if (port < 0) {
if (checkServer(-port, false)) {
System.err.println(
"CTP appears to be running.\nPlease stop CTP and run the installer again.");
System.exit(0);
}
}
// Now install the files and report the results.
int count =
unpackZipFile(installer, "CTP", directory.getAbsolutePath(), suppressFirstPathElement);
if (count > 0) {
// Create the service installer batch files.
updateWindowsServiceInstaller();
updateLinuxServiceInstaller();
// If this was a new installation, set up the config file and set the port
installConfigFile(port);
// Make any necessary changes in the config file to reflect schema evolution
fixConfigSchema();
System.out.println("Installation complete.");
System.out.println(programName + " has been installed successfully.");
System.out.println(count + " files were installed.");
} else {
System.err.println("Installation failed.");
System.err.println(programName + " could not be fully installed.");
}
if (!programName.equals("ISN") && startRunner(new File(directory, "CTP"))) System.exit(0);
}
private void cleanup(File directory) {
// Clean up from old installations, removing or renaming files.
// Note that directory is the parent of the CTP directory
// unless the original installation was done by Bill Weadock's
// all-in-one installer for Windows.
// Get a file pointing to the CTP directory.
// This might be the current directory, or
// it might be the CTP child.
File dir;
if (directory.getName().equals("RSNA")) dir = directory;
else dir = new File(directory, "CTP");
// If CTP.jar exists in this directory, it is a really
// old CTP main file - not used anymore
File ctp = new File(dir, "CTP.jar");
if (ctp.exists()) ctp.delete();
// These are old names for the Launcher.jar file
File launcher = new File(dir, "CTP-launcher.jar");
if (launcher.exists()) launcher.delete();
launcher = new File(dir, "TFS-launcher.jar");
if (launcher.exists()) launcher.delete();
// Delete the obsolete CTP-runner.jar file
File runner = new File(dir, "CTP-runner.jar");
if (runner.exists()) runner.delete();
// Delete the obsolete MIRC-copier.jar file
File copier = new File(dir, "MIRC-copier.jar");
if (copier.exists()) copier.delete();
// Rename the old versions of the properties files
File oldprops = new File(dir, "CTP-startup.properties");
File newprops = new File(dir, "CTP-launcher.properties");
File correctprops = new File(dir, "Launcher.properties");
if (oldprops.exists()) {
if (newprops.exists() || correctprops.exists()) oldprops.delete();
else oldprops.renameTo(correctprops);
}
if (newprops.exists()) {
if (correctprops.exists()) newprops.delete();
else newprops.renameTo(correctprops);
}
// Get rid of obsolete startup and shutdown programs
File startup = new File(dir, "CTP-startup.jar");
if (startup.exists()) startup.delete();
File shutdown = new File(dir, "CTP-shutdown.jar");
if (shutdown.exists()) shutdown.delete();
// Get rid of the obsolete linux directory
File linux = new File(dir, "linux");
if (linux.exists()) {
startup = new File(linux, "CTP-startup.jar");
if (startup.exists()) startup.delete();
shutdown = new File(linux, "CTP-shutdown.jar");
if (shutdown.exists()) shutdown.delete();
linux.delete();
}
// clean up the libraries directory
File libraries = new File(dir, "libraries");
if (libraries.exists()) {
// remove obsolete versions of the slf4j libraries
// and the dcm4che-imageio libraries
File[] files = libraries.listFiles();
for (File file : files) {
if (file.isFile()) {
String name = file.getName();
if (name.startsWith("slf4j-") || name.startsWith("dcm4che-imageio-rle")) {
file.delete();
}
}
}
// remove the email subdirectory
File email = new File(libraries, "email");
deleteAll(email);
// remove the xml subdirectory
File xml = new File(libraries, "xml");
deleteAll(xml);
// remove the sftp subdirectory
File sftp = new File(libraries, "sftp");
deleteAll(xml);
// move edtftpj.jar to the ftp directory
File edtftpj = new File(libraries, "edtftpj.jar");
if (edtftpj.exists()) {
File ftp = new File(libraries, "ftp");
ftp.mkdirs();
File ftpedtftpj = new File(ftp, "edtftpj.jar");
edtftpj.renameTo(ftpedtftpj);
}
}
// remove the obsolete xml library under dir
File xml = new File(dir, "xml");
deleteAll(xml);
// remove the dicom profiles so any
// obsolete files will disappear
File profiles = new File(dir, "profiles");
File dicom = new File(profiles, "dicom");
deleteAll(dicom);
dicom.mkdirs();
// Remove the index.html file so it will be rebuilt from
// example-index.html when the system next starts.
File root = new File(dir, "ROOT");
if (root.exists()) {
File index = new File(root, "index.html");
index.delete();
}
}
/** runs the BB features */
public void runBB() {
File f = new File(sourceFile);
String sourceFileName = f.getName();
f = new File(targetFile);
String targetFileName = f.getName();
String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";
File file = new File(resourceManager.getString("output"));
if (!file.exists()) {
System.err.println("Creating dir: " + resourceManager.getString("output"));
Logger.log("Creating dir: " + resourceManager.getString("output"));
file.mkdirs();
} else {
Logger.log("output dir exists: " + resourceManager.getString("output"));
}
String out = resourceManager.getString("output") + File.separator + outputFileName;
System.out.println("Output will be: " + out);
String pplSourcePath =
resourceManager.getString("input")
+ File.separator
+ sourceLang
+ File.separator
+ sourceFileName
+ resourceManager.getString("tools.ngram.output.ext");
String pplTargetPath =
resourceManager.getString("input")
+ File.separator
+ targetLang
+ File.separator
+ targetFileName
+ resourceManager.getString("tools.ngram.output.ext");
String pplPOSTargetPath =
resourceManager.getString("input")
+ File.separator
+ targetLang
+ File.separator
+ targetFileName
+ PosTagger.getXPOS()
+ resourceManager.getString("tools.ngram.output.ext");
runNGramPPL();
FileModel fm = new FileModel(sourceFile, resourceManager.getString(sourceLang + ".corpus"));
// FileModel fm = new FileModel(sourceFile,
// resourceManager.getString("source" + ".corpus"));
PPLProcessor pplProcSource =
new PPLProcessor(pplSourcePath, new String[] {"logprob", "ppl", "ppl1"});
PPLProcessor pplProcTarget =
new PPLProcessor(pplTargetPath, new String[] {"logprob", "ppl", "ppl1"});
String sourcePosOutput = null;
String targetPosOutput = null;
PPLProcessor pplPosTarget = null;
if (!isBaseline) {
sourcePosOutput = runPOS(sourceFile, sourceLang, "source");
targetPosOutput = runPOS(targetFile, targetLang, "target");
String targetPPLPos = runNGramPPLPos(targetPosOutput + PosTagger.getXPOS());
System.out.println("---------TARGET PPLPOS: " + targetPPLPos);
pplPosTarget =
new PPLProcessor(targetPPLPos, new String[] {"poslogprob", "posppl", "posppl1"});
}
loadGiza();
processNGrams();
boolean gl = false;
String temp0 = resourceManager.getString("GL");
if (null != temp0 && temp0.equals("1")) {
gl = true;
}
if (gl) {
loadGlobalLexicon();
}
// Preparing the indices for IR_similarity_features
Lucene sourceLuc = null;
Lucene targetLuc = null;
if (featureManager.hasFeature("1700")) {
// The indices reside under lang_resources path
String lang_resources = workDir + File.separator + "lang_resources";
// Indices are saved under: luceneIndex folder
String source_lucene_path =
lang_resources + File.separator + sourceLang + File.separator + "luceneIndex";
// The corpus to index
String source_lucene_corpus = source_lucene_path + File.separator + sourceLang + ".corpus";
// System.out.println("SOURCE: " + source_lucene_path + " ||| " + source_lucene_corpus);
try {
sourceLuc = new Lucene(source_lucene_path, source_lucene_corpus, true, true, "Source");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String target_lucene_path =
lang_resources + File.separator + targetLang + File.separator + "luceneIndex";
String target_lucene_corpus = target_lucene_path + File.separator + targetLang + ".corpus";
// System.out.println("TARGET: " + target_lucene_path + " ||| " + target_lucene_corpus);
try {
targetLuc = new Lucene(target_lucene_path, target_lucene_corpus, true, true, "Target");
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// MQM kicks in
MQMManager.getInstance().initialize(resourceManager);
Context context = new Context();
context.setSourceFilePath(sourceFile);
context.setTargetFilePath(targetFile);
MQMManager.getInstance().globalProcessing(context);
try {
BufferedReader brSource = new BufferedReader(new FileReader(sourceFile));
BufferedReader brTarget = new BufferedReader(new FileReader(targetFile));
BufferedWriter output = new BufferedWriter(new FileWriter(out));
BufferedReader posSource = null;
BufferedReader posTarget = null;
boolean posSourceExists = ResourceManager.isRegistered("sourcePosTagger");
boolean posTargetExists = ResourceManager.isRegistered("targetPosTagger");
POSProcessor posSourceProc = null;
POSProcessor posTargetProc = null;
// lefterav: Berkeley parser modifications start here
// Check if user has defined the grammar files for source
// and target language
// if ( ResourceManager.isRegistered("BParser")){
boolean bp = false;
String temp = resourceManager.getString("BP");
if (null != temp && temp.equals("1")) {
bp = true;
}
BParserProcessor sourceParserProcessor = null;
BParserProcessor targetParserProcessor = null;
if (bp) {
sourceParserProcessor = new BParserProcessor();
targetParserProcessor = new BParserProcessor();
sourceParserProcessor.initialize(sourceFile, resourceManager, sourceLang);
targetParserProcessor.initialize(targetFile, resourceManager, targetLang);
}
// }
/** BEGIN: Added by Raphael Rubino for the Topic Model Features */
boolean tm = false;
String temp1 = resourceManager.getString("TM");
if (temp1 != null && temp1.equals("1")) {
tm = true;
}
TopicDistributionProcessor sourceTopicDistributionProcessor = null;
TopicDistributionProcessor targetTopicDistributionProcessor = null;
if (tm) {
String sourceTopicDistributionFile =
resourceManager.getString(sourceLang + ".topic.distribution");
String targetTopicDistributionFile =
resourceManager.getString(targetLang + ".topic.distribution");
sourceTopicDistributionProcessor =
new TopicDistributionProcessor(sourceTopicDistributionFile, "sourceTopicDistribution");
targetTopicDistributionProcessor =
new TopicDistributionProcessor(targetTopicDistributionFile, "targetTopicDistribution");
}
/* END: Added by Raphael Rubino for the Topic Model Features
*/
if (!isBaseline) {
if (posSourceExists) {
posSourceProc = new POSProcessor(sourcePosOutput);
posSource =
new BufferedReader(
new InputStreamReader(new FileInputStream(sourcePosOutput), "utf-8"));
}
if (posTargetExists) {
posTargetProc = new POSProcessor(targetPosOutput);
posTarget =
new BufferedReader(new InputStreamReader(new FileInputStream(targetPosOutput)));
}
}
ResourceManager.printResources();
Sentence sourceSent;
Sentence targetSent;
int sentCount = 0;
String lineSource = brSource.readLine();
String lineTarget = brTarget.readLine();
/** Triggers (by David Langlois) */
boolean tr = false;
String temp2 = resourceManager.getString("TR");
if (temp2 != null && temp2.equals("1")) {
tr = true;
}
Triggers itl_target = null;
TriggersProcessor itl_target_p = null;
Triggers itl_source = null;
TriggersProcessor itl_source_p = null;
// TriggersProcessor itl_source_p = null;
Triggers itl_source_target = null;
TriggersProcessor itl_source_target_p = null;
if (tr) {
itl_target =
new Triggers(
resourceManager.getString("target.intra.triggers.file"),
Integer.parseInt(resourceManager.getString("nb.max.triggers.target.intra")),
resourceManager.getString("phrase.separator"));
itl_target_p = new TriggersProcessor(itl_target);
itl_source =
new Triggers(
resourceManager.getString("source.intra.triggers.file"),
Integer.parseInt(resourceManager.getString("nb.max.triggers.source.intra")),
resourceManager.getString("phrase.separator"));
itl_source_p = new TriggersProcessor(itl_source);
itl_source_target =
new Triggers(
resourceManager.getString("source.target.inter.triggers.file"),
Integer.parseInt(resourceManager.getString("nb.max.triggers.source.target.inter")),
resourceManager.getString("phrase.separator"));
itl_source_target_p = new TriggersProcessor(itl_source_target);
}
/*
* End modification for Triggers
*/
// read in each line from the source and target files
// create a sentence from each
// process each sentence
// run the features on the sentences
while ((lineSource != null) && (lineTarget != null)) {
// lineSource = lineSource.trim().substring(lineSource.indexOf(" ")).replace("+", "");
sourceSent = new Sentence(lineSource, sentCount);
targetSent = new Sentence(lineTarget, sentCount);
// System.out.println("Processing sentence "+sentCount);
// System.out.println("SORCE: " + sourceSent.getText());
// System.out.println("TARGET: " + targetSent.getText());
if (posSourceExists) {
posSourceProc.processSentence(sourceSent);
}
if (posTargetExists) {
posTargetProc.processSentence(targetSent);
}
sourceSent.computeNGrams(3);
targetSent.computeNGrams(3);
pplProcSource.processNextSentence(sourceSent);
pplProcTarget.processNextSentence(targetSent);
if (!isBaseline) {
pplPosTarget.processNextSentence(targetSent);
}
// lefterav: Parse code here
if (bp) {
sourceParserProcessor.processNextSentence(sourceSent);
targetParserProcessor.processNextSentence(targetSent);
}
if (tm) {
sourceTopicDistributionProcessor.processNextSentence(sourceSent);
targetTopicDistributionProcessor.processNextSentence(targetSent);
}
// modified by David
if (tr) {
itl_source_p.processNextSentence(sourceSent);
itl_target_p.processNextSentence(targetSent);
itl_source_target_p.processNextParallelSentences(sourceSent, targetSent);
}
// end modification by David
// MQM kicks in
MQMManager.getInstance().processNextParallelSentences(sourceSent, targetSent);
// Ergun
if (featureManager.hasFeature("1700")) {
sourceLuc.processNextSentence(sourceSent);
targetLuc.processNextSentence(targetSent);
}
++sentCount;
output.write(featureManager.runFeatures(sourceSent, targetSent));
output.newLine();
lineSource = brSource.readLine();
lineTarget = brTarget.readLine();
}
if (posSource != null) {
posSource.close();
}
if (posTarget != null) {
posTarget.close();
}
brSource.close();
brTarget.close();
output.close();
Logger.close();
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* constructs the folders required by the application. These are, typically: <br>
*
* <ul>
* <li>/input and subfolders
* <ul>
* <li>/input/<i>sourceLang</i>, /input/<i>targetLang</i> (for storing the results of
* processing the input files with various tools, such as pos tagger, transliterator,
* morphological analyser),<br>
* <li>/input/systems/<i>systemName</i> (for storing system specific resources - for
* example, the compiled and processed word lattices in the case of the IBM system
* </ul>
* <li>/output (for storing the resulting feature files),
* </ul>
*/
public void constructFolders() {
File f = new File(input);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + sourceLang);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + targetLang);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + targetLang + File.separator + "temp");
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
/*
f = new File(input + File.separator + "systems");
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + "systems" + File.separator
+ "IBM");
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
f = new File(input + File.separator + "systems" + File.separator
+ "MOSES");
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
*/
String output = resourceManager.getString("output");
f = new File(output);
if (!f.exists()) {
f.mkdirs();
System.out.println("folder created " + f.getPath());
}
if (featureManager.hasFeature("1700")) {
String lang_resources = workDir + File.separator + "lang_resources";
f = new File(lang_resources);
if (!f.exists()) {
System.out.println("For Lucene features, lang_resources are needed.");
System.exit(0);
}
String source_lang_resources = lang_resources + File.separator + sourceLang;
f = new File(source_lang_resources);
if (!f.exists()) {
System.out.println("For Lucene features, source lang_resources are needed.");
System.exit(0);
}
String source_lucene_path =
lang_resources + File.separator + sourceLang + File.separator + "luceneIndex";
f = new File(source_lucene_path);
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
String source_lucene_corpus = source_lucene_path + File.separator + sourceLang + ".corpus";
try {
Runtime.getRuntime()
.exec(
"ln -s "
+ workDir
+ File.separator
+ resourceManager.getString(sourceLang + ".corpus")
+ " "
+ source_lucene_corpus);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// Indexing the target
String target_lang_resources = lang_resources + File.separator + targetLang;
f = new File(target_lang_resources);
if (!f.exists()) {
System.out.println("For Lucene features, target lang_resources are needed.");
System.exit(0);
}
String target_lucene_path =
lang_resources + File.separator + targetLang + File.separator + "luceneIndex";
f = new File(target_lucene_path);
if (!f.exists()) {
f.mkdir();
System.out.println("folder created " + f.getPath());
}
String target_lucene_corpus = target_lucene_path + File.separator + targetLang + ".corpus";
try {
Runtime.getRuntime()
.exec(
"ln -s "
+ workDir
+ File.separator
+ resourceManager.getString(targetLang + ".corpus")
+ " "
+ target_lucene_corpus);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}