Exemplo n.º 1
0
 public void run() {
   int baseKey = (index * keyCount) % keys.size();
   for (int i = 0; i < requests; i++) {
     int keyIndex = (baseKey + (i % keyCount)) % keys.size();
     String key = keys.get(keyIndex);
     Transaction txn = fac.create();
     long start = System.currentTimeMillis();
     try {
       txn.begin();
       byte[] value = txn.read(key);
       txn.commit();
       timeElapsed += System.currentTimeMillis() - start;
       if (!silent.get()) {
         System.out.println("[Thread-" + index + "] " + key + ": " + new String(value));
       }
       success++;
     } catch (Exception e) {
       timeElapsed += System.currentTimeMillis() - start;
       if (!silent.get()) {
         System.out.println("[Thread-" + index + "] Error: " + e.getMessage());
       }
       failure++;
     }
   }
 }
Exemplo n.º 2
0
  public static void toLibSVM(List<TrainingSample<BxZoneLabel>> trainingElements, String filePath)
      throws IOException {
    BufferedWriter svmDataFile = null;
    try {
      FileWriter fstream = new FileWriter(filePath);
      svmDataFile = new BufferedWriter(fstream);
      for (TrainingSample<BxZoneLabel> elem : trainingElements) {
        if (elem.getLabel() == null) {
          continue;
        }
        svmDataFile.write(String.valueOf(elem.getLabel().ordinal()));
        svmDataFile.write(" ");

        Integer featureCounter = 1;
        for (Double value : elem.getFeatureVector().getValues()) {
          StringBuilder sb = new StringBuilder();
          Formatter formatter = new Formatter(sb, Locale.US);
          formatter.format("%d:%.5f", featureCounter++, value);
          svmDataFile.write(sb.toString());
          svmDataFile.write(" ");
        }
        svmDataFile.write("\n");
      }
      svmDataFile.close();
    } catch (Exception e) {
      System.err.println("Error: " + e.getMessage());
      return;
    } finally {
      if (svmDataFile != null) {
        svmDataFile.close();
      }
    }

    System.out.println("Done.");
  }
Exemplo n.º 3
0
 private void initConfigDecorator() {
   InputStream in =
       Q2.class
           .getClassLoader()
           .getResourceAsStream("META-INF/org/jpos/config/Q2-decorator.properties");
   try {
     if (in != null) {
       PropertyResourceBundle bundle = new PropertyResourceBundle(in);
       String ccdClass = bundle.getString("config-decorator-class");
       if (log != null) log.info("Initializing config decoration provider: " + ccdClass);
       decorator =
           (ConfigDecorationProvider) Q2.class.getClassLoader().loadClass(ccdClass).newInstance();
       decorator.initialize(getDeployDir());
     }
   } catch (IOException e) {
   } catch (Exception e) {
     if (log != null) log.error(e);
     else {
       e.printStackTrace();
     }
   } finally {
     if (in != null) {
       try {
         in.close();
       } catch (IOException e) {
       }
     }
   }
 }
  public static void main(String args[]) throws IOException {
    LoaderOptions options = LoaderOptions.parseArgs(args);
    try {
      SSTableLoader loader =
          new SSTableLoader(options.directory, new ExternalClient(options), options);
      SSTableLoader.LoaderFuture future = loader.stream(options.ignores);

      if (options.noProgress) {
        future.get();
      } else {
        ProgressIndicator indicator = new ProgressIndicator(future.getPendingFiles());
        indicator.start();
        System.out.println("");
        while (!future.isDone()) {
          if (indicator.printProgress()) {
            // We're done with streaming
            System.out.println("\nWaiting for targets to rebuild indexes ...");
            future.get();
            assert future.isDone();
          } else {
            try {
              Thread.sleep(1000L);
            } catch (Exception e) {
            }
          }
        }
      }

      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
Exemplo n.º 5
0
 private Hashtable<String, String> getJarManifestAttributes(String path) {
   Hashtable<String, String> h = new Hashtable<String, String>();
   JarInputStream jis = null;
   try {
     cp.appendln(Color.black, "Looking for " + path);
     InputStream is = getClass().getResourceAsStream(path);
     if (is == null) {
       if (!path.endsWith("/MIRC.jar")) {
         cp.appendln(Color.red, "...could not find it.");
       } else {
         cp.appendln(
             Color.black,
             "...could not find it. [OK, this is a " + programName + " installation]");
       }
       return null;
     }
     jis = new JarInputStream(is);
     Manifest manifest = jis.getManifest();
     h = getManifestAttributes(manifest);
   } catch (Exception ex) {
     ex.printStackTrace();
   }
   if (jis != null) {
     try {
       jis.close();
     } catch (Exception ignore) {
     }
   }
   return h;
 }
Exemplo n.º 6
0
  /** runs the GB features */
  public void runGB() {
    MTOutputProcessor mtop = null;

    if (gbMode == 1) gbXML = initialiseGBResources();

    String nbestSentPath =
        resourceManager.getString("input") + File.separator + targetLang + File.separator + "temp";
    String ngramExecPath = resourceManager.getString("tools.ngram.path");

    mtop = new MTOutputProcessor(gbXML, nbestSentPath, ngramExecPath, ngramSize);
    //		MorphAnalysisProcessor map = new MorphAnalysisProcessor(madaFile);

    File f = new File(sourceFile);
    String sourceFileName = f.getName();
    f = new File(targetFile);
    String targetFileName = f.getName();

    String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";

    String out = resourceManager.getString("output") + File.separator + getMod() + outputFileName;
    System.out.println("Output will be: " + out);

    String lineTarget;

    try {
      BufferedReader brSource = new BufferedReader(new FileReader(sourceFile));
      BufferedReader brTarget = new BufferedReader(new FileReader(targetFile));
      BufferedWriter output = new BufferedWriter(new FileWriter(out));

      ResourceManager.printResources();

      Sentence targetSent;
      Sentence sourceSent;
      int sentCount = 0;

      String lineSource;

      while (((lineSource = brSource.readLine()) != null)
          && ((lineTarget = brTarget.readLine()) != null)) {

        lineSource = lineSource.trim().substring(lineSource.indexOf(" "));
        sourceSent = new Sentence(lineSource, sentCount);
        targetSent = new Sentence(lineTarget, sentCount);

        // map.processNextSentence(sourceSent);
        mtop.processNextSentence(sourceSent);

        ++sentCount;
        output.write(featureManager.runFeatures(sourceSent, targetSent));
        output.write("\r\n");
      }
      brSource.close();
      brTarget.close();
      output.close();
      featureManager.printFeatureIndeces();
      Logger.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Exemplo n.º 7
0
  /**
   * Load the Configuration file with the values from the command line and config files, and place
   * stuff in the DistrubutedCacheService as needed
   *
   * @param conf The Configuration to populate
   * @param args The command line arguments
   * @throws Exception
   */
  public static void initalizeConf(Configuration conf, String[] args) throws Exception {
    // Parse the arguments and make sure the required args are there
    final CommandLine commandLine;
    final Options options = constructGnuOptions();
    try {
      commandLine = new GnuParser().parse(options, args);
    } catch (MissingOptionException ex) {
      HelpFormatter help = new HelpFormatter();
      help.printHelp("hadoop jar <jarFile> " + FrameworkDriver.class.getCanonicalName(), options);
      return;
    } catch (Exception ex) {
      ex.printStackTrace();
      return;
    }

    final String userConfFilePath = commandLine.getOptionValue("amino_config_file_path", "");
    final String aminoDefaultConfigPath = commandLine.getOptionValue("amino_default_config_path");
    final String baseDir = commandLine.getOptionValue("base_dir");

    stopOnFirstPhase = commandLine.hasOption("stop");

    // Set the base dir config value if it was provided.
    if (StringUtils.isNotEmpty(baseDir)) {
      conf.set(AminoConfiguration.BASE_DIR, baseDir);
    }

    conf.set(AminoConfiguration.DEFAULT_CONFIGURATION_PATH_KEY, aminoDefaultConfigPath);

    // create a single DistributedCacheService so that multiple cache entries are deduped.
    // Cache files are added after each config is loaded in case the the property value changes.
    final DistributedCacheService distributedCacheService = new DistributedCacheService();

    // 1. load AminoDefaults
    AminoConfiguration.loadAndMergeWithDefault(conf, true);
    distributedCacheService.addFilesToDistributedCache(conf);

    // 2. load user config files, allowing them to overwrite
    if (!StringUtils.isEmpty(userConfFilePath)) {
      for (String path : getUserConfigFiles(userConfFilePath)) {
        Configuration userConf = new Configuration(false);
        logger.info("Grabbing configuration information from: " + path);
        userConf.addResource(new FileInputStream(path));
        HadoopConfigurationUtils.mergeConfs(conf, userConf);
      }
    }
    distributedCacheService.addFilesToDistributedCache(conf);

    // 3. load command line arguments as properties, allowing them to overwrite
    final Properties propertyOverrides = commandLine.getOptionProperties("property_override");
    for (Object key : propertyOverrides.keySet()) {
      conf.set((String) key, (String) propertyOverrides.get(key));
    }
    distributedCacheService.addFilesToDistributedCache(conf);
  }
Exemplo n.º 8
0
  /**
   * converts a string into a valid CBR
   *
   * @param string the string to parse
   * @return the CBR to use
   * @throws Exception
   */
  private CBR parseCBR(String string) throws Exception {

    try {
      return (string == null) ? (null) : (new CBR(this)).parse(string);
    } catch (Exception e) {
      System.err.println("error parsing CBR, exiting.");
      e.printStackTrace();
      System.exit(5);
      return null;
    }
  }
Exemplo n.º 9
0
 // Take a tree of files starting in a directory in a zip file
 // and copy them to a disk directory, recreating the tree.
 private int unpackZipFile(
     File inZipFile, String directory, String parent, boolean suppressFirstPathElement) {
   int count = 0;
   if (!inZipFile.exists()) return count;
   parent = parent.trim();
   if (!parent.endsWith(File.separator)) parent += File.separator;
   if (!directory.endsWith(File.separator)) directory += File.separator;
   File outFile = null;
   try {
     ZipFile zipFile = new ZipFile(inZipFile);
     Enumeration zipEntries = zipFile.entries();
     while (zipEntries.hasMoreElements()) {
       ZipEntry entry = (ZipEntry) zipEntries.nextElement();
       String name = entry.getName().replace('/', File.separatorChar);
       if (name.startsWith(directory)) {
         if (suppressFirstPathElement) name = name.substring(directory.length());
         outFile = new File(parent + name);
         // Create the directory, just in case
         if (name.indexOf(File.separatorChar) >= 0) {
           String p = name.substring(0, name.lastIndexOf(File.separatorChar) + 1);
           File dirFile = new File(parent + p);
           dirFile.mkdirs();
         }
         if (!entry.isDirectory()) {
           System.out.println("Installing " + outFile);
           // Copy the file
           BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outFile));
           BufferedInputStream in = new BufferedInputStream(zipFile.getInputStream(entry));
           int size = 1024;
           int n = 0;
           byte[] b = new byte[size];
           while ((n = in.read(b, 0, size)) != -1) out.write(b, 0, n);
           in.close();
           out.flush();
           out.close();
           // Count the file
           count++;
         }
       }
     }
     zipFile.close();
   } catch (Exception e) {
     System.err.println("...an error occured while installing " + outFile);
     e.printStackTrace();
     System.err.println("Error copying " + outFile.getName() + "\n" + e.getMessage());
     return -count;
   }
   System.out.println(count + " files were installed.");
   return count;
 }
Exemplo n.º 10
0
 private boolean startLauncher(File dir) {
   try {
     Runtime rt = Runtime.getRuntime();
     ArrayList<String> command = new ArrayList<String>();
     command.add("java");
     command.add("-jar");
     command.add("Launcher.jar");
     String[] cmdarray = command.toArray(new String[command.size()]);
     Process proc = rt.exec(cmdarray, null, dir);
     return true;
   } catch (Exception ex) {
     System.err.println("Unable to start the Launcher program.\n" + ex.getMessage());
     return false;
   }
 }
Exemplo n.º 11
0
  private void updateLinuxServiceInstaller() {
    try {
      File dir = new File(directory, "CTP");
      if (suppressFirstPathElement) dir = dir.getParentFile();
      Properties props = new Properties();
      String ctpHome = dir.getAbsolutePath();
      cp.appendln(Color.black, "...CTP_HOME: " + ctpHome);
      ctpHome = ctpHome.replaceAll("\\\\", "\\\\\\\\");
      props.put("CTP_HOME", ctpHome);
      File javaHome = new File(System.getProperty("java.home"));
      String javaBin = (new File(javaHome, "bin")).getAbsolutePath();
      cp.appendln(Color.black, "...JAVA_BIN: " + javaBin);
      javaBin = javaBin.replaceAll("\\\\", "\\\\\\\\");
      props.put("JAVA_BIN", javaBin);

      File linux = new File(dir, "linux");
      File install = new File(linux, "ctpService-ubuntu.sh");
      cp.appendln(Color.black, "Linux service installer:");
      cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
      String bat = getFileText(install);
      bat = replace(bat, props); // do the substitutions
      bat = bat.replace("\r", "");
      setFileText(install, bat);

      // If this is an ISN installation, put the script in the correct place.
      String osName = System.getProperty("os.name").toLowerCase();
      if (programName.equals("ISN") && !osName.contains("windows")) {
        install = new File(linux, "ctpService-red.sh");
        cp.appendln(Color.black, "ISN service installer:");
        cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
        bat = getFileText(install);
        bat = replace(bat, props); // do the substitutions
        bat = bat.replace("\r", "");
        File initDir = new File("/etc/init.d");
        File initFile = new File(initDir, "ctpService");
        if (initDir.exists()) {
          setOwnership(initDir, "edge", "edge");
          setFileText(initFile, bat);
          initFile.setReadable(true, false); // everybody can read //Java 1.6
          initFile.setWritable(true); // only the owner can write //Java 1.6
          initFile.setExecutable(true, false); // everybody can execute //Java 1.6
        }
      }
    } catch (Exception ex) {
      ex.printStackTrace();
      System.err.println("Unable to update the Linux service ctpService.sh file");
    }
  }
Exemplo n.º 12
0
 public static String getLicensee() {
   InputStream is = Q2.class.getResourceAsStream(LICENSEE);
   ByteArrayOutputStream baos = new ByteArrayOutputStream();
   if (is != null) {
     BufferedReader br = new BufferedReader(new InputStreamReader(is));
     PrintStream p = new PrintStream(baos);
     p.println();
     p.println();
     try {
       while (br.ready()) p.println(br.readLine());
     } catch (Exception e) {
       e.printStackTrace(p);
     }
   }
   return baos.toString();
 }
  public static void main(String[] args) {
    Options options = KeyGenCLI.buildOptions();
    HelpFormatter helpFormatter = new HelpFormatter();
    CommandLineParser parser = new GnuParser();

    try {
      CommandLine commandLine = parser.parse(options, args);
      String service = commandLine.getOptionValue("service");
      String key = Crypto.generateAes128KeyWithSeed(service);

      if (key == null) {
        throw new Exception("Key was not generated!");
      }

      JSONObject keyObject = new JSONObject();
      keyObject.put("name", service);
      keyObject.put("data", key);

      System.out.println(String.format("Key: `%s`\n", key));

      System.out.println("Key object:");
      System.out.println(keyObject.toString());
    } catch (MissingOptionException e) {
      System.out.println("Missing required option(s)!");
      helpFormatter.printHelp(
          "\nKeyGenCLI",
          "This tool generates a unique key and prints the result.",
          options,
          "",
          true);
    } catch (UnrecognizedOptionException e) {
      System.out.println("Unrecognized option!");
      helpFormatter.printHelp(
          "\nKeyGenCLI",
          "This tool generates a unique key and prints the result.",
          options,
          "",
          true);
    } catch (ParseException e) {
      e.printStackTrace();
    } catch (JSONException e) {
      e.printStackTrace();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Exemplo n.º 14
0
  private void parseCmdLine(String[] args) {
    CommandLineParser parser = new PosixParser();

    Options options = new Options();
    options.addOption("v", "version", false, "Q2's version");
    options.addOption("d", "deploydir", true, "Deployment directory");
    options.addOption("r", "recursive", false, "Deploy subdirectories recursively");
    options.addOption("h", "help", false, "Usage information");
    options.addOption("C", "config", true, "Configuration bundle");
    options.addOption("e", "encrypt", true, "Encrypt configuration bundle");
    options.addOption("i", "cli", false, "Command Line Interface");
    options.addOption("c", "command", true, "Command to execute");

    try {
      CommandLine line = parser.parse(options, args);
      if (line.hasOption("v")) {
        displayVersion();
        System.exit(0);
      }
      if (line.hasOption("h")) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp("Q2", options);
        System.exit(0);
      }
      if (line.hasOption("c")) {
        cli = new CLI(this, line.getOptionValue("c"), line.hasOption("i"));
      } else if (line.hasOption("i")) cli = new CLI(this, null, true);

      String dir = DEFAULT_DEPLOY_DIR;
      if (line.hasOption("d")) {
        dir = line.getOptionValue("d");
      }
      recursive = line.hasOption("r");
      this.deployDir = new File(dir);
      if (line.hasOption("C")) deployBundle(new File(line.getOptionValue("C")), false);
      if (line.hasOption("e")) deployBundle(new File(line.getOptionValue("e")), true);
    } catch (MissingArgumentException e) {
      System.out.println("ERROR: " + e.getMessage());
      System.exit(1);
    } catch (Exception e) {
      e.printStackTrace();
      System.exit(1);
    }
  }
Exemplo n.º 15
0
  @SuppressWarnings("unused")
  private static void callFederatorAsWS()
      throws MalformedURLException, SNAAExceptionException, AuthenticationExceptionException {

    SNAA port = WisebedServiceHelper.getSNAAService("http://*****:*****@wisebed1.itm.uni-luebeck.de");
    auth1.setPassword("xxx");
    authTriples.add(auth1);

    AuthenticationTriple auth2 = new AuthenticationTriple();
    auth2.setUrnPrefix("urn:wisebed:dummy1");
    auth2.setUsername("*****@*****.**");
    auth2.setPassword("xxx");
    authTriples.add(auth2);

    AuthenticationTriple auth3 = new AuthenticationTriple();
    auth3.setUrnPrefix("urn:wisebed:dummy2");
    auth3.setUsername("*****@*****.**");
    auth3.setPassword("xxx");
    authTriples.add(auth3);

    AuthenticationTriple auth4 = new AuthenticationTriple();
    auth4.setUrnPrefix("urn:wisebed:shib2");
    auth4.setUsername("*****@*****.**");
    auth4.setPassword("xxx");
    authTriples.add(auth4);

    try {
      List<SecretAuthenticationKey> keys = port.authenticate(authTriples);
      log.info("Got authentication keys: " + keys);
      Action action = new Action();
      action.setAction("sth");
      boolean b = port.isAuthorized(new ArrayList<SecretAuthenticationKey>(), action);
      log.info("Is authorized: " + b);
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Exemplo n.º 16
0
  private void updateWindowsServiceInstaller() {
    try {
      File dir = new File(directory, "CTP");
      if (suppressFirstPathElement) dir = dir.getParentFile();
      File windows = new File(dir, "windows");
      File install = new File(windows, "install.bat");
      cp.appendln(Color.black, "Windows service installer:");
      cp.appendln(Color.black, "...file: " + install.getAbsolutePath());
      String bat = getFileText(install);
      Properties props = new Properties();
      String home = dir.getAbsolutePath();
      cp.appendln(Color.black, "...home: " + home);
      home = home.replaceAll("\\\\", "\\\\\\\\");
      props.put("home", home);

      bat = replace(bat, props);
      setFileText(install, bat);
    } catch (Exception ex) {
      ex.printStackTrace();
      System.err.println("Unable to update the windows service install.barfile.");
    }
  }
Exemplo n.º 17
0
 /**
  * runs the part of speech tagger
  *
  * @param file input file
  * @param lang language
  * @param type source or target
  * @return path to the output file of the POS tagger
  */
 public String runPOS(String file, String lang, String type) {
   String posName = resourceManager.getString(lang + ".postagger");
   String langResPath = input + File.separator + lang;
   File f = new File(file);
   String absoluteSourceFilePath = f.getAbsolutePath();
   String fileName = f.getName();
   String relativeFilePath = langResPath + File.separator + fileName + ".pos";
   String absoluteOutputFilePath = (new File(relativeFilePath)).getAbsolutePath();
   String posSourceTaggerPath = resourceManager.getString(lang + ".postagger.exePath");
   String outPath = "";
   try {
     Class c = Class.forName(posName);
     PosTagger tagger = (PosTagger) c.newInstance();
     tagger.setParameters(
         type, posName, posSourceTaggerPath, absoluteSourceFilePath, absoluteOutputFilePath);
     PosTagger.ForceRun(forceRun);
     outPath = tagger.run();
   } catch (Exception e) {
     e.printStackTrace();
   }
   // returns the path of the output file; this is for convenience only so
   // we do't have to calculate it again
   return outPath;
 }
Exemplo n.º 18
0
  public static void main(String args[]) {
    Config.setClientMode(true);
    LoaderOptions options = LoaderOptions.parseArgs(args);
    OutputHandler handler = new OutputHandler.SystemOutput(options.verbose, options.debug);
    SSTableLoader loader =
        new SSTableLoader(
            options.directory,
            new ExternalClient(
                options.hosts,
                options.rpcPort,
                options.user,
                options.passwd,
                options.transportFactory,
                options.storagePort,
                options.sslStoragePort,
                options.serverEncOptions),
            handler);
    DatabaseDescriptor.setStreamThroughputOutboundMegabitsPerSec(options.throttle);
    StreamResultFuture future = null;
    try {
      if (options.noProgress) future = loader.stream(options.ignores);
      else future = loader.stream(options.ignores, new ProgressIndicator());
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (e.getCause() != null) System.err.println(e.getCause());
      if (options.debug) e.printStackTrace(System.err);
      else System.err.println("Run with --debug to get full stack trace or --help to get help.");
      System.exit(1);
    }

    handler.output(String.format("Streaming session ID: %s", future.planId));

    try {
      future.get();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println("Streaming to the following hosts failed:");
      System.err.println(loader.getFailedHosts());
      System.err.println(e);
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
Exemplo n.º 19
0
  public void runAll() {
    File f = new File(sourceFile);
    String sourceFileName = f.getName();
    f = new File(targetFile);
    String targetFileName = f.getName();
    String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";
    String out = resourceManager.getString("output") + File.separator + outputFileName;
    System.out.println("Output will be: " + out);

    MTOutputProcessor mtop = null;

    if (gbMode == 1) gbXML = initialiseGBResources();

    String nbestSentPath =
        resourceManager.getString("input") + File.separator + targetLang + File.separator + "temp";
    String ngramExecPath = resourceManager.getString("tools.ngram.path");

    mtop = new MTOutputProcessor(gbXML, nbestSentPath, ngramExecPath, ngramSize);

    // wlv.mt.features.coherence.Coherence coh = new wlv.mt.features.coherence.Coherence(
    //		getTargetFile());

    String pplSourcePath =
        resourceManager.getString("input")
            + File.separator
            + sourceLang
            + File.separator
            + sourceFileName
            + resourceManager.getString("tools.ngram.output.ext");
    String pplTargetPath =
        resourceManager.getString("input")
            + File.separator
            + targetLang
            + File.separator
            + targetFileName
            + resourceManager.getString("tools.ngram.output.ext");

    String pplPOSTargetPath =
        resourceManager.getString("input")
            + File.separator
            + targetLang
            + File.separator
            + targetFileName
            + PosTagger.getXPOS()
            + resourceManager.getString("tools.ngram.output.ext");
    runNGramPPL();

    PPLProcessor pplProcSource =
        new PPLProcessor(pplSourcePath, new String[] {"logprob", "ppl", "ppl1"});
    PPLProcessor pplProcTarget =
        new PPLProcessor(pplTargetPath, new String[] {"logprob", "ppl", "ppl1"});

    FileModel fm = new FileModel(sourceFile, resourceManager.getString(sourceLang + ".corpus"));
    String sourcePosOutput = runPOS(sourceFile, sourceLang, "source");
    String targetPosOutput = runPOS(targetFile, targetLang, "target");

    String targetPPLPos = runNGramPPLPos(targetPosOutput + PosTagger.getXPOS());
    System.out.println("---------TARGET PPLPOS: " + targetPPLPos);
    PPLProcessor pplPosTarget =
        new PPLProcessor(targetPPLPos, new String[] {"poslogprob", "posppl", "posppl1"});

    loadGiza();
    processNGrams();

    try {
      BufferedReader brSource = new BufferedReader(new FileReader(sourceFileName));
      BufferedReader brTarget = new BufferedReader(new FileReader(targetFileName));
      BufferedWriter output = new BufferedWriter(new FileWriter(out));
      BufferedReader posSource = null;
      BufferedReader posTarget = null;
      boolean posSourceExists = ResourceManager.isRegistered("sourcePosTagger");
      boolean posTargetExists = ResourceManager.isRegistered("targetPosTagger");
      POSProcessor posSourceProc = null;
      POSProcessor posTargetProc = null;
      if (posSourceExists) {
        posSourceProc = new POSProcessor(sourcePosOutput);
        posSource =
            new BufferedReader(
                new InputStreamReader(new FileInputStream(sourcePosOutput), "utf-8"));
      }
      if (posTargetExists) {
        posTargetProc = new POSProcessor(targetPosOutput);
        posTarget = new BufferedReader(new InputStreamReader(new FileInputStream(targetPosOutput)));
      }
      ResourceManager.printResources();

      Sentence targetSent;
      // HACK
      Sentence sourceSent;
      int sentCount = 0;
      // HACK
      String lineSource = brSource.readLine();
      String lineTarget = brTarget.readLine();
      // HACK
      int result;

      while ((lineSource != null) && (lineTarget != null)) {

        // the MADA-tokenised files contain start each sentence with the setence ID. We put it there
        // (why?) - no we've got to remove it

        lineSource = lineSource.trim().substring(lineSource.indexOf(" ")).replace("+", "");
        sourceSent = new Sentence(lineSource, sentCount);
        targetSent = new Sentence(lineTarget, sentCount);
        System.out.println("Processing sentence " + sentCount);
        if (posSourceExists) {

          posSourceProc.processSentence(sourceSent);
        }
        if (posTargetExists) {

          posTargetProc.processSentence(targetSent);
        }

        sourceSent.computeNGrams(3);
        targetSent.computeNGrams(3);

        pplProcSource.processNextSentence(sourceSent);

        pplProcTarget.processNextSentence(targetSent);

        pplPosTarget.processNextSentence(targetSent);

        //				coh.processNextSentence(targetSent);

        mtop.processNextSentence(sourceSent);

        ++sentCount;
        output.write(featureManager.runFeatures(sourceSent, targetSent));
        output.write("\r\n");

        lineSource = brSource.readLine();
        lineTarget = brTarget.readLine();
      }
      //		featureManager.printFeatureIndeces();
      if (posSource != null) {
        posSource.close();
      }
      if (posTarget != null) {
        posTarget.close();
      }

      brSource.close();
      brTarget.close();
      output.close();

      Logger.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Exemplo n.º 20
0
  /**
   * Performs some basic processing of the input source and target files For English, this consists
   * of converting the input to lower case and tokenizing For Arabic, this consists of
   * transliteration and tokenization. Please note that the current tools used for tokenizing Arabic
   * also perform POS tagging and morphological analysis Although we could separate the tokenization
   * process from the more in-depth text analysis performed by these tools, for efficiency reasons
   * this is not desirable The input files are also copied to the /input folder. This is necessary
   * because the MADA analyser produces its output in the same folder as the input file, which may
   * cause problems if the right access rights are not available for that particular folder
   */
  private static void preprocessing() {
    String sourceInputFolder = input + File.separator + sourceLang;
    String targetInputFolder = input + File.separator + targetLang;
    File origSourceFile = new File(sourceFile);
    File inputSourceFile = new File(sourceInputFolder + File.separator + origSourceFile.getName());

    System.out.println("source input:" + sourceFile);
    System.out.println("target input:" + targetFile);
    File origTargetFile = new File(targetFile);
    File inputTargetFile = new File(targetInputFolder + File.separator + origTargetFile.getName());
    try {
      System.out.println("copying input to " + inputSourceFile.getPath());
      copyFile(origSourceFile, inputSourceFile);
      System.out.println("copying input to " + inputTargetFile.getPath());
      copyFile(origTargetFile, inputTargetFile);
    } catch (Exception e) {
      e.printStackTrace();
    }

    // run tokenizer for source (English)
    System.out.println("running tokenizer");

    String src_abbr = "";
    if (sourceLang.equalsIgnoreCase("english")) src_abbr = "en";
    else if (sourceLang.equalsIgnoreCase("spanish")) src_abbr = "es";
    else if (sourceLang.equalsIgnoreCase("french")) src_abbr = "fr";
    else if (sourceLang.equalsIgnoreCase("german")) src_abbr = "de";
    else if (targetLang.equalsIgnoreCase("dutch")) src_abbr = "nl";
    else if (targetLang.equalsIgnoreCase("portuguese")) src_abbr = "pt";
    else if (targetLang.equalsIgnoreCase("czech")) tgt_abbr = "cs";
    else System.out.println("Don't recognise the source language");

    String tgt_abbr = "";
    if (targetLang.equalsIgnoreCase("english")) tgt_abbr = "en";
    else if (targetLang.equalsIgnoreCase("spanish")) tgt_abbr = "es";
    else if (targetLang.equalsIgnoreCase("french")) tgt_abbr = "fr";
    else if (targetLang.equalsIgnoreCase("german")) tgt_abbr = "de";
    else if (targetLang.equalsIgnoreCase("dutch")) tgt_abbr = "nl";
    else if (targetLang.equalsIgnoreCase("portuguese")) tgt_abbr = "pt";
    else if (targetLang.equalsIgnoreCase("czech")) tgt_abbr = "cs";
    else System.out.println("Don't recognise the target language");

    String truecasePath = "";
    if (null != resourceManager.getProperty(sourceLang + ".lowercase")) {
      truecasePath = resourceManager.getProperty(sourceLang + ".lowercase") + " -q ";
    } else {
      truecasePath =
          resourceManager.getString(sourceLang + ".truecase")
              + " --model "
              + resourceManager.getString(sourceLang + ".truecase.model");
    }
    Tokenizer enTok =
        new Tokenizer(
            inputSourceFile.getPath(),
            inputSourceFile.getPath() + ".tok",
            truecasePath,
            resourceManager.getString(sourceLang + ".tokenizer"),
            src_abbr,
            forceRun);

    // Tokenizer enTok = new Tokenizer(inputSourceFile.getPath(), inputSourceFile.getPath() +
    // ".tok", resourceManager.getString("english.lowercase"),
    // resourceManager.getString("english.tokenizer"), "en", forceRun);
    enTok.run();
    sourceFile = enTok.getTok();
    System.out.println(sourceFile);
    // run tokenizer for target (Spanish)
    System.out.println("running tokenizer");
    //        Tokenizer esTok = new Tokenizer(inputTargetFile.getPath(), inputTargetFile.getPath() +
    // ".tok", resourceManager.getString("spanish.lowercase"),
    // resourceManager.getString("spanish.tokenizer"), "es", forceRun);

    if (null != resourceManager.getProperty(targetLang + ".lowercase")) {
      truecasePath = resourceManager.getProperty(targetLang + ".lowercase") + " -q ";
    } else {
      truecasePath =
          resourceManager.getString(targetLang + ".truecase")
              + " --model "
              + resourceManager.getString(targetLang + ".truecase.model");
    }
    Tokenizer esTok =
        new Tokenizer(
            inputTargetFile.getPath(),
            inputTargetFile.getPath() + ".tok",
            truecasePath,
            resourceManager.getString(targetLang + ".tokenizer"),
            tgt_abbr,
            forceRun);

    esTok.run();
    targetFile = esTok.getTok();
    System.out.println(targetFile);

    // Normalize files to avoid strange characters in UTF-8 that may break the PoS tagger
    // normalize_utf8();
  }
Exemplo n.º 21
0
 public void run() {
   started = true;
   Thread.currentThread().setName("Q2-" + getInstanceId().toString());
   try {
     /*
      * The following code determines whether a MBeanServer exists
      * already. If so then the first one in the list is used.
      * I have not yet find a way to interrogate the server for
      * information other than MBeans so to pick a specific one
      * would be difficult.
      */
     ArrayList mbeanServerList = MBeanServerFactory.findMBeanServer(null);
     if (mbeanServerList.isEmpty()) {
       server = MBeanServerFactory.createMBeanServer(JMX_NAME);
     } else {
       server = (MBeanServer) mbeanServerList.get(0);
     }
     final ObjectName loaderName = new ObjectName(Q2_CLASS_LOADER);
     try {
       loader =
           (QClassLoader)
               java.security.AccessController.doPrivileged(
                   new java.security.PrivilegedAction() {
                     public Object run() {
                       return new QClassLoader(server, libDir, loaderName, mainClassLoader);
                     }
                   });
       server.registerMBean(loader, loaderName);
       loader = loader.scan(false);
     } catch (Throwable t) {
       if (log != null) log.error("initial-scan", t);
       else t.printStackTrace();
     }
     factory = new QFactory(loaderName, this);
     initSystemLogger();
     addShutdownHook();
     q2Thread = Thread.currentThread();
     q2Thread.setContextClassLoader(loader);
     if (cli != null) cli.start();
     initConfigDecorator();
     for (int i = 1; !shutdown; i++) {
       try {
         boolean forceNewClassLoader = scan();
         QClassLoader oldClassLoader = loader;
         loader = loader.scan(forceNewClassLoader);
         if (loader != oldClassLoader) {
           oldClassLoader = null; // We want't this to be null so it gets GCed.
           System.gc(); // force a GC
           log.info(
               "new classloader ["
                   + Integer.toString(loader.hashCode(), 16)
                   + "] has been created");
         }
         deploy();
         checkModified();
         relax(SCAN_INTERVAL);
         if (i % (3600000 / SCAN_INTERVAL) == 0) logVersion();
       } catch (Throwable t) {
         log.error("start", t);
         relax();
       }
     }
     undeploy();
     try {
       server.unregisterMBean(loaderName);
     } catch (InstanceNotFoundException e) {
       log.error(e);
     }
     if (decorator != null) {
       decorator.uninitialize();
     }
     if (exit && !shuttingDown) System.exit(0);
   } catch (Exception e) {
     if (log != null) log.error(e);
     else e.printStackTrace();
     System.exit(1);
   }
 }
Exemplo n.º 22
0
  /** loads [additional] policies from commandline (either -p or -f) */
  private void loadCLPolicies() {
    // we already checked to make sure we have one of the options avaliable
    File pLoc = null;
    PolicyObject p = null;

    if (genProps.getProperty("p3pLocation", null) != null) {
      pLoc = new File(genProps.getProperty("p3pLocation"));
      if (!pLoc.exists()) {
        System.err.println(
            "no file found at p3p policy location specified by the -p3p option: "
                + genProps.getProperty("p3pLocation"));
        System.err.println("current location is " + System.getProperty("user.dir"));
        System.exit(1);
      }
      try {
        p = (new P3PParser()).parse(pLoc.getAbsolutePath());
        if (p.getContextDomain() == null) {
          p.setContext(
              new Context(
                  new Date(System.currentTimeMillis()),
                  new Date(System.currentTimeMillis()),
                  genProps.getProperty("p3pLocation")));
        }
        pdb.addPolicy(p);
      } catch (Exception e) {
        System.err.println("Error with parsing or database");
        e.printStackTrace();
        // System.exit(5);
      }
    }
    if (genProps.getProperty("p3pDirLocation", null) != null) {
      pLoc = new File(genProps.getProperty("p3pDirLocation"));
      File[] pfiles = pLoc.listFiles();

      // System.err.println("pfiles for p3pDirLocation: "+pfiles);
      for (int i = 0; i < pfiles.length; i++) {

        pLoc = (pfiles[i]);
        if (!pLoc.exists()) {
          System.err.println(
              "no file found at p3p policy location specified by the -p3pDirLocation option, "
                  + genProps.getProperty("p3pDirLocation"));
          System.exit(1);
        }
        try {

          p = (new P3PParser()).parse(pLoc.getAbsolutePath());
          if (p.getContext().getDomain() == null) {
            p.setContext(
                new Context(
                    new Date(System.currentTimeMillis()),
                    new Date(System.currentTimeMillis()),
                    pfiles[i].getAbsolutePath()));
          }
          pdb.addPolicy(p);
        } catch (Exception e) {
          System.err.println("Einar needs to fix this parsing error.");
          e.printStackTrace();
        }
      }
    }
  }
Exemplo n.º 23
0
  public static void main(String args[]) throws IOException {
    Options options = Options.parseArgs(args);
    try {
      // load keyspace descriptions.
      DatabaseDescriptor.loadSchemas(false);

      if (Schema.instance.getCFMetaData(options.keyspace, options.cf) == null)
        throw new IllegalArgumentException(
            String.format("Unknown keyspace/columnFamily %s.%s", options.keyspace, options.cf));

      Keyspace keyspace = Keyspace.openWithoutSSTables(options.keyspace);
      ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(options.cf);

      OutputHandler handler = new OutputHandler.SystemOutput(false, options.debug);
      Directories.SSTableLister lister = cfs.directories.sstableLister();
      if (options.snapshot != null) lister.onlyBackups(true).snapshots(options.snapshot);
      else lister.includeBackups(false);

      Collection<SSTableReader> readers = new ArrayList<SSTableReader>();

      // Upgrade sstables
      for (Map.Entry<Descriptor, Set<Component>> entry : lister.list().entrySet()) {
        Set<Component> components = entry.getValue();
        if (!components.contains(Component.DATA) || !components.contains(Component.PRIMARY_INDEX))
          continue;

        try {
          SSTableReader sstable =
              SSTableReader.openNoValidation(entry.getKey(), components, cfs.metadata);
          if (sstable.descriptor.version.equals(Descriptor.Version.CURRENT)) continue;
          readers.add(sstable);
        } catch (Exception e) {
          JVMStabilityInspector.inspectThrowable(e);
          System.err.println(String.format("Error Loading %s: %s", entry.getKey(), e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);

          continue;
        }
      }

      int numSSTables = readers.size();
      handler.output("Found " + numSSTables + " sstables that need upgrading.");

      for (SSTableReader sstable : readers) {
        try {
          Upgrader upgrader = new Upgrader(cfs, sstable, handler);
          upgrader.upgrade();

          if (!options.keepSource) {
            // Remove the sstable (it's been copied by upgrade)
            System.out.format("Deleting table %s.%n", sstable.descriptor.baseFilename());
            sstable.markObsolete();
            sstable.selfRef().release();
          }
        } catch (Exception e) {
          System.err.println(String.format("Error upgrading %s: %s", sstable, e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      CompactionManager.instance.finishCompactionsAndShutdown(5, TimeUnit.MINUTES);
      SSTableDeletingTask.waitForDeletions();
      System.exit(0);
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
Exemplo n.º 24
0
  public static void main(String args[]) throws IOException {
    Options options = Options.parseArgs(args);
    try {
      // load keyspace descriptions.
      DatabaseDescriptor.loadSchemas();

      String ksName = null;
      String cfName = null;
      Map<Descriptor, Set<Component>> parsedFilenames = new HashMap<Descriptor, Set<Component>>();
      for (String filename : options.filenames) {
        File file = new File(filename);
        if (!file.exists()) {
          System.out.println("Skipping inexisting file " + file);
          continue;
        }

        Pair<Descriptor, Component> pair =
            SSTable.tryComponentFromFilename(file.getParentFile(), file.getName());
        if (pair == null) {
          System.out.println("Skipping non sstable file " + file);
          continue;
        }
        Descriptor desc = pair.left;

        if (ksName == null) ksName = desc.ksname;
        else if (!ksName.equals(desc.ksname))
          throw new IllegalArgumentException("All sstables must be part of the same keyspace");

        if (cfName == null) cfName = desc.cfname;
        else if (!cfName.equals(desc.cfname))
          throw new IllegalArgumentException("All sstables must be part of the same column family");

        Set<Component> components =
            new HashSet<Component>(
                Arrays.asList(
                    new Component[] {
                      Component.DATA,
                      Component.PRIMARY_INDEX,
                      Component.FILTER,
                      Component.COMPRESSION_INFO,
                      Component.STATS
                    }));

        Iterator<Component> iter = components.iterator();
        while (iter.hasNext()) {
          Component component = iter.next();
          if (!(new File(desc.filenameFor(component)).exists())) iter.remove();
        }
        parsedFilenames.put(desc, components);
      }

      if (ksName == null || cfName == null) {
        System.err.println("No valid sstables to split");
        System.exit(1);
      }

      // Do not load sstables since they might be broken
      Table table = Table.openWithoutSSTables(ksName);
      ColumnFamilyStore cfs = table.getColumnFamilyStore(cfName);

      String snapshotName = "pre-split-" + System.currentTimeMillis();

      List<SSTableReader> sstables = new ArrayList<SSTableReader>();
      for (Map.Entry<Descriptor, Set<Component>> fn : parsedFilenames.entrySet()) {
        try {
          SSTableReader sstable =
              SSTableReader.openNoValidation(fn.getKey(), fn.getValue(), cfs.metadata);
          sstables.add(sstable);

          if (options.snapshot) {
            File snapshotDirectory =
                Directories.getSnapshotDirectory(sstable.descriptor, snapshotName);
            sstable.createLinks(snapshotDirectory.getPath());
          }

        } catch (Exception e) {
          System.err.println(String.format("Error Loading %s: %s", fn.getKey(), e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      if (options.snapshot)
        System.out.println(
            String.format("Pre-split sstables snapshotted into snapshot %s", snapshotName));

      cfs.getDataTracker().markCompacting(sstables);
      for (SSTableReader sstable : sstables) {
        try {
          new SSTableSplitter(cfs, sstable, options.sizeInMB).split();

          // Remove the sstable
          sstable.markCompacted();
          sstable.releaseReference();
        } catch (Exception e) {
          System.err.println(String.format("Error splitting %s: %s", sstable, e.getMessage()));
          if (options.debug) e.printStackTrace(System.err);
        }
      }
      SSTableDeletingTask.waitForDeletions();
      System.exit(0); // We need that to stop non daemonized threads
    } catch (Exception e) {
      System.err.println(e.getMessage());
      if (options.debug) e.printStackTrace(System.err);
      System.exit(1);
    }
  }
Exemplo n.º 25
0
  public void parse() {
    PerfPrediction predict = new PerfPrediction();
    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = null;
    try {
      cmd = parser.parse(options, args);

      if (cmd.hasOption("h")) {
        help();
      }

      if (cmd.getOptions().length == 0) {
        try (BufferedReader reader = new BufferedReader(new InputStreamReader(System.in))) {
          if (reader.ready()) {
            predict.add(reader, "");
            predict.printFixedValues();
          } else {
            help();
          }
        } catch (Exception e) {
          e.printStackTrace();
        }
        return;
      }

      if (cmd.hasOption("cs") && cmd.getOptionValue("cs").equals("sqlite")) {
        if (cmd.hasOption("f")) {
          File location = new File(cmd.getOptionValue("f"));
          if (location.isDirectory()) {
            setSQLitePredictionScenarioNo(location);
            if (cmd.hasOption("im")) {
              if (isValidMode(cmd.getOptionValue("im"))) {
                if (cmd.hasOption("pm")) {
                  if (isValidMode(cmd.getOptionValue("pm"))) {
                    this.INPUT_MODE = cmd.getOptionValue("im");
                    this.PREDICT_MODE = cmd.getOptionValue("pm");
                    try {
                      predict.addPrediction(location, this.INPUT_MODE);
                    } catch (Exception e) {
                      e.printStackTrace();
                    }
                    predict.predict(this.PREDICT_MODE, getSQLiteDir(), location);

                  } else {
                    log.log(
                        Level.SEVERE, "invalid mode for option '-pm " + validModesToString() + "'");
                  }
                } else {
                  log.log(
                      Level.SEVERE, "missing prediction mode '-pm " + validModesToString() + "'");
                }
              } else {
                log.log(Level.SEVERE, "invalid mode for option '-im " + validModesToString() + "'");
              }
            } else {
              log.log(Level.SEVERE, "missing input mode '-im " + validModesToString() + "'");
            }

          } else {
            log.log(
                Level.SEVERE,
                "'-f <location>' does not point to a folder",
                cmd.getOptionValue("f"));
          }
        } else {
          log.log(Level.SEVERE, "Use option '-f <location>'", cmd.getOptionValue("f"));
        }
      }
      if (cmd.hasOption("rc")) {
        GenerateRandomConfig.generateConfigs(cmd.getOptionValue("rc"));
      }
      if (cmd.hasOption("fo")) {
        try (BufferedReader reader = new BufferedReader(new FileReader(cmd.getOptionValue("fo")))) {
          predict.add(reader, "");
          predict.printFixedValues();
        } catch (Exception e) {
          e.printStackTrace();
        }
      }
      if (cmd.hasOption("md")) {
        Accumulator.enableMedian();
      }
      if (cmd.hasOption("acc")) {
        Accumulator.start(new File(cmd.getOptionValue("acc")));
      }

    } catch (ParseException e) {
      log.log(Level.SEVERE, "Failed to parse comand line properties", e);
      help();
    }
  }
Exemplo n.º 26
0
  public static void main(String[] args) {
    options = new Options();
    options.addOption("h", "help", false, "display this help message");
    options.addOption("v", "version", false, "display the version number");
    options.addOption("i", "importer", true, "run the importer");
    options.addOption("w", "webapi", true, "run the web api server");
    options.addOption("s", "simulator", true, "run the simulator");
    options.addOption("c", "converter", true, "run the converter");
    options.addOption("u", "usage", false, "display heap usage");
    options.addOption("", "host", true, "Hostname for REST server (e.g.: 'http://199.116.235.225'");

    // Return value
    int r = 0;

    try {
      CommandLineParser parser = new BasicParser();
      CommandLine line = parser.parse(options, args);

      //// Options that don't need any requirements should be listed first
      if (line.hasOption("h")) {
        // Help
        print_help();
        return;
      }

      if (line.hasOption("u")) {
        // Heap Usage
        print_heap_usage();
        return;
      }

      //// Options that require arguments
      // Check that only one of the importer, webapi, or simulator params are set
      boolean[] proper_options = {
        line.hasOption("i"), line.hasOption("w"), line.hasOption("s"), line.hasOption("c")
      };
      int proper_option_count = 0;
      for (boolean proper_option : proper_options) {
        if (proper_option) {
          proper_option_count += 1;
        }
      }
      if (proper_option_count != 1) {
        print_help();
        return;
      }

      if (line.hasOption("i")) {
        // Run the importer
        try {
          Importer.import_from_file(line.getOptionValue("i"));
        } catch (XMLStreamException e) {
          e.printStackTrace();
        }
      } else if (line.hasOption("w")) {
        // Run the webapi
        String raw_port = line.getOptionValue("w");
        WebApi.start_web_api(raw_port);
      } else if (line.hasOption("s")) {
        // Run the simulator
        r = Simulator.run(line.getOptionValue('s'));
        if (r != 0) {
          // Kiewan complains about this, but I disagree with Kiewan.
          // If the program failed, I should be able to return a non-0 exit code.
          System.exit(r);
        }
      } else if (line.hasOption("c")) {
        if (!line.hasOption("host")) {
          System.err.println("Run converter with the option --host point to the REST API");
        }
        // Run the converter
        Converter.run(line.getOptionValue('c'), line.getOptionValue("host"));
      }
    } catch (ParseException exp) {
      System.err.println("Parsing failed.  Reason: " + exp.getMessage());
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
Exemplo n.º 27
0
  /** runs the BB features */
  public void runBB() {
    File f = new File(sourceFile);
    String sourceFileName = f.getName();
    f = new File(targetFile);
    String targetFileName = f.getName();
    String outputFileName = sourceFileName + "_to_" + targetFileName + ".out";

    File file = new File(resourceManager.getString("output"));
    if (!file.exists()) {
      System.err.println("Creating dir: " + resourceManager.getString("output"));
      Logger.log("Creating dir: " + resourceManager.getString("output"));
      file.mkdirs();
    } else {
      Logger.log("output dir exists: " + resourceManager.getString("output"));
    }

    String out = resourceManager.getString("output") + File.separator + outputFileName;
    System.out.println("Output will be: " + out);

    String pplSourcePath =
        resourceManager.getString("input")
            + File.separator
            + sourceLang
            + File.separator
            + sourceFileName
            + resourceManager.getString("tools.ngram.output.ext");
    String pplTargetPath =
        resourceManager.getString("input")
            + File.separator
            + targetLang
            + File.separator
            + targetFileName
            + resourceManager.getString("tools.ngram.output.ext");

    String pplPOSTargetPath =
        resourceManager.getString("input")
            + File.separator
            + targetLang
            + File.separator
            + targetFileName
            + PosTagger.getXPOS()
            + resourceManager.getString("tools.ngram.output.ext");
    runNGramPPL();

    FileModel fm = new FileModel(sourceFile, resourceManager.getString(sourceLang + ".corpus"));

    // FileModel fm = new FileModel(sourceFile,
    //     resourceManager.getString("source" + ".corpus"));

    PPLProcessor pplProcSource =
        new PPLProcessor(pplSourcePath, new String[] {"logprob", "ppl", "ppl1"});
    PPLProcessor pplProcTarget =
        new PPLProcessor(pplTargetPath, new String[] {"logprob", "ppl", "ppl1"});

    String sourcePosOutput = null;
    String targetPosOutput = null;
    PPLProcessor pplPosTarget = null;
    if (!isBaseline) {
      sourcePosOutput = runPOS(sourceFile, sourceLang, "source");
      targetPosOutput = runPOS(targetFile, targetLang, "target");

      String targetPPLPos = runNGramPPLPos(targetPosOutput + PosTagger.getXPOS());
      System.out.println("---------TARGET PPLPOS: " + targetPPLPos);
      pplPosTarget =
          new PPLProcessor(targetPPLPos, new String[] {"poslogprob", "posppl", "posppl1"});
    }

    loadGiza();
    processNGrams();
    boolean gl = false;
    String temp0 = resourceManager.getString("GL");
    if (null != temp0 && temp0.equals("1")) {
      gl = true;
    }

    if (gl) {
      loadGlobalLexicon();
    }

    // Preparing the indices for IR_similarity_features
    Lucene sourceLuc = null;
    Lucene targetLuc = null;
    if (featureManager.hasFeature("1700")) {
      // The indices reside under lang_resources path
      String lang_resources = workDir + File.separator + "lang_resources";
      // Indices are saved under: luceneIndex folder
      String source_lucene_path =
          lang_resources + File.separator + sourceLang + File.separator + "luceneIndex";
      // The corpus to index
      String source_lucene_corpus = source_lucene_path + File.separator + sourceLang + ".corpus";
      //			System.out.println("SOURCE: " + source_lucene_path + " ||| " + source_lucene_corpus);
      try {
        sourceLuc = new Lucene(source_lucene_path, source_lucene_corpus, true, true, "Source");
      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
      String target_lucene_path =
          lang_resources + File.separator + targetLang + File.separator + "luceneIndex";
      String target_lucene_corpus = target_lucene_path + File.separator + targetLang + ".corpus";
      //			System.out.println("TARGET: " + target_lucene_path + " ||| " + target_lucene_corpus);
      try {
        targetLuc = new Lucene(target_lucene_path, target_lucene_corpus, true, true, "Target");
      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }

    // MQM kicks in
    MQMManager.getInstance().initialize(resourceManager);
    Context context = new Context();
    context.setSourceFilePath(sourceFile);
    context.setTargetFilePath(targetFile);
    MQMManager.getInstance().globalProcessing(context);

    try {
      BufferedReader brSource = new BufferedReader(new FileReader(sourceFile));
      BufferedReader brTarget = new BufferedReader(new FileReader(targetFile));
      BufferedWriter output = new BufferedWriter(new FileWriter(out));
      BufferedReader posSource = null;
      BufferedReader posTarget = null;
      boolean posSourceExists = ResourceManager.isRegistered("sourcePosTagger");
      boolean posTargetExists = ResourceManager.isRegistered("targetPosTagger");
      POSProcessor posSourceProc = null;
      POSProcessor posTargetProc = null;

      // lefterav: Berkeley parser modifications start here
      // Check if user has defined the grammar files for source
      // and target language

      //   if ( ResourceManager.isRegistered("BParser")){
      boolean bp = false;
      String temp = resourceManager.getString("BP");
      if (null != temp && temp.equals("1")) {
        bp = true;
      }

      BParserProcessor sourceParserProcessor = null;
      BParserProcessor targetParserProcessor = null;

      if (bp) {
        sourceParserProcessor = new BParserProcessor();
        targetParserProcessor = new BParserProcessor();
        sourceParserProcessor.initialize(sourceFile, resourceManager, sourceLang);
        targetParserProcessor.initialize(targetFile, resourceManager, targetLang);
      }
      // }

      /** BEGIN: Added by Raphael Rubino for the Topic Model Features */
      boolean tm = false;
      String temp1 = resourceManager.getString("TM");
      if (temp1 != null && temp1.equals("1")) {
        tm = true;
      }
      TopicDistributionProcessor sourceTopicDistributionProcessor = null;
      TopicDistributionProcessor targetTopicDistributionProcessor = null;
      if (tm) {
        String sourceTopicDistributionFile =
            resourceManager.getString(sourceLang + ".topic.distribution");
        String targetTopicDistributionFile =
            resourceManager.getString(targetLang + ".topic.distribution");
        sourceTopicDistributionProcessor =
            new TopicDistributionProcessor(sourceTopicDistributionFile, "sourceTopicDistribution");
        targetTopicDistributionProcessor =
            new TopicDistributionProcessor(targetTopicDistributionFile, "targetTopicDistribution");
      }
      /* END: Added by Raphael Rubino for the Topic Model Features
       */

      if (!isBaseline) {
        if (posSourceExists) {
          posSourceProc = new POSProcessor(sourcePosOutput);
          posSource =
              new BufferedReader(
                  new InputStreamReader(new FileInputStream(sourcePosOutput), "utf-8"));
        }
        if (posTargetExists) {
          posTargetProc = new POSProcessor(targetPosOutput);
          posTarget =
              new BufferedReader(new InputStreamReader(new FileInputStream(targetPosOutput)));
        }
      }
      ResourceManager.printResources();
      Sentence sourceSent;
      Sentence targetSent;
      int sentCount = 0;

      String lineSource = brSource.readLine();
      String lineTarget = brTarget.readLine();

      /** Triggers (by David Langlois) */
      boolean tr = false;
      String temp2 = resourceManager.getString("TR");
      if (temp2 != null && temp2.equals("1")) {
        tr = true;
      }

      Triggers itl_target = null;
      TriggersProcessor itl_target_p = null;
      Triggers itl_source = null;
      TriggersProcessor itl_source_p = null;
      // TriggersProcessor itl_source_p = null;
      Triggers itl_source_target = null;
      TriggersProcessor itl_source_target_p = null;

      if (tr) {

        itl_target =
            new Triggers(
                resourceManager.getString("target.intra.triggers.file"),
                Integer.parseInt(resourceManager.getString("nb.max.triggers.target.intra")),
                resourceManager.getString("phrase.separator"));
        itl_target_p = new TriggersProcessor(itl_target);

        itl_source =
            new Triggers(
                resourceManager.getString("source.intra.triggers.file"),
                Integer.parseInt(resourceManager.getString("nb.max.triggers.source.intra")),
                resourceManager.getString("phrase.separator"));
        itl_source_p = new TriggersProcessor(itl_source);

        itl_source_target =
            new Triggers(
                resourceManager.getString("source.target.inter.triggers.file"),
                Integer.parseInt(resourceManager.getString("nb.max.triggers.source.target.inter")),
                resourceManager.getString("phrase.separator"));
        itl_source_target_p = new TriggersProcessor(itl_source_target);
      }
      /*
       * End modification for Triggers
       */

      // read in each line from the source and target files
      // create a sentence from each
      // process each sentence
      // run the features on the sentences
      while ((lineSource != null) && (lineTarget != null)) {

        // lineSource = lineSource.trim().substring(lineSource.indexOf(" ")).replace("+", "");
        sourceSent = new Sentence(lineSource, sentCount);
        targetSent = new Sentence(lineTarget, sentCount);

        //       System.out.println("Processing sentence "+sentCount);
        //     System.out.println("SORCE: " + sourceSent.getText());
        //   System.out.println("TARGET: " + targetSent.getText());

        if (posSourceExists) {
          posSourceProc.processSentence(sourceSent);
        }
        if (posTargetExists) {
          posTargetProc.processSentence(targetSent);
        }
        sourceSent.computeNGrams(3);
        targetSent.computeNGrams(3);
        pplProcSource.processNextSentence(sourceSent);
        pplProcTarget.processNextSentence(targetSent);
        if (!isBaseline) {
          pplPosTarget.processNextSentence(targetSent);
        }

        // lefterav: Parse code here

        if (bp) {
          sourceParserProcessor.processNextSentence(sourceSent);
          targetParserProcessor.processNextSentence(targetSent);
        }

        if (tm) {

          sourceTopicDistributionProcessor.processNextSentence(sourceSent);
          targetTopicDistributionProcessor.processNextSentence(targetSent);
        }

        // modified by David
        if (tr) {
          itl_source_p.processNextSentence(sourceSent);
          itl_target_p.processNextSentence(targetSent);
          itl_source_target_p.processNextParallelSentences(sourceSent, targetSent);
        }
        // end modification by David

        // MQM kicks in
        MQMManager.getInstance().processNextParallelSentences(sourceSent, targetSent);

        // Ergun
        if (featureManager.hasFeature("1700")) {
          sourceLuc.processNextSentence(sourceSent);
          targetLuc.processNextSentence(targetSent);
        }

        ++sentCount;
        output.write(featureManager.runFeatures(sourceSent, targetSent));
        output.newLine();
        lineSource = brSource.readLine();
        lineTarget = brTarget.readLine();
      }
      if (posSource != null) {
        posSource.close();
      }
      if (posTarget != null) {
        posTarget.close();
      }

      brSource.close();
      brTarget.close();
      output.close();
      Logger.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }