private static void index_h(String prefix, File file, IndexWriter indexWriter)
throws IOException {
Document doc = null;
if (file.isDirectory()) {
File files[] = file.listFiles();
for (File file1 : files) {
index_h(prefix + FILE_SEPARATOR + file.getName(), file1, indexWriter);
}
} else {
String content = FileUtils.readFileToString(file, "utf-8");
System.out.println("==============================================================");
System.out.println("index_h " + content);
System.out.println("==============================================================");
String filename = prefix + FILE_SEPARATOR + file.getName();
String path = file.getAbsolutePath();
doc = new Document();
doc.add(new Field("content", content, Field.Store.YES, Field.Index.ANALYZED));
doc.add(new Field("relative_path", filename, Field.Store.YES, Field.Index.NOT_ANALYZED));
indexWriter.addDocument(doc);
}
}
private boolean register(File f) {
boolean rc = false;
if (f.isDirectory()) {
File file[] = f.listFiles(this);
for (int i = 0; i < file.length; i++) {
if (register(file[i])) rc = true;
}
} else if (dirMap.get(f) == null) {
dirMap.put(f, new QEntry());
rc = true;
}
return rc;
}
public static void deleteRecursive(File file) {
if (!file.exists()) {
return;
}
if (file.isFile()) {
file.delete();
} else if (file.isDirectory()) {
File[] files = file.listFiles();
for (int i = 0; i < files.length; i++) {
deleteRecursive(files[i]);
}
file.delete();
}
}
public static boolean deleteAll(File file) {
boolean b = true;
if ((file != null) && file.exists()) {
if (file.isDirectory()) {
try {
File[] files = file.listFiles();
for (File f : files) b &= deleteAll(f);
} catch (Exception e) {
return false;
}
}
b &= file.delete();
}
return b;
}
public void process() {
File inFolder = new File(vectorFolder);
if (!inFolder.isDirectory()) {
System.out.println("In should be a folder: " + vectorFolder);
return;
}
boolean clusterSaved = false;
this.invertedFixedClases = loadFixedClasses(fixedClassesFile);
if (!histogram) {
Map<String, List<String>> sectors = loadStockSectors(sectorFile);
sectorToClazz = convertSectorsToClazz(sectors);
if (!clusterSaved) {
changeClassLabels();
clusterSaved = true;
}
for (Map.Entry<String, Integer> entry : sectorToClazz.entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue());
}
}
for (File inFile : inFolder.listFiles()) {
String fileName = inFile.getName();
String fileNameWithOutExt = FilenameUtils.removeExtension(fileName);
if (histogram) {
sectorToClazz.clear();
invertedSectors.clear();
Map<String, List<String>> sectors =
loadHistoSectors(sectorFile + "/" + fileNameWithOutExt + ".csv");
sectorToClazz = convertSectorsToClazz(sectors);
if (!clusterSaved) {
changeClassLabels();
clusterSaved = true;
}
for (Map.Entry<String, Integer> entry : sectorToClazz.entrySet()) {
System.out.println(entry.getKey() + " : " + entry.getValue());
}
}
processFile(fileNameWithOutExt);
}
}
public boolean accept(File f) {
return f.canRead()
&& (f.getName().endsWith(".xml")
|| (recursive && f.isDirectory() && !"lib".equalsIgnoreCase(f.getName())));
}
/**
* Parses the command line arguments and sets the respective fields accordingly. This function
* sets the input source and target files, the source and target language, the running mode (gb or
* bb), the additional files required by the GB feature extractor, the rebuild and log options
*
* @param args The command line arguments
*/
public void parseArguments(String[] args) {
Option help =
OptionBuilder.withArgName("help")
.hasArg()
.withDescription("print project help information")
.isRequired(false)
.create("help");
Option input = OptionBuilder.withArgName("input").hasArgs(3).isRequired(true).create("input");
Option lang = OptionBuilder.withArgName("lang").hasArgs(2).isRequired(false).create("lang");
Option feat = OptionBuilder.withArgName("feat").hasArgs(1).isRequired(false).create("feat");
Option gb =
OptionBuilder.withArgName("gb")
.withDescription("GlassBox input files")
.hasOptionalArgs(2)
.hasArgs(3)
.create("gb");
Option mode =
OptionBuilder.withArgName("mode")
.withDescription("blackbox features, glassbox features or both")
.hasArgs(1)
.isRequired(true)
.create("mode");
Option config =
OptionBuilder.withArgName("config")
.withDescription("cofiguration file")
.hasArgs(1)
.isRequired(false)
.create("config");
Option rebuild = new Option("rebuild", "run all preprocessing tools");
rebuild.setRequired(false);
// separate 17 BB from 79 BB
Option baseline = new Option("baseline", "only 17 baseline feature will be calculated");
baseline.setRequired(false);
CommandLineParser parser = new PosixParser();
Options options = new Options();
options.addOption(help);
options.addOption(input);
options.addOption(mode);
options.addOption(lang);
options.addOption(feat);
options.addOption(gb);
options.addOption(rebuild);
options.addOption(config);
options.addOption(baseline);
try {
// parse the command line arguments
CommandLine line = parser.parse(options, args);
if (line.hasOption("config")) {
resourceManager = new PropertiesManager(line.getOptionValue("config"));
} else {
resourceManager = new PropertiesManager();
}
if (line.hasOption("input")) {
// print the value of block-size
String[] files = line.getOptionValues("input");
sourceFile = files[0];
targetFile = files[1];
}
if (line.hasOption("lang")) {
String[] langs = line.getOptionValues("lang");
sourceLang = langs[0];
targetLang = langs[1];
} else {
sourceLang = resourceManager.getString("sourceLang.default");
targetLang = resourceManager.getString("targetLang.default");
}
if (line.hasOption("gb")) {
String[] gbOpt = line.getOptionValues("gb");
for (String s : gbOpt) System.out.println(s);
if (gbOpt.length > 1) {
mtSys = MOSES;
nbestInput = gbOpt[0];
onebestPhrases = gbOpt[1];
onebestLog = gbOpt[2];
gbMode = 1;
} else {
File f = new File(gbOpt[0]);
if (f.isDirectory()) {
mtSys = IBM;
wordLattices = gbOpt[0];
gbMode = 1;
} else {
gbMode = 0;
gbXML = gbOpt[0];
}
}
}
if (line.hasOption("mode")) {
String[] modeOpt = line.getOptionValues("mode");
setMod(modeOpt[0].trim());
System.out.println(getMod());
configPath = resourceManager.getString("featureConfig." + getMod());
System.out.println("feature config:" + configPath);
featureManager = new FeatureManager(configPath);
}
if (line.hasOption("feat")) {
// print the value of block-size
features = line.getOptionValue("feat");
featureManager.setFeatureList(features);
} else {
featureManager.setFeatureList("all");
}
if (line.hasOption("rebuild")) {
forceRun = true;
}
if (line.hasOption("baseline")) {
isBaseline = true;
}
} catch (ParseException exp) {
System.out.println("Unexpected exception:" + exp.getMessage());
}
}