public void addNumbers(DataHolder dataholderHandler) {
PropertyConfigurator.configure("conf/log4j.properties");
Logger myLogger = Logger.getLogger("learn.addNumbers");
myLogger.trace("Add numbers");
List<String> nums = new ArrayList<String>();
nums.addAll(Arrays.asList(this.myLearnerUtility.getConstant().NUMBER.split("\\|")));
// System.out.println(nums);
// System.out.println(this.myLearnerUtility.getConstant().NUMBER);
for (int i = 0; i < nums.size(); i++) {
String word = nums.get(i);
// String reg="\\b("+this.myLearnerUtility.getConstant().FORBIDDEN+")\\b";
// boolean f = word.matches(reg);
if (word.matches("\\b(" + this.myLearnerUtility.getConstant().FORBIDDEN + ")\\b")) {
continue;
}
dataholderHandler.updateDataHolder(word, "b", "*", "wordpos", 0);
// this.getWordPOSHolder().put(new WordPOSKey(word, "b"), new
// WordPOSValue("*", 0, 0, null, null));
// System.out.println("add Number: " + word);
}
dataholderHandler.updateDataHolder("NUM", "b", "*", "wordpos", 0);
// this.getWordPOSHolder().put(new WordPOSKey("NUM", "b"), new
// WordPOSValue("*",0, 0, null, null));
}
public void addProperNouns(DataHolder dataholderHandler) {
PropertyConfigurator.configure("conf/log4j.properties");
Logger myLogger = Logger.getLogger("learn.addProperNouns");
myLogger.trace("Add proper nouns");
List<String> ppnouns = new ArrayList<String>();
ppnouns.addAll(Arrays.asList(Constant.PROPERNOUN.split("\\|")));
for (int i = 0; i < ppnouns.size(); i++) {
String word = ppnouns.get(i);
if (word.matches("\\b(" + this.myLearnerUtility.getConstant().FORBIDDEN + ")\\b")) {
continue;
}
dataholderHandler.updateDataHolder(word, "b", "*", "wordpos", 0);
// this.getWordPOSHolder().put(new WordPOSKey(word, "z"), new
// WordPOSValue("*", 0, 0, null, null));
// System.out.println("Add ProperNoun: " + word);
}
}
public void addClusterStrings(DataHolder dataholderHandler) {
PropertyConfigurator.configure("conf/log4j.properties");
Logger myLogger = Logger.getLogger("learn.addClusterstrings");
myLogger.trace("Add clusterstrings");
List<String> cltstrs = new ArrayList<String>();
cltstrs.addAll(Arrays.asList(this.myLearnerUtility.getConstant().CLUSTERSTRING.split("\\|")));
// System.out.println(cltstrs);
// System.out.println(this.myLearnerUtility.getConstant().CLUSTERSTRING);
for (int i = 0; i < cltstrs.size(); i++) {
String word = cltstrs.get(i);
if (word.matches("\\b(" + this.myLearnerUtility.getConstant().FORBIDDEN + ")\\b")) {
continue;
}
dataholderHandler.updateDataHolder(word, "b", "*", "wordpos", 0);
// this.getWordPOSHolder().put(new WordPOSKey(word, "b"), new
// WordPOSValue("*", 1, 1, null, null));
// System.out.println("addClusterString: " + word);
}
}
public void addStopWords(DataHolder dataholderHandler) {
PropertyConfigurator.configure("conf/log4j.properties");
Logger myLogger = Logger.getLogger("learn.addStopWords");
myLogger.trace("Add stop words");
List<String> stops = new ArrayList<String>();
stops.addAll(Arrays.asList(this.myLearnerUtility.getConstant().STOP.split("\\|")));
stops.addAll(Arrays.asList(new String[] {"NUM", "(", "[", "{", ")", "]", "}", "d+"}));
myLogger.trace("Stop Words: " + stops);
for (int i = 0; i < stops.size(); i++) {
String word = stops.get(i);
if (word.matches("\\b(" + this.myLearnerUtility.getConstant().FORBIDDEN + ")\\b")) {
continue;
}
dataholderHandler.updateDataHolder(word, "b", "*", "wordpos", 0);
myLogger.trace(String.format("(\"%s\", \"b\", \"*\", \"wordpos\", 0) added\n", word));
// this.getWordPOSHolder().put(new WordPOSKey(word, "b"), new
// WordPOSValue("*", 0, 0, null, null));
// System.out.println("Add Stop Word: " + word+"\n");
}
myLogger.trace("Quite\n");
}