@Before
public void setUp() throws Exception {
sessionFactory.getCache().evictEntityRegion(Hypothesis.class);
sessionFactory.getCache().evictEntityRegion(Helicopter.class);
log.info("예제용 데이터 추가");
Hypothesis socrates = new Hypothesis();
socrates.setId("13");
socrates.setDescription(
"There are more than two dimensions over the shadows we see out of the cave");
socrates.setPosition(1);
getSession().saveOrUpdate(socrates);
Hypothesis peano = new Hypothesis();
peano.setId("14");
peano.setDescription(
"Peano's curve and then Hilbert's space filling curve proof the connection from mono-dimensional to bi-dimensional space");
peano.setPosition(2);
getSession().saveOrUpdate(peano);
Hypothesis sanne = new Hypothesis();
sanne.setId("15");
sanne.setDescription(
"Hilbert's proof of connection to 2 dimensions can be induced to reason on N dimensions");
sanne.setPosition(3);
getSession().saveOrUpdate(sanne);
Hypothesis shortOne = new Hypothesis();
shortOne.setId("16");
shortOne.setDescription("stuff works");
shortOne.setPosition(4);
getSession().saveOrUpdate(shortOne);
Helicopter helicopter = new Helicopter();
helicopter.setName("No creative clue ");
getSession().saveOrUpdate(helicopter);
getSession().flush();
getSession().clear();
}
private void loadAndShowHyps() {
if (selectHypFileUI.getSelectedHypFile() != null) {
hypotheses = selectHypFileUI.getLoadedHyps();
Iterator hypothesisIterator = hypotheses.iterator();
showLoadedHypsUI.createTable(hypotheses.size());
int rowNumber = 0;
while (hypothesisIterator.hasNext()) {
Hypothesis hypothesis = (Hypothesis) hypothesisIterator.next();
showLoadedHypsUI.addHyp(
rowNumber,
hypothesis.getNumber(),
hypothesis.getHypothesisText(),
hypothesis.getScore());
rowNumber++;
}
showLoadedHypsUI.setInfoLabelText(
"You selected "
+ selectHypFileUI.getSelectedHypFile().getName().toString()
+ " as the input file."
+ " I found "
+ hypotheses.size()
+ " hypotheses.");
} else {
showLoadedHypsUI.setInfoLabelText("No hypothesis file selected.");
}
}
/**
* Classifies the input.
*
* @param input the value for each feature.
* @return the probability associated with each classification.
*/
public HashMap<Integer, Double> classify(double[] input) {
HashMap<Integer, Double> p = new HashMap<Integer, Double>();
double tProb = 0;
for (Hypothesis h : hypothesis) {
double prob = h.predict(input);
tProb += prob;
p.put(h.getClassification(), prob);
}
for (Integer c : p.keySet()) p.put(c, p.get(c) / tProb);
return p;
}
public void saveTriplesAsArff() {
TreeMap tripleMap = new TreeMap();
if (hypotheses == null) {
saveWordTriplesAsArffUI.setInfoLabelText("No hyps loaded!");
hypotheses = getHypsFromFile();
}
if ((wordCodeMap == null) || (wordCodeMap.size() == 0)) {
wordCodeMap = new HashMap();
saveWordTriplesAsArffUI.setInfoLabelText("No word list loaded!");
ArrayList wordList = getWordListFromFile();
for (int i = 0; i < wordList.size(); i++) {
Word word = (Word) wordList.get(i);
wordCodeMap.put(word.getText(), new Integer(word.getCode()));
}
}
if (showWordTriplesUI.getTripleMap().size() == 0) {
saveWordTriplesAsArffUI.setInfoLabelText("No word triples loaded!");
tripleMap = getTriplesFromFile();
} else {
tripleMap = showWordTriplesUI.getTripleMap();
}
Iterator hypIterator = hypotheses.iterator();
TreeMap scoreMap = new TreeMap();
TreeMap scoreCounts = new TreeMap();
while (hypIterator.hasNext()) {
Hypothesis hypothesis = (Hypothesis) hypIterator.next();
Integer score = new Integer(hypothesis.getScore());
scoreMap.put(score, score);
if (!scoreCounts.containsKey(score)) {
scoreCounts.put(score, new Integer(1));
} else {
int oldCount = ((Integer) scoreCounts.get(score)).intValue();
scoreCounts.put(score, new Integer(oldCount + 1));
}
}
saveWordTriplesAsArffUI.createTable(scoreMap.size());
Iterator scoreIterator = scoreMap.keySet().iterator();
int rowNumber = 0;
while (scoreIterator.hasNext()) {
Integer currentScore = (Integer) scoreIterator.next();
saveWordTriplesAsArffUI.addScore(
rowNumber, currentScore.intValue(), ((Integer) scoreCounts.get(currentScore)).intValue());
rowNumber++;
}
saveWordTriplesAsArffUI.setHypsAndScores(hypotheses, wordCodeMap, tripleMap);
saveWordTriplesAsArffUI.setInfoLabelText("Optionally edit the scores and" + " save the file.");
}
public void calculateWordPairs() {
numCodes = wordCodeMap.keySet().size() + 1;
pairs = new int[numCodes][numCodes];
Iterator hypIterator = hypotheses.iterator();
while (hypIterator.hasNext()) {
Hypothesis hyp = (Hypothesis) hypIterator.next();
int[] codes = hyp.getCodeList(wordCodeMap);
for (int i = 0; i < (codes.length - 1); i++) {
int firstWordCode = codes[i];
int secondWordCode = codes[i + 1];
pairs[firstWordCode][secondWordCode]++;
}
}
calculateWordDoublesUI.setPairs(pairs);
}
public void calculateWordTriples() {
numCodes = wordCodeMap.keySet().size() + 1;
triples = new SparseDoubleMatrix3D(numCodes, numCodes, numCodes);
Iterator hypIterator = hypotheses.iterator();
while (hypIterator.hasNext()) {
Hypothesis hyp = (Hypothesis) hypIterator.next();
int[] codes = hyp.getCodeList(wordCodeMap);
for (int i = 0; i < (codes.length - 2); i++) {
int firstWordCode = codes[i];
int secondWordCode = codes[i + 1];
int thirdWordCode = codes[i + 2];
double oldVal = triples.getQuick(firstWordCode, secondWordCode, thirdWordCode);
triples.setQuick(firstWordCode, secondWordCode, thirdWordCode, (oldVal + 1));
}
}
calculateWordTriplesUI.setTriples(triples, numCodes);
}
private void scoreBySingleWords() {
if (hypotheses == null) {
scoreHypsByWordsUI.setInfoLabelText("No hyps loaded!");
hypotheses = getHypsFromFile();
}
if (showWordsUI.getWordListTableModel() == null) {
scoreHypsByWordsUI.setInfoLabelText("No word list loaded!");
ArrayList allWords = getWordListFromFile();
showWordsUI.createTable(allWords.size());
for (int i = 0; i < allWords.size(); i++) {
Word word = (Word) allWords.get(i);
showWordsUI.addWord(i, word.getText(), word.getCount(), word.getGroup(), word.getCode());
}
showWordsUI.setInfoLabelText("I found " + allWords.size() + " words.");
}
// load the scoring table as a HashMap
Object[][] wordListTable = showWordsUI.getWordListTableModel().getAllWords();
int wordCount = showWordsUI.getWordListTableModel().getRowCount();
wordCodeMap = new HashMap();
for (int i = 0; i < wordCount; i++) {
if (((Integer) wordListTable[i][3]).intValue() != 0) {
wordCodeMap.put(wordListTable[i][0], wordListTable[i][3]);
}
}
// make list of words
words = new String[wordCodeMap.size() + 1];
Iterator wordIterator = wordCodeMap.keySet().iterator();
while (wordIterator.hasNext()) {
String currentWord = (String) wordIterator.next();
int code = ((Integer) wordCodeMap.get(currentWord)).intValue();
words[code] = currentWord;
}
// score the hyps and display
scoreHypsByWordsUI.createTable(hypotheses.size(), words);
for (int i = 0; i < hypotheses.size(); i++) {
Hypothesis hypothesis = (Hypothesis) hypotheses.get(i);
int[] singleWordScores = hypothesis.scoreBySingleWords(wordCodeMap);
scoreHypsByWordsUI.addHyp(i, hypothesis.getHypothesisText(), singleWordScores);
}
scoreHypsByWordsUI.setInfoLabelText(
"I scored " + hypotheses.size() + " hypotheses with " + wordCodeMap.size() + " words.");
}
private void showAndEditWords() {
if (hypotheses != null) {
Iterator hypothesisIterator = hypotheses.iterator();
wordsAndCounts = new HashMap();
while (hypothesisIterator.hasNext()) {
Hypothesis hypothesis = (Hypothesis) hypothesisIterator.next();
ArrayList wordSet = hypothesis.getWordSet();
Iterator wordIterator = wordSet.iterator();
while (wordIterator.hasNext()) {
String wordText = (String) wordIterator.next();
if (wordsAndCounts.containsKey(wordText)) {
int oldCount = ((Integer) wordsAndCounts.get(wordText)).intValue();
wordsAndCounts.put(wordText, new Integer(oldCount + 1));
} else {
wordsAndCounts.put(wordText, new Integer(1));
}
}
}
Iterator wordListIterator = wordsAndCounts.keySet().iterator();
showWordsUI.createTable(wordsAndCounts.size());
int rowNumber = 0;
while (wordListIterator.hasNext()) {
String wordText = (String) wordListIterator.next();
int count = ((Integer) wordsAndCounts.get(wordText)).intValue();
showWordsUI.addWord(rowNumber, wordText, count, "", 0);
rowNumber++;
}
showWordsUI.setInfoLabelText("I found " + rowNumber + " words.");
} else {
showWordsUI.setInfoLabelText("No hypotheses loaded.");
ArrayList allWords = getWordListFromFile();
showWordsUI.createTable(allWords.size());
for (int i = 0; i < allWords.size(); i++) {
Word word = (Word) allWords.get(i);
showWordsUI.addWord(i, word.getText(), word.getCount(), word.getGroup(), word.getCode());
}
showWordsUI.setInfoLabelText("I found " + allWords.size() + " words.");
}
}
/**
* Runs gradient decent to tune the parameters of each hypothesis.
*
* @param iterations the number of times to run gradient decent
*/
public void tune(int iterations) {
for (Hypothesis h : hypothesis) {
// construct a new training set using One vs. Rest
// if the training example has the same value as the
// hypothesis then set the answer to 1
// otherwise set the answer to 0.
TrainingExample[] tSet = new TrainingExample[trainingSet.length];
int answer;
int i = 0;
for (TrainingExample t : trainingSet) {
if (t.getAnswer() == h.getClassification()) answer = 1;
else answer = 0;
tSet[i] = new TrainingExample(t.getInput(), answer);
++i;
}
for (i = 0; i < iterations; ++i) {
h.gradientDecent(tSet);
}
}
}
public void loadFromFile(File f) {
ArrayList<Hypothesis> hl = new ArrayList<Hypothesis>();
String s;
try {
BufferedReader reader = new BufferedReader(new FileReader(f));
Hypothesis hyp;
while ((s = reader.readLine()) != null) {
if (s.equals("")) continue;
hyp = new Hypothesis(0, 0);
hyp.loadFromString(s);
hl.add(hyp);
}
} catch (Exception e) {
e.printStackTrace();
}
hypothesis = new Hypothesis[hl.size()];
int i = 0;
for (Hypothesis h : hl) {
hypothesis[i] = h;
++i;
}
}
/**
* Calculates the cost of the <code>trainingSet</code>.
*
* @param hyp the hypothesis to use in calculating the cost.
* @return the cost associated with the hypothesis.
*/
public double defaultCostFunction(Hypothesis hyp) {
double error = 0;
double h;
int answer;
for (TrainingExample t : trainingSet) {
try {
h = (Double) hyp.predict(t.getInput());
} catch (Exception e) {
e.printStackTrace();
continue;
}
answer = t.getAnswer();
error -= answer * log(h) + (1 - answer) * log(1 - h);
}
double regError = 0;
for (int i = 0; i < hyp.getNumFeatures(); ++i) {
regError += pow(hyp.getParameter(i), 2);
}
error += regError / regularizationParam;
return error / (2 * trainingSet.length);
}
public void saveSinglesAsArff() {
if (scoreHypsByWordsUI.getScoresTableModel() != null) {
// tally up all the possible scores
Iterator hypIterator = hypotheses.iterator();
TreeMap scoreMap = new TreeMap();
TreeMap scoreCounts = new TreeMap();
while (hypIterator.hasNext()) {
Hypothesis hypothesis = (Hypothesis) hypIterator.next();
Integer score = new Integer(hypothesis.getScore());
scoreMap.put(score, score);
if (!scoreCounts.containsKey(score)) {
scoreCounts.put(score, new Integer(1));
} else {
int oldCount = ((Integer) scoreCounts.get(score)).intValue();
scoreCounts.put(score, new Integer(oldCount + 1));
}
}
saveSingleWordAsArffUI.createTable(scoreMap.size());
Iterator scoreIterator = scoreMap.keySet().iterator();
int rowNumber = 0;
while (scoreIterator.hasNext()) {
Integer currentScore = (Integer) scoreIterator.next();
saveSingleWordAsArffUI.addScore(
rowNumber,
currentScore.intValue(),
((Integer) scoreCounts.get(currentScore)).intValue());
rowNumber++;
}
saveSingleWordAsArffUI.setHypsAndScores(hypotheses, scoreHypsByWordsUI.getScoresTableModel());
saveSingleWordAsArffUI.setInfoLabelText("Optionally edit the scores and" + " save the file.");
} else {
saveSingleWordAsArffUI.setInfoLabelText("No hyps have been scored yet!");
}
}
/** Prints each hypothesis along with its classification to stdout. */
public void show() {
for (Hypothesis h : hypothesis) {
System.out.print(h.getClassification() + ": ");
h.show();
}
}
public void writeToFile(File f) {
for (Hypothesis h : hypothesis) {
h.writeToFile(f);
}
}
