/**
* @param index
* @param word
*/
@Override
public synchronized void addWordToIndex(int index, String word) {
if (word == null || word.isEmpty())
throw new IllegalArgumentException("Word can't be empty or null");
if (!tokens.containsKey(word)) {
VocabWord token = new VocabWord(1.0, word);
tokens.put(word, token);
wordFrequencies.incrementCount(word, 1.0);
}
/*
If we're speaking about adding any word to index directly, it means it's going to be vocab word, not token
*/
if (!vocabs.containsKey(word)) {
VocabWord vw = tokenFor(word);
vw.setIndex(index);
vocabs.put(word, vw);
vw.setIndex(index);
}
if (!wordFrequencies.containsKey(word)) wordFrequencies.incrementCount(word, 1);
wordIndex.add(word, index);
}
@Override
public int hashCode() {
int result = wordIndex != null ? wordIndex.hashCode() : 0;
result = 31 * result + (wordFrequencies != null ? wordFrequencies.hashCode() : 0);
result = 31 * result + (docFrequencies != null ? docFrequencies.hashCode() : 0);
result = 31 * result + (vocabs != null ? vocabs.hashCode() : 0);
result = 31 * result + (tokens != null ? tokens.hashCode() : 0);
result = 31 * result + (totalWordOccurrences != null ? totalWordOccurrences.hashCode() : 0);
result = 31 * result + numDocs;
return result;
}
public DoubleMatrix getScoreMatrix(File file) {
Counter<String> docWords = new Counter<String>();
try {
LineIterator iter = FileUtils.lineIterator(file);
while (iter.hasNext()) {
Tokenizer t =
tokenizerFactory.create((new InputHomogenization(iter.nextLine()).transform()));
while (t.hasMoreTokens()) {
docWords.incrementCount(t.nextToken(), 1.0);
}
}
iter.close();
} catch (IOException e) {
throw new IllegalStateException("Unable to read file", e);
}
DoubleMatrix ret = new DoubleMatrix(1, currVocab.size());
for (int i = 0; i < currVocab.size(); i++) {
if (docWords.getCount(currVocab.get(i).toString()) > 0) {
ret.put(i, wordScores.getCount(currVocab.get(i).toString()));
}
}
return ret;
}
/**
* Creates an index of the top size words based on tf-idf metrics
*
* @param size the number of words in the vocab
* @return the index of the words
* @throws IOException
*/
public Index createVocab(int size) {
Index vocab = new Index();
// bootstrapping
calcWordFrequencies();
// term frequency has every word
for (String word : tf.keySet()) {
double tfVal = MathUtils.tf((int) documentWordFrequencies.getCount(word));
double idfVal = MathUtils.idf(numFiles, idf.getCount(word));
double tfidfVal = MathUtils.tfidf(tfVal, idfVal);
java.util.regex.Matcher m = punct.matcher(word);
if (!stopWords.contains(word) && !m.matches()) tfidf.setCount(word, tfidfVal);
}
Counter<String> aggregate = tfidf;
// keep top size keys via tfidf rankings
aggregate.keepTopNKeys(size - 1);
log.info("Created vocab of size " + aggregate.size());
wordScores = aggregate;
// add words that made it via rankings
for (String word : aggregate.keySet()) {
if (vocab.indexOf(word) < 0) vocab.add(word);
}
// cache the vocab
currVocab = vocab;
return vocab;
}
/**
* Transforms the matrix
*
* @param text
* @return
*/
@Override
public DoubleMatrix transform(String text) {
Tokenizer tokenizer = tokenizerFactory.create(text);
List<String> tokens = tokenizer.getTokens();
DoubleMatrix input = new DoubleMatrix(1, vocab.size());
for (int i = 0; i < tokens.size(); i++) {
int idx = vocab.indexOf(tokens.get(i));
if (vocab.indexOf(tokens.get(i)) >= 0) input.put(idx, wordCounts.getCount(tokens.get(i)));
}
return input;
}
private void process() {
while (sentenceIter.hasNext()) {
Tokenizer tokenizer = tokenizerFactory.create(sentenceIter.nextSentence());
List<String> tokens = tokenizer.getTokens();
for (String token : tokens)
if (!stopWords.contains(token)) {
wordCounts.incrementCount(token, 1.0);
if (vocab.indexOf(token) < 0) vocab.add(token);
}
}
}
/**
* Vectorizes the passed in text treating it as one document
*
* @param text the text to vectorize
* @param label the label of the text
* @return a dataset with a applyTransformToDestination of weights(relative to impl; could be word
* counts or tfidf scores)
*/
@Override
public DataSet vectorize(String text, String label) {
Tokenizer tokenizer = tokenizerFactory.create(text);
List<String> tokens = tokenizer.getTokens();
DoubleMatrix input = new DoubleMatrix(1, vocab.size());
for (int i = 0; i < tokens.size(); i++) {
int idx = vocab.indexOf(tokens.get(i));
if (vocab.indexOf(tokens.get(i)) >= 0) input.put(idx, wordCounts.getCount(tokens.get(i)));
}
DoubleMatrix labelMatrix = MatrixUtil.toOutcomeVector(labels.indexOf(label), labels.size());
return new DataSet(input, labelMatrix);
}
/**
* Loads the google binary model Credit to:
* https://github.com/NLPchina/Word2VEC_java/blob/master/src/com/ansj/vec/Word2VEC.java
*
* @param path path to model
* @throws IOException
*/
public static Word2Vec loadGoogleModel(String path) throws IOException {
DataInputStream dis = null;
BufferedInputStream bis = null;
double len = 0;
float vector = 0;
Word2Vec ret = new Word2Vec();
Index wordIndex = new Index();
FloatMatrix wordVectors = null;
try {
bis =
new BufferedInputStream(
path.endsWith(".gz")
? new GZIPInputStream(new FileInputStream(path))
: new FileInputStream(path));
dis = new DataInputStream(bis);
Map<String, FloatMatrix> wordMap = new HashMap<>();
// number of words
int words = Integer.parseInt(readString(dis));
// word vector size
int size = Integer.parseInt(readString(dis));
wordVectors = new FloatMatrix(words, size);
String word;
float[] vectors = null;
for (int i = 0; i < words; i++) {
word = readString(dis);
log.info("Loaded " + word);
vectors = new float[size];
len = 0;
for (int j = 0; j < size; j++) {
vector = readFloat(dis);
len += vector * vector;
vectors[j] = vector;
}
len = Math.sqrt(len);
for (int j = 0; j < size; j++) {
vectors[j] /= len;
}
wordIndex.add(word);
wordVectors.putRow(i, new FloatMatrix(vectors));
}
} finally {
bis.close();
dis.close();
}
ret.setWordIndex(wordIndex);
ret.setSyn0(wordVectors);
return ret;
}
/**
* @param index
* @param word
*/
@Override
public synchronized void addWordToIndex(int index, String word) {
if (word == null || word.isEmpty())
throw new IllegalArgumentException("Word can't be empty or null");
if (!wordFrequencies.containsKey(word)) wordFrequencies.incrementCount(word, 1);
wordIndex.add(word, index);
}
/** @param word */
@Override
public synchronized void putVocabWord(String word) {
if (word == null || word.isEmpty())
throw new IllegalArgumentException("Word can't be empty or null");
// STOP and UNK are not added as tokens
if (word.equals("STOP") || word.equals("UNK")) return;
VocabWord token = tokenFor(word);
if (token == null)
throw new IllegalStateException("Word " + word + " not found as token in vocab");
int ind = token.getIndex();
addWordToIndex(ind, word);
if (!hasToken(word))
throw new IllegalStateException("Unable to add token " + word + " when not already a token");
vocabs.put(word, token);
wordIndex.add(word, token.getIndex());
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
InMemoryLookupCache that = (InMemoryLookupCache) o;
if (numDocs != that.numDocs) return false;
if (wordIndex != null ? !wordIndex.equals(that.wordIndex) : that.wordIndex != null)
return false;
if (wordFrequencies != null
? !wordFrequencies.equals(that.wordFrequencies)
: that.wordFrequencies != null) return false;
if (docFrequencies != null
? !docFrequencies.equals(that.docFrequencies)
: that.docFrequencies != null) return false;
if (vocabWords().equals(that.vocabWords())) return true;
return true;
}
/**
* Returns the index of a given word
*
* @param word the index of a given word
* @return the index of a given word or -1 if not found
*/
@Override
public synchronized int indexOf(String word) {
return wordIndex.indexOf(word);
}
/**
* Returns the word contained at the given index or null
*
* @param index the index of the word to get
* @return the word at the given index
*/
@Override
public synchronized String wordAtIndex(int index) {
return (String) wordIndex.get(index);
}
/**
* Returns the index of a given word
*
* @param word the index of a given word
* @return the index of a given word or -1 if not found
*/
@Override
public int indexOf(String word) {
return wordIndex.indexOf(word);
}
/**
* Returns the word contained at the given index or null
*
* @param index the index of the word to get
* @return the word at the given index
*/
@Override
public String wordAtIndex(int index) {
return (String) wordIndex.get(index);
}
/**
* @param index
* @param word
*/
@Override
public void addWordToIndex(int index, String word) {
if (!wordFrequencies.containsKey(word)) wordFrequencies.incrementCount(word, 1);
if (!vocabs.containsKey(word)) vocabs.put(word, new VocabWord(1, vectorLength));
wordIndex.add(word);
}