public static void extremeValues(double[][] values, int order, MapSymbolTable symbolTable) {
ObjectToDoubleMap<String> topVals = new ObjectToDoubleMap<String>();
for (int i = 0; i < values.length; ++i) {
String token = symbolTable.idToSymbol(i);
topVals.set(token, values[i][order]);
}
List<String> tokensByValue = topVals.keysOrderedByValueList();
int size = tokensByValue.size();
for (int i = 0; i < 10 && i < size; ++i) {
String token = tokensByValue.get(i);
double value = topVals.getValue(token);
System.out.printf(" %6d %-15s % 5.3f\n", i, token, value);
}
System.out.println("...");
for (int i = 10; --i >= 0; ) {
String token = tokensByValue.get(size - i - 1);
double value = topVals.getValue(token);
System.out.printf(" %6d %-15s % 5.3f\n", size - i - 1, token, value);
}
}
public static void reportSvd(double[][] values, SvdMatrix matrix, MapSymbolTable symbolTable) {
double[] singularValues = matrix.singularValues();
double[][] leftSingularVectors = matrix.leftSingularVectors();
double[][] rightSingularVectors = matrix.rightSingularVectors();
for (int order = 0; order < singularValues.length; ++order) {
System.out.println("\n\nORDER=" + order + " singular value=" + singularValues[order]);
System.out.println("Extreme Left Values");
extremeValues(leftSingularVectors, order, symbolTable);
System.out.println("\nExtreme Right Values");
extremeValues(rightSingularVectors, order, symbolTable);
}
ObjectToDoubleMap<String> topPairCounts = new ObjectToDoubleMap<String>();
int numSymbols = symbolTable.numSymbols();
for (int i = 0; i < numSymbols; ++i) {
for (int j = 0; j < numSymbols; ++j) {
if (values[i][j] != 0) {
topPairCounts.set(
symbolTable.idToSymbol(i) + "," + symbolTable.idToSymbol(j), values[i][j]);
}
}
}
int numPairs = topPairCounts.size();
System.out.println("#unique pairs=" + numPairs);
List<String> pairsByCount = topPairCounts.keysOrderedByValueList();
for (int i = 0; i < 25; ++i) {
String pair = pairsByCount.get(i);
System.out.println(" " + pair + " count=" + topPairCounts.getValue(pair));
}
System.out.println("\nRECONSTRUCTED TOP COUNTS");
System.out.println("LeftToken,RightToken OriginalValue SvdValue");
for (int i = 0; i < 25; ++i) {
String pair = pairsByCount.get(i);
String[] tokenPair = pair.split(",");
String leftToken = tokenPair[0];
String rightToken = tokenPair[1];
int leftSymbol = symbolTable.symbolToID(leftToken);
int rightSymbol = symbolTable.symbolToID(rightToken);
double originalValue = topPairCounts.getValue(pair);
double reconstructedValue = matrix.value(leftSymbol, rightSymbol);
System.out.println(pair + " " + originalValue + " " + reconstructedValue);
}
}
public Map<String, ? extends Number> nodeFeatures(int n) {
ObjectToDoubleMap<String> feats = new ObjectToDoubleMap<String>();
boolean bos = n == 0;
boolean eos = (n + 1) >= numTokens();
String tokenCat = tokenCat(n);
String prevTokenCat = bos ? null : tokenCat(n - 1);
String nextTokenCat = eos ? null : tokenCat(n + 1);
String token = normedToken(n);
String prevToken = bos ? null : normedToken(n - 1);
String nextToken = eos ? null : normedToken(n + 1);
String posTag = mPosTagging.tag(n);
String prevPosTag = bos ? null : mPosTagging.tag(n - 1);
String nextPosTag = eos ? null : mPosTagging.tag(n + 1);
if (bos) feats.set("BOS", 1.0);
if (eos) feats.set("EOS", 1.0);
if (!bos && !eos) feats.set("!BOS!EOS", 1.0);
feats.set("TOK_" + token, 1.0);
if (!bos) feats.set("TOK_PREV_" + prevToken, 1.0);
if (!eos) feats.set("TOK_NEXT_" + nextToken, 1.0);
feats.set("TOK_CAT_" + tokenCat, 1.0);
if (!bos) feats.set("TOK_CAT_PREV_" + prevTokenCat, 1.0);
if (!eos) feats.set("TOK_CAT_NEXT_" + nextToken, 1.0);
feats.set("POS_" + posTag, 1.0);
if (!bos) feats.set("POS_PREV_" + prevPosTag, 1.0);
if (!eos) feats.set("POS_NEXT_" + nextPosTag, 1.0);
for (String suffix : suffixes(token)) feats.set("SUFF_" + suffix, 1.0);
if (!bos) for (String suffix : suffixes(prevToken)) feats.set("SUFF_PREV_" + suffix, 1.0);
if (!eos) for (String suffix : suffixes(nextToken)) feats.set("SUFF_NEXT_" + suffix, 1.0);
for (String prefix : prefixes(token)) feats.set("PREF_" + prefix, 1.0);
if (!bos) for (String prefix : prefixes(prevToken)) feats.set("PREF_PREV_" + prefix, 1.0);
if (!eos) for (String prefix : prefixes(nextToken)) feats.set("PREF_NEXT_" + prefix, 1.0);
return feats;
}
public Map<String, ? extends Number> edgeFeatures(int n, int k) {
ObjectToDoubleMap<String> feats = new ObjectToDoubleMap<String>();
feats.set("PREV_TAG_" + tag(k), 1.0);
feats.set("PREV_TAG_TOKEN_CAT_" + tag(k) + "_" + tokenCat(n - 1), 1.0);
return feats;
}