public Exp validate(Exp exp, boolean scalar) {
Exp resolved;
try {
resolved = (Exp) resolvedNodes.get(exp);
} catch (ClassCastException e) {
// A classcast exception will occur if there is a String
// placeholder in the map. This is an internal error -- should
// not occur for any query, valid or invalid.
throw Util.newInternal(
e, "Infinite recursion encountered while validating '" + Util.unparse(exp) + "'");
}
if (resolved == null) {
try {
stack.push((QueryPart) exp);
// To prevent recursion, put in a placeholder while we're
// resolving.
resolvedNodes.put((QueryPart) exp, placeHolder);
resolved = exp.accept(this);
Util.assertTrue(resolved != null);
resolvedNodes.put((QueryPart) exp, (QueryPart) resolved);
} finally {
stack.pop();
}
}
if (scalar) {
final Type type = resolved.getType();
if (!TypeUtil.canEvaluate(type)) {
String exprString = Util.unparse(resolved);
throw MondrianResource.instance().MdxMemberExpIsSet.ex(exprString);
}
}
return resolved;
}
public void allPreds(int arity, List result) {
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
e.allPreds(arity, result);
}
}
public void extractPTypeExps(List l) {
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
e.extractPTypeExps(l);
}
}
public boolean removeUnscoped(List vars) {
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
Exp e = i.next();
if (e.removeUnscoped(vars)) i.remove();
}
return false;
}
void getOuterRefs(Exp e, List<Exp> refs) {
// now, recurse
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
Exp ex = i.next();
ex.getOuterRefs(e, refs);
}
}
public void allLits(int arity, List result, boolean b) {
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
e.allLits(arity, result, b);
}
}
public void allSubExps(String type, List result) {
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
if (e.getClass().getName().equals(type)) result.add(e);
e.allSubExps(type, result);
}
}
public void allSubExps(List result) {
result.add(this);
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
e.allSubExps(result);
}
}
public Exp deleteExp(Exp l) {
IntSet result = new IntSet();
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
Exp e = i.next();
if (e != l) result.exps.add(e.deleteExp(l));
}
return result;
}
public void allSubExps(Type type, List result) {
if (type == null || type().equals(type)) result.add(this);
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
e.allSubExps(type, result);
}
}
public int predCount(Object p) {
int result = 0;
Exp e;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
e = i.next();
result += e.predCount(p);
}
return result;
}
public int expCount(int eq, Exp e) {
int count = 0;
if (equals(eq, e)) count++;
Iterator<Exp> i = exps.iterator();
while (i.hasNext()) {
Exp ex = i.next();
count += ex.expCount(eq, e);
}
return count;
}
public Type inferType(List<Var> vars, List<List<Type>> varTypes) {
for (Exp e : exps) {
Type t = e.inferType(vars, varTypes);
if (t == null || !t.subType(PType.E)) {
inferedType = null; // update cache
return null;
}
}
inferedType = FType.ET; // update cache
return FType.ET; // could we infer subtypes here?
}
public Exp instReplace(Exp olde, Exp newe) {
if (this == olde) return newe;
Exp e;
for (int i = 0; i < exps.size(); i++) {
e = exps.get(i);
exps.set(i, e.instReplace(olde, newe));
e = exps.get(i);
if (e == null) return null;
}
return this;
}
public int expCount(int id) {
int result = 0;
Exp ex;
Iterator<Exp> i;
i = exps.iterator();
while (i.hasNext()) {
ex = i.next();
result += ex.expCount(id);
}
return result;
}
public int repeatPredCount(int t, Object p) {
int result = 0;
Exp e;
Iterator<Exp> i;
i = exps.iterator();
while (i.hasNext()) {
e = i.next();
result += e.repeatPredCount(t, p);
}
return result;
}
public int repeatExpCount(int t, Exp e) {
int result = 0;
Exp ex;
Iterator<Exp> i;
i = exps.iterator();
while (i.hasNext()) {
ex = i.next();
result += ex.repeatExpCount(t, e);
}
return result;
}
public Exp replace(Exp olde, Exp newe) {
// System.out.println("Exps: "+exps);
if (equals(olde)) return newe;
Exp e;
for (int i = 0; i < exps.size(); i++) {
e = exps.get(i);
if (e.equals(olde)) exps.set(i, newe);
else exps.set(i, e.replace(olde, newe));
e = exps.get(i);
if (e == null) return null;
}
return this;
}
public Equals(String input, Map vars) {
LispReader lr = new LispReader(new StringReader(input));
String t = lr.next(); // read type
if (!t.equals("=")) {
System.err.println("= not found in Equals constructor");
System.exit(-1);
}
left = Exp.makeExp(lr.next(), vars);
right = Exp.makeExp(lr.next(), vars);
if (!wellTyped()) {
System.out.println("MISTYPED in Equals.java: " + this);
}
addToNameMap(Arrays.asList(left, right));
}
public void extractFuncts(List functors, List functees, Exp orig) {
List vars = body.freeVars();
if (vars.size() == 1) {
Var v = (Var) vars.get(0);
Exp functee = new Funct(v, body.copy());
Var v2 = new Var(functee.type());
Appl a = new Appl(v2, v);
Exp e = body;
body = a;
Exp functorbody = orig.copy();
body = e;
Exp functor = new Funct(v2, functorbody);
functors.add(functor);
functees.add(functee);
}
body.extractFuncts(functors, functees, orig);
}
public Count(String input, Map vars) {
LispReader lr = new LispReader(new StringReader(input));
String stype = lr.next(); // read 'count';
// always count entities
arg = new Var(PType.E);
String argname = lr.next();
vars.put(argname, arg);
body = Exp.makeExp(lr.next(), vars);
// body = body.replace(a,arg);
}
public boolean wellTyped() {
Iterator<Exp> i = exps.iterator();
Exp e;
while (i.hasNext()) {
e = i.next();
if (e == null || !e.wellTyped()) {
// System.out.println("not well typed:"+e);
return false;
}
// only allow sets of Integers
if (!PType.I.matches(e.type())) {
System.out.println("not type T:" + e);
return false;
}
// System.out.println("well typed: "+e);
}
// System.out.println("conj well typed");
return true;
}
public IntSet(String input, Map vars) {
exps = new LinkedList();
LispReader lr = new LispReader(new StringReader(input));
String t = lr.next(); // read the identifier "set"
if (!t.equals("intset")) {
System.err.println("Bad IntSet identifier: " + t);
System.exit(-1);
}
while (lr.hasNext()) {
exps.add(Exp.makeExp(lr.next(), vars));
}
}
/**
* Returns the hierarchies on a non-filter axis.
*
* @return List of hierarchies, never null
*/
private List<Hierarchy> getHierarchiesNonFilter() {
final Exp exp = queryAxis.getSet();
if (exp == null) {
return Collections.emptyList();
}
Type type = exp.getType();
if (type instanceof SetType) {
type = ((SetType) type).getElementType();
}
final MondrianOlap4jConnection olap4jConnection =
cellSetMetaData.olap4jStatement.olap4jConnection;
if (type instanceof TupleType) {
final TupleType tupleType = (TupleType) type;
List<Hierarchy> hierarchyList = new ArrayList<Hierarchy>();
for (Type elementType : tupleType.elementTypes) {
hierarchyList.add(olap4jConnection.toOlap4j(elementType.getHierarchy()));
}
return hierarchyList;
} else {
return Collections.singletonList((Hierarchy) olap4jConnection.toOlap4j(type.getHierarchy()));
}
}
public void extractFuncts(List functors, List functees, Exp orig) {
for (int i = 0; i < exps.size(); i++) {
Exp e = exps.get(i);
e.extractFuncts(functors, functees, orig);
List vars = e.freeVars();
if (vars.size() == 1) {
Var v = (Var) vars.get(0);
Exp functee = new Funct(v, e.copy());
Var v2 = new Var(functee.type());
Appl a = new Appl(v2, v);
exps.set(i, a);
Exp functorbody = orig.copy();
exps.set(i, e);
Exp functor = new Funct(v2, functorbody);
functors.add(functor);
functees.add(functee);
}
}
}
public double avgDepth(int d) {
double total = 0.0;
for (Exp e : exps) total += e.avgDepth(d + 1);
return total / exps.size();
}
public void getConstStrings(List<String> result) {
result.add("intset");
for (Exp e : exps) e.getConstStrings(result);
}
public boolean isCorrect(String words, Exp sem, Parser parser) {
List<ParseResult> parses = parser.bestParses();
if (parses.size() > 0) {
noAnswer = false;
} else {
noAnswer = true;
}
if (parses.size() == 1) {
ParseResult p = parses.get(0);
Exp e = p.getExp();
e = e.copy();
e.simplify();
List l = p.getLexEntries();
parsed++;
if (e.equals(sem)) {
if (verbose) {
System.out.println("CORRECT");
printLex(l);
}
int lits = sem.allLitsCount();
correctParses++;
return true;
} else {
// one parse, it was wrong... oh well...
if (verbose) {
System.out.println("WRONG");
System.out.println(parses.size() + " parses: " + parses);
printLex(l);
}
wrongParses++;
boolean hasCorrect = parser.hasParseFor(sem);
if (verbose) {
System.out.println("Had correct parse: " + hasCorrect);
System.out.print("Feats: ");
Exp eb = parser.bestSem();
Chart c = parser.getChart();
HashVector h = c.computeExpFeatVals(eb);
h.divideBy(c.computeNorm(eb));
h.dropSmallEntries();
System.out.println(h);
}
}
} else {
noParses++;
if (parses.size() > 1) {
// There are more than one equally high scoring
// logical forms. If this is the case, we abstain
// from returning a result.
if (verbose) {
System.out.println("too many parses");
System.out.println(parses.size() + " parses: " + parses);
}
Exp e = parses.get(0).getExp();
ParseResult p = parses.get(0);
boolean hasCorrect = parser.hasParseFor(sem);
if (verbose) System.out.println("Had correct parse: " + hasCorrect);
} else {
// no parses, potentially reparse with word skipping
if (verbose) System.out.println("no parses");
if (emptyTest) {
List<LexEntry> emps = new LinkedList<LexEntry>();
for (int j = 0; j < Globals.tokens.size(); j++) {
List l = Globals.tokens.subList(j, j + 1);
LexEntry le = new LexEntry(l, Cat.EMP);
emps.add(le);
}
parser.setTempLexicon(new Lexicon(emps));
String mes = null;
if (verbose) mes = "EMPTY";
parser.parseTimed(words, null, mes);
parser.setTempLexicon(null);
parses = parser.bestParses();
if (parses.size() == 1) {
ParseResult p = parses.get(0);
List l = p.getLexEntries();
Exp e = p.getExp();
e = e.copy();
e.simplify();
int noEmpty = p.noEmpty();
if (e.equals(sem)) {
if (verbose) {
System.out.println("CORRECT");
printLex(l);
}
emptyCorrect++;
} else {
// one parse, but wrong
if (verbose) {
System.out.println("WRONG: " + e);
printLex(l);
boolean hasCorrect = parser.hasParseFor(sem);
System.out.println("Had correct parse: " + hasCorrect);
}
}
} else {
// too many parses or no parses
emptyNoParses++;
if (verbose) {
System.out.println("WRONG:" + parses);
boolean hasCorrect = parser.hasParseFor(sem);
System.out.println("Had correct parse: " + hasCorrect);
}
}
}
}
}
return false;
}
public boolean canConvert(
int ordinal, Exp fromExp, int to, List<Resolver.Conversion> conversions) {
return TypeUtil.canConvert(ordinal, fromExp.getType(), to, conversions);
}
public void stocGradTrain(Parser parser, boolean testEachRound) {
int numUpdates = 0;
List<LexEntry> fixedEntries = new LinkedList<LexEntry>();
fixedEntries.addAll(parser.returnLex().getLexicon());
// add all sentential lexical entries.
for (int l = 0; l < trainData.size(); l++) {
parser.addLexEntries(trainData.getDataSet(l).makeSentEntries());
}
parser.setGlobals();
DataSet data = null;
// for each pass over the data
for (int j = 0; j < EPOCHS; j++) {
System.out.println("Training, iteration " + j);
int total = 0, correct = 0, wrong = 0, looCorrect = 0, looWrong = 0;
for (int l = 0; l < trainData.size(); l++) {
// the variables to hold the current training example
String words = null;
Exp sem = null;
data = trainData.getDataSet(l);
if (verbose) System.out.println("---------------------");
String filename = trainData.getFilename(l);
if (verbose) System.out.println("DataSet: " + filename);
if (verbose) System.out.println("---------------------");
// loop through the training examples
// try to create lexical entries for each training example
for (int i = 0; i < data.size(); i++) {
// print running stats
if (verbose) {
if (total != 0) {
double r = (double) correct / total;
double p = (double) correct / (correct + wrong);
System.out.print(i + ": =========== r:" + r + " p:" + p);
System.out.println(" (epoch:" + j + " file:" + l + " " + filename + ")");
} else System.out.println(i + ": ===========");
}
// get the training example
words = data.sent(i);
sem = data.sem(i);
if (verbose) {
System.out.println(words);
System.out.println(sem);
}
List<String> tokens = Parser.tokenize(words);
if (tokens.size() > maxSentLen) continue;
total++;
String mes = null;
boolean hasCorrect = false;
// first, get all possible lexical entries from
// a manipulation of the best parse.
List<LexEntry> lex = makeLexEntriesChart(words, sem, parser);
if (verbose) {
System.out.println("Adding:");
for (LexEntry le : lex) {
System.out.println(le + " : " + LexiconFeatSet.initialWeight(le));
}
}
parser.addLexEntries(lex);
if (verbose) System.out.println("Lex Size: " + parser.returnLex().size());
// first parse to see if we are currently correct
if (verbose) mes = "First";
parser.parseTimed(words, null, mes);
Chart firstChart = parser.getChart();
Exp best = parser.bestSem();
// this just collates and outputs the training
// accuracy.
if (sem.equals(best)) {
// System.out.println(parser.bestParses().get(0));
if (verbose) {
System.out.println("CORRECT:" + best);
lex = parser.getMaxLexEntriesFor(sem);
System.out.println("Using:");
printLex(lex);
if (lex.size() == 0) {
System.out.println("ERROR: empty lex");
}
}
correct++;
} else {
if (verbose) {
System.out.println("WRONG: " + best);
lex = parser.getMaxLexEntriesFor(best);
System.out.println("Using:");
printLex(lex);
if (best != null && lex.size() == 0) {
System.out.println("ERROR: empty lex");
}
}
wrong++;
}
// compute first half of parameter update:
// subtract the expectation of parameters
// under the distribution that is conditioned
// on the sentence alone.
double norm = firstChart.computeNorm();
HashVector update = new HashVector();
HashVector firstfeats = null, secondfeats = null;
if (norm != 0.0) {
firstfeats = firstChart.computeExpFeatVals();
firstfeats.divideBy(norm);
firstfeats.dropSmallEntries();
firstfeats.addTimesInto(-1.0, update);
} else continue;
firstChart = null;
if (verbose) mes = "Second";
parser.parseTimed(words, sem, mes);
hasCorrect = parser.hasParseFor(sem);
// compute second half of parameter update:
// add the expectation of parameters
// under the distribution that is conditioned
// on the sentence and correct logical form.
if (!hasCorrect) continue;
Chart secondChart = parser.getChart();
double secnorm = secondChart.computeNorm(sem);
if (norm != 0.0) {
secondfeats = secondChart.computeExpFeatVals(sem);
secondfeats.divideBy(secnorm);
secondfeats.dropSmallEntries();
secondfeats.addTimesInto(1.0, update);
lex = parser.getMaxLexEntriesFor(sem);
data.setBestLex(i, lex);
if (verbose) {
System.out.println("Best LexEntries:");
printLex(lex);
if (lex.size() == 0) {
System.out.println("ERROR: empty lex");
}
}
} else continue;
// now do the update
double scale = alpha_0 / (1.0 + c * numUpdates);
if (verbose) System.out.println("Scale: " + scale);
update.multiplyBy(scale);
update.dropSmallEntries();
numUpdates++;
if (verbose) {
System.out.println("Update:");
System.out.println(update);
}
if (!update.isBad()) {
if (!update.valuesInRange(-100, 100)) {
System.out.println("WARNING: large update");
System.out.println("first feats: " + firstfeats);
System.out.println("second feats: " + secondfeats);
}
parser.updateParams(update);
} else {
System.out.println(
"ERROR: Bad Update: " + update + " -- norm: " + norm + " -- feats: ");
parser.getParams().printValues(update);
System.out.println();
}
} // end for each training example
} // end for each data set
double r = (double) correct / total;
// we can prune the lexical items that were not used
// in a max scoring parse.
if (pruneLex) {
Lexicon cur = new Lexicon();
cur.addLexEntries(fixedEntries);
cur.addLexEntries(data.getBestLex());
parser.setLexicon(cur);
}
if (testEachRound) {
System.out.println("Testing");
test(parser, false);
}
} // end epochs loop
}
