public void WriteHeader(){
out.println("\t\t"+Parser.StartingTag("ValidichroHeader"));
out.println("\t\t\t"+Parser.Tag("Date",experiment_date));
out.println("\t\t\t"+Parser.Tag("Generic",generic));
//...
out.println("\t\t"+Parser.EndingTag("ValidichroHeader"));
}
public Expr dropNoncompArgs(Context ctxt) {
if (ctxt.getFlag("debug_drop_noncomp")) {
ctxt.w.println("Dropping non-comp arguments from term-app " + toString(ctxt));
ctxt.w.flush();
}
ArrayList nX = new ArrayList();
ArrayList no = new ArrayList();
boolean changed = false;
Boolean b = new Boolean(false);
for (int i = 0, iend = X.length; i < iend; i++) {
if (X[i].isProof(ctxt) || specarg[i]) changed = true;
else {
nX.add(X[i]);
no.add(b);
}
}
Expr ret = this;
if (changed) {
if (nX.size() == 0) ret = head;
else ret = new TermApp(head, Parser.toExprArray(nX), Parser.toBooleanArray(no));
}
if (ctxt.getFlag("debug_drop_noncomp")) {
ctxt.w.println("Returning " + ret.toString(ctxt));
ctxt.w.flush();
}
return ret;
}
public void parseStyleDeclaration(CSSStyleDeclaration sd, InputSource source) throws IOException {
Stack nodeStack = new Stack();
nodeStack.push(sd);
CSSOMHandler handler = new CSSOMHandler(nodeStack);
_parser.setDocumentHandler(handler);
_parser.parseStyleDeclaration(source);
}
public void WriteExported(){
out.println(Parser.getFileHeader());
out.println("\t"+Parser.StartingTag("ValidichroData"));
WriteHeader();
WriteBody();
WriteUnparsed();
out.println("\t"+Parser.EndingTag("ValidichroData"));
}
/** Gets the index of the first occurence of any string of array <i>ss</i> ignoring case. */
public int indexOfIgnoreCase(String[] ss) {
Parser par = new Parser(str, index);
while (par.hasMore()) {
if (par.startsWithIgnoreCase(ss)) return par.getPos();
else par.skipChar();
}
return -1;
}
public void WriteBody(){
//@ToDo : Make this proper java!
//@ToDo Important: Add missing elements and check existing ones
// - Update:, now refers to values as Unknown1,Uknown2... It's just unknown to me, can't remember what these values represent!
out.println("\t\t"+Parser.StartingTag("ValidichroBody"));
for(String line :spectral_table){
out.println("\t\t\t"+Parser.Tag("SpectralEntry",line));
}
out.println("\t\t"+Parser.EndingTag("ValidichroBody"));
}
private Parser createJsonParser(String queryScope, String input) {
Parser parser = new JsonParser(input);
if (!HttpAdapter.createResponseWrangling(queryScope, this).getRecordSelector().isEmpty()) {
((JsonParser) parser)
.setRootElement(
parser.parse(
HttpAdapter.createResponseWrangling(queryScope, this).getRecordSelector(),
JsonElement.class));
}
return parser;
}
// 得到的tokens包含源代码的由词法分析器分析出的所有单词，但不包含注释
private static ArrayList<JavaScriptToken> parse(
Reader in, ErrorReporter reporter) // 返回tokens（保存的是源文件中出现的javascript关键字和NAME，REGEXP，STRING等类型）
throws IOException, EvaluatorException {
CompilerEnvirons env = new CompilerEnvirons(); // 创建编译环境对象
env.setLanguageVersion(Context.VERSION_1_7); // 设置语言版本
Parser parser = new Parser(env, reporter); // 创建解释器对象
parser.parse(in, null, 1); // 解释输入流
String source = parser.getEncodedSource(); // 获得已编码的源码（词法分析阶段通常是把从源程序中识别出的各个单词的词文
// 转换为某种内部表示
int offset = 0;
int length = source.length();
ArrayList<JavaScriptToken> tokens = new ArrayList<JavaScriptToken>();
StringBuffer sb = new StringBuffer();
while (offset < length) {
int tt = source.charAt(offset++); // 获取特定位置上的字符，并转化为ASCII编码
switch (tt) {
case Token.CONDCOMMENT: // 条件注释
case Token.KEEPCOMMENT: // 注释
case Token.NAME: //
case Token.REGEXP: // 正则表达式类型
case Token.STRING: // String类型,js程序中双引号或单引号括起来的字符串
sb.setLength(0);
offset = printSourceString(source, offset, sb);
tokens.add(new JavaScriptToken(tt, sb.toString()));
break;
case Token.NUMBER: // Number类型
sb.setLength(0);
offset = printSourceNumber(source, offset, sb);
tokens.add(new JavaScriptToken(tt, sb.toString()));
break;
default:
String literal = literals.get(new Integer(tt));
if (literal != null) { // 若不为空，说明哈希表literals中含有键new
// Integer(tt)所对应的值
tokens.add(new JavaScriptToken(tt, literal)); // 将此关键字保存到数组列表tokens中
}
break;
}
}
/*
* //begin Iterator<JavaScriptToken> iterator = tokens.iterator();
* JavaScriptToken token; while(iterator.hasNext()) { token =
* iterator.next();
* System.out.println(token.getType()+"\t"+token.getValue()); } //end
*/
return tokens;
}
/*.................................................................................................................*/
public Object doCommand(String commandName, String arguments, CommandChecker checker) {
if (checker.compare(
this.getClass(), "Sets the username", "[username]", commandName, "setUsername")) {
username = parser.getFirstToken(arguments);
}
return null;
}
public void ParseSpectralEntry(String line){
//@ToDo, take all values (there are 7)
// Update - now takes all values, but calls them Unknown1,2,3,4 I need to find out what these values represent!
//Not production safe!
String table_ent=Parser.getTableRow(line,new String[]{"WaveLength","Smooth","Unsmooth","Unknown1","Unknown2","Uknown3","Unknown4"});
spectral_table.add(table_ent);
}
public void evaluate() {
try {
// clear problems and console messages
problemsView.setText("");
consoleView.setText("");
// update status view
statusView.setText(" Parsing ...");
tabbedPane.setSelectedIndex(0);
LispExpr root = Parser.parse(textView.getText());
statusView.setText(" Running ...");
tabbedPane.setSelectedIndex(1);
// update run button
runButton.setIcon(stopImage);
runButton.setActionCommand("Stop");
// start run thread
runThread = new RunThread(root);
runThread.start();
} catch (SyntaxError e) {
tabbedPane.setSelectedIndex(0);
System.err.println(
"Syntax Error at " + e.getLine() + ", " + e.getColumn() + " : " + e.getMessage());
} catch (Error e) {
// parsing error
System.err.println(e.getMessage());
statusView.setText(" Errors.");
}
}
/**
* Parses an expression. Returns a object of type Node, does not catch errors. Does not set the
* topNode variable of the JEP instance. This method should generally be used with the {@link
* #evaluate evaluate} method rather than getValueAsObject.
*
* @param expression represented as a string.
* @return The top node of a tree representing the parsed expression.
* @throws ParseException
* @since 2.3.0 alpha
* @since 2.3.0 beta - will raise exception if errorList non empty
*/
public Node parse(String expression) throws ParseException {
java.io.StringReader sr = new java.io.StringReader(expression);
errorList.removeAllElements();
Node node = parser.parseStream(sr, this);
if (this.hasError()) throw new ParseException(getErrorInfo());
return node;
}
public ThrowsTagImpl(String text, ClassDocImpl contextClass, MemberDocImpl contextMember) {
super(text);
char[] textarr = text.toCharArray();
int i = 0;
for (; i < textarr.length; ++i) {
if (!Parser.isWhitespace(textarr[i])) break;
}
for (; i < textarr.length; ++i) {
if (Parser.isWhitespace(textarr[i])) {
this.exceptionName = new String(textarr, 0, i).trim();
this.exceptionComment = new String(textarr, i, textarr.length - i).trim();
break;
}
}
if (null != exceptionName) {
if (contextClass == null) {
this.exception = Main.getRootDoc().classNamed(exceptionName);
} else {
this.exception = contextClass.findClass(exceptionName);
}
if (exception != null) this.exceptionName = exception.qualifiedName();
else {
if (text.trim().startsWith("<")) {
Main.getRootDoc()
.printWarning(
"Expected exception name but got '"
+ text
+ "' in class "
+ contextClass.getClassName());
}
}
} else {
Main.getRootDoc()
.printWarning(
"@throws tag in comment for "
+ contextClass.qualifiedName()
+ "."
+ contextMember.name()
+ " doesn't specify an exception.");
}
if (this.exceptionComment != null) {
setBody(this.exceptionComment, contextClass, contextMember);
}
}
public static HashMap<String, Test> processScriptFile(File file) {
HashMap<String, Test> tests = new HashMap<String, Test>();
try {
Parser parser = new Parser(new Lexer(new PushbackReader(new FileReader(file))));
Start root = parser.parse();
AScript a1 = (AScript) root.getPScript();
for (PScriptTail tail : a1.getScriptTail()) {
AScriptTail a2 = (AScriptTail) tail;
Test test = TestBuilder.buildTest(a2.getTest());
tests.put(test.getName(), test);
}
System.out.println("SUCESSFULLY PARSED!");
} catch (Exception e) {
System.err.println(e.getMessage());
}
return tests;
}
public void reset() {
this.lookupEnvironment.reset();
// GROOVY start: give the parser a chance to reset as well
parser.reset();
// GROOVY end
this.parser.scanner.source = null;
this.unitsToProcess = null;
if (DebugRequestor != null) DebugRequestor.reset();
this.problemReporter.reset();
}
public KObject eval(CTX ctx, String script, long uline) {
ctx.gamma = new Gamma();
ctx.gamma.ks = this;
List<Token> tls = new ArrayList<Token>();
int pos = tls.size();
tokenize(ctx, script, uline, tls);
Block bk = Parser.newBlock(ctx, this, null, tls, pos, tls.size(), ';');
KArray.clear(tls, pos);
return evalBlock(ctx, bk);
}
@Test
public void testWarnings() throws ParseException, TokeniserException {
ArrayList<Warning> warnings = new ArrayList<>();
Policy p1 = Parser.parse("frame-src aaa", "https://origin", warnings);
assertEquals("frame-src aaa", p1.show());
assertEquals(1, warnings.size());
assertEquals(
"The frame-src directive is deprecated as of CSP version 1.1. Authors who wish to govern nested browsing contexts SHOULD use the child-src directive instead.",
warnings.iterator().next().message);
}
private void parseMapsXml() {
try {
String fileName = LocationLoader.getLocationDir(NavigineApp.AppContext, null) + "/maps.xml";
List<LocationInfo> infoList = Parser.parseMapsXml(NavigineApp.AppContext, fileName);
mInfoList = new ArrayList<LocationInfo>();
if (infoList != null) mInfoList = infoList;
mAdapter.updateList();
new File(fileName).delete();
} catch (Throwable e) {
Log.e(TAG, Log.getStackTraceString(e));
}
}
/*.................................................................................................................*/
public Object doCommand(String commandName, String arguments, CommandChecker checker) {
if (checker.compare(
this.getClass(),
"Sets the running file path",
"[file path]",
commandName,
"setRunningFilePath")) {
runningFilePath = parser.getFirstToken(arguments);
} else if (checker.compare(
this.getClass(),
"Sets the output file paths",
"[file paths]",
commandName,
"setOutputFilePaths")) {
int num = parser.getNumberOfTokens(arguments);
outputFilePaths = new String[num];
if (num > 0) outputFilePaths[0] = parser.getFirstToken();
for (int i = 1; i < num; i++) outputFilePaths[i] = parser.getNextToken();
}
return null;
}
public JavaProgram createJavaProgram(
List sourceFiles,
String libraryPath,
String mainClassname,
String projectType,
String policyType)
throws Exception {
// Debug.setOn(true);
long l = Debug.getTime();
LibraryData libraryData = new LibraryData(new File(libraryPath), projectType, language, false);
Library library = new Library(libraryData);
Parser parser =
Parser.createParser(this.language, sourceFiles, library, new TranslationPolicy(policyType));
long lll = Debug.getTime();
System.out.println("Library Loaded in " + (lll - l) + "ms");
ParseTree tree = parser.parse();
TranslationReport report = tree.getTranslationReport();
long ll = Debug.getTime();
System.out.println("Code parsed in " + (ll - lll) + "ms");
Debug.setOn(true);
Translator tt = new Translator(this.language);
JavaProgram jp = tt.createJavaProgram(tree, mainClassname, projectType);
if (report.hasTypeResolveErrors() || report.hasTranslationWarnings()) {
List errors = new ArrayList();
errors.addAll(report.getTypeResolveExceptions());
report.doReport(errors, report.getTranslationWarnings());
}
l = Debug.getTime();
System.out.println("Translated in " + (l - ll) + "ms");
jp.setTypeResolved(!report.hasTypeResolveErrors()); // resolved if no errors
return jp;
}
private static void testParser(Parser parser, List<Tree<String>> testTrees) {
EnglishPennTreebankParseEvaluator.LabeledConstituentEval<String> eval =
new EnglishPennTreebankParseEvaluator.LabeledConstituentEval<String>(
Collections.singleton("ROOT"),
new HashSet<String>(Arrays.asList(new String[] {"''", "``", ".", ":", ","})));
for (Tree<String> testTree : testTrees) {
List<String> testSentence = testTree.getYield();
if (testSentence.size() > MAX_LENGTH) continue;
Tree<String> guessedTree = parser.getBestParse(testSentence);
System.out.println("Guess:\n" + Trees.PennTreeRenderer.render(guessedTree));
System.out.println("Gold:\n" + Trees.PennTreeRenderer.render(testTree));
eval.evaluate(guessedTree, testTree);
}
eval.display(true);
}
public void prettyPrint() {
try {
// clear old problem messages
problemsView.setText("");
// update status view
statusView.setText(" Parsing ...");
LispExpr root = Parser.parse(textView.getText());
statusView.setText(" Pretty Printing ...");
String newText = PrettyPrinter.prettyPrint(root);
textView.setText(newText);
statusView.setText(" Done.");
} catch (Error e) {
System.err.println(e.getMessage());
statusView.setText(" Errors.");
}
}
public static void test1() {
Sequence[] seqs = Parser.test1();
double[][] distances = (new GappedHammingDistance()).getDistanceMatrix(seqs);
DecimalFormat f = new DecimalFormat("0000.000");
System.out.println(" " + seqs.length);
for (int i = 0; i < seqs.length; i++) {
System.out.print(seqs[i].name);
for (int k = 0; k < 10 - seqs[i].name.length(); k++) {
System.out.print(" ");
}
System.out.print(" ");
int cols = 1;
for (int j = 0; j < seqs.length; j++) {
if (i != j) {
int di = i;
int dj = j;
// force symmetric matrix - arg - what the???
// testing!!!
if (j > i) {
di = j;
dj = i;
}
double d = distances[di][dj];
// testing
if (d > 2000) {
System.err.println("ARGH");
}
System.out.print(f.format(d));
} else {
System.out.print(f.format(0));
}
System.out.print(" ");
cols++;
if (cols >= 7) {
System.out.println();
System.out.print(" ");
cols = 0;
}
}
if (cols != 7) {
System.out.println();
}
}
}
/**
* Parses the expression. If there are errors in the expression, they are added to the <code>
* errorList</code> member. Errors can be obtained through <code>getErrorInfo()</code>.
*
* @param expression_in The input expression string
* @return the top node of the expression tree if the parse was successful, <code>null</code>
* otherwise
*/
public Node parseExpression(String expression_in) {
Reader reader = new StringReader(expression_in);
try {
// try parsing
errorList.removeAllElements();
topNode = parser.parseStream(reader, this);
// if there is an error in the list, the parse failed
// so set topNode to null
if (hasError()) topNode = null;
} catch (Throwable e) {
// an exception was thrown, so there is no parse tree
topNode = null;
// check the type of error
if (e instanceof ParseException) {
// the ParseException object contains additional error
// information
errorList.addElement(((ParseException) e).getMessage());
// getErrorInfo());
} else {
// if the exception was not a ParseException, it was most
// likely a syntax error
if (debug) {
System.out.println(e.getMessage());
e.printStackTrace();
}
errorList.addElement("Syntax error");
}
}
// If traversing is enabled, print a dump of the tree to
// standard output
if (traverse && !hasError()) {
ParserVisitor v = new ParserDumpVisitor();
try {
topNode.jjtAccept(v, null);
} catch (ParseException e) {
errorList.addElement(e.getMessage());
}
}
return topNode;
}
protected PatternExpr getSequencePatternExpr(String name, boolean copy) {
Object obj = variables.get(name);
if (obj != null) {
if (obj instanceof PatternExpr) {
PatternExpr pe = (PatternExpr) obj;
return (copy) ? pe.copy() : pe;
} else if (obj instanceof NodePattern) {
return new NodePatternExpr((NodePattern) obj);
} else if (obj instanceof String) {
try {
return parser.parseSequence(this, (String) obj);
} catch (Exception pex) {
throw new RuntimeException("Error parsing " + obj + " to sequence pattern", pex);
}
} else {
throw new Error("Invalid sequence pattern variable class: " + obj.getClass());
}
}
return null;
}
protected NodePattern getNodePattern(String name) {
Object obj = variables.get(name);
if (obj != null) {
if (obj instanceof NodePatternExpr) {
NodePatternExpr pe = (NodePatternExpr) obj;
return pe.nodePattern;
} else if (obj instanceof NodePattern) {
return (NodePattern) obj;
} else if (obj instanceof String) {
try {
NodePatternExpr pe = (NodePatternExpr) parser.parseNode(this, (String) obj);
return pe.nodePattern;
} catch (Exception pex) {
throw new RuntimeException("Error parsing " + obj + " to node pattern", pex);
}
} else {
throw new Error("Invalid node pattern variable class: " + obj.getClass());
}
}
return null;
}
private void test(Parser p, DataSet d) {
int dsize = d.size();
testSize = dsize;
for (int i = 0; i < dsize; i++) {
String words = d.sent(i);
Exp sem = d.sem(i);
if (verbose) {
System.out.println(
i + ": ==================(" + correctParses + " -- " + wrongParses + ")");
System.out.println(words);
System.out.println(sem);
}
String mes = null;
if (verbose) mes = "Test";
p.parseTimed(words, null, mes);
isCorrect(words, sem, p);
}
}
// bindKeyToCommand binds a given key sequence to a Lisp command
// so that pressing the key sequence executes the command.
public void bindKeyToCommand(String keySequence, String cmd) {
bindKeyToCommand(keySequence, Parser.parse(cmd));
}
public CSSRule parseRule(InputSource source) throws IOException {
CSSOMHandler handler = new CSSOMHandler();
_parser.setDocumentHandler(handler);
_parser.parseRule(source);
return (CSSRule) handler.getRoot();
}
public static void main(String[] args) {
// set up default options ..............................................
Map<String, String> options = new HashMap<String, String>();
options.put("--path", "../data/parser/");
options.put("--data", "masc");
options.put("--parser", "nlpclass.assignments.PCFGParserTester$BaselineParser");
options.put("--maxLength", "20");
// let command-line options supersede defaults .........................
options.putAll(CommandLineUtils.simpleCommandLineParser(args));
System.out.println("PCFGParserTester options:");
for (Map.Entry<String, String> entry : options.entrySet()) {
System.out.printf(" %-12s: %s%n", entry.getKey(), entry.getValue());
}
System.out.println();
MAX_LENGTH = Integer.parseInt(options.get("--maxLength"));
Parser parser;
try {
Class parserClass = Class.forName(options.get("--parser"));
parser = (Parser) parserClass.newInstance();
} catch (Exception e) {
throw new RuntimeException(e);
}
System.out.println("Using parser: " + parser);
String basePath = options.get("--path");
String preBasePath = basePath;
String dataSet = options.get("--data");
if (!basePath.endsWith("/")) {
basePath += "/";
}
// basePath += dataSet;
System.out.println("Data will be loaded from: " + basePath + "\n");
List<Tree<String>> trainTrees = new ArrayList<Tree<String>>();
List<Tree<String>> validationTrees = new ArrayList<Tree<String>>();
List<Tree<String>> testTrees = new ArrayList<Tree<String>>();
if (dataSet.equals("miniTest")) {
// training data: first 3 of 4 datums
basePath += "parser/" + dataSet;
System.out.println("Loading training trees...");
trainTrees = readTrees(basePath, 1, 3);
System.out.println("done.");
// test data: last of 4 datums
System.out.println("Loading test trees...");
testTrees = readTrees(basePath, 4, 4);
System.out.println("done.");
} else if (dataSet.equals("masc")) {
basePath += "parser/";
// training data: MASC train
System.out.println("Loading MASC training trees... from: " + basePath + "masc/train");
trainTrees.addAll(readMASCTrees(basePath + "masc/train", 0, 38));
System.out.println("done.");
System.out.println("Train trees size: " + trainTrees.size());
System.out.println("First train tree: " + Trees.PennTreeRenderer.render(trainTrees.get(0)));
System.out.println(
"Last train tree: "
+ Trees.PennTreeRenderer.render(trainTrees.get(trainTrees.size() - 1)));
// test data: MASC devtest
System.out.println("Loading MASC test trees...");
testTrees.addAll(readMASCTrees(basePath + "masc/devtest", 0, 11));
// testTrees.addAll(readMASCTrees(basePath+"masc/blindtest", 0, 8));
System.out.println("Test trees size: " + testTrees.size());
System.out.println("done.");
System.out.println("First test tree: " + Trees.PennTreeRenderer.render(testTrees.get(0)));
System.out.println(
"Last test tree: " + Trees.PennTreeRenderer.render(testTrees.get(testTrees.size() - 1)));
} else if (!dataSet.equals("miniTest") && !dataSet.equals("masc")) {
throw new RuntimeException("Bad data set: " + dataSet + ": use miniTest or masc.");
}
System.out.println("\nTraining parser...");
parser.train(trainTrees);
System.out.println("\nTesting parser...");
testParser(parser, testTrees);
}
