private RootNode parse(String markdown) {
Parser parser = Parboiled.createParser(DocumentrParser.class);
PegDownProcessor proc = new PegDownProcessor(parser);
RootNode rootNode = proc.parseMarkdown(markdown.toCharArray());
fixParaNodes(rootNode);
return rootNode;
}
public ParseResult parse(String sourceCode) {
GrammarDefinition grammar = Parboiled.createParser(GrammarDefinition.class);
ParsingResult<DOMNode> result =
new RecoveringParseRunner<DOMNode>(grammar.program()).run(sourceCode);
if (result.hasErrors()) {
return ParseResult.fail(result.inputBuffer);
}
return ParseResult.ok(result.inputBuffer, (DOMProgram) result.resultValue);
}
@Test
public void test() {
Parser parser = Parboiled.createParser(Parser.class);
// threw IllegalStateException in 0.9.9.0
testWithRecovery(parser.Clause(), "a.b;")
.hasErrors(
""
+ "Invalid input 'EOI', expected '.', ANY or ';' (line 1, pos 5):\n"
+ "a.b;\n"
+ " ^\n");
}
/**
* The domain parser converts a provided domain string representation into a {@code StructuredType}.
* If the domain string defines a simple {@code Vector} based representation of {@code Numeric}
* types, then a {@code Vector} is returned.
*/
public final class DomainParser {
private static final DomainParserGrammar.ExpandingParser EXPANDING_PARSER =
Parboiled.createParser(DomainParserGrammar.ExpandingParser.class);
private static final DomainParserGrammar.DomainGrammar DOMAIN_PARSER =
Parboiled.createParser(DomainParserGrammar.DomainGrammar.class);
private DomainParser() {}
/**
* Parse the provided domain string and return the constructed representation.
*
* @param <E> The structured type.
* @param domain The string to parse.
* @return A {@code TypeList} is returned by default, but if the type is defined to consist of
* {@code Numeric} types, a {@code Vector} instance is returned.
*/
public static <E extends StructuredType<? extends Type>> E parse(String domain) {
final ReportingParseRunner<String> expander =
new ReportingParseRunner<String>(EXPANDING_PARSER.Expansion());
final ParsingResult<String> d = expander.run(domain.replaceAll(" ", ""));
if (d.hasErrors()) {
StringBuilder strBuilder = new StringBuilder();
for (ParseError e : d.parseErrors) {
strBuilder.append(e.getInputBuffer().extract(e.getStartIndex(), e.getEndIndex()));
}
throw new RuntimeException(
"Error in expanding domain: " + domain + " near: " + strBuilder.toString());
}
final String expanded = Joiner.on(",").join(d.valueStack);
final ReportingParseRunner<?> runner = new ReportingParseRunner(DOMAIN_PARSER.Domain());
final ParsingResult<Type> result = (ParsingResult<Type>) runner.run(expanded);
if (result.hasErrors()) {
StringBuilder strBuilder = new StringBuilder();
for (ParseError e : result.parseErrors) {
strBuilder.append(e.getInputBuffer().extract(e.getStartIndex(), e.getEndIndex()));
}
throw new RuntimeException(
"Error in parsing domain: "
+ expanded
+ ". Ensure that the domain is a valid domain string and contains no whitespace.\nError occured near: "
+ strBuilder.toString());
}
List<Type> l = Lists.newArrayList(result.valueStack);
if (isVector(l)) {
@SuppressWarnings("unchecked")
E vector = (E) toVector(l);
return vector;
}
return (E) toTypeList(l);
}
private static TypeList toTypeList(List<Type> l) {
TypeList list = new TypeList();
for (Type t : l) {
list.add(t);
}
return list;
}
/**
* Convert the {@code TypeList} into a {@code Vector} if all elements are {@code Numeric} types.
*
* @param representation The current data structure of the representation.
* @return {@code true} if the structure should really be a {@code Vector}, {@code false}
* otherwise.
*/
private static boolean isVector(List<Type> representation) {
for (Type type : representation) {
if (!(type instanceof Numeric)) {
return false;
}
}
return true;
}
/**
* Convert the provided {@code representation} into a {@code Vector}.
*
* @param representation The {@code StructuredType} to convert.
* @return The converted vector object.
*/
private static Vector toVector(List<Type> representation) {
Vector.Builder vector = Vector.newBuilder();
for (Type type : representation) {
vector.add((Numeric) type);
}
return vector.build();
}
}
public class SparqlTest {
enum Result {
OK,
FAIL
}
class TextInfo {
Result result;
String text;
public TextInfo(Result result, String text) {
super();
this.result = result;
this.text = text;
}
}
SparqlParser parser = Parboiled.createParser(SparqlParser.class);
@Test
public void test() throws Exception {
int failures = 0;
for (TextInfo textInfo : getTextInfos()) {
ParsingResult result = RecoveringParseRunner.run(parser.Query(), textInfo.text);
boolean passed = result.hasErrors() == (textInfo.result == Result.FAIL);
if (!passed) {
failures++;
// System.err.println(textInfo.text + " --> " + textInfo.result + "\n\n");
}
}
Assert.assertEquals(
failures, 12); // currently 12 tests require semantic validation and therefore fail
}
protected List<TextInfo> getTextInfos() throws Exception {
List<TextInfo> textInfos = new ArrayList<TextInfo>();
BufferedReader in =
new BufferedReader(
new InputStreamReader(
ClassLoader.getSystemClassLoader().getResourceAsStream("SparqlTest.test"),
Charset.forName("UTF8")));
while (in.ready()) {
if (in.read() == '<' && in.ready() && in.read() == '<') {
in.readLine();
textInfos.add(parseText(in));
}
}
return textInfos;
}
private TextInfo parseText(BufferedReader in) throws Exception {
Pattern unicodes = Pattern.compile("\\\\u([0-9a-fA-F]{4})");
StringBuffer text = new StringBuffer();
while (in.ready()) {
String line = in.readLine();
if (line.startsWith(">>")) {
return new TextInfo(
line.toLowerCase().contains("ok") ? Result.OK : Result.FAIL, text.toString());
} else {
Matcher matcher = unicodes.matcher(line);
while (matcher.find()) {
matcher.appendReplacement(
text, Character.toString((char) Integer.parseInt(matcher.group(1), 16)));
}
matcher.appendTail(text);
text.append('\n');
}
}
throw new NoSuchElementException("Expected \">>\"");
}
}
@SuppressWarnings({"ConstantConditions"})
protected void run(String[] args) {
System.out.println("parboiled Java parser, performance test");
System.out.println("---------------------------------------");
System.out.print("Creating parser... :");
long start = System.currentTimeMillis();
Parboiled.createParser(JavaParser.class);
time(start);
System.out.print("Creating 100 more parser instances... :");
JavaParser parser = null;
start = System.currentTimeMillis();
for (int i = 0; i < 100; i++) {
parser = Parboiled.createParser(JavaParser.class);
}
time(start);
System.out.print("Creating 100 more parser instances using BaseParser.newInstance() ... :");
start = System.currentTimeMillis();
for (int i = 0; i < 100; i++) {
parser = parser.newInstance();
}
time(start);
start = System.currentTimeMillis();
File baseDir = args.length == 1 ? new File(args[0]) : null;
if (baseDir == null || !baseDir.exists()) baseDir = new File(".");
System.out.printf("Retrieving file list from '%s'", baseDir);
List<File> sources = recursiveGetAllJavaSources(baseDir, new ArrayList<File>());
time(start);
System.out.printf("Parsing all %s given java sources", sources.size());
Rule rootRule =
parser.CompilationUnit().suppressNode(); // we want to see the parse-tree-less performance
start = System.currentTimeMillis();
long lines = 0, characters = 0;
for (File sourceFile : sources) {
long dontCountStart = System.currentTimeMillis();
String sourceText = readAllText(sourceFile);
start +=
System.currentTimeMillis()
- dontCountStart; // do not count the time for reading the text file
ParsingResult<?> result = null;
try {
result = run(rootRule, sourceText);
} catch (Exception e) {
System.out.printf("\nException while parsing file '%s':\n%s", sourceFile, e);
System.exit(1);
}
if (!result.matched) {
System.out.printf(
"\nParse error(s) in file '%s':\n%s", sourceFile, printParseErrors(result));
System.exit(1);
} else {
System.out.print('.');
}
lines += result.inputBuffer.getLineCount();
characters += sourceText.length();
}
long time = time(start);
System.out.println("Parsing performance:");
System.out.printf(
" %6d Files -> %6.2f Files/sec\n", sources.size(), sources.size() * 1000.0 / time);
System.out.printf(" %6d Lines -> %6d Lines/sec\n", lines, lines * 1000 / time);
System.out.printf(" %6d Chars -> %6d Chars/sec\n", characters, characters * 1000 / time);
}
