/** @author Patrice Lopez */
public class FullTextParser extends AbstractParser {
private static final Logger LOGGER = LoggerFactory.getLogger(FullTextParser.class);
private LanguageUtilities languageUtilities = LanguageUtilities.getInstance();
private HeaderParser headerParser = null;
private CitationParser citationParser = null;
// private String tmpPathName = null;
private Document doc = null;
private File tmpPath = null;
private String pathXML = null;
private BiblioItem resHeader = null;
/** TODO some documentation... */
public FullTextParser() {
super(GrobidModels.FULLTEXT);
tmpPath = GrobidProperties.getInstance().getTempPath();
}
/**
* TODO some documentation...
*
* @param input filename of pdf file
* @param consolidateHeader if consolidate header
* @param consolidateCitations if consolidate citations
* @return result
*/
public String processing(String input, boolean consolidateHeader, boolean consolidateCitations) {
if (input == null) {
throw new GrobidResourceException("Cannot process pdf file, because input file was null.");
}
File inputFile = new File(input);
if (!inputFile.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because input file '"
+ inputFile.getAbsolutePath()
+ "' does not exists.");
}
if (tmpPath == null) {
throw new GrobidResourceException("Cannot process pdf file, because temp path is null.");
}
if (!tmpPath.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because temp path '"
+ tmpPath.getAbsolutePath()
+ "' does not exists.");
}
doc = new Document(input, tmpPath.getAbsolutePath());
try {
int startPage = -1;
int endPage = -1;
pathXML =
doc.pdf2xml(true, false, startPage, endPage, input, tmpPath.getAbsolutePath(), false);
// with timeout,
// no force pdf reloading
// input is the pdf absolute path, tmpPath is the temp. directory for the temp. lxml file,
// path is the resource path
// and we don't extract images in the PDF file
if (pathXML == null) {
throw new GrobidResourceException(
"PDF parsing fails, " + "because path of where to store xml file is null.");
}
doc.setPathXML(pathXML);
// doc.addFeaturesDocument();
if (headerParser == null) {
headerParser = new HeaderParser();
}
if (citationParser == null) {
citationParser = new CitationParser();
}
String tei =
doc.toTEI(
headerParser,
citationParser,
consolidateHeader,
consolidateCitations,
false,
null,
false,
false);
resHeader = doc.getResHeader();
LOGGER.debug(tei);
return tei;
} catch (Exception e) {
throw new GrobidException("An exception occured while running Grobid.", e);
} finally {
// keep it clean when leaving...
doc.cleanLxmlFile(pathXML, false);
}
}
public String processing2(String input, boolean consolidateHeader, boolean consolidateCitations)
throws Exception {
if (input == null) {
throw new GrobidResourceException("Cannot process pdf file, because input file was null.");
}
File inputFile = new File(input);
if (!inputFile.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because input file '"
+ inputFile.getAbsolutePath()
+ "' does not exists.");
}
if (tmpPath == null) {
throw new GrobidResourceException("Cannot process pdf file, because temp path is null.");
}
if (!tmpPath.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because temp path '"
+ tmpPath.getAbsolutePath()
+ "' does not exists.");
}
doc = new Document(input, tmpPath.getAbsolutePath());
try {
int startPage = -1;
int endPage = -1;
pathXML =
doc.pdf2xml(true, false, startPage, endPage, input, tmpPath.getAbsolutePath(), true);
// with timeout,
// no force pdf reloading
// input is the pdf absolute path, tmpPath is the temp. directory for the temp. lxml file,
// path is the resource path
// and we process images in the pdf file
if (pathXML == null) {
throw new Exception("PDF parsing fails");
}
doc.setPathXML(pathXML);
ArrayList<String> tokenizations = doc.addFeaturesDocument();
if (doc.getBlocks() == null) {
throw new Exception("PDF parsing resulted in empty content");
}
String fulltext = doc.getFulltextFeatured(true, true);
StringTokenizer st = new StringTokenizer(fulltext, "\n");
feedTaggerAndParse(st);
StringBuilder res = new StringBuilder();
for (int i = 0; i < tagger.size(); i++) {
for (int j = 0; j < tagger.xsize(); j++) {
res.append(tagger.x(i, j)).append("\t");
}
res.append(tagger.y2(i));
res.append("\n");
}
// buffer for the fulltext block
String rese = res.toString();
// set the different sections of the Document object
doc = BasicStructureBuilder.resultSegmentation(doc, rese, tokenizations);
// header processing
if (headerParser == null) {
headerParser = new HeaderParser();
}
resHeader = new BiblioItem();
headerParser.processingHeaderBlock(consolidateHeader, doc, resHeader);
// the language identification is normally done during the header parsing, but only
// based on header information.
// LanguageUtilities languageUtilities = LanguageUtilities.getInstance();
Language langu = languageUtilities.runLanguageId(resHeader.getTitle() + "\n" + doc.getBody());
if (langu != null) {
String lang = langu.getLangId();
doc.setLanguage(lang);
resHeader.setLanguage(lang);
}
// citation processing
if (citationParser == null) {
citationParser = new CitationParser();
}
ArrayList<BibDataSet> resCitations;
// ArrayList<String> tokenizationsRef = doc.getTokenizationsReferences();
// System.out.println(tokenizationsRef.toString());
// resCitations = BasicStructureBuilder.giveReferenceSegments(doc);
resCitations = doc.bibDataSets;
if (resCitations != null) {
for (BibDataSet bds : resCitations) {
String marker = bds.getRefSymbol();
if (marker != null) {
marker = marker.replace(".", "");
marker = marker.replace(" ", "");
bds.setRefSymbol(marker);
}
BiblioItem bib = citationParser.processing(bds.getRawBib(), consolidateCitations);
bds.setResBib(bib);
}
}
// final combination
return toTEI(doc, rese, tokenizations, resHeader, false, null, false);
} catch (Exception e) {
throw new GrobidException("An exception occured while running Grobid.", e);
} finally {
// keep it clean when leaving...
doc.cleanLxmlFile(pathXML, false);
}
}
/**
* Process the full text of the specified pdf and format the result as training data.
*
* @param inputFile input file
* @param pathFullText path to fulltext
* @param pathTEI path to TEI
* @param id id
*/
public void createTrainingFullText(
String inputFile, String pathFullText, String pathTEI, int id) {
if (tmpPath == null)
throw new GrobidResourceException("Cannot process pdf file, because temp path is null.");
if (!tmpPath.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because temp path '"
+ tmpPath.getAbsolutePath()
+ "' does not exists.");
}
doc = new Document(inputFile, tmpPath.getAbsolutePath());
try {
int startPage = -1;
int endPage = -1;
File file = new File(inputFile);
if (!file.exists()) {
throw new GrobidResourceException(
"Cannot train for fulltext, becuase file '"
+ file.getAbsolutePath()
+ "' does not exists.");
}
String PDFFileName = file.getName();
pathXML =
doc.pdf2xml(true, false, startPage, endPage, inputFile, tmpPath.getAbsolutePath(), true);
// with timeout,
// no force pdf reloading
// pathPDF is the pdf file, tmpPath is the tmp directory for the lxml file,
// path is the resource path
// and we don't extract images in the pdf file
if (pathXML == null) {
throw new Exception("PDF parsing fails");
}
doc.setPathXML(pathXML);
doc.addFeaturesDocument();
if (doc.getBlocks() == null) {
throw new Exception("PDF parsing resulted in empty content");
}
String fulltext = doc.getFulltextFeatured(true, true);
ArrayList<String> tokenizations = doc.getTokenizationsFulltext();
// we write the full text untagged
String outPathFulltext =
pathFullText + "/" + PDFFileName.replace(".pdf", ".training.fulltext");
Writer writer =
new OutputStreamWriter(new FileOutputStream(new File(outPathFulltext), false), "UTF-8");
writer.write(fulltext + "\n");
writer.close();
StringTokenizer st = new StringTokenizer(fulltext, "\n");
feedTaggerAndParse(st);
StringBuilder res = new StringBuilder();
for (int i = 0; i < tagger.size(); i++) {
for (int j = 0; j < tagger.xsize(); j++) {
res.append(tagger.x(i, j)).append("\t");
}
res.append(tagger.y2(i));
res.append("\n");
}
// buffer for the fulltext block
String rese = res.toString();
StringBuffer bufferFulltext = trainingExtraction(rese, tokenizations);
// write the TEI file to reflect the extract layout of the text as extracted from the pdf
writer =
new OutputStreamWriter(
new FileOutputStream(
new File(
pathTEI + "/" + PDFFileName.replace(".pdf", ".training.fulltext.tei.xml")),
false),
"UTF-8");
writer.write(
"<?xml version=\"1.0\" ?>\n<tei>\n\t<teiHeader>\n\t\t<fileDesc xml:id=\""
+ id
+ "\"/>\n\t</teiHeader>\n\t<text xml:lang=\"en\">\n");
writer.write(bufferFulltext.toString());
writer.write("\n\t</text>\n</tei>\n");
writer.close();
// output of the identified citations as traning date
// buffer for the reference block
StringBuilder allBufferReference = new StringBuilder();
// we need to rebuild the found citation string as it appears
String input = "";
ArrayList<String> inputs = new ArrayList<String>();
int q = 0;
st = new StringTokenizer(rese, "\n");
while (st.hasMoreTokens() && (q < tokenizations.size())) {
String line = st.nextToken();
String theTotalTok = tokenizations.get(q);
String theTok = tokenizations.get(q);
while (theTok.equals(" ") || theTok.equals("\t") || theTok.equals("\n")) {
q++;
theTok = tokenizations.get(q);
theTotalTok += theTok;
}
if (line.endsWith("I-<reference>")) {
if (input.trim().length() > 1) {
inputs.add(input.trim());
input = "";
}
input += "\n" + theTotalTok;
} else if (line.endsWith("<reference>")) {
input += theTotalTok;
}
q++;
}
if (input.trim().length() > 1) {
inputs.add(input.trim());
if (citationParser == null) {
citationParser = new CitationParser();
}
for (String inpu : inputs) {
ArrayList<String> inpus = new ArrayList<String>();
inpus.add(inpu);
StringBuilder bufferReference = citationParser.trainingExtraction(inpus);
if (bufferReference != null) {
allBufferReference.append(bufferReference.toString()).append("\n");
}
}
}
if (allBufferReference != null) {
if (allBufferReference.length() > 0) {
Writer writerReference =
new OutputStreamWriter(
new FileOutputStream(
new File(
pathTEI + "/" + PDFFileName.replace(".pdf", ".training.references.xml")),
false),
"UTF-8");
writerReference.write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
writerReference.write("<citations>\n");
writerReference.write(allBufferReference.toString());
writerReference.write("</citations>\n");
writerReference.close();
}
}
} catch (Exception e) {
throw new GrobidException(
"An exception occured while running Grobid training" + " data generation for full text.",
e);
} finally {
doc.cleanLxmlFile(pathXML, true);
}
}
/** Return the Document object of the last processed pdf file. */
public Document getDoc() {
return doc;
}
/** Return the Biblio object corresponding to the last processed pdf file. */
public BiblioItem getResHeader() {
return resHeader;
}
/**
* Extract results from a labelled full text in the training format without any string
* modification.
*
* @param result reult
* @param tokenizations toks
* @return extraction
*/
private StringBuffer trainingExtraction(String result, ArrayList<String> tokenizations) {
// this is the main buffer for the whole full text
StringBuffer buffer = new StringBuffer();
try {
StringTokenizer st = new StringTokenizer(result, "\n");
String s1 = null;
String s2 = null;
String lastTag = null;
// current token position
int p = 0;
boolean start = true;
boolean openFigure = false;
boolean headFigure = false;
boolean descFigure = false;
boolean tableBlock = false;
while (st.hasMoreTokens()) {
boolean addSpace = false;
String tok = st.nextToken().trim();
if (tok.length() == 0) {
continue;
}
StringTokenizer stt = new StringTokenizer(tok, " \t");
ArrayList<String> localFeatures = new ArrayList<String>();
int i = 0;
boolean newLine = false;
int ll = stt.countTokens();
while (stt.hasMoreTokens()) {
String s = stt.nextToken().trim();
if (i == 0) {
s2 = TextUtilities.HTMLEncode(s); // lexical token
boolean strop = false;
while ((!strop) && (p < tokenizations.size())) {
String tokOriginal = tokenizations.get(p);
if (tokOriginal.equals(" ")) {
addSpace = true;
} else if (tokOriginal.equals(s)) {
strop = true;
}
p++;
}
} else if (i == ll - 1) {
s1 = s; // current tag
} else {
if (s.equals("LINESTART")) newLine = true;
localFeatures.add(s);
}
i++;
}
if (newLine && !start) {
buffer.append("<lb/>");
}
String lastTag0 = null;
if (lastTag != null) {
if (lastTag.startsWith("I-")) {
lastTag0 = lastTag.substring(2, lastTag.length());
} else {
lastTag0 = lastTag;
}
}
String currentTag0 = null;
if (s1 != null) {
if (s1.startsWith("I-")) {
currentTag0 = s1.substring(2, s1.length());
} else {
currentTag0 = s1;
}
}
boolean closeParagraph = false;
if (lastTag != null) {
closeParagraph = testClosingTag(buffer, currentTag0, lastTag0, s1);
}
boolean output = false;
if (!currentTag0.equals("<table>")
&& !currentTag0.equals("<trash>")
&& !currentTag0.equals("<figure_head>")
&& !currentTag0.equals("<label>")) {
if (openFigure) {
buffer.append("\n\t\t\t</figure>\n\n");
}
openFigure = false;
headFigure = false;
descFigure = false;
tableBlock = false;
}
output = writeField(buffer, s1, lastTag0, s2, "<header>", "<front>", addSpace, 3);
if (!output) {
output =
writeField(buffer, s1, lastTag0, s2, "<other>", "<note type=\"other\">", addSpace, 3);
}
// for paragraph we must distinguish starting and closing tags
if (!output) {
if (closeParagraph) {
output = writeFieldBeginEnd(buffer, s1, "", s2, "<paragraph>", "<p>", addSpace, 3);
} else {
output = writeFieldBeginEnd(buffer, s1, lastTag, s2, "<paragraph>", "<p>", addSpace, 3);
}
}
if (!output) {
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<page_header>",
"<note place=\"headnote\">",
addSpace,
3);
}
if (!output) {
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<page_footnote>",
"<note place=\"footnote\">",
addSpace,
3);
}
if (!output) {
output = writeField(buffer, s1, lastTag0, s2, "<page>", "<page>", addSpace, 3);
}
if (!output) {
output =
writeFieldBeginEnd(buffer, s1, lastTag0, s2, "<reference>", "<bibl>", addSpace, 3);
}
if (!output) {
if (closeParagraph) {
output = writeField(buffer, s1, "", s2, "<reference_marker>", "<label>", addSpace, 3);
} else
output =
writeField(buffer, s1, lastTag0, s2, "<reference_marker>", "<label>", addSpace, 3);
}
if (!output) {
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<citation_marker>",
"<ref type=\"biblio\">",
addSpace,
3);
}
if (!output) {
output = writeField(buffer, s1, lastTag0, s2, "<section>", "<head>", addSpace, 3);
}
if (!output) {
output = writeField(buffer, s1, lastTag0, s2, "<subsection>", "<head>", addSpace, 3);
}
if (!output) {
if (openFigure) {
output = writeField(buffer, s1, lastTag0, s2, "<trash>", "<trash>", addSpace, 4);
} else {
// output = writeField(buffer, s1, lastTag0, s2, "<trash>", "<figure>\n\t\t\t\t<trash>",
output = writeField(buffer, s1, lastTag0, s2, "<trash>", "<trash>", addSpace, 3);
if (output) {
openFigure = true;
}
}
}
if (!output) {
output = writeField(buffer, s1, lastTag0, s2, "<equation>", "<formula>", addSpace, 3);
}
if (!output) {
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<figure_marker>",
"<ref type=\"figure\">",
addSpace,
3);
}
if (!output) {
if (openFigure) {
if (tableBlock && (!lastTag0.equals("<table>")) && (currentTag0.equals("<table>"))) {
buffer.append("\n\t\t\t</figure>\n\n");
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<figure>\n\t\t\t\t<table>",
"<figure>",
addSpace,
3);
if (output) {
tableBlock = true;
descFigure = false;
headFigure = false;
}
} else {
output = writeField(buffer, s1, lastTag0, s2, "<table>", "<table>", addSpace, 4);
if (output) {
tableBlock = true;
}
}
} else {
output =
writeField(
buffer, s1, lastTag0, s2, "<table>", "<figure>\n\t\t\t\t<table>", addSpace, 3);
if (output) {
openFigure = true;
tableBlock = true;
}
}
}
if (!output) {
if (openFigure) {
if (descFigure && (!lastTag0.equals("<label>")) && (currentTag0.equals("<label>"))) {
buffer.append("\n\t\t\t</figure>\n\n");
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<label>",
"<figure>\n\t\t\t\t<figDesc>",
addSpace,
3);
if (output) {
descFigure = true;
tableBlock = false;
headFigure = false;
}
} else {
output = writeField(buffer, s1, lastTag0, s2, "<label>", "<figDesc>", addSpace, 4);
if (output) {
descFigure = true;
}
}
} else {
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<label>",
"<figure>\n\t\t\t\t<figDesc>",
addSpace,
3);
if (output) {
openFigure = true;
descFigure = true;
}
}
}
if (!output) {
if (openFigure) {
if (headFigure
&& (!lastTag0.equals("<figure_head>"))
&& (currentTag0.equals("<figure_head>"))) {
buffer.append("\n\t\t\t</figure>\n\n");
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<figure_head>",
"<figure>\n\t\t\t\t<head>",
addSpace,
3);
if (output) {
descFigure = false;
tableBlock = false;
headFigure = true;
}
} else {
output = writeField(buffer, s1, lastTag0, s2, "<figure_head>", "<head>", addSpace, 4);
if (output) {
headFigure = true;
}
}
} else {
output =
writeField(
buffer,
s1,
lastTag0,
s2,
"<figure_head>",
"<figure>\n\t\t\t\t<head>",
addSpace,
3);
if (output) {
openFigure = true;
headFigure = true;
}
}
}
// for item we must distinguish starting and closing tags
if (!output) {
output = writeFieldBeginEnd(buffer, s1, lastTag, s2, "<item>", "<item>", addSpace, 3);
}
lastTag = s1;
if (!st.hasMoreTokens()) {
if (lastTag != null) {
testClosingTag(buffer, "", currentTag0, s1);
}
if (openFigure) {
buffer.append("\n\t\t\t</figure>\n\n");
}
}
if (start) {
start = false;
}
}
return buffer;
} catch (Exception e) {
throw new GrobidException("An exception occured while running Grobid.", e);
}
}
/**
* TODO some documentation...
*
* @param buffer
* @param s1
* @param lastTag0
* @param s2
* @param field
* @param outField
* @param addSpace
* @param nbIndent
* @return
*/
private boolean writeField(
StringBuffer buffer,
String s1,
String lastTag0,
String s2,
String field,
String outField,
boolean addSpace,
int nbIndent) {
boolean result = false;
if ((s1.equals(field)) || (s1.equals("I-" + field))) {
result = true;
if (s1.equals(lastTag0) || s1.equals("I-" + lastTag0)) {
if (addSpace) buffer.append(" " + s2);
else buffer.append(s2);
}
/*else if (lastTag0 == null) {
for(int i=0; i<nbIndent; i++) {
buffer.append("\t");
}
buffer.append(outField+s2);
}*/
else if (field.equals("<citation_marker>")) {
if (addSpace) buffer.append(" " + outField + s2);
else buffer.append(outField + s2);
} else if (field.equals("<figure_marker>")) {
if (addSpace) buffer.append(" " + outField + s2);
else buffer.append(outField + s2);
} else if (field.equals("<reference_marker>")) {
if (!lastTag0.equals("<reference>") && !lastTag0.equals("<reference_marker>")) {
for (int i = 0; i < nbIndent; i++) {
buffer.append("\t");
}
buffer.append("<bibl>");
}
if (addSpace) buffer.append(" " + outField + s2);
else buffer.append(outField + s2);
} else if (lastTag0 == null) {
for (int i = 0; i < nbIndent; i++) {
buffer.append("\t");
}
buffer.append(outField + s2);
} else if (!lastTag0.equals("<citation_marker>")
&& !lastTag0.equals("<figure_marker>")
&& !lastTag0.equals("<figure>")) {
for (int i = 0; i < nbIndent; i++) {
buffer.append("\t");
}
buffer.append(outField + s2);
} else {
if (addSpace) buffer.append(" " + s2);
else buffer.append(s2);
}
}
return result;
}
/**
* This is for writing fields for fields where begin and end of field matter, like paragraph or
* item
*
* @param buffer
* @param s1
* @param lastTag0
* @param s2
* @param field
* @param outField
* @param addSpace
* @param nbIndent
* @return
*/
private boolean writeFieldBeginEnd(
StringBuffer buffer,
String s1,
String lastTag0,
String s2,
String field,
String outField,
boolean addSpace,
int nbIndent) {
boolean result = false;
if ((s1.equals(field)) || (s1.equals("I-" + field))) {
result = true;
if (lastTag0.equals("I-" + field)) {
if (addSpace) buffer.append(" " + s2);
else buffer.append(s2);
} else if (lastTag0.equals(field) && s1.equals(field)) {
if (addSpace) buffer.append(" " + s2);
else buffer.append(s2);
} else if (!lastTag0.equals("<citation_marker>")
&& !lastTag0.equals("<figure_marker>")
&& !lastTag0.equals("<figure>")
&& !lastTag0.equals("<reference_marker>")) {
for (int i = 0; i < nbIndent; i++) {
buffer.append("\t");
}
buffer.append(outField + s2);
} else {
if (addSpace) buffer.append(" " + s2);
else buffer.append(s2);
}
}
return result;
}
/**
* TODO some documentation
*
* @param buffer
* @param currentTag0
* @param lastTag0
* @param currentTag
* @return
*/
private boolean testClosingTag(
StringBuffer buffer, String currentTag0, String lastTag0, String currentTag) {
boolean res = false;
// reference_marker and citation_marker are two exceptions because they can be embedded
if (!currentTag0.equals(lastTag0)
|| currentTag.equals("I-<paragraph>")
|| currentTag.equals("I-<item>")) {
if (currentTag0.equals("<citation_marker>") || currentTag0.equals("<figure_marker>")) {
return res;
}
res = false;
// we close the current tag
if (lastTag0.equals("<other>")) {
buffer.append("</note>\n\n");
} else if (lastTag0.equals("<header>")) {
buffer.append("</front>\n\n");
} else if (lastTag0.equals("<page_header>")) {
buffer.append("</note>\n\n");
} else if (lastTag0.equals("<page_footnote>")) {
buffer.append("</note>\n\n");
} else if (lastTag0.equals("<reference>")) {
buffer.append("</bibl>\n\n");
res = true;
} else if (lastTag0.equals("<paragraph>")) {
buffer.append("</p>\n\n");
res = true;
} else if (lastTag0.equals("<section>")) {
buffer.append("</head>\n\n");
} else if (lastTag0.equals("<subsection>")) {
buffer.append("</head>\n\n");
} else if (lastTag0.equals("<equation>")) {
buffer.append("</formula>\n\n");
} else if (lastTag0.equals("<table>")) {
buffer.append("</table>\n");
} else if (lastTag0.equals("<label>")) {
buffer.append("</figDesc>\n");
} else if (lastTag0.equals("<figure_head>")) {
buffer.append("</head>\n\n");
} else if (lastTag0.equals("<item>")) {
buffer.append("</item>\n\n");
} else if (lastTag0.equals("<trash>")) {
buffer.append("</trash>\n\n");
} else if (lastTag0.equals("<reference_marker>")) {
buffer.append("</label>");
} else if (lastTag0.equals("<citation_marker>")) {
buffer.append("</ref>");
} else if (lastTag0.equals("<figure_marker>")) {
buffer.append("</ref>");
} else if (lastTag0.equals("<page>")) {
buffer.append("</page>\n\n");
} else {
res = false;
}
}
return res;
}
/**
* Create the TEI representation for a document based on the parsed header, references and body
* sections.
*/
private String toTEI(
Document doc,
String rese,
ArrayList<String> tokenizations,
BiblioItem resHeader,
boolean peer,
BiblioItem catalogue,
boolean withStyleSheet) {
if (doc.blocks == null) {
return null;
}
TEIFormater teiFormater = new TEIFormater(doc);
StringBuffer tei;
try {
tei = teiFormater.toTEIHeader(resHeader, peer, withStyleSheet, null);
tei = teiFormater.toTEIBodyML(tei, rese, resHeader, doc.bibDataSets, tokenizations, doc);
tei = teiFormater.toTEIReferences(tei, doc.bibDataSets);
tei.append("\t\t</back>\n");
tei.append("\t</text>\n");
tei.append("</TEI>\n");
} catch (Exception e) {
throw new GrobidException("An exception occured while running Grobid.", e);
}
return tei.toString();
}
@Override
public void close() throws IOException {
super.close();
headerParser.close();
headerParser = null;
citationParser.close();
citationParser = null;
}
}
public String processing2(String input, boolean consolidateHeader, boolean consolidateCitations)
throws Exception {
if (input == null) {
throw new GrobidResourceException("Cannot process pdf file, because input file was null.");
}
File inputFile = new File(input);
if (!inputFile.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because input file '"
+ inputFile.getAbsolutePath()
+ "' does not exists.");
}
if (tmpPath == null) {
throw new GrobidResourceException("Cannot process pdf file, because temp path is null.");
}
if (!tmpPath.exists()) {
throw new GrobidResourceException(
"Cannot process pdf file, because temp path '"
+ tmpPath.getAbsolutePath()
+ "' does not exists.");
}
doc = new Document(input, tmpPath.getAbsolutePath());
try {
int startPage = -1;
int endPage = -1;
pathXML =
doc.pdf2xml(true, false, startPage, endPage, input, tmpPath.getAbsolutePath(), true);
// with timeout,
// no force pdf reloading
// input is the pdf absolute path, tmpPath is the temp. directory for the temp. lxml file,
// path is the resource path
// and we process images in the pdf file
if (pathXML == null) {
throw new Exception("PDF parsing fails");
}
doc.setPathXML(pathXML);
ArrayList<String> tokenizations = doc.addFeaturesDocument();
if (doc.getBlocks() == null) {
throw new Exception("PDF parsing resulted in empty content");
}
String fulltext = doc.getFulltextFeatured(true, true);
StringTokenizer st = new StringTokenizer(fulltext, "\n");
feedTaggerAndParse(st);
StringBuilder res = new StringBuilder();
for (int i = 0; i < tagger.size(); i++) {
for (int j = 0; j < tagger.xsize(); j++) {
res.append(tagger.x(i, j)).append("\t");
}
res.append(tagger.y2(i));
res.append("\n");
}
// buffer for the fulltext block
String rese = res.toString();
// set the different sections of the Document object
doc = BasicStructureBuilder.resultSegmentation(doc, rese, tokenizations);
// header processing
if (headerParser == null) {
headerParser = new HeaderParser();
}
resHeader = new BiblioItem();
headerParser.processingHeaderBlock(consolidateHeader, doc, resHeader);
// the language identification is normally done during the header parsing, but only
// based on header information.
// LanguageUtilities languageUtilities = LanguageUtilities.getInstance();
Language langu = languageUtilities.runLanguageId(resHeader.getTitle() + "\n" + doc.getBody());
if (langu != null) {
String lang = langu.getLangId();
doc.setLanguage(lang);
resHeader.setLanguage(lang);
}
// citation processing
if (citationParser == null) {
citationParser = new CitationParser();
}
ArrayList<BibDataSet> resCitations;
// ArrayList<String> tokenizationsRef = doc.getTokenizationsReferences();
// System.out.println(tokenizationsRef.toString());
// resCitations = BasicStructureBuilder.giveReferenceSegments(doc);
resCitations = doc.bibDataSets;
if (resCitations != null) {
for (BibDataSet bds : resCitations) {
String marker = bds.getRefSymbol();
if (marker != null) {
marker = marker.replace(".", "");
marker = marker.replace(" ", "");
bds.setRefSymbol(marker);
}
BiblioItem bib = citationParser.processing(bds.getRawBib(), consolidateCitations);
bds.setResBib(bib);
}
}
// final combination
return toTEI(doc, rese, tokenizations, resHeader, false, null, false);
} catch (Exception e) {
throw new GrobidException("An exception occured while running Grobid.", e);
} finally {
// keep it clean when leaving...
doc.cleanLxmlFile(pathXML, false);
}
}