public static String extractText(InputStream src) throws IOException {
StringBuilder text = new StringBuilder();
COSDocument cosDoc = null;
PDDocument pdDoc = null;
try {
PDFParser parser = new PDFParser(src);
parser.parse();
cosDoc = parser.getDocument();
PDFTextStripper stripper = new PDFTextStripper();
pdDoc = new PDDocument(cosDoc);
int nbPages = pdDoc.getDocumentCatalog().getPages().getCount();
for (int i = 0; i < nbPages; i++) {
stripper.setStartPage(i + 1);
stripper.setEndPage(i + 1);
text.append(stripper.getText(pdDoc));
}
} finally {
try {
if (cosDoc != null) {
cosDoc.close();
}
} catch (IOException e) {
// Do nada
}
try {
if (pdDoc != null) {
pdDoc.close();
}
} catch (IOException e) {
// Do nada
}
}
return text.toString();
}
private String pdfToText(InputStream in) {
PDFParser parser = null;
PDDocument pdDoc = null;
COSDocument cosDoc = null;
PDFTextStripper pdfStripper;
try {
parser = new PDFParser(in);
parser.parse();
cosDoc = parser.getDocument();
pdfStripper = new PDFTextStripper();
pdDoc = new PDDocument(cosDoc);
return pdfStripper.getText(pdDoc);
// System.out.println(parsedText.replaceAll("[^A-Za-z0-9. ]+", ""));
} catch (Exception e) {
e.printStackTrace();
try {
if (cosDoc != null) cosDoc.close();
if (pdDoc != null) pdDoc.close();
} catch (Exception e1) {
e.printStackTrace();
}
}
return null;
}
public static String getContent(PDFParser parser) throws IOException {
parser.parse();
COSDocument cosDoc = parser.getDocument();
PDFTextStripper pdfStripper = new PDFTextStripper();
PDDocument pdDoc = new PDDocument(cosDoc);
String content = pdfStripper.getText(pdDoc);
cosDoc.close();
pdDoc.close();
return content;
}
/** Extracts the textual contents from a PDF file as one long string. */
public String extractPDFContents(File f) throws IOException {
FileInputStream fi = new FileInputStream(f);
PDFParser parser = new PDFParser(fi);
parser.parse();
fi.close();
COSDocument cd = parser.getDocument();
PDFTextStripper stripper = new PDFTextStripper();
String result = stripper.getText(new PDDocument(cd));
cd.close();
return result;
}
public static String extract(File pdfFile) throws IOException {
checkNotNull(pdfFile, "pdfFile");
PDFParser parser = new PDFParser(new FileInputStream(pdfFile));
parser.parse();
COSDocument cosDoc = parser.getDocument();
PDFTextStripper pdfStripper = new PDFTextStripper();
PDDocument pdDoc = new PDDocument(cosDoc);
pdfStripper.setStartPage(1);
pdfStripper.setEndPage(pdDoc.getNumberOfPages());
pdfStripper.setSortByPosition(true);
String pdfText = pdfStripper.getText(pdDoc);
pdDoc.close();
cosDoc.close();
return pdfText;
}
public static String parsePdfToString(String filename) {
String text = null;
try {
System.out.println("Parsing the PDF file..");
RandomAccessRead rar = new RandomAccessFile(new File(filename), "r");
PDFParser parser = new PDFParser(rar);
parser.parse();
System.out.println("Parsing Done!");
PDDocument document = parser.getPDDocument();
PDFTextStripper stripper = new PDFTextStripper();
text = stripper.getText(document);
System.out.println(text);
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return text;
}
private static void extract(InputStream in) throws Exception {
PDDocument document = null;
try {
PDFParser parser = new PDFParser(in);
parser.parse();
document = parser.getPDDocument();
if (document.isEncrypted()) {
System.err.println("Document is Encrypted!");
}
PDDocumentCatalog cat = document.getDocumentCatalog();
PDMetadata metadata = cat.getMetadata();
if (metadata != null) {
// System.out.println(metadata.getStream().getStreamTokens());
// Levantamos la MetaData
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
Document doc = dBuilder.parse(metadata.createInputStream());
// Buscamos el tag de SEmployee y el Element -> CUIT
NodeList nList = doc.getElementsByTagName("foaf:SEmployee");
Element elem = (Element) nList.item(0);
String cuit = elem.getElementsByTagName("foaf:cuit").item(0).getTextContent();
System.out.println(cuit);
System.out.println("---");
System.out.println(metadata.getInputStreamAsString());
}
} catch (Exception err) {
throw err;
} finally {
if (document != null)
try {
document.close();
} catch (Throwable err2) {
}
}
}
String pdftoText(String fileName) {
System.out.println("Parsing text from PDF file " + fileName + "....");
File f = new File(fileName);
if (!f.isFile()) {
System.out.println("File " + fileName + " does not exist.");
return null;
}
try {
parser = new PDFParser(new FileInputStream(f));
} catch (Exception e) {
System.out.println("Unable to open PDF Parser.");
return null;
}
try {
parser.parse();
cosDoc = parser.getDocument();
pdfStripper = new PDFTextStripper();
pdDoc = new PDDocument(cosDoc);
parsedText = pdfStripper.getText(pdDoc);
} catch (Exception e) {
System.out.println("An exception occured in parsing the PDF Document.");
e.printStackTrace();
try {
if (cosDoc != null) cosDoc.close();
if (pdDoc != null) pdDoc.close();
} catch (Exception e1) {
e.printStackTrace();
}
return null;
}
System.out.println("Done.");
return parsedText;
}
@Override
public Document[] parse(
final AnchorURL location,
final String mimeType,
final String charset,
final VocabularyScraper scraper,
final int timezoneOffset,
final InputStream source)
throws Parser.Failure, InterruptedException {
// check memory for parser
if (!MemoryControl.request(200 * 1024 * 1024, false))
throw new Parser.Failure(
"Not enough Memory available for pdf parser: " + MemoryControl.available(), location);
// create a pdf parser
PDDocument pdfDoc;
try {
Thread.currentThread().setPriority(Thread.MIN_PRIORITY); // the pdfparser is a big pain
// pdfDoc = PDDocument.load(source);
final PDFParser pdfParser = new PDFParser(source);
pdfParser.setTempDirectory(new File(System.getProperty("java.io.tmpdir")));
pdfParser.parse();
pdfDoc = pdfParser.getPDDocument();
} catch (final IOException e) {
throw new Parser.Failure(e.getMessage(), location);
} finally {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
}
if (pdfDoc.isEncrypted()) {
try {
pdfDoc.openProtection(new StandardDecryptionMaterial(""));
} catch (final BadSecurityHandlerException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (1): " + e.getMessage(), location);
} catch (final IOException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (2): " + e.getMessage(), location);
} catch (final CryptographyException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (3): " + e.getMessage(), location);
}
final AccessPermission perm = pdfDoc.getCurrentAccessPermission();
if (perm == null || !perm.canExtractContent()) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted and cannot be decrypted", location);
}
}
// extracting some metadata
PDDocumentInformation info = pdfDoc.getDocumentInformation();
String docTitle = null,
docSubject = null,
docAuthor = null,
docPublisher = null,
docKeywordStr = null;
Date docDate = new Date();
if (info != null) {
docTitle = info.getTitle();
docSubject = info.getSubject();
docAuthor = info.getAuthor();
docPublisher = info.getProducer();
if (docPublisher == null || docPublisher.isEmpty()) docPublisher = info.getCreator();
docKeywordStr = info.getKeywords();
try {
if (info.getModificationDate() != null) docDate = info.getModificationDate().getTime();
} catch (IOException e) {
}
// unused:
// info.getTrapped());
}
info = null;
if (docTitle == null || docTitle.isEmpty()) {
docTitle = MultiProtocolURL.unescape(location.getFileName());
}
if (docTitle == null) {
docTitle = docSubject;
}
String[] docKeywords = null;
if (docKeywordStr != null) {
docKeywords = docKeywordStr.split(" |,");
}
Collection<AnchorURL>[] pdflinks = null;
Document[] result = null;
try {
// get the links
pdflinks = extractPdfLinks(pdfDoc);
// get the fulltext (either per document or for each page)
final PDFTextStripper stripper = new PDFTextStripper(StandardCharsets.UTF_8.name());
if (individualPages) {
// this is a hack which stores individual pages of the source pdf into individual index
// documents
// the new documents will get a virtual link with a post argument page=X appended to the
// original url
// collect text
int pagecount = pdfDoc.getNumberOfPages();
String[] pages = new String[pagecount];
for (int page = 1; page <= pagecount; page++) {
stripper.setStartPage(page);
stripper.setEndPage(page);
pages[page - 1] = stripper.getText(pdfDoc);
// System.out.println("PAGE " + page + ": " + pages[page - 1]);
}
// create individual documents for each page
assert pages.length == pdflinks.length
: "pages.length = " + pages.length + ", pdflinks.length = " + pdflinks.length;
result = new Document[Math.min(pages.length, pdflinks.length)];
String loc = location.toNormalform(true);
for (int page = 0; page < result.length; page++) {
result[page] =
new Document(
new AnchorURL(
loc
+ (loc.indexOf('?') > 0 ? '&' : '?')
+ individualPagePropertyname
+ '='
+ (page
+ 1)), // these are virtual new pages; we cannot combine them with '#'
// as that would be removed when computing the urlhash
mimeType,
StandardCharsets.UTF_8.name(),
this,
null,
docKeywords,
singleList(docTitle),
docAuthor,
docPublisher,
null,
null,
0.0f,
0.0f,
pages == null || page > pages.length ? new byte[0] : UTF8.getBytes(pages[page]),
pdflinks == null || page >= pdflinks.length ? null : pdflinks[page],
null,
null,
false,
docDate);
}
} else {
// collect the whole text at once
final CharBuffer writer = new CharBuffer(odtParser.MAX_DOCSIZE);
byte[] contentBytes = new byte[0];
stripper.setEndPage(3); // get first 3 pages (always)
writer.append(stripper.getText(pdfDoc));
contentBytes = writer.getBytes(); // remember text in case of interrupting thread
if (pdfDoc.getNumberOfPages() > 3) { // spare creating/starting thread if all pages read
stripper.setStartPage(4); // continue with page 4 (terminated, resulting in no text)
stripper.setEndPage(Integer.MAX_VALUE); // set to default
// we start the pdf parsing in a separate thread to ensure that it can be terminated
final PDDocument pdfDocC = pdfDoc;
final Thread t =
new Thread() {
@Override
public void run() {
Thread.currentThread().setName("pdfParser.getText:" + location);
try {
writer.append(stripper.getText(pdfDocC));
} catch (final Throwable e) {
}
}
};
t.start();
t.join(3000); // pdfbox likes to forget to terminate ... (quite often)
if (t.isAlive()) t.interrupt();
}
contentBytes = writer.getBytes(); // get final text before closing writer
Collection<AnchorURL> pdflinksCombined = new HashSet<AnchorURL>();
for (Collection<AnchorURL> pdflinksx : pdflinks)
if (pdflinksx != null) pdflinksCombined.addAll(pdflinksx);
result =
new Document[] {
new Document(
location,
mimeType,
StandardCharsets.UTF_8.name(),
this,
null,
docKeywords,
singleList(docTitle),
docAuthor,
docPublisher,
null,
null,
0.0f,
0.0f,
contentBytes,
pdflinksCombined,
null,
null,
false,
docDate)
};
}
} catch (final Throwable e) {
// close the writer (in finally)
// throw new Parser.Failure(e.getMessage(), location);
} finally {
try {
pdfDoc.close();
} catch (final Throwable e) {
}
}
// clear resources in pdfbox. they say that is resolved but it's not. see:
// https://issues.apache.org/jira/browse/PDFBOX-313
// https://issues.apache.org/jira/browse/PDFBOX-351
// https://issues.apache.org/jira/browse/PDFBOX-441
// the pdfbox still generates enormeous number of object allocations and don't delete these
// the following Object are statically stored and never flushed:
// COSFloat, COSArray, COSInteger, COSObjectKey, COSObject, COSDictionary,
// COSStream, COSString, COSName, COSDocument, COSInteger[], COSNull
// the great number of these objects can easily be seen in Java Visual VM
// we try to get this shit out of the memory here by forced clear calls, hope the best the
// rubbish gets out.
pdfDoc = null;
clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes();
return result;
}
