public static String extractText(InputStream src) throws IOException {
StringBuilder text = new StringBuilder();
COSDocument cosDoc = null;
PDDocument pdDoc = null;
try {
PDFParser parser = new PDFParser(src);
parser.parse();
cosDoc = parser.getDocument();
PDFTextStripper stripper = new PDFTextStripper();
pdDoc = new PDDocument(cosDoc);
int nbPages = pdDoc.getDocumentCatalog().getPages().getCount();
for (int i = 0; i < nbPages; i++) {
stripper.setStartPage(i + 1);
stripper.setEndPage(i + 1);
text.append(stripper.getText(pdDoc));
}
} finally {
try {
if (cosDoc != null) {
cosDoc.close();
}
} catch (IOException e) {
// Do nada
}
try {
if (pdDoc != null) {
pdDoc.close();
}
} catch (IOException e) {
// Do nada
}
}
return text.toString();
}
public static String getFileContent(File file) throws FileNotFoundException, IOException {
String ext = FilenameUtils.getExtension(file.getName());
String outContent = "";
try {
if (ext.toLowerCase().equals("doc")) {
if (file != null) {
WordExtractor we = new WordExtractor(new FileInputStream(file));
outContent = we.getText();
} else {
logger.warning("file not found : " + file);
}
} else if (ext.toLowerCase().equals("pdf")) {
PDDocument doc = PDDocument.load(file);
PDFTextStripper text = new PDFTextStripper();
outContent = text.getText(doc);
doc.close();
} else if (StringHelper.isHTML(file.getName())) {
return loadStringFromFile(file);
}
} catch (Throwable t) {
logger.warning("error when read : " + file + "+ [" + t.getMessage() + "]");
t.printStackTrace();
}
return outContent;
}
private String pdfToText(InputStream in) {
PDFParser parser = null;
PDDocument pdDoc = null;
COSDocument cosDoc = null;
PDFTextStripper pdfStripper;
try {
parser = new PDFParser(in);
parser.parse();
cosDoc = parser.getDocument();
pdfStripper = new PDFTextStripper();
pdDoc = new PDDocument(cosDoc);
return pdfStripper.getText(pdDoc);
// System.out.println(parsedText.replaceAll("[^A-Za-z0-9. ]+", ""));
} catch (Exception e) {
e.printStackTrace();
try {
if (cosDoc != null) cosDoc.close();
if (pdDoc != null) pdDoc.close();
} catch (Exception e1) {
e.printStackTrace();
}
}
return null;
}
/**
* Extracts text from a PDF and stores it in the document. Takes an input stream rather than a
* file name.
*
* @param filesInputStream An input stream pointing to a PDF file.
* @throws IOException
*/
private static char[] loadPDF(InputStream filesInputStream) throws IOException {
PDDocument doc = PDDocument.load(filesInputStream);
PDFTextStripper pdfStripper = new PDFTextStripper();
pdfStripper.setSortByPosition(false);
char[] origText = pdfStripper.getText(doc).toCharArray();
doc.close();
return origText;
}
public static String getContent(PDFParser parser) throws IOException {
parser.parse();
COSDocument cosDoc = parser.getDocument();
PDFTextStripper pdfStripper = new PDFTextStripper();
PDDocument pdDoc = new PDDocument(cosDoc);
String content = pdfStripper.getText(pdDoc);
cosDoc.close();
pdDoc.close();
return content;
}
/** Extracts the textual contents from a PDF file as one long string. */
public String extractPDFContents(File f) throws IOException {
FileInputStream fi = new FileInputStream(f);
PDFParser parser = new PDFParser(fi);
parser.parse();
fi.close();
COSDocument cd = parser.getDocument();
PDFTextStripper stripper = new PDFTextStripper();
String result = stripper.getText(new PDDocument(cd));
cd.close();
return result;
}
public static void main(String[] args) {
PDDocument pd;
try {
File input = new File("pdf/1.pdf"); // The PDF file from where you would like to extract
pd = PDDocument.load(input);
int numberOfPages = pd.getNumberOfPages();
PDFTextStripper stripper = new PDFTextStripper();
String fullText = stripper.getText(pd);
int indexReferences = fullText.lastIndexOf("References\n");
String textOutReferences =
fullText.substring(0, indexReferences > 0 ? indexReferences : fullText.length());
String textOutStop = removeStopWords(textOutReferences);
findMoreCiteds(textOutStop);
extractReferences(fullText);
stripper.setEndPage(3);
String startText = stripper.getText(pd);
System.out.println("Autores");
extractAuthor(startText);
System.out.println("Objetivos");
extractObjective(startText);
System.out.println("\n\nProblemas");
extractProblem(startText);
System.out.println("\n\nMetodologia");
extractMethodology(fullText);
System.out.println("\n\nContribuições");
extractContributes(fullText);
pd.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public static String extract(File pdfFile) throws IOException {
checkNotNull(pdfFile, "pdfFile");
PDFParser parser = new PDFParser(new FileInputStream(pdfFile));
parser.parse();
COSDocument cosDoc = parser.getDocument();
PDFTextStripper pdfStripper = new PDFTextStripper();
PDDocument pdDoc = new PDDocument(cosDoc);
pdfStripper.setStartPage(1);
pdfStripper.setEndPage(pdDoc.getNumberOfPages());
pdfStripper.setSortByPosition(true);
String pdfText = pdfStripper.getText(pdDoc);
pdDoc.close();
cosDoc.close();
return pdfText;
}
ExtractPageContent(String filePath) {
this.filePath = filePath;
try {
reader = new PdfReader(filePath);
parser = new PdfReaderContentParser(reader);
getContents();
} catch (Exception e) {
try {
PDDocument doc = PDDocument.load(filePath);
PDFTextStripper stripper = new PDFTextStripper();
this.fileContents = stripper.getText(doc);
doc.close();
} catch (IOException e1) {
// TODO Auto-generated catch block
// e1.printStackTrace();
}
}
}
/**
* Método para la indexación individual de cada fichero PDF
*
* @param f el fichero PDF
* @param writer el IndexWriter
* @throws IOException
*/
public static void indexFile(File f, IndexWriter writer) throws IOException {
// Cargamos el fichero mediante PDFBox
PDDocument pddDocument = PDDocument.load(f.getAbsolutePath());
PDFTextStripper textStripper = new PDFTextStripper();
int numPages = pddDocument.getNumberOfPages();
String pageContent;
// Declaramos un Field propio
FieldType fieldText = new FieldType();
fieldText.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
fieldText.setStored(false);
fieldText.setStoreTermVectorOffsets(true);
fieldText.setStoreTermVectorPositions(true);
fieldText.setStoreTermVectors(true);
// Recorremos e indexamos cada una de las páginas del fichero, almacenando el número de página y
// el título del fichero, e indexando el contenido
for (int i = 0; i < numPages; i++) {
if (i == 0) {
i++;
}
textStripper.setStartPage(i);
textStripper.setEndPage(i);
// coger una página
pageContent = textStripper.getText(pddDocument);
if (pageContent != null && !pageContent.isEmpty()) {
pageContent = pageContent.toLowerCase();
}
if (pageContent != null) {
// Declaramos el documento a indexar para esa página
// Número de página
// Contenido de la página
// Título del fichero
// Añadimos el documento
}
}
// Cerramos el fichero PDF
}
public static void main(String[] args) throws Exception {
File file = new File("C:/Users/jatin.goyal/Desktop/demoexcel.pdf");
PDDocument pd = PDDocument.load(file);
System.out.println(pd.getNumberOfPages());
PDFTextStripper st = new PDFTextStripper();
st.setStartPage(1);
// st.setEndPage(4);
// PDFTextStripperByArea stripper = new PDFTextStripperByArea();
// stripper.setSortByPosition( true );
// Rectangle rect1 = new Rectangle( 50, 140, 60, 20 );
// Rectangle rect2 = new Rectangle( 110, 140, 20, 20 );
// stripper.addRegion( "row1column1", rect1 );
// stripper.addRegion( "row1column2", rect2 );
// List allPages = pd.getDocumentCatalog().getAllPages();
// PDPage firstPage = (PDPage)allPages.get( 0 );
// stripper.extractRegions( firstPage );
// System.out.println(stripper.getTextForRegion( "row1column1" ));
// System.out.println(stripper.getTextForRegion( "row1column2" ));
System.out.println(st.getText(pd));
}
String pdftoText(String fileName) {
System.out.println("Parsing text from PDF file " + fileName + "....");
File f = new File(fileName);
if (!f.isFile()) {
System.out.println("File " + fileName + " does not exist.");
return null;
}
try {
parser = new PDFParser(new FileInputStream(f));
} catch (Exception e) {
System.out.println("Unable to open PDF Parser.");
return null;
}
try {
parser.parse();
cosDoc = parser.getDocument();
pdfStripper = new PDFTextStripper();
pdDoc = new PDDocument(cosDoc);
parsedText = pdfStripper.getText(pdDoc);
} catch (Exception e) {
System.out.println("An exception occured in parsing the PDF Document.");
e.printStackTrace();
try {
if (cosDoc != null) cosDoc.close();
if (pdDoc != null) pdDoc.close();
} catch (Exception e1) {
e.printStackTrace();
}
return null;
}
System.out.println("Done.");
return parsedText;
}
public void PDF2TextPreProssesd(String filename) {
try {
stripper = new PDFTextStripper();
stripper.setParagraphStart("&*&");
stripper.setLineSeparator("#%#");
stripper.setPageSeparator("#%#");
String fulltxt = stripper.getText(pd);
String paras[] = fulltxt.split("&*&");
File file = new File(filename);
try {
BufferedWriter out = new BufferedWriter(new FileWriter(file));
int i = 0;
while (i < paras.length) {
if (paras[i].length() > 200) {
String para = paras[i].replace("#%#", " ");
out.write(para + "\r\n");
}
i++;
}
out.close();
} catch (IOException ex) {
ex.printStackTrace();
}
if (pd != null) {
pd.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public Document[] parse(
final AnchorURL location,
final String mimeType,
final String charset,
final VocabularyScraper scraper,
final int timezoneOffset,
final InputStream source)
throws Parser.Failure, InterruptedException {
// check memory for parser
if (!MemoryControl.request(200 * 1024 * 1024, false))
throw new Parser.Failure(
"Not enough Memory available for pdf parser: " + MemoryControl.available(), location);
// create a pdf parser
PDDocument pdfDoc;
try {
Thread.currentThread().setPriority(Thread.MIN_PRIORITY); // the pdfparser is a big pain
// pdfDoc = PDDocument.load(source);
final PDFParser pdfParser = new PDFParser(source);
pdfParser.setTempDirectory(new File(System.getProperty("java.io.tmpdir")));
pdfParser.parse();
pdfDoc = pdfParser.getPDDocument();
} catch (final IOException e) {
throw new Parser.Failure(e.getMessage(), location);
} finally {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY);
}
if (pdfDoc.isEncrypted()) {
try {
pdfDoc.openProtection(new StandardDecryptionMaterial(""));
} catch (final BadSecurityHandlerException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (1): " + e.getMessage(), location);
} catch (final IOException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (2): " + e.getMessage(), location);
} catch (final CryptographyException e) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted (3): " + e.getMessage(), location);
}
final AccessPermission perm = pdfDoc.getCurrentAccessPermission();
if (perm == null || !perm.canExtractContent()) {
try {
pdfDoc.close();
} catch (final IOException ee) {
}
throw new Parser.Failure("Document is encrypted and cannot be decrypted", location);
}
}
// extracting some metadata
PDDocumentInformation info = pdfDoc.getDocumentInformation();
String docTitle = null,
docSubject = null,
docAuthor = null,
docPublisher = null,
docKeywordStr = null;
Date docDate = new Date();
if (info != null) {
docTitle = info.getTitle();
docSubject = info.getSubject();
docAuthor = info.getAuthor();
docPublisher = info.getProducer();
if (docPublisher == null || docPublisher.isEmpty()) docPublisher = info.getCreator();
docKeywordStr = info.getKeywords();
try {
if (info.getModificationDate() != null) docDate = info.getModificationDate().getTime();
} catch (IOException e) {
}
// unused:
// info.getTrapped());
}
info = null;
if (docTitle == null || docTitle.isEmpty()) {
docTitle = MultiProtocolURL.unescape(location.getFileName());
}
if (docTitle == null) {
docTitle = docSubject;
}
String[] docKeywords = null;
if (docKeywordStr != null) {
docKeywords = docKeywordStr.split(" |,");
}
Collection<AnchorURL>[] pdflinks = null;
Document[] result = null;
try {
// get the links
pdflinks = extractPdfLinks(pdfDoc);
// get the fulltext (either per document or for each page)
final PDFTextStripper stripper = new PDFTextStripper(StandardCharsets.UTF_8.name());
if (individualPages) {
// this is a hack which stores individual pages of the source pdf into individual index
// documents
// the new documents will get a virtual link with a post argument page=X appended to the
// original url
// collect text
int pagecount = pdfDoc.getNumberOfPages();
String[] pages = new String[pagecount];
for (int page = 1; page <= pagecount; page++) {
stripper.setStartPage(page);
stripper.setEndPage(page);
pages[page - 1] = stripper.getText(pdfDoc);
// System.out.println("PAGE " + page + ": " + pages[page - 1]);
}
// create individual documents for each page
assert pages.length == pdflinks.length
: "pages.length = " + pages.length + ", pdflinks.length = " + pdflinks.length;
result = new Document[Math.min(pages.length, pdflinks.length)];
String loc = location.toNormalform(true);
for (int page = 0; page < result.length; page++) {
result[page] =
new Document(
new AnchorURL(
loc
+ (loc.indexOf('?') > 0 ? '&' : '?')
+ individualPagePropertyname
+ '='
+ (page
+ 1)), // these are virtual new pages; we cannot combine them with '#'
// as that would be removed when computing the urlhash
mimeType,
StandardCharsets.UTF_8.name(),
this,
null,
docKeywords,
singleList(docTitle),
docAuthor,
docPublisher,
null,
null,
0.0f,
0.0f,
pages == null || page > pages.length ? new byte[0] : UTF8.getBytes(pages[page]),
pdflinks == null || page >= pdflinks.length ? null : pdflinks[page],
null,
null,
false,
docDate);
}
} else {
// collect the whole text at once
final CharBuffer writer = new CharBuffer(odtParser.MAX_DOCSIZE);
byte[] contentBytes = new byte[0];
stripper.setEndPage(3); // get first 3 pages (always)
writer.append(stripper.getText(pdfDoc));
contentBytes = writer.getBytes(); // remember text in case of interrupting thread
if (pdfDoc.getNumberOfPages() > 3) { // spare creating/starting thread if all pages read
stripper.setStartPage(4); // continue with page 4 (terminated, resulting in no text)
stripper.setEndPage(Integer.MAX_VALUE); // set to default
// we start the pdf parsing in a separate thread to ensure that it can be terminated
final PDDocument pdfDocC = pdfDoc;
final Thread t =
new Thread() {
@Override
public void run() {
Thread.currentThread().setName("pdfParser.getText:" + location);
try {
writer.append(stripper.getText(pdfDocC));
} catch (final Throwable e) {
}
}
};
t.start();
t.join(3000); // pdfbox likes to forget to terminate ... (quite often)
if (t.isAlive()) t.interrupt();
}
contentBytes = writer.getBytes(); // get final text before closing writer
Collection<AnchorURL> pdflinksCombined = new HashSet<AnchorURL>();
for (Collection<AnchorURL> pdflinksx : pdflinks)
if (pdflinksx != null) pdflinksCombined.addAll(pdflinksx);
result =
new Document[] {
new Document(
location,
mimeType,
StandardCharsets.UTF_8.name(),
this,
null,
docKeywords,
singleList(docTitle),
docAuthor,
docPublisher,
null,
null,
0.0f,
0.0f,
contentBytes,
pdflinksCombined,
null,
null,
false,
docDate)
};
}
} catch (final Throwable e) {
// close the writer (in finally)
// throw new Parser.Failure(e.getMessage(), location);
} finally {
try {
pdfDoc.close();
} catch (final Throwable e) {
}
}
// clear resources in pdfbox. they say that is resolved but it's not. see:
// https://issues.apache.org/jira/browse/PDFBOX-313
// https://issues.apache.org/jira/browse/PDFBOX-351
// https://issues.apache.org/jira/browse/PDFBOX-441
// the pdfbox still generates enormeous number of object allocations and don't delete these
// the following Object are statically stored and never flushed:
// COSFloat, COSArray, COSInteger, COSObjectKey, COSObject, COSDictionary,
// COSStream, COSString, COSName, COSDocument, COSInteger[], COSNull
// the great number of these objects can easily be seen in Java Visual VM
// we try to get this shit out of the memory here by forced clear calls, hope the best the
// rubbish gets out.
pdfDoc = null;
clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes();
return result;
}
