public static String extractText(InputStream src) throws IOException { StringBuilder text = new StringBuilder(); COSDocument cosDoc = null; PDDocument pdDoc = null; try { PDFParser parser = new PDFParser(src); parser.parse(); cosDoc = parser.getDocument(); PDFTextStripper stripper = new PDFTextStripper(); pdDoc = new PDDocument(cosDoc); int nbPages = pdDoc.getDocumentCatalog().getPages().getCount(); for (int i = 0; i < nbPages; i++) { stripper.setStartPage(i + 1); stripper.setEndPage(i + 1); text.append(stripper.getText(pdDoc)); } } finally { try { if (cosDoc != null) { cosDoc.close(); } } catch (IOException e) { // Do nada } try { if (pdDoc != null) { pdDoc.close(); } } catch (IOException e) { // Do nada } } return text.toString(); }
public static String getFileContent(File file) throws FileNotFoundException, IOException { String ext = FilenameUtils.getExtension(file.getName()); String outContent = ""; try { if (ext.toLowerCase().equals("doc")) { if (file != null) { WordExtractor we = new WordExtractor(new FileInputStream(file)); outContent = we.getText(); } else { logger.warning("file not found : " + file); } } else if (ext.toLowerCase().equals("pdf")) { PDDocument doc = PDDocument.load(file); PDFTextStripper text = new PDFTextStripper(); outContent = text.getText(doc); doc.close(); } else if (StringHelper.isHTML(file.getName())) { return loadStringFromFile(file); } } catch (Throwable t) { logger.warning("error when read : " + file + "+ [" + t.getMessage() + "]"); t.printStackTrace(); } return outContent; }
private String pdfToText(InputStream in) { PDFParser parser = null; PDDocument pdDoc = null; COSDocument cosDoc = null; PDFTextStripper pdfStripper; try { parser = new PDFParser(in); parser.parse(); cosDoc = parser.getDocument(); pdfStripper = new PDFTextStripper(); pdDoc = new PDDocument(cosDoc); return pdfStripper.getText(pdDoc); // System.out.println(parsedText.replaceAll("[^A-Za-z0-9. ]+", "")); } catch (Exception e) { e.printStackTrace(); try { if (cosDoc != null) cosDoc.close(); if (pdDoc != null) pdDoc.close(); } catch (Exception e1) { e.printStackTrace(); } } return null; }
/** * Extracts text from a PDF and stores it in the document. Takes an input stream rather than a * file name. * * @param filesInputStream An input stream pointing to a PDF file. * @throws IOException */ private static char[] loadPDF(InputStream filesInputStream) throws IOException { PDDocument doc = PDDocument.load(filesInputStream); PDFTextStripper pdfStripper = new PDFTextStripper(); pdfStripper.setSortByPosition(false); char[] origText = pdfStripper.getText(doc).toCharArray(); doc.close(); return origText; }
public static String getContent(PDFParser parser) throws IOException { parser.parse(); COSDocument cosDoc = parser.getDocument(); PDFTextStripper pdfStripper = new PDFTextStripper(); PDDocument pdDoc = new PDDocument(cosDoc); String content = pdfStripper.getText(pdDoc); cosDoc.close(); pdDoc.close(); return content; }
/** Extracts the textual contents from a PDF file as one long string. */ public String extractPDFContents(File f) throws IOException { FileInputStream fi = new FileInputStream(f); PDFParser parser = new PDFParser(fi); parser.parse(); fi.close(); COSDocument cd = parser.getDocument(); PDFTextStripper stripper = new PDFTextStripper(); String result = stripper.getText(new PDDocument(cd)); cd.close(); return result; }
public static void main(String[] args) { PDDocument pd; try { File input = new File("pdf/1.pdf"); // The PDF file from where you would like to extract pd = PDDocument.load(input); int numberOfPages = pd.getNumberOfPages(); PDFTextStripper stripper = new PDFTextStripper(); String fullText = stripper.getText(pd); int indexReferences = fullText.lastIndexOf("References\n"); String textOutReferences = fullText.substring(0, indexReferences > 0 ? indexReferences : fullText.length()); String textOutStop = removeStopWords(textOutReferences); findMoreCiteds(textOutStop); extractReferences(fullText); stripper.setEndPage(3); String startText = stripper.getText(pd); System.out.println("Autores"); extractAuthor(startText); System.out.println("Objetivos"); extractObjective(startText); System.out.println("\n\nProblemas"); extractProblem(startText); System.out.println("\n\nMetodologia"); extractMethodology(fullText); System.out.println("\n\nContribuições"); extractContributes(fullText); pd.close(); } catch (Exception e) { e.printStackTrace(); } }
public static String extract(File pdfFile) throws IOException { checkNotNull(pdfFile, "pdfFile"); PDFParser parser = new PDFParser(new FileInputStream(pdfFile)); parser.parse(); COSDocument cosDoc = parser.getDocument(); PDFTextStripper pdfStripper = new PDFTextStripper(); PDDocument pdDoc = new PDDocument(cosDoc); pdfStripper.setStartPage(1); pdfStripper.setEndPage(pdDoc.getNumberOfPages()); pdfStripper.setSortByPosition(true); String pdfText = pdfStripper.getText(pdDoc); pdDoc.close(); cosDoc.close(); return pdfText; }
ExtractPageContent(String filePath) { this.filePath = filePath; try { reader = new PdfReader(filePath); parser = new PdfReaderContentParser(reader); getContents(); } catch (Exception e) { try { PDDocument doc = PDDocument.load(filePath); PDFTextStripper stripper = new PDFTextStripper(); this.fileContents = stripper.getText(doc); doc.close(); } catch (IOException e1) { // TODO Auto-generated catch block // e1.printStackTrace(); } } }
/** * Método para la indexación individual de cada fichero PDF * * @param f el fichero PDF * @param writer el IndexWriter * @throws IOException */ public static void indexFile(File f, IndexWriter writer) throws IOException { // Cargamos el fichero mediante PDFBox PDDocument pddDocument = PDDocument.load(f.getAbsolutePath()); PDFTextStripper textStripper = new PDFTextStripper(); int numPages = pddDocument.getNumberOfPages(); String pageContent; // Declaramos un Field propio FieldType fieldText = new FieldType(); fieldText.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); fieldText.setStored(false); fieldText.setStoreTermVectorOffsets(true); fieldText.setStoreTermVectorPositions(true); fieldText.setStoreTermVectors(true); // Recorremos e indexamos cada una de las páginas del fichero, almacenando el número de página y // el título del fichero, e indexando el contenido for (int i = 0; i < numPages; i++) { if (i == 0) { i++; } textStripper.setStartPage(i); textStripper.setEndPage(i); // coger una página pageContent = textStripper.getText(pddDocument); if (pageContent != null && !pageContent.isEmpty()) { pageContent = pageContent.toLowerCase(); } if (pageContent != null) { // Declaramos el documento a indexar para esa página // Número de página // Contenido de la página // Título del fichero // Añadimos el documento } } // Cerramos el fichero PDF }
public static void main(String[] args) throws Exception { File file = new File("C:/Users/jatin.goyal/Desktop/demoexcel.pdf"); PDDocument pd = PDDocument.load(file); System.out.println(pd.getNumberOfPages()); PDFTextStripper st = new PDFTextStripper(); st.setStartPage(1); // st.setEndPage(4); // PDFTextStripperByArea stripper = new PDFTextStripperByArea(); // stripper.setSortByPosition( true ); // Rectangle rect1 = new Rectangle( 50, 140, 60, 20 ); // Rectangle rect2 = new Rectangle( 110, 140, 20, 20 ); // stripper.addRegion( "row1column1", rect1 ); // stripper.addRegion( "row1column2", rect2 ); // List allPages = pd.getDocumentCatalog().getAllPages(); // PDPage firstPage = (PDPage)allPages.get( 0 ); // stripper.extractRegions( firstPage ); // System.out.println(stripper.getTextForRegion( "row1column1" )); // System.out.println(stripper.getTextForRegion( "row1column2" )); System.out.println(st.getText(pd)); }
String pdftoText(String fileName) { System.out.println("Parsing text from PDF file " + fileName + "...."); File f = new File(fileName); if (!f.isFile()) { System.out.println("File " + fileName + " does not exist."); return null; } try { parser = new PDFParser(new FileInputStream(f)); } catch (Exception e) { System.out.println("Unable to open PDF Parser."); return null; } try { parser.parse(); cosDoc = parser.getDocument(); pdfStripper = new PDFTextStripper(); pdDoc = new PDDocument(cosDoc); parsedText = pdfStripper.getText(pdDoc); } catch (Exception e) { System.out.println("An exception occured in parsing the PDF Document."); e.printStackTrace(); try { if (cosDoc != null) cosDoc.close(); if (pdDoc != null) pdDoc.close(); } catch (Exception e1) { e.printStackTrace(); } return null; } System.out.println("Done."); return parsedText; }
public void PDF2TextPreProssesd(String filename) { try { stripper = new PDFTextStripper(); stripper.setParagraphStart("&*&"); stripper.setLineSeparator("#%#"); stripper.setPageSeparator("#%#"); String fulltxt = stripper.getText(pd); String paras[] = fulltxt.split("&*&"); File file = new File(filename); try { BufferedWriter out = new BufferedWriter(new FileWriter(file)); int i = 0; while (i < paras.length) { if (paras[i].length() > 200) { String para = paras[i].replace("#%#", " "); out.write(para + "\r\n"); } i++; } out.close(); } catch (IOException ex) { ex.printStackTrace(); } if (pd != null) { pd.close(); } } catch (Exception e) { e.printStackTrace(); } }
@Override public Document[] parse( final AnchorURL location, final String mimeType, final String charset, final VocabularyScraper scraper, final int timezoneOffset, final InputStream source) throws Parser.Failure, InterruptedException { // check memory for parser if (!MemoryControl.request(200 * 1024 * 1024, false)) throw new Parser.Failure( "Not enough Memory available for pdf parser: " + MemoryControl.available(), location); // create a pdf parser PDDocument pdfDoc; try { Thread.currentThread().setPriority(Thread.MIN_PRIORITY); // the pdfparser is a big pain // pdfDoc = PDDocument.load(source); final PDFParser pdfParser = new PDFParser(source); pdfParser.setTempDirectory(new File(System.getProperty("java.io.tmpdir"))); pdfParser.parse(); pdfDoc = pdfParser.getPDDocument(); } catch (final IOException e) { throw new Parser.Failure(e.getMessage(), location); } finally { Thread.currentThread().setPriority(Thread.NORM_PRIORITY); } if (pdfDoc.isEncrypted()) { try { pdfDoc.openProtection(new StandardDecryptionMaterial("")); } catch (final BadSecurityHandlerException e) { try { pdfDoc.close(); } catch (final IOException ee) { } throw new Parser.Failure("Document is encrypted (1): " + e.getMessage(), location); } catch (final IOException e) { try { pdfDoc.close(); } catch (final IOException ee) { } throw new Parser.Failure("Document is encrypted (2): " + e.getMessage(), location); } catch (final CryptographyException e) { try { pdfDoc.close(); } catch (final IOException ee) { } throw new Parser.Failure("Document is encrypted (3): " + e.getMessage(), location); } final AccessPermission perm = pdfDoc.getCurrentAccessPermission(); if (perm == null || !perm.canExtractContent()) { try { pdfDoc.close(); } catch (final IOException ee) { } throw new Parser.Failure("Document is encrypted and cannot be decrypted", location); } } // extracting some metadata PDDocumentInformation info = pdfDoc.getDocumentInformation(); String docTitle = null, docSubject = null, docAuthor = null, docPublisher = null, docKeywordStr = null; Date docDate = new Date(); if (info != null) { docTitle = info.getTitle(); docSubject = info.getSubject(); docAuthor = info.getAuthor(); docPublisher = info.getProducer(); if (docPublisher == null || docPublisher.isEmpty()) docPublisher = info.getCreator(); docKeywordStr = info.getKeywords(); try { if (info.getModificationDate() != null) docDate = info.getModificationDate().getTime(); } catch (IOException e) { } // unused: // info.getTrapped()); } info = null; if (docTitle == null || docTitle.isEmpty()) { docTitle = MultiProtocolURL.unescape(location.getFileName()); } if (docTitle == null) { docTitle = docSubject; } String[] docKeywords = null; if (docKeywordStr != null) { docKeywords = docKeywordStr.split(" |,"); } Collection<AnchorURL>[] pdflinks = null; Document[] result = null; try { // get the links pdflinks = extractPdfLinks(pdfDoc); // get the fulltext (either per document or for each page) final PDFTextStripper stripper = new PDFTextStripper(StandardCharsets.UTF_8.name()); if (individualPages) { // this is a hack which stores individual pages of the source pdf into individual index // documents // the new documents will get a virtual link with a post argument page=X appended to the // original url // collect text int pagecount = pdfDoc.getNumberOfPages(); String[] pages = new String[pagecount]; for (int page = 1; page <= pagecount; page++) { stripper.setStartPage(page); stripper.setEndPage(page); pages[page - 1] = stripper.getText(pdfDoc); // System.out.println("PAGE " + page + ": " + pages[page - 1]); } // create individual documents for each page assert pages.length == pdflinks.length : "pages.length = " + pages.length + ", pdflinks.length = " + pdflinks.length; result = new Document[Math.min(pages.length, pdflinks.length)]; String loc = location.toNormalform(true); for (int page = 0; page < result.length; page++) { result[page] = new Document( new AnchorURL( loc + (loc.indexOf('?') > 0 ? '&' : '?') + individualPagePropertyname + '=' + (page + 1)), // these are virtual new pages; we cannot combine them with '#' // as that would be removed when computing the urlhash mimeType, StandardCharsets.UTF_8.name(), this, null, docKeywords, singleList(docTitle), docAuthor, docPublisher, null, null, 0.0f, 0.0f, pages == null || page > pages.length ? new byte[0] : UTF8.getBytes(pages[page]), pdflinks == null || page >= pdflinks.length ? null : pdflinks[page], null, null, false, docDate); } } else { // collect the whole text at once final CharBuffer writer = new CharBuffer(odtParser.MAX_DOCSIZE); byte[] contentBytes = new byte[0]; stripper.setEndPage(3); // get first 3 pages (always) writer.append(stripper.getText(pdfDoc)); contentBytes = writer.getBytes(); // remember text in case of interrupting thread if (pdfDoc.getNumberOfPages() > 3) { // spare creating/starting thread if all pages read stripper.setStartPage(4); // continue with page 4 (terminated, resulting in no text) stripper.setEndPage(Integer.MAX_VALUE); // set to default // we start the pdf parsing in a separate thread to ensure that it can be terminated final PDDocument pdfDocC = pdfDoc; final Thread t = new Thread() { @Override public void run() { Thread.currentThread().setName("pdfParser.getText:" + location); try { writer.append(stripper.getText(pdfDocC)); } catch (final Throwable e) { } } }; t.start(); t.join(3000); // pdfbox likes to forget to terminate ... (quite often) if (t.isAlive()) t.interrupt(); } contentBytes = writer.getBytes(); // get final text before closing writer Collection<AnchorURL> pdflinksCombined = new HashSet<AnchorURL>(); for (Collection<AnchorURL> pdflinksx : pdflinks) if (pdflinksx != null) pdflinksCombined.addAll(pdflinksx); result = new Document[] { new Document( location, mimeType, StandardCharsets.UTF_8.name(), this, null, docKeywords, singleList(docTitle), docAuthor, docPublisher, null, null, 0.0f, 0.0f, contentBytes, pdflinksCombined, null, null, false, docDate) }; } } catch (final Throwable e) { // close the writer (in finally) // throw new Parser.Failure(e.getMessage(), location); } finally { try { pdfDoc.close(); } catch (final Throwable e) { } } // clear resources in pdfbox. they say that is resolved but it's not. see: // https://issues.apache.org/jira/browse/PDFBOX-313 // https://issues.apache.org/jira/browse/PDFBOX-351 // https://issues.apache.org/jira/browse/PDFBOX-441 // the pdfbox still generates enormeous number of object allocations and don't delete these // the following Object are statically stored and never flushed: // COSFloat, COSArray, COSInteger, COSObjectKey, COSObject, COSDictionary, // COSStream, COSString, COSName, COSDocument, COSInteger[], COSNull // the great number of these objects can easily be seen in Java Visual VM // we try to get this shit out of the memory here by forced clear calls, hope the best the // rubbish gets out. pdfDoc = null; clean_up_idiotic_PDFParser_font_cache_which_eats_up_tons_of_megabytes(); return result; }