@NotNull public static char[] adaptiveLoadText(@NotNull Reader reader) throws IOException { char[] chars = new char[4096]; List<char[]> buffers = null; int count = 0; int total = 0; while (true) { int n = reader.read(chars, count, chars.length - count); if (n <= 0) break; count += n; if (total > 1024 * 1024 * 10) throw new FileTooBigException("File too big " + reader); total += n; if (count == chars.length) { if (buffers == null) { buffers = new ArrayList<char[]>(); } buffers.add(chars); int newLength = Math.min(1024 * 1024, chars.length * 2); chars = new char[newLength]; count = 0; } } char[] result = new char[total]; if (buffers != null) { for (char[] buffer : buffers) { System.arraycopy(buffer, 0, result, result.length - total, buffer.length); total -= buffer.length; } } System.arraycopy(chars, 0, result, result.length - total, total); return result; }
/** * Obtains streams from the Clob and makes sure we can always read the last char in the Clob. * * <p>See DERBY-4060. * * @param id id of the Clob to use * @param length the length of the Clob * @param alphabet the alphabet used to create the content * @throws IOException if reading from a stream fails * @throws SQLException if something goes wrong */ private void getCharacterStreamLongLastChar(int id, int length, CharAlphabet alphabet) throws IOException, SQLException { // Get last char from the source stream. Reader cmpReader = new LoopingAlphabetReader(length, alphabet); cmpReader.skip(length - 1); char srcLastChar = (char) cmpReader.read(); assertTrue(cmpReader.read() == -1); PreparedStatement ps = prepareStatement("select CLOBDATA from BLOBCLOB where ID=?"); ps.setInt(1, id); // Read everything first. int charsToRead = length; ResultSet rs = ps.executeQuery(); rs.next(); Reader reader = rs.getClob(1).getCharacterStream(length - charsToRead + 1, charsToRead); // Drain the stream, and make sure we are able to read the last char. char lastCharRead = getLastCharInStream(reader, charsToRead); assertEquals(srcLastChar, lastCharRead); reader.close(); rs.close(); // Read a portion of the stream. charsToRead = length / 4; rs = ps.executeQuery(); rs.next(); reader = rs.getClob(1).getCharacterStream(length - charsToRead + 1, charsToRead); lastCharRead = getLastCharInStream(reader, charsToRead); assertEquals(srcLastChar, lastCharRead); reader.close(); rs.close(); // Read a very small portion of the stream. charsToRead = 1; rs = ps.executeQuery(); rs.next(); reader = rs.getClob(1).getCharacterStream(length - charsToRead + 1, charsToRead); lastCharRead = getLastCharInStream(reader, charsToRead); assertEquals(srcLastChar, lastCharRead); reader.close(); rs.close(); }
public void runImpl() throws Exception { // create the URL corresponding to the date URL u = new URL(ADDRESS_ + dateFormatter1_.format(date)); Messages.debug(3, "Rag::runImpl - reading data from URL=%1", u); // fetch the page into the buffer buffer_ = new char[BUF_SIZE_]; Reader r = new InputStreamReader(u.openStream()); int size = 1; charsread_ = 0; while (charsread_ < BUF_SIZE_ && size > 0) { size = r.read(buffer_, 0, BUF_SIZE_ - charsread_); if (size != -1) charsread_ += size; // Messages.debug(3, "Rag::runImpl - read %1 chars", String.valueOf(size)); if (!r.ready()) break; } r.close(); // Messages.debug(3, "Rag::runImpl - buffer=\"%1\"XXXXXXX", new String(buffer_, 0, charsread_)); // create the results createResults_(); }
private String getStringFromStreamSource(StreamSource src, int length) throws Exception { byte buf[] = null; if (src == null) return null; InputStream outStr = src.getInputStream(); if (outStr != null) { int len = outStr.available(); if (outStr.markSupported()) outStr.reset(); buf = new byte[len]; outStr.read(buf, 0, len); // System.out.println("From inputstream: "+new String(buf)); return new String(buf); } else { char buf1[] = new char[length]; Reader r = src.getReader(); if (r == null) return null; r.reset(); r.read(buf1); // System.out.println("From Reader: "+new String(buf)); return new String(buf1); } }
public static String readToEnd(Reader reader) throws IOException { StringBuilder builder = new StringBuilder(); char[] buffer = new char[1024 * 4]; int count = reader.read(buffer); while (count != -1) { builder.append(buffer, 0, count); count = reader.read(buffer); } reader.close(); return builder.toString(); }
/** * Creates a new <code>DictionaryNameFactory</code>. * * @param file the file from which the names can be read. * @param nameFactory the name factory from which names will be retrieved if the list of read * names has been exhausted. */ public DictionaryNameFactory(File file, NameFactory nameFactory) throws IOException { this.names = new ArrayList(); this.nameFactory = nameFactory; Reader reader = new FileReader(file); try { StringBuffer buffer = new StringBuffer(); while (true) { // Read the next character. int c = reader.read(); // Is it a valid identifier character? if (c != -1 && (buffer.length() == 0 ? Character.isJavaIdentifierStart((char) c) : Character.isJavaIdentifierPart((char) c))) { // Append it to the current identifier. buffer.append((char) c); } else { // Did we collect a new identifier? if (buffer.length() > 0) { // Add the completed name to the list of names, if it's // not in it yet. String name = buffer.toString(); if (!names.contains(name)) { names.add(name); } // Clear the buffer. buffer.setLength(0); } // Is this the beginning of a comment line? if (c == COMMENT_CHARACTER) { // Skip all characters till the end of the line. do { c = reader.read(); } while (c != -1 && c != '\n' && c != '\r'); } // Is this the end of the file? if (c == -1) { // Just return. return; } } } } finally { reader.close(); } }
public static String getResponseBodyContent(final HttpEntity entity) throws IOException, ParseException { InputStream instream = entity.getContent(); if (entity.getContentLength() > Integer.MAX_VALUE) { String errorMsg = "HTTP entity too large to be buffered in memory."; Log.e(TAG, errorMsg); throw new IllegalArgumentException(errorMsg); } String charset = getContentCharSet(entity); if (charset == null) { charset = HTTP.DEFAULT_CONTENT_CHARSET; } Reader reader = new InputStreamReader(instream, charset); StringBuilder buffer = new StringBuilder(); try { char[] bufferSize = new char[1024]; int length; while ((length = reader.read(bufferSize)) != -1) { buffer.append(bufferSize, 0, length); } } finally { reader.close(); } return buffer.toString(); }
static String outputOf(Reader r) throws IOException { final StringBuilder sb = new StringBuilder(); final char[] buf = new char[1024]; int n; while ((n = r.read(buf)) > 0) sb.append(buf, 0, n); return sb.toString(); }
/** * Drains the stream and returns the last char read from the stream. * * @param reader stream to drain * @param expectedCount expected number of chars (remaining) in the stream * @return The last char read. * @throws AssertionError if there are too many/few chars in the stream * @throws IOException if reading from the stream fails */ public static char getLastCharInStream(Reader reader, int expectedCount) throws IOException { int read = 0; final char[] buf = new char[256]; assertTrue(buf.length > 0); // Do not allow an infinite loop here. while (true) { int readThisTime = reader.read(buf, 0, buf.length); // -1 is expected, but catch all cases with a negative return value. if (readThisTime < 0) { assertEquals("Invalid return value from stream", -1, readThisTime); fail("Reached EOF prematurely, expected " + expectedCount + ", got " + read); } else if (readThisTime == 0) { // Another special case that should not happen. fail("Stream breaks contract, read zero chars: " + reader); } read += readThisTime; if (read == expectedCount) { return buf[readThisTime - 1]; } else if (read > expectedCount) { fail( "Too many chars in stream, expected " + expectedCount + "have " + read + "(EOF not reached/confirmed)"); } } }
private void copy(Reader in, Writer out) throws IOException { final char[] buf = new char[10000]; int read; while ((read = in.read(buf)) > 0) { out.write(buf, 0, read); } }
int read() throws Exception { current = reader.read(); if (digest != null) { digest.update((byte) (current / 256)); digest.update((byte) (current % 256)); } return current; }
private String read(Reader in) throws IOException { StringBuilder builder = new StringBuilder(); char[] buffer = new char[1048]; while (true) { int nBytes = in.read(buffer); if (nBytes == -1) break; builder.append(new String(buffer, 0, nBytes)); } return builder.toString(); }
/** * Read characters from a Reader and generate SAX characters events. * * <p>The caller has to close the Reader if needed. */ public static void readerToCharacters(Reader reader, ContentHandler contentHandler) { try { // Work with buffered Reader reader = new BufferedReader(reader); // Read and write in chunks char[] buf = new char[1024]; int count; while ((count = reader.read(buf)) != -1) contentHandler.characters(buf, 0, count); } catch (Exception e) { throw new OXFException(e); } }
@Override public void run() { try { // initialize the statusbar status.removeAll(); JProgressBar progress = new JProgressBar(); progress.setMinimum(0); progress.setMaximum((int) f.length()); status.add(progress); status.revalidate(); // try to start reading Reader in = new FileReader(f); char[] buff = new char[4096]; int nch; while ((nch = in.read(buff, 0, buff.length)) != -1) { doc.insertString(doc.getLength(), new String(buff, 0, nch), null); progress.setValue(progress.getValue() + nch); } } catch (IOException e) { final String msg = e.getMessage(); SwingUtilities.invokeLater( new Runnable() { public void run() { JOptionPane.showMessageDialog( getFrame(), "Could not open file: " + msg, "Error opening file", JOptionPane.ERROR_MESSAGE); } }); } catch (BadLocationException e) { System.err.println(e.getMessage()); } doc.addUndoableEditListener(undoHandler); // we are done... get rid of progressbar status.removeAll(); status.revalidate(); resetUndoManager(); if (elementTreePanel != null) { SwingUtilities.invokeLater( new Runnable() { public void run() { elementTreePanel.setEditor(getEditor()); } }); } }
public static String get_stream(Reader isr) throws IOException { char[] buffer = new char[BUFF_SZ]; StringBuilder out = new StringBuilder(); logm("DBG", 9, "STREAM GET", isr + ""); for (; ; ) { int rsz = isr.read(buffer, 0, buffer.length); logm("DBG", 9, "STREAM GET READ", rsz); if (rsz < 0) break; out.append(buffer, 0, rsz); } String s = out.toString(); logm("DBG", 9, "STREAM GET RESULT", s); return s; }
public static int copy(Reader input, Writer output) throws IOException { char[] buffer = new char[8 * 1024]; int count = 0; int n = 0; try { while (-1 != (n = input.read(buffer))) { output.write(buffer, 0, n); count += n; } } finally { output.flush(); output.close(); } return count; }
public static String readResource(File input) { try { Reader stream = new FileReader(input); StringBuilder output = new StringBuilder(); char[] buffer = new char[1024]; int read; while ((read = stream.read(buffer)) >= 0) { output.append(buffer, 0, read); } stream.close(); return output.toString(); } catch (Exception e) { throw new RuntimeException(e); } }
// Utility methods // --------------- // Get the next character, throwing an unchecked exception on error. private char next() { try { int i = (saved != 0) ? saved : r.read(); saved = 0; if (i <= -1) { if (body_only) return ']'; // Just a little hacky. throw eof; } return (char) i; } catch (RuntimeException e) { throw e; } catch (Exception e) { throw new RuntimeException(e); } }
/** Copies the contents of in into out */ protected static int copy(Reader in, Writer out) { int count = 0; char[] buf = new char[1024]; while (true) { try { int num = in.read(buf, 0, buf.length); if (num == -1) break; out.write(buf, 0, num); count += num; } catch (IOException e) { break; } } return count; }
public void run() { try { // An arbitrary buffer size. char[] chbuf = new char[4096]; while (keepRunning) { int numchars = reader.read(chbuf); if (numchars == -1) { keepRunning = false; } else if (keepRunning) { writer.write(chbuf, 0, numchars); if (!reader.ready()) { writer.flush(); } } } } catch (IOException ex) { println("Error when linking JVM output to terminal window input."); } }
private static short[] load(String name, Reader input, Model.Translator translator) throws java.io.IOException { short[] sequence = null; char[] buffer = new char[10240]; int len; while ((len = input.read(buffer)) > 0) { for (int i = len - 1; i >= 0; i--) { if (Character.isWhitespace(buffer[i])) { if (i == (len - 1)) { len--; } else { System.arraycopy(buffer, i + 1, buffer, i, len - i - 1); } } } int start; if (sequence == null) { sequence = new short[len]; start = 0; } else { start = sequence.length; short[] newseq = new short[sequence.length + len]; System.arraycopy(sequence, 0, newseq, 0, sequence.length); sequence = newseq; } for (int i = 0; i < len; i++) { sequence[start + i] = translator.translate(buffer[i]); if (sequence[start + i] < 0) { throw new IOException( "Sequence character '" + buffer[i] + "' (" + ((int) buffer[i]) + ") at offset " + (start + i) + " is not in the model's lexicon."); } } } return sequence; }
public void run() { try { // initialize the statusbar status.removeAll(); JProgressBar progress = new JProgressBar(); progress.setMinimum(0); progress.setMaximum((int) f2.length()); status.add(progress); status.revalidate(); // try to start reading Reader in = new FileReader(f2); char[] buff = new char[4096]; int nch; while ((nch = in.read(buff, 0, buff.length)) != -1) { doc2.insertString(doc2.getLength(), new String(buff, 0, nch), null); progress.setValue(progress.getValue() + nch); } // we are done... get rid of progressbar // doc2.addUndoableEditListener(undoHandler); status.removeAll(); status.revalidate(); // resetUndoManager(); } catch (IOException e) { System.err.println("TextViewer:FileLoader " + e.toString()); } catch (BadLocationException e) { System.err.println("TextViewer:FileLoader " + e.getMessage()); } /* aa if (elementTreePanel != null) { SwingUtilities.invokeLater(new Runnable() { public void run() { elementTreePanel.setEditor(getEditor()); } }); } */ }
private static String getBody(HttpServletRequest req) { try { // Try reading the post body using characters. // This might throw an exception if something on the // server side already called getInputStream(). // In that case we'll pull as bytes. Reader reader = null; try { reader = new BufferedReader(req.getReader()); } catch (IOException e) { reader = new BufferedReader(new InputStreamReader(req.getInputStream(), "UTF-8")); } StringBuffer sbuf = new StringBuffer(); char[] cbuf = new char[4096]; int count = 0; while ((count = reader.read(cbuf)) != -1) { sbuf.append(cbuf, 0, count); } return sbuf.toString(); } catch (IOException e2) { throw new ServerProblemException("IOException in reading POST body: " + e2.getMessage()); } }
public static void copyStream(Reader reader, Writer writer) throws IOException { int count; char[] buffer = new char[1024]; while ((count = reader.read(buffer)) > 0) writer.write(buffer, 0, count); }
private static void read(Reader in) throws IOException { int read; int var; while ((read = in.read()) != -1) var = read; }
/** Builds the model from the files */ public void extractKeyphrases(Hashtable stems) throws Exception { Vector stats = new Vector(); // Check whether there is actually any data if (stems.size() == 0) { throw new Exception("Couldn't find any data!"); } FastVector atts = new FastVector(2); atts.addElement(new Attribute("doc", (FastVector) null)); atts.addElement(new Attribute("keyphrases", (FastVector) null)); Instances data = new Instances("keyphrase_training_data", atts, 0); // Extract keyphrases Enumeration elem = stems.keys(); while (elem.hasMoreElements()) { String str = (String) elem.nextElement(); double[] newInst = new double[2]; try { File txt = new File(m_dirName + "/" + str + ".txt"); Reader is; if (!m_encoding.equals("default")) { is = new BomStrippingInputStreamReader(new FileInputStream(txt), m_encoding); } else { is = new BomStrippingInputStreamReader(new FileInputStream(txt)); } StringBuffer txtStr = new StringBuffer(); int c; while ((c = is.read()) != -1) { txtStr.append((char) c); } newInst[0] = (double) data.attribute(0).addStringValue(txtStr.toString()); } catch (Exception e) { if (m_debug) { System.err.println("Can't read document " + str + ".txt"); } newInst[0] = Instance.missingValue(); } try { File key = new File(m_dirName + "/" + str + ".key"); Reader is; if (!m_encoding.equals("default")) { is = new BomStrippingInputStreamReader(new FileInputStream(key), m_encoding); } else { is = new BomStrippingInputStreamReader(new FileInputStream(key)); } StringBuffer keyStr = new StringBuffer(); int c; while ((c = is.read()) != -1) { keyStr.append((char) c); } newInst[1] = (double) data.attribute(1).addStringValue(keyStr.toString()); } catch (Exception e) { if (m_debug) { System.err.println("No keyphrases for stem " + str + "."); } newInst[1] = Instance.missingValue(); } data.add(new Instance(1.0, newInst)); m_KEAFilter.input(data.instance(0)); data = data.stringFreeStructure(); if (m_debug) { System.err.println("-- Document: " + str); } Instance[] topRankedInstances = new Instance[m_numPhrases]; Instance inst; while ((inst = m_KEAFilter.output()) != null) { int index = (int) inst.value(m_KEAFilter.getRankIndex()) - 1; if (index < m_numPhrases) { topRankedInstances[index] = inst; } } if (m_debug) { System.err.println("-- Keyphrases and feature values:"); } FileOutputStream out = null; PrintWriter printer = null; File key = new File(m_dirName + "/" + str + ".key"); if (!key.exists()) { out = new FileOutputStream(m_dirName + "/" + str + ".key"); if (!m_encoding.equals("default")) { printer = new PrintWriter(new OutputStreamWriter(out, m_encoding)); } else { printer = new PrintWriter(out); } } double numExtracted = 0, numCorrect = 0; for (int i = 0; i < m_numPhrases; i++) { if (topRankedInstances[i] != null) { if (!topRankedInstances[i].isMissing(topRankedInstances[i].numAttributes() - 1)) { numExtracted += 1.0; } if ((int) topRankedInstances[i].value(topRankedInstances[i].numAttributes() - 1) == topRankedInstances[i] .attribute(topRankedInstances[i].numAttributes() - 1) .indexOfValue("True")) { numCorrect += 1.0; } if (printer != null) { printer.print(topRankedInstances[i].stringValue(m_KEAFilter.getUnstemmedPhraseIndex())); if (m_AdditionalInfo) { printer.print("\t"); printer.print(topRankedInstances[i].stringValue(m_KEAFilter.getStemmedPhraseIndex())); printer.print("\t"); printer.print( Utils.doubleToString( topRankedInstances[i].value(m_KEAFilter.getProbabilityIndex()), 4)); } printer.println(); } if (m_debug) { System.err.println(topRankedInstances[i]); } } } if (numExtracted > 0) { if (m_debug) { System.err.println("-- " + numCorrect + " correct"); } stats.addElement(new Double(numCorrect)); } if (printer != null) { printer.flush(); printer.close(); out.close(); } } double[] st = new double[stats.size()]; for (int i = 0; i < stats.size(); i++) { st[i] = ((Double) stats.elementAt(i)).doubleValue(); } double avg = Utils.mean(st); double stdDev = Math.sqrt(Utils.variance(st)); System.err.println( "Avg. number of correct keyphrases: " + Utils.doubleToString(avg, 2) + " +/- " + Utils.doubleToString(stdDev, 2)); System.err.println("Based on " + stats.size() + " documents"); m_KEAFilter.batchFinished(); }