@NotNull
public static char[] adaptiveLoadText(@NotNull Reader reader) throws IOException {
char[] chars = new char[4096];
List<char[]> buffers = null;
int count = 0;
int total = 0;
while (true) {
int n = reader.read(chars, count, chars.length - count);
if (n <= 0) break;
count += n;
if (total > 1024 * 1024 * 10) throw new FileTooBigException("File too big " + reader);
total += n;
if (count == chars.length) {
if (buffers == null) {
buffers = new ArrayList<char[]>();
}
buffers.add(chars);
int newLength = Math.min(1024 * 1024, chars.length * 2);
chars = new char[newLength];
count = 0;
}
}
char[] result = new char[total];
if (buffers != null) {
for (char[] buffer : buffers) {
System.arraycopy(buffer, 0, result, result.length - total, buffer.length);
total -= buffer.length;
}
}
System.arraycopy(chars, 0, result, result.length - total, total);
return result;
}
/**
* Obtains streams from the Clob and makes sure we can always read the last char in the Clob.
*
* <p>See DERBY-4060.
*
* @param id id of the Clob to use
* @param length the length of the Clob
* @param alphabet the alphabet used to create the content
* @throws IOException if reading from a stream fails
* @throws SQLException if something goes wrong
*/
private void getCharacterStreamLongLastChar(int id, int length, CharAlphabet alphabet)
throws IOException, SQLException {
// Get last char from the source stream.
Reader cmpReader = new LoopingAlphabetReader(length, alphabet);
cmpReader.skip(length - 1);
char srcLastChar = (char) cmpReader.read();
assertTrue(cmpReader.read() == -1);
PreparedStatement ps = prepareStatement("select CLOBDATA from BLOBCLOB where ID=?");
ps.setInt(1, id);
// Read everything first.
int charsToRead = length;
ResultSet rs = ps.executeQuery();
rs.next();
Reader reader = rs.getClob(1).getCharacterStream(length - charsToRead + 1, charsToRead);
// Drain the stream, and make sure we are able to read the last char.
char lastCharRead = getLastCharInStream(reader, charsToRead);
assertEquals(srcLastChar, lastCharRead);
reader.close();
rs.close();
// Read a portion of the stream.
charsToRead = length / 4;
rs = ps.executeQuery();
rs.next();
reader = rs.getClob(1).getCharacterStream(length - charsToRead + 1, charsToRead);
lastCharRead = getLastCharInStream(reader, charsToRead);
assertEquals(srcLastChar, lastCharRead);
reader.close();
rs.close();
// Read a very small portion of the stream.
charsToRead = 1;
rs = ps.executeQuery();
rs.next();
reader = rs.getClob(1).getCharacterStream(length - charsToRead + 1, charsToRead);
lastCharRead = getLastCharInStream(reader, charsToRead);
assertEquals(srcLastChar, lastCharRead);
reader.close();
rs.close();
}
public void runImpl() throws Exception {
// create the URL corresponding to the date
URL u = new URL(ADDRESS_ + dateFormatter1_.format(date));
Messages.debug(3, "Rag::runImpl - reading data from URL=%1", u);
// fetch the page into the buffer
buffer_ = new char[BUF_SIZE_];
Reader r = new InputStreamReader(u.openStream());
int size = 1;
charsread_ = 0;
while (charsread_ < BUF_SIZE_ && size > 0) {
size = r.read(buffer_, 0, BUF_SIZE_ - charsread_);
if (size != -1) charsread_ += size;
// Messages.debug(3, "Rag::runImpl - read %1 chars", String.valueOf(size));
if (!r.ready()) break;
}
r.close();
// Messages.debug(3, "Rag::runImpl - buffer=\"%1\"XXXXXXX", new String(buffer_, 0, charsread_));
// create the results
createResults_();
}
private String getStringFromStreamSource(StreamSource src, int length) throws Exception {
byte buf[] = null;
if (src == null) return null;
InputStream outStr = src.getInputStream();
if (outStr != null) {
int len = outStr.available();
if (outStr.markSupported()) outStr.reset();
buf = new byte[len];
outStr.read(buf, 0, len);
// System.out.println("From inputstream: "+new String(buf));
return new String(buf);
} else {
char buf1[] = new char[length];
Reader r = src.getReader();
if (r == null) return null;
r.reset();
r.read(buf1);
// System.out.println("From Reader: "+new String(buf));
return new String(buf1);
}
}
public static String readToEnd(Reader reader) throws IOException {
StringBuilder builder = new StringBuilder();
char[] buffer = new char[1024 * 4];
int count = reader.read(buffer);
while (count != -1) {
builder.append(buffer, 0, count);
count = reader.read(buffer);
}
reader.close();
return builder.toString();
}
/**
* Creates a new <code>DictionaryNameFactory</code>.
*
* @param file the file from which the names can be read.
* @param nameFactory the name factory from which names will be retrieved if the list of read
* names has been exhausted.
*/
public DictionaryNameFactory(File file, NameFactory nameFactory) throws IOException {
this.names = new ArrayList();
this.nameFactory = nameFactory;
Reader reader = new FileReader(file);
try {
StringBuffer buffer = new StringBuffer();
while (true) {
// Read the next character.
int c = reader.read();
// Is it a valid identifier character?
if (c != -1
&& (buffer.length() == 0
? Character.isJavaIdentifierStart((char) c)
: Character.isJavaIdentifierPart((char) c))) {
// Append it to the current identifier.
buffer.append((char) c);
} else {
// Did we collect a new identifier?
if (buffer.length() > 0) {
// Add the completed name to the list of names, if it's
// not in it yet.
String name = buffer.toString();
if (!names.contains(name)) {
names.add(name);
}
// Clear the buffer.
buffer.setLength(0);
}
// Is this the beginning of a comment line?
if (c == COMMENT_CHARACTER) {
// Skip all characters till the end of the line.
do {
c = reader.read();
} while (c != -1 && c != '\n' && c != '\r');
}
// Is this the end of the file?
if (c == -1) {
// Just return.
return;
}
}
}
} finally {
reader.close();
}
}
public static String getResponseBodyContent(final HttpEntity entity)
throws IOException, ParseException {
InputStream instream = entity.getContent();
if (entity.getContentLength() > Integer.MAX_VALUE) {
String errorMsg = "HTTP entity too large to be buffered in memory.";
Log.e(TAG, errorMsg);
throw new IllegalArgumentException(errorMsg);
}
String charset = getContentCharSet(entity);
if (charset == null) {
charset = HTTP.DEFAULT_CONTENT_CHARSET;
}
Reader reader = new InputStreamReader(instream, charset);
StringBuilder buffer = new StringBuilder();
try {
char[] bufferSize = new char[1024];
int length;
while ((length = reader.read(bufferSize)) != -1) {
buffer.append(bufferSize, 0, length);
}
} finally {
reader.close();
}
return buffer.toString();
}
static String outputOf(Reader r) throws IOException {
final StringBuilder sb = new StringBuilder();
final char[] buf = new char[1024];
int n;
while ((n = r.read(buf)) > 0) sb.append(buf, 0, n);
return sb.toString();
}
/**
* Drains the stream and returns the last char read from the stream.
*
* @param reader stream to drain
* @param expectedCount expected number of chars (remaining) in the stream
* @return The last char read.
* @throws AssertionError if there are too many/few chars in the stream
* @throws IOException if reading from the stream fails
*/
public static char getLastCharInStream(Reader reader, int expectedCount) throws IOException {
int read = 0;
final char[] buf = new char[256];
assertTrue(buf.length > 0); // Do not allow an infinite loop here.
while (true) {
int readThisTime = reader.read(buf, 0, buf.length);
// -1 is expected, but catch all cases with a negative return value.
if (readThisTime < 0) {
assertEquals("Invalid return value from stream", -1, readThisTime);
fail("Reached EOF prematurely, expected " + expectedCount + ", got " + read);
} else if (readThisTime == 0) {
// Another special case that should not happen.
fail("Stream breaks contract, read zero chars: " + reader);
}
read += readThisTime;
if (read == expectedCount) {
return buf[readThisTime - 1];
} else if (read > expectedCount) {
fail(
"Too many chars in stream, expected "
+ expectedCount
+ "have "
+ read
+ "(EOF not reached/confirmed)");
}
}
}
private void copy(Reader in, Writer out) throws IOException {
final char[] buf = new char[10000];
int read;
while ((read = in.read(buf)) > 0) {
out.write(buf, 0, read);
}
}
int read() throws Exception {
current = reader.read();
if (digest != null) {
digest.update((byte) (current / 256));
digest.update((byte) (current % 256));
}
return current;
}
private String read(Reader in) throws IOException {
StringBuilder builder = new StringBuilder();
char[] buffer = new char[1048];
while (true) {
int nBytes = in.read(buffer);
if (nBytes == -1) break;
builder.append(new String(buffer, 0, nBytes));
}
return builder.toString();
}
/**
* Read characters from a Reader and generate SAX characters events.
*
* <p>The caller has to close the Reader if needed.
*/
public static void readerToCharacters(Reader reader, ContentHandler contentHandler) {
try {
// Work with buffered Reader
reader = new BufferedReader(reader);
// Read and write in chunks
char[] buf = new char[1024];
int count;
while ((count = reader.read(buf)) != -1) contentHandler.characters(buf, 0, count);
} catch (Exception e) {
throw new OXFException(e);
}
}
@Override
public void run() {
try {
// initialize the statusbar
status.removeAll();
JProgressBar progress = new JProgressBar();
progress.setMinimum(0);
progress.setMaximum((int) f.length());
status.add(progress);
status.revalidate();
// try to start reading
Reader in = new FileReader(f);
char[] buff = new char[4096];
int nch;
while ((nch = in.read(buff, 0, buff.length)) != -1) {
doc.insertString(doc.getLength(), new String(buff, 0, nch), null);
progress.setValue(progress.getValue() + nch);
}
} catch (IOException e) {
final String msg = e.getMessage();
SwingUtilities.invokeLater(
new Runnable() {
public void run() {
JOptionPane.showMessageDialog(
getFrame(),
"Could not open file: " + msg,
"Error opening file",
JOptionPane.ERROR_MESSAGE);
}
});
} catch (BadLocationException e) {
System.err.println(e.getMessage());
}
doc.addUndoableEditListener(undoHandler);
// we are done... get rid of progressbar
status.removeAll();
status.revalidate();
resetUndoManager();
if (elementTreePanel != null) {
SwingUtilities.invokeLater(
new Runnable() {
public void run() {
elementTreePanel.setEditor(getEditor());
}
});
}
}
public static String get_stream(Reader isr) throws IOException {
char[] buffer = new char[BUFF_SZ];
StringBuilder out = new StringBuilder();
logm("DBG", 9, "STREAM GET", isr + "");
for (; ; ) {
int rsz = isr.read(buffer, 0, buffer.length);
logm("DBG", 9, "STREAM GET READ", rsz);
if (rsz < 0) break;
out.append(buffer, 0, rsz);
}
String s = out.toString();
logm("DBG", 9, "STREAM GET RESULT", s);
return s;
}
public static int copy(Reader input, Writer output) throws IOException {
char[] buffer = new char[8 * 1024];
int count = 0;
int n = 0;
try {
while (-1 != (n = input.read(buffer))) {
output.write(buffer, 0, n);
count += n;
}
} finally {
output.flush();
output.close();
}
return count;
}
public static String readResource(File input) {
try {
Reader stream = new FileReader(input);
StringBuilder output = new StringBuilder();
char[] buffer = new char[1024];
int read;
while ((read = stream.read(buffer)) >= 0) {
output.append(buffer, 0, read);
}
stream.close();
return output.toString();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// Utility methods
// ---------------
// Get the next character, throwing an unchecked exception on error.
private char next() {
try {
int i = (saved != 0) ? saved : r.read();
saved = 0;
if (i <= -1) {
if (body_only) return ']'; // Just a little hacky.
throw eof;
}
return (char) i;
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
}
/** Copies the contents of in into out */
protected static int copy(Reader in, Writer out) {
int count = 0;
char[] buf = new char[1024];
while (true) {
try {
int num = in.read(buf, 0, buf.length);
if (num == -1) break;
out.write(buf, 0, num);
count += num;
} catch (IOException e) {
break;
}
}
return count;
}
public void run() {
try {
// An arbitrary buffer size.
char[] chbuf = new char[4096];
while (keepRunning) {
int numchars = reader.read(chbuf);
if (numchars == -1) {
keepRunning = false;
} else if (keepRunning) {
writer.write(chbuf, 0, numchars);
if (!reader.ready()) {
writer.flush();
}
}
}
} catch (IOException ex) {
println("Error when linking JVM output to terminal window input.");
}
}
private static short[] load(String name, Reader input, Model.Translator translator)
throws java.io.IOException {
short[] sequence = null;
char[] buffer = new char[10240];
int len;
while ((len = input.read(buffer)) > 0) {
for (int i = len - 1; i >= 0; i--) {
if (Character.isWhitespace(buffer[i])) {
if (i == (len - 1)) {
len--;
} else {
System.arraycopy(buffer, i + 1, buffer, i, len - i - 1);
}
}
}
int start;
if (sequence == null) {
sequence = new short[len];
start = 0;
} else {
start = sequence.length;
short[] newseq = new short[sequence.length + len];
System.arraycopy(sequence, 0, newseq, 0, sequence.length);
sequence = newseq;
}
for (int i = 0; i < len; i++) {
sequence[start + i] = translator.translate(buffer[i]);
if (sequence[start + i] < 0) {
throw new IOException(
"Sequence character '"
+ buffer[i]
+ "' ("
+ ((int) buffer[i])
+ ") at offset "
+ (start + i)
+ " is not in the model's lexicon.");
}
}
}
return sequence;
}
public void run() {
try {
// initialize the statusbar
status.removeAll();
JProgressBar progress = new JProgressBar();
progress.setMinimum(0);
progress.setMaximum((int) f2.length());
status.add(progress);
status.revalidate();
// try to start reading
Reader in = new FileReader(f2);
char[] buff = new char[4096];
int nch;
while ((nch = in.read(buff, 0, buff.length)) != -1) {
doc2.insertString(doc2.getLength(), new String(buff, 0, nch), null);
progress.setValue(progress.getValue() + nch);
}
// we are done... get rid of progressbar
// doc2.addUndoableEditListener(undoHandler);
status.removeAll();
status.revalidate();
// resetUndoManager();
} catch (IOException e) {
System.err.println("TextViewer:FileLoader " + e.toString());
} catch (BadLocationException e) {
System.err.println("TextViewer:FileLoader " + e.getMessage());
}
/* aa
if (elementTreePanel != null) {
SwingUtilities.invokeLater(new Runnable() {
public void run() {
elementTreePanel.setEditor(getEditor());
}
});
}
*/
}
private static String getBody(HttpServletRequest req) {
try {
// Try reading the post body using characters.
// This might throw an exception if something on the
// server side already called getInputStream().
// In that case we'll pull as bytes.
Reader reader = null;
try {
reader = new BufferedReader(req.getReader());
} catch (IOException e) {
reader = new BufferedReader(new InputStreamReader(req.getInputStream(), "UTF-8"));
}
StringBuffer sbuf = new StringBuffer();
char[] cbuf = new char[4096];
int count = 0;
while ((count = reader.read(cbuf)) != -1) {
sbuf.append(cbuf, 0, count);
}
return sbuf.toString();
} catch (IOException e2) {
throw new ServerProblemException("IOException in reading POST body: " + e2.getMessage());
}
}
public static void copyStream(Reader reader, Writer writer) throws IOException {
int count;
char[] buffer = new char[1024];
while ((count = reader.read(buffer)) > 0) writer.write(buffer, 0, count);
}
private static void read(Reader in) throws IOException {
int read;
int var;
while ((read = in.read()) != -1) var = read;
}
/** Builds the model from the files */
public void extractKeyphrases(Hashtable stems) throws Exception {
Vector stats = new Vector();
// Check whether there is actually any data
if (stems.size() == 0) {
throw new Exception("Couldn't find any data!");
}
FastVector atts = new FastVector(2);
atts.addElement(new Attribute("doc", (FastVector) null));
atts.addElement(new Attribute("keyphrases", (FastVector) null));
Instances data = new Instances("keyphrase_training_data", atts, 0);
// Extract keyphrases
Enumeration elem = stems.keys();
while (elem.hasMoreElements()) {
String str = (String) elem.nextElement();
double[] newInst = new double[2];
try {
File txt = new File(m_dirName + "/" + str + ".txt");
Reader is;
if (!m_encoding.equals("default")) {
is = new BomStrippingInputStreamReader(new FileInputStream(txt), m_encoding);
} else {
is = new BomStrippingInputStreamReader(new FileInputStream(txt));
}
StringBuffer txtStr = new StringBuffer();
int c;
while ((c = is.read()) != -1) {
txtStr.append((char) c);
}
newInst[0] = (double) data.attribute(0).addStringValue(txtStr.toString());
} catch (Exception e) {
if (m_debug) {
System.err.println("Can't read document " + str + ".txt");
}
newInst[0] = Instance.missingValue();
}
try {
File key = new File(m_dirName + "/" + str + ".key");
Reader is;
if (!m_encoding.equals("default")) {
is = new BomStrippingInputStreamReader(new FileInputStream(key), m_encoding);
} else {
is = new BomStrippingInputStreamReader(new FileInputStream(key));
}
StringBuffer keyStr = new StringBuffer();
int c;
while ((c = is.read()) != -1) {
keyStr.append((char) c);
}
newInst[1] = (double) data.attribute(1).addStringValue(keyStr.toString());
} catch (Exception e) {
if (m_debug) {
System.err.println("No keyphrases for stem " + str + ".");
}
newInst[1] = Instance.missingValue();
}
data.add(new Instance(1.0, newInst));
m_KEAFilter.input(data.instance(0));
data = data.stringFreeStructure();
if (m_debug) {
System.err.println("-- Document: " + str);
}
Instance[] topRankedInstances = new Instance[m_numPhrases];
Instance inst;
while ((inst = m_KEAFilter.output()) != null) {
int index = (int) inst.value(m_KEAFilter.getRankIndex()) - 1;
if (index < m_numPhrases) {
topRankedInstances[index] = inst;
}
}
if (m_debug) {
System.err.println("-- Keyphrases and feature values:");
}
FileOutputStream out = null;
PrintWriter printer = null;
File key = new File(m_dirName + "/" + str + ".key");
if (!key.exists()) {
out = new FileOutputStream(m_dirName + "/" + str + ".key");
if (!m_encoding.equals("default")) {
printer = new PrintWriter(new OutputStreamWriter(out, m_encoding));
} else {
printer = new PrintWriter(out);
}
}
double numExtracted = 0, numCorrect = 0;
for (int i = 0; i < m_numPhrases; i++) {
if (topRankedInstances[i] != null) {
if (!topRankedInstances[i].isMissing(topRankedInstances[i].numAttributes() - 1)) {
numExtracted += 1.0;
}
if ((int) topRankedInstances[i].value(topRankedInstances[i].numAttributes() - 1)
== topRankedInstances[i]
.attribute(topRankedInstances[i].numAttributes() - 1)
.indexOfValue("True")) {
numCorrect += 1.0;
}
if (printer != null) {
printer.print(topRankedInstances[i].stringValue(m_KEAFilter.getUnstemmedPhraseIndex()));
if (m_AdditionalInfo) {
printer.print("\t");
printer.print(topRankedInstances[i].stringValue(m_KEAFilter.getStemmedPhraseIndex()));
printer.print("\t");
printer.print(
Utils.doubleToString(
topRankedInstances[i].value(m_KEAFilter.getProbabilityIndex()), 4));
}
printer.println();
}
if (m_debug) {
System.err.println(topRankedInstances[i]);
}
}
}
if (numExtracted > 0) {
if (m_debug) {
System.err.println("-- " + numCorrect + " correct");
}
stats.addElement(new Double(numCorrect));
}
if (printer != null) {
printer.flush();
printer.close();
out.close();
}
}
double[] st = new double[stats.size()];
for (int i = 0; i < stats.size(); i++) {
st[i] = ((Double) stats.elementAt(i)).doubleValue();
}
double avg = Utils.mean(st);
double stdDev = Math.sqrt(Utils.variance(st));
System.err.println(
"Avg. number of correct keyphrases: "
+ Utils.doubleToString(avg, 2)
+ " +/- "
+ Utils.doubleToString(stdDev, 2));
System.err.println("Based on " + stats.size() + " documents");
m_KEAFilter.batchFinished();
}
