public void endElement(java.lang.String uri, java.lang.String localName, java.lang.String qName)
throws SAXException {
try {
if (qName.equals("p") || qName.equals("description")) {
writer.write(getText());
accumulator.setLength(0);
}
if (qName.equals("description")) {
counting = false;
}
if (!counting) {
writer.write(getText());
accumulator.setLength(0);
writer.write("</" + qName + ">\n");
} else {
if (qName.equals("row")) {
accumulator.append(" ");
}
if (qName.equals("p")) {
writer.write("\n");
accumulator.append(" ");
}
}
} catch (Exception e) {
// e.printStackTrace();
throw new GrobidException("An exception occured while running Grobid.", e);
}
}
/**
* Parses identifiers until <code>extraChar</code> is encountered, returning the ending token,
* which will be IDENTIFIER if extraChar is found.
*/
private int parseIdentifiers(char extraChar, boolean wantsBlocks) throws IOException {
int nextToken;
int ubl;
unitBuffer.setLength(0);
for (; ; ) {
nextToken = nextToken(extraChar);
switch (nextToken) {
case IDENTIFIER:
if (tokenBufferLength > 0) {
if (tokenBuffer[tokenBufferLength - 1] == extraChar) {
if (--tokenBufferLength > 0) {
if (readWS && unitBuffer.length() > 0) {
unitBuffer.append(' ');
}
unitBuffer.append(tokenBuffer, 0, tokenBufferLength);
}
return IDENTIFIER;
}
if (readWS && unitBuffer.length() > 0) {
unitBuffer.append(' ');
}
unitBuffer.append(tokenBuffer, 0, tokenBufferLength);
}
break;
case BRACKET_OPEN:
case BRACE_OPEN:
case PAREN_OPEN:
ubl = unitBuffer.length();
if (wantsBlocks) {
unitBuffer.append(charMapping[nextToken]);
}
parseTillClosed(nextToken);
if (!wantsBlocks) {
unitBuffer.setLength(ubl);
}
break;
case BRACE_CLOSE:
// No need to throw for these two, we return token and
// caller can do whatever.
case BRACKET_CLOSE:
case PAREN_CLOSE:
case END:
// Hit the end
return nextToken;
}
}
}
/**
* Output the specified {@link Collection} in proper columns.
*
* @param stuff the stuff to print
*/
public void printColumns(final Collection stuff) throws IOException {
if ((stuff == null) || (stuff.size() == 0)) {
return;
}
int width = getTermwidth();
int maxwidth = 0;
for (Iterator i = stuff.iterator();
i.hasNext();
maxwidth = Math.max(maxwidth, i.next().toString().length())) {;
}
StringBuffer line = new StringBuffer();
int showLines;
if (usePagination) showLines = getTermheight() - 1; // page limit
else showLines = Integer.MAX_VALUE;
for (Iterator i = stuff.iterator(); i.hasNext(); ) {
String cur = (String) i.next();
if ((line.length() + maxwidth) > width) {
printString(line.toString().trim());
printNewline();
line.setLength(0);
if (--showLines == 0) { // Overflow
printString(loc.getString("display-more"));
flushConsole();
int c = readVirtualKey();
if (c == '\r' || c == '\n') showLines = 1; // one step forward
else if (c != 'q') showLines = getTermheight() - 1; // page forward
back(loc.getString("display-more").length());
if (c == 'q') break; // cancel
}
}
pad(cur, maxwidth + 3, line);
}
if (line.length() > 0) {
printString(line.toString().trim());
printNewline();
line.setLength(0);
}
}
/**
* @return the clipboard content as a String (DataFlavor.stringFlavor) Code snippet adapted from
* jEdit (Registers.java), http://www.jedit.org. Returns null if clipboard is empty.
*/
public static String getClipboardStringContent(Clipboard clipboard) {
// Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
try {
String selection =
(String) (clipboard.getContents(null).getTransferData(DataFlavor.stringFlavor));
if (selection == null) return null;
boolean trailingEOL =
(selection.endsWith("\n") || selection.endsWith(System.getProperty("line.separator")));
// Some Java versions return the clipboard contents using the native line separator,
// so have to convert it here , see jEdit's "registers.java"
BufferedReader in = new BufferedReader(new StringReader(selection));
StringBuffer buf = new StringBuffer();
String line;
while ((line = in.readLine()) != null) {
buf.append(line);
buf.append('\n');
}
// remove trailing \n
if (!trailingEOL) buf.setLength(buf.length() - 1);
return buf.toString();
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/** Copies all data that it can read from the given reader to the given writer. */
protected void copyData(Reader reader, Writer writer) throws IOException {
StringBuffer word = new StringBuffer();
while (true) {
int i = reader.read();
if (i < 0) {
break;
}
// Is the character part of a word?
char c = (char) i;
if (Character.isJavaIdentifierPart(c) || c == '.' || c == '-') {
// Collect the characters in this word.
word.append(c);
} else {
// Write out the updated word, if any.
writeUpdatedWord(writer, word.toString());
word.setLength(0);
// Write out the character that terminated it.
writer.write(c);
}
}
// Write out the final word.
writeUpdatedWord(writer, word.toString());
}
/**
* Reads an array of strings from the TIFF file.
*
* @param count Number of strings to read
* @param value Offset from which to read
*/
protected String[] readASCIIArray(long count, long value) throws IOException {
_raf.seek(value);
int nstrs = 0;
List list = new LinkedList();
byte[] buf = new byte[(int) count];
_raf.read(buf);
StringBuffer strbuf = new StringBuffer();
for (int i = 0; i < count; i++) {
int b = buf[i];
if (b == 0) {
list.add(strbuf.toString());
strbuf.setLength(0);
} else {
strbuf.append((char) b);
}
}
/* We can't use ArrayList.toArray because that returns an
Object[], not a String[] ... sigh. */
String[] strs = new String[nstrs];
ListIterator iter = list.listIterator();
for (int i = 0; i < nstrs; i++) {
strs[i] = (String) iter.next();
}
return strs;
}
public void run() {
StringBuffer data = new StringBuffer();
Print.logDebug("Client:InputThread started");
while (true) {
data.setLength(0);
boolean timeout = false;
try {
if (this.readTimeout > 0L) {
this.socket.setSoTimeout((int) this.readTimeout);
}
ClientSocketThread.socketReadLine(this.socket, -1, data);
} catch (InterruptedIOException ee) { // SocketTimeoutException ee) {
// error("Read interrupted (timeout) ...");
if (getRunStatus() != THREAD_RUNNING) {
break;
}
timeout = true;
// continue;
} catch (Throwable t) {
Print.logError("Client:InputThread - " + t);
t.printStackTrace();
break;
}
if (!timeout || (data.length() > 0)) {
ClientSocketThread.this.handleMessage(data.toString());
}
}
synchronized (this.threadLock) {
this.isRunning = false;
Print.logDebug("Client:InputThread stopped");
this.threadLock.notify();
}
}
/**
* Gets the next statement, returning false if the end is reached. A statement is either an @rule,
* or a ruleset.
*/
private boolean getNextStatement() throws IOException {
unitBuffer.setLength(0);
int token = nextToken((char) 0);
switch (token) {
case IDENTIFIER:
if (tokenBufferLength > 0) {
if (tokenBuffer[0] == '@') {
parseAtRule();
} else {
encounteredRuleSet = true;
parseRuleSet();
}
}
return true;
case BRACKET_OPEN:
case BRACE_OPEN:
case PAREN_OPEN:
parseTillClosed(token);
return true;
case BRACKET_CLOSE:
case BRACE_CLOSE:
case PAREN_CLOSE:
// Shouldn't happen...
throw new RuntimeException("Unexpected top level block close");
case END:
return false;
}
return true;
}
private int printThreadGroup(OutputSink out, ThreadGroupReference tg, int iThread) {
out.println("Group " + tg.name() + ":");
List<ThreadReference> tlist = tg.threads();
int maxId = 0;
int maxName = 0;
for (int i = 0; i < tlist.size(); i++) {
ThreadReference thr = tlist.get(i);
int len = Utils.description(thr).length();
if (len > maxId) {
maxId = len;
}
String name = thr.name();
int iDot = name.lastIndexOf('.');
if (iDot >= 0 && name.length() > iDot) {
name = name.substring(iDot + 1);
}
if (name.length() > maxName) {
maxName = name.length();
}
}
String maxNumString = String.valueOf(iThread + tlist.size());
int maxNumDigits = maxNumString.length();
for (int i = 0; i < tlist.size(); i++) {
ThreadReference thr = tlist.get(i);
char buf[] = new char[80];
for (int j = 0; j < 79; j++) {
buf[j] = ' ';
}
buf[79] = '\0';
StringBuffer sbOut = new StringBuffer();
sbOut.append(buf);
// Right-justify the thread number at start of output string
String numString = String.valueOf(iThread + i + 1);
sbOut.insert(maxNumDigits - numString.length(), numString);
sbOut.insert(maxNumDigits, ".");
int iBuf = maxNumDigits + 2;
sbOut.insert(iBuf, Utils.description(thr));
iBuf += maxId + 1;
String name = thr.name();
int iDot = name.lastIndexOf('.');
if (iDot >= 0 && name.length() > iDot) {
name = name.substring(iDot + 1);
}
sbOut.insert(iBuf, name);
iBuf += maxName + 1;
sbOut.insert(iBuf, Utils.getStatus(thr));
sbOut.setLength(79);
out.println(sbOut.toString());
}
for (ThreadGroupReference tg0 : tg.threadGroups()) {
if (!tg.equals(tg0)) { // TODO ref mgt
iThread += printThreadGroup(out, tg0, iThread + tlist.size());
}
}
return tlist.size();
}
public static void BeginRedirect(
int target, StringBuffer buffer, int buffersize, RD_Flusher flush) {
if (0 == target || null == buffer || 0 == buffersize || null == flush) return;
rd_target = target;
rd_buffer = buffer;
rd_buffersize = buffersize;
rd_flusher = flush;
rd_buffer.setLength(0);
}
/**
* Creates a new <code>DictionaryNameFactory</code>.
*
* @param file the file from which the names can be read.
* @param nameFactory the name factory from which names will be retrieved if the list of read
* names has been exhausted.
*/
public DictionaryNameFactory(File file, NameFactory nameFactory) throws IOException {
this.names = new ArrayList();
this.nameFactory = nameFactory;
Reader reader = new FileReader(file);
try {
StringBuffer buffer = new StringBuffer();
while (true) {
// Read the next character.
int c = reader.read();
// Is it a valid identifier character?
if (c != -1
&& (buffer.length() == 0
? Character.isJavaIdentifierStart((char) c)
: Character.isJavaIdentifierPart((char) c))) {
// Append it to the current identifier.
buffer.append((char) c);
} else {
// Did we collect a new identifier?
if (buffer.length() > 0) {
// Add the completed name to the list of names, if it's
// not in it yet.
String name = buffer.toString();
if (!names.contains(name)) {
names.add(name);
}
// Clear the buffer.
buffer.setLength(0);
}
// Is this the beginning of a comment line?
if (c == COMMENT_CHARACTER) {
// Skip all characters till the end of the line.
do {
c = reader.read();
} while (c != -1 && c != '\n' && c != '\r');
}
// Is this the end of the file?
if (c == -1) {
// Just return.
return;
}
}
}
} finally {
reader.close();
}
}
private static void setupType(
JaxType currentType,
StringBuffer currentTypeName,
ClassDoc containingClass,
JAXDoclet<?> doclet)
throws InvalidJaxTypeException {
if (currentTypeName.length() == 0) {
throw new InvalidJaxTypeException();
}
currentType.typeName = currentTypeName.toString();
currentType.type = resolveType(currentType.typeName, containingClass, doclet);
currentTypeName.setLength(0);
}
/**
* @param cssFilePath - path of the cssFile
* @param contextPath - web app context path or custom context path
* @param inputStream - input stream
* @param outputStream - output stream
* @throws java.io.IOException - throws exception in case something woes wrong (IO read/write)
*/
private void processCSS(
String contextPath, String cssFilePath, InputStream inputStream, OutputStream outputStream)
throws IOException {
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
String line;
StringBuffer buffer = new StringBuffer();
while ((line = bufferedReader.readLine()) != null) {
buffer.setLength(0);
buffer.append(line);
line = this.processCSSLine(context, contextPath, cssFilePath, buffer);
outputStream.write((line + "\n").getBytes());
}
}
/** Parses a set of selectors, returning false if the end of the stream is reached. */
private boolean parseSelectors() throws IOException {
// Parse the selectors
int nextToken;
if (tokenBufferLength > 0) {
callback.handleSelector(new String(tokenBuffer, 0, tokenBufferLength));
}
unitBuffer.setLength(0);
for (; ; ) {
while ((nextToken = nextToken((char) 0)) == IDENTIFIER) {
if (tokenBufferLength > 0) {
callback.handleSelector(new String(tokenBuffer, 0, tokenBufferLength));
}
}
switch (nextToken) {
case BRACE_OPEN:
return true;
case BRACKET_OPEN:
case PAREN_OPEN:
parseTillClosed(nextToken);
// Not too sure about this, how we handle this isn't very
// well spec'd.
unitBuffer.setLength(0);
break;
case BRACKET_CLOSE:
case BRACE_CLOSE:
case PAREN_CLOSE:
throw new RuntimeException("Unexpected block close in selector");
case END:
// Prematurely hit end.
return false;
}
}
}
/** Prints out messages, which can also be redirected to a remote client. */
public static void Printf(String fmt, Vargs vargs) {
String msg = sprintf(_debugContext + fmt, vargs);
if (rd_target != 0) {
if ((msg.length() + rd_buffer.length()) > (rd_buffersize - 1)) {
rd_flusher.rd_flush(rd_target, rd_buffer);
rd_buffer.setLength(0);
}
rd_buffer.append(msg);
return;
}
Console.Print(msg);
// also echo to debugging console
Sys.ConsoleOutput(msg);
// logfile
if (Globals.logfile_active != null && Globals.logfile_active.value != 0) {
String name;
if (Globals.logfile == null) {
name = QuakeFileSystem.Gamedir() + "/qconsole.log";
if (Globals.logfile_active.value > 2)
try {
Globals.logfile = new RandomAccessFile(name, "rw");
Globals.logfile.seek(Globals.logfile.length());
} catch (Exception e) {
// TODO: do quake2 error handling!
e.printStackTrace();
}
else
try {
Globals.logfile = new RandomAccessFile(name, "rw");
} catch (FileNotFoundException e1) {
// TODO: do quake2 error handling!
e1.printStackTrace();
}
}
if (Globals.logfile != null)
try {
Globals.logfile.writeChars(msg);
} catch (IOException e) {
// TODO: do quake2 error handling!
e.printStackTrace();
}
if (Globals.logfile_active.value > 1) ; // do nothing
// fflush (logfile); // force it to save every time
}
}
public static StringBuffer buildSaveBuffer(XMLElement auctionsData, XMLElement deletedData) {
synchronized (_saveBuf) {
_saveBuf.setLength(0);
_saveBuf.append("<?xml version=\"1.0\"?>\n\n");
_saveBuf.append(Constants.XML_SAVE_DOCTYPE);
_saveBuf.append('\n');
_saveBuf.append("<jbidwatcher format=\"0101\">\n");
auctionsData.toStringBuffer(_saveBuf, 1);
if (deletedData != null) {
deletedData.toStringBuffer(_saveBuf, 1);
}
_saveBuf.append("</jbidwatcher>");
}
return _saveBuf;
}
/**
* Update the SQL string and replace the ? markers with parameter names eg @P0, @P1 etc.
*
* @param sql the SQL containing markers to substitute
* @param list the parameter list
* @return the modified SQL as a <code>String</code>
*/
static String substituteParamMarkers(String sql, ParamInfo[] list) {
// A parameter can have at most 8 characters: " @P" plus at most 4
// digits plus " ". We subtract the "?" placeholder, that's at most
// 7 extra characters needed for each parameter.
char[] buf = new char[sql.length() + list.length * 7];
int bufferPtr = 0; // Output buffer pointer
int start = 0; // Input string pointer
StringBuffer number = new StringBuffer(4);
for (int i = 0; i < list.length; i++) {
int pos = list[i].markerPos;
if (pos > 0) {
sql.getChars(start, pos, buf, bufferPtr);
bufferPtr += (pos - start);
start = pos + 1;
// Append " @P"
buf[bufferPtr++] = ' ';
buf[bufferPtr++] = '@';
buf[bufferPtr++] = 'P';
// Append parameter number
// Rather complicated, but it's the only way in which no
// unnecessary objects are created
number.setLength(0);
number.append(i);
number.getChars(0, number.length(), buf, bufferPtr);
bufferPtr += number.length();
// Append " "
buf[bufferPtr++] = ' ';
}
}
if (start < sql.length()) {
sql.getChars(start, sql.length(), buf, bufferPtr);
bufferPtr += (sql.length() - start);
}
return new String(buf, 0, bufferPtr);
}
/**
* parse: break the input String into fields
*
* @return java.util.Iterator containing each field from the original as a String, in order.
*/
public List parse(String line) {
StringBuffer sb = new StringBuffer();
list.clear(); // recycle to initial state
int i = 0;
if (line.length() == 0) {
list.add(line);
return list;
}
do {
sb.setLength(0);
if (i < line.length() && line.charAt(i) == '"') i = advQuoted(line, sb, ++i); // skip quote
else i = advPlain(line, sb, i);
list.add(sb.toString());
i++;
} while (i < line.length());
return list;
}
/**
* Parse end of element from document. Collects character data to the end tag and returns it with
* whitespace stripped. Throws an exception if a start tag is seen before an end tag, or if the
* end tag seen does not match the expected name.
*
* @param tag element name expected
* @return content text with whitespace stripped
* @throws IOException if error reading document
* @throws XmlPullParserException if expected element not found, or if other parse error
*/
protected String parseEndTag(String tag) throws IOException, XmlPullParserException {
m_buffer.setLength(0);
while (true) {
switch (m_parser.next()) {
case XmlPullParser.CONTENT:
m_buffer.append(m_parser.readContent());
break;
case XmlPullParser.END_TAG:
m_parser.readEndTag(m_endTag);
if (m_endTag.getLocalName().equals(tag)) {
return m_buffer.toString().trim();
}
// fall through for error handling
case XmlPullParser.START_TAG:
case XmlPullParser.END_DOCUMENT:
throw new XmlPullParserException("Missing expected end tag " + tag);
}
}
}
private void checkBodyEnd(String endTag, Map replaces) {
int index = test.length() - endTag.length();
if (index >= 0) {
test = test.substring(index);
if (test.equalsIgnoreCase(endTag)) {
if (index > 0) {
buf.setLength(index);
String temp = buf.toString();
if (replaces != null) temp = StringUtils.replaceStrings(temp, replaces);
unitList.add(new ParseUnit(temp, mode));
}
mode = NORMAL;
} else {
test = null;
}
} else {
test = null;
}
}
private byte[] readMultiPartChunk(ServletInputStream requestStream, String contentType)
throws IOException {
// fast forward stream past multi-part header
int boundaryOff = contentType.indexOf("boundary="); // $NON-NLS-1$
String boundary = contentType.substring(boundaryOff + 9);
BufferedReader reader =
new BufferedReader(new InputStreamReader(requestStream, "ISO-8859-1")); // $NON-NLS-1$
StringBuffer out = new StringBuffer();
// skip headers up to the first blank line
String line = reader.readLine();
while (line != null && line.length() > 0) line = reader.readLine();
// now process the file
char[] buf = new char[1000];
int read;
while ((read = reader.read(buf)) > 0) {
out.append(buf, 0, read);
}
// remove the boundary from the output (end of input is \r\n--<boundary>--\r\n)
out.setLength(out.length() - (boundary.length() + 8));
return out.toString().getBytes("ISO-8859-1"); // $NON-NLS-1$
}
/** EdiDialog constructor comment. */
public void keyReleased(java.awt.event.KeyEvent e) {
if (e.getSource() == list) {
switch (e.getKeyCode()) {
case KeyEvent.VK_DELETE:
case KeyEvent.VK_BACK_SPACE:
if (keys.length() > 0) keys.setLength(keys.length() - 1);
break;
case KeyEvent.VK_ESCAPE:
dispose();
break;
case KeyEvent.VK_ENTER:
dispose();
actionOK();
break;
case KeyEvent.VK_SPACE:
actionAdd();
break;
default:
// keys.append((char) e.getKeyChar());
list.ensureIndexIsVisible(list.getSelectedIndex());
}
// if (debug)
// System.out.println("keys: " + keys);
return;
}
switch (e.getKeyCode()) {
case KeyEvent.VK_ENTER:
case KeyEvent.VK_ESCAPE:
dispose();
break;
default:
break;
}
super.keyReleased(e);
}
public void startElement(String namespaceURI, String localName, String qName, Attributes atts)
throws SAXException {
try {
// we output the remaining text
if (!counting) {
writer.write(getText());
accumulator.setLength(0);
}
if (!counting) {
writer.write("<" + qName);
int length = atts.getLength();
// Process each attribute
for (int i = 0; i < length; i++) {
// Get names and values for each attribute
String name = atts.getQName(i);
String value = atts.getValue(i);
if ((name != null) && (value != null)) {
writer.write(" " + name + "=\"" + value + "\"");
}
}
writer.write(">");
}
if (qName.equals("description")) {
offset = 0;
counting = true;
} else if (qName.equals("patent-document")) {
counting = false;
}
} catch (Exception e) {
// e.printStackTrace();
throw new GrobidException("An exception occured while running Grobid.", e);
}
}
private static String FilterID(String id) {
if (id == null) {
return null;
} else {
StringBuffer newID = new StringBuffer();
int st = 0;
for (int i = 0; i < id.length(); i++) {
char ch = Character.toLowerCase(id.charAt(i));
if (Character.isLetterOrDigit(ch)) {
newID.append(ch);
st = 1;
} else if (st == 1) {
newID.append("_");
st = 0;
} else {
// ignore char
}
}
while ((newID.length() > 0) && (newID.charAt(newID.length() - 1) == '_')) {
newID.setLength(newID.length() - 1);
}
return newID.toString();
}
}
private void parseStream(BufferedReader in, Map replaces) throws IOException {
int c;
ParseUnit unit;
unitList.clear();
mode = NORMAL;
buf = new StringBuffer();
c = in.read();
while (c >= 0) {
switch (c) {
case '<':
if (mode == NORMAL && buf.length() > 0) {
test = buf.toString();
if (test.startsWith("<!--")) {
mode = COMMENT;
} else if (test.startsWith("<%--")) {
mode = JSP_COMMENT;
} else if (test.startsWith("<%")) {
mode = JSP;
} else {
if (replaces != null) test = StringUtils.replaceStrings(test, replaces);
// test = StringUtils.stripEnterSymbol(test);
unitList.add(new ParseUnit(test, mode));
buf.setLength(0);
}
}
buf.append((char) c);
break;
case '>':
buf.append((char) c);
test = buf.toString();
if (mode == NORMAL) {
if (test.startsWith("<!--")) {
mode = COMMENT;
} else if (test.startsWith("<%--")) {
mode = JSP_COMMENT;
} else if (test.startsWith("<%")) {
mode = JSP;
}
}
switch (mode) {
case COMMENT:
if (!test.endsWith("-->")) {
test = null;
}
break;
case JSP_COMMENT:
if (!test.endsWith("--%>")) {
test = null;
}
break;
case JSP:
if (!test.endsWith("%>")) {
test = null;
}
break;
case SCRIPT_BODY:
checkBodyEnd("</SCRIPT>", replaces);
break;
case STYLE_BODY:
checkBodyEnd("</STYLE>", replaces);
break;
}
if (test != null) {
//
if (replaces != null) test = StringUtils.replaceStrings(test, replaces);
unit = new ParseUnit(test, mode);
unitList.add(unit);
buf.setLength(0);
mode = NORMAL;
if (unit.isStartTag()) {
test = unit.getTagName();
if (test.equals("script")) {
mode = SCRIPT_BODY;
} else if (test.equals("style")) {
mode = STYLE_BODY;
}
}
}
break;
default:
buf.append((char) c);
}
c = in.read();
}
if (buf.length() > 0) {
String temp = buf.toString();
if (replaces != null) temp = StringUtils.replaceStrings(temp, replaces);
unitList.add(new ParseUnit(temp, mode));
}
units = new ParseUnit[unitList.size()];
unitList.toArray(units);
}
/**
* Resumes scanning until the next regular expression is matched, the end of input is encountered
* or an I/O-Error occurs.
*
* @return the next token
* @exception java.io.IOException if any I/O-Error occurs
*/
public XiSymbol next_token() throws java.io.IOException {
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
char[] zzBufferL = zzBuffer;
char[] zzCMapL = ZZ_CMAP;
int[] zzTransL = ZZ_TRANS;
int[] zzRowMapL = ZZ_ROWMAP;
int[] zzAttrL = ZZ_ATTRIBUTE;
while (true) {
zzMarkedPosL = zzMarkedPos;
yychar += zzMarkedPosL - zzStartRead;
boolean zzR = false;
for (zzCurrentPosL = zzStartRead; zzCurrentPosL < zzMarkedPosL; zzCurrentPosL++) {
switch (zzBufferL[zzCurrentPosL]) {
case '\u000B':
case '\u000C':
case '\u0085':
case '\u2028':
case '\u2029':
yyline++;
yycolumn = 0;
zzR = false;
break;
case '\r':
yyline++;
yycolumn = 0;
zzR = true;
break;
case '\n':
if (zzR) zzR = false;
else {
yyline++;
yycolumn = 0;
}
break;
default:
zzR = false;
yycolumn++;
}
}
if (zzR) {
// peek one character ahead if it is \n (if we have counted one line too much)
boolean zzPeek;
if (zzMarkedPosL < zzEndReadL) zzPeek = zzBufferL[zzMarkedPosL] == '\n';
else if (zzAtEOF) zzPeek = false;
else {
boolean eof = zzRefill();
zzEndReadL = zzEndRead;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
if (eof) zzPeek = false;
else zzPeek = zzBufferL[zzMarkedPosL] == '\n';
}
if (zzPeek) yyline--;
}
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
zzState = ZZ_LEXSTATE[zzLexicalState];
zzForAction:
{
while (true) {
if (zzCurrentPosL < zzEndReadL) zzInput = zzBufferL[zzCurrentPosL++];
else if (zzAtEOF) {
zzInput = YYEOF;
break zzForAction;
} else {
// store back cached positions
zzCurrentPos = zzCurrentPosL;
zzMarkedPos = zzMarkedPosL;
boolean eof = zzRefill();
// get translated positions and possibly new buffer
zzCurrentPosL = zzCurrentPos;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
zzEndReadL = zzEndRead;
if (eof) {
zzInput = YYEOF;
break zzForAction;
} else {
zzInput = zzBufferL[zzCurrentPosL++];
}
}
int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
if (zzNext == -1) break zzForAction;
zzState = zzNext;
int zzAttributes = zzAttrL[zzState];
if ((zzAttributes & 1) == 1) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
if ((zzAttributes & 8) == 8) break zzForAction;
}
}
}
// store back cached position
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 1:
{
/* ignore */
}
case 72:
break;
case 19:
{
return xiSymbol(Sym.OPEN_BRACKET, yytext());
}
case 73:
break;
case 29:
{
throw new RuntimeException("Unterminated string at end of line:" + yyline);
}
case 74:
break;
case 68:
{
return xiSymbol(Sym.LENGTH, yytext());
}
case 75:
break;
case 16:
{
return xiSymbol(Sym.GT, yytext());
}
case 76:
break;
case 41:
{
string.append('\f');
}
case 77:
break;
case 49:
{
return xiSymbol(Sym.NEW, yytext());
}
case 78:
break;
case 65:
{
return xiSymbol(Sym.WHILE, yytext());
}
case 79:
break;
case 35:
{
return xiSymbol(Sym.EQUAL, yytext());
}
case 80:
break;
case 11:
{
return xiSymbol(Sym.DIVIDE, yytext());
}
case 81:
break;
case 47:
{
return xiSymbol(Sym.INT, yytext());
}
case 82:
break;
case 42:
{
string.append('\b');
}
case 83:
break;
case 60:
{
return xiSymbol(Sym.BOOL, yytext());
}
case 84:
break;
case 15:
{
return xiSymbol(Sym.GETS, yytext());
}
case 85:
break;
case 3:
{
return xiSymbol(Sym.INTEGER_LITERAL, Long.parseLong(yytext()));
}
case 86:
break;
case 46:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character(yytext().charAt(0)), 1);
}
case 87:
break;
case 2:
{
return xiSymbol(Sym.IDENTIFIER, yytext());
}
case 88:
break;
case 44:
{
string.append('\t');
}
case 89:
break;
case 5:
{
yybegin(CHARACTER);
}
case 90:
break;
case 25:
{
return xiSymbol(Sym.COLON, yytext());
}
case 91:
break;
case 38:
{
string.append('\'');
}
case 92:
break;
case 48:
{
return xiSymbol(Sym.USE, yytext());
}
case 93:
break;
case 20:
{
return xiSymbol(Sym.CLOSE_BRACKET, yytext());
}
case 94:
break;
case 10:
{
return xiSymbol(Sym.TIMES, yytext());
}
case 95:
break;
case 34:
{
return xiSymbol(Sym.LEQ, yytext());
}
case 96:
break;
case 22:
{
return xiSymbol(Sym.CLOSE_PAREN, yytext());
}
case 97:
break;
case 7:
{
return xiSymbol(Sym.PERIOD, yytext());
}
case 98:
break;
case 69:
{
return xiSymbol(Sym.RETURN, yytext());
}
case 99:
break;
case 13:
{
return xiSymbol(Sym.NOT, yytext());
}
case 100:
break;
case 71:
{
return xiSymbol(Sym.CONTINUE, yytext());
}
case 101:
break;
case 33:
{
return xiSymbol(Sym.NOT_EQUAL, yytext());
}
case 102:
break;
case 8:
{
return xiSymbol(Sym.PLUS, yytext());
}
case 103:
break;
case 70:
{
return xiSymbol(Sym.EXTENDS, yytext());
}
case 104:
break;
case 14:
{
return xiSymbol(Sym.LT, yytext());
}
case 105:
break;
case 64:
{
return xiSymbol(Sym.FALSE, yytext());
}
case 106:
break;
case 4:
{
return xiSymbol(Sym.UNDERSCORE, yytext());
}
case 107:
break;
case 9:
{
return xiSymbol(Sym.MINUS, yytext());
}
case 108:
break;
case 18:
{
return xiSymbol(Sym.OR, yytext());
}
case 109:
break;
case 30:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.STRING_LITERAL, string.toString(), string.length());
}
case 110:
break;
case 21:
{
return xiSymbol(Sym.OPEN_PAREN, yytext());
}
case 111:
break;
case 26:
{
return xiSymbol(Sym.COMMA, yytext());
}
case 112:
break;
case 61:
{
return xiSymbol(Sym.THIS, yytext());
}
case 113:
break;
case 66:
{
return xiSymbol(Sym.BREAK, yytext());
}
case 114:
break;
case 40:
{
string.append('\"');
}
case 115:
break;
case 17:
{
return xiSymbol(Sym.AND, yytext());
}
case 116:
break;
case 24:
{
return xiSymbol(Sym.CLOSE_BRACE, yytext());
}
case 117:
break;
case 12:
{
return xiSymbol(Sym.MODULO, yytext());
}
case 118:
break;
case 31:
{
throw new RuntimeException("Unterminated character literal at end of line");
}
case 119:
break;
case 53:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\"'), 1);
}
case 120:
break;
case 28:
{
string.append(yytext());
}
case 121:
break;
case 51:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\''), 1);
}
case 122:
break;
case 36:
{
return xiSymbol(Sym.GEQ, yytext());
}
case 123:
break;
case 63:
{
return xiSymbol(Sym.NULL, yytext());
}
case 124:
break;
case 43:
{
string.append('\r');
}
case 125:
break;
case 62:
{
return xiSymbol(Sym.TRUE, yytext());
}
case 126:
break;
case 39:
{
string.append('\\');
}
case 127:
break;
case 6:
{
yybegin(STRING);
string.setLength(0);
}
case 128:
break;
case 59:
{
return xiSymbol(Sym.ELSE, yytext());
}
case 129:
break;
case 45:
{
string.append('\n');
}
case 130:
break;
case 52:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\\'), 1);
}
case 131:
break;
case 23:
{
return xiSymbol(Sym.OPEN_BRACE, yytext());
}
case 132:
break;
case 54:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\f'), 1);
}
case 133:
break;
case 32:
{
return xiSymbol(Sym.IF, yytext());
}
case 134:
break;
case 67:
{
return xiSymbol(Sym.CLASS, yytext());
}
case 135:
break;
case 55:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\b'), 1);
}
case 136:
break;
case 50:
{
/* IGNORE */
}
case 137:
break;
case 27:
{
return xiSymbol(Sym.SEMICOLON, yytext());
}
case 138:
break;
case 56:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\r'), 1);
}
case 139:
break;
case 58:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\n'), 1);
}
case 140:
break;
case 57:
{
yybegin(YYINITIAL);
return xiSymbol(Sym.CHAR_LITERAL, new Character('\t'), 1);
}
case 141:
break;
case 37:
{
throw new RuntimeException("Illegal escape sequence \"" + yytext() + "\"");
}
case 142:
break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
zzDoEOF();
{
return new XiSymbol(Sym.EOF);
}
} else {
zzScanError(ZZ_NO_MATCH);
}
}
}
}
protected void newTokenHere() {
lexbuf.setLength(0);
pbeg.copyFrom(loc);
}
protected void backupChar() throws LexException {
ibuf.backup();
lexbuf.setLength(lexbuf.length() - 1);
}
public static void main(String[] args) {
// String inFile = "C:/My Documents/HMM/Chunk/wsj_15_18_train.log";
String inFile = chunkDir + "wsj_15_18_train.log";
// String featureFile = "C:/My Documents/HMM/Chunk/chunk features.txt";
String featureFile = chunkDir + "chunk features.txt";
try {
BufferedReader reader = new BufferedReader(new FileReader(inFile));
PrintStream writer = new PrintStream(new FileOutputStream(featureFile));
String line;
String prevToken = "";
String prevPOS = "";
String prevTag = "";
String currentToken = "";
String currentPOS = "";
String currentTag = "";
String nextToken = "";
String nextPOS = "";
String nextTag = "";
String followingToken = "";
String followingPOS = "";
String followingTag = "";
StringBuffer features = new StringBuffer(200);
boolean inGroup = false;
boolean firstToken = true;
while ((line = reader.readLine()) != null) {
StringTokenizer st = new StringTokenizer(line);
int count = st.countTokens();
if (count == 0) {
// blank line -- end of sentence
followingToken = "";
followingPOS = "";
followingTag = "";
} else if (count >= 3) {
// token
followingToken = st.nextToken();
followingPOS = st.nextToken();
followingTag = st.nextToken();
} else {
System.out.println("Error: invalid input line: " + line);
}
if (currentToken != "") {
features.setLength(0);
features.append("prevPOS=" + prevPOS + " ");
features.append("currPOS=" + currentPOS + " ");
features.append("nextPOS=" + nextPOS + " ");
if (nextToken == "") features.append("POS012=" + currentPOS + ":: ");
else features.append("POS012=" + currentPOS + ":" + nextPOS + ":" + followingPOS + " ");
features.append("prevTag=" + prevTag + " ");
features.append("currWord=" + currentToken + " ");
features.append("W-1W0=" + prevToken + ":" + currentToken + " ");
features.append("W0W1=" + currentToken + ":" + nextToken + " ");
features.append(currentTag);
writer.println(features);
}
prevToken = currentToken;
prevPOS = currentPOS;
prevTag = currentTag;
currentToken = nextToken;
currentPOS = nextPOS;
currentTag = nextTag;
nextToken = followingToken;
nextPOS = followingPOS;
nextTag = followingTag;
}
} catch (IOException e) {
System.out.println(e);
}
}
/** Performs the subsequent ReTrace operations. */
public void execute() throws IOException {
// Read the mapping file.
MappingReader mappingReader = new MappingReader(mappingFile);
mappingReader.pump(this);
StringBuffer expressionBuffer = new StringBuffer(regularExpression.length() + 32);
char[] expressionTypes = new char[32];
int expressionTypeCount = 0;
int index = 0;
while (true) {
int nextIndex = regularExpression.indexOf('%', index);
if (nextIndex < 0
|| nextIndex == regularExpression.length() - 1
|| expressionTypeCount == expressionTypes.length) {
break;
}
expressionBuffer.append(regularExpression.substring(index, nextIndex));
expressionBuffer.append('(');
char expressionType = regularExpression.charAt(nextIndex + 1);
switch (expressionType) {
case 'c':
expressionBuffer.append(REGEX_CLASS);
break;
case 'C':
expressionBuffer.append(REGEX_CLASS_SLASH);
break;
case 'l':
expressionBuffer.append(REGEX_LINE_NUMBER);
break;
case 't':
expressionBuffer.append(REGEX_TYPE);
break;
case 'f':
expressionBuffer.append(REGEX_MEMBER);
break;
case 'm':
expressionBuffer.append(REGEX_MEMBER);
break;
case 'a':
expressionBuffer.append(REGEX_ARGUMENTS);
break;
}
expressionBuffer.append(')');
expressionTypes[expressionTypeCount++] = expressionType;
index = nextIndex + 2;
}
expressionBuffer.append(regularExpression.substring(index));
Pattern pattern = Pattern.compile(expressionBuffer.toString());
// Read the stack trace file.
LineNumberReader reader =
new LineNumberReader(
stackTraceFile == null
? (Reader) new InputStreamReader(System.in)
: (Reader) new BufferedReader(new FileReader(stackTraceFile)));
try {
StringBuffer outLine = new StringBuffer(256);
List extraOutLines = new ArrayList();
String className = null;
// Read the line in the stack trace.
while (true) {
String line = reader.readLine();
if (line == null) {
break;
}
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
int lineNumber = 0;
String type = null;
String arguments = null;
// Figure out a class name, line number, type, and
// arguments beforehand.
for (int expressionTypeIndex = 0;
expressionTypeIndex < expressionTypeCount;
expressionTypeIndex++) {
int startIndex = matcher.start(expressionTypeIndex + 1);
if (startIndex >= 0) {
String match = matcher.group(expressionTypeIndex + 1);
char expressionType = expressionTypes[expressionTypeIndex];
switch (expressionType) {
case 'c':
className = originalClassName(match);
break;
case 'C':
className = originalClassName(ClassUtil.externalClassName(match));
break;
case 'l':
lineNumber = Integer.parseInt(match);
break;
case 't':
type = originalType(match);
break;
case 'a':
arguments = originalArguments(match);
break;
}
}
}
// Actually construct the output line.
int lineIndex = 0;
outLine.setLength(0);
extraOutLines.clear();
for (int expressionTypeIndex = 0;
expressionTypeIndex < expressionTypeCount;
expressionTypeIndex++) {
int startIndex = matcher.start(expressionTypeIndex + 1);
if (startIndex >= 0) {
int endIndex = matcher.end(expressionTypeIndex + 1);
String match = matcher.group(expressionTypeIndex + 1);
// Copy a literal piece of input line.
outLine.append(line.substring(lineIndex, startIndex));
char expressionType = expressionTypes[expressionTypeIndex];
switch (expressionType) {
case 'c':
className = originalClassName(match);
outLine.append(className);
break;
case 'C':
className = originalClassName(ClassUtil.externalClassName(match));
outLine.append(ClassUtil.internalClassName(className));
break;
case 'l':
lineNumber = Integer.parseInt(match);
outLine.append(match);
break;
case 't':
type = originalType(match);
outLine.append(type);
break;
case 'f':
originalFieldName(className, match, type, outLine, extraOutLines);
break;
case 'm':
originalMethodName(
className, match, lineNumber, type, arguments, outLine, extraOutLines);
break;
case 'a':
arguments = originalArguments(match);
outLine.append(arguments);
break;
}
// Skip the original element whose processed version
// has just been appended.
lineIndex = endIndex;
}
}
// Copy the last literal piece of input line.
outLine.append(line.substring(lineIndex));
// Print out the main line.
System.out.println(outLine);
// Print out any additional lines.
for (int extraLineIndex = 0; extraLineIndex < extraOutLines.size(); extraLineIndex++) {
System.out.println(extraOutLines.get(extraLineIndex));
}
} else {
// Print out the original line.
System.out.println(line);
}
}
} catch (IOException ex) {
throw new IOException("Can't read stack trace (" + ex.getMessage() + ")");
} finally {
if (stackTraceFile != null) {
try {
reader.close();
} catch (IOException ex) {
// This shouldn't happen.
}
}
}
}
