public void endElement(java.lang.String uri, java.lang.String localName, java.lang.String qName)
      throws SAXException {

    try {
      if (qName.equals("p") || qName.equals("description")) {
        writer.write(getText());
        accumulator.setLength(0);
      }

      if (qName.equals("description")) {
        counting = false;
      }

      if (!counting) {
        writer.write(getText());
        accumulator.setLength(0);
        writer.write("</" + qName + ">\n");
      } else {
        if (qName.equals("row")) {
          accumulator.append(" ");
        }
        if (qName.equals("p")) {
          writer.write("\n");
          accumulator.append(" ");
        }
      }
    } catch (Exception e) {
      //		    e.printStackTrace();
      throw new GrobidException("An exception occured while running Grobid.", e);
    }
  }
示例#2
0
  /**
   * Parses identifiers until <code>extraChar</code> is encountered, returning the ending token,
   * which will be IDENTIFIER if extraChar is found.
   */
  private int parseIdentifiers(char extraChar, boolean wantsBlocks) throws IOException {
    int nextToken;
    int ubl;

    unitBuffer.setLength(0);
    for (; ; ) {
      nextToken = nextToken(extraChar);

      switch (nextToken) {
        case IDENTIFIER:
          if (tokenBufferLength > 0) {
            if (tokenBuffer[tokenBufferLength - 1] == extraChar) {
              if (--tokenBufferLength > 0) {
                if (readWS && unitBuffer.length() > 0) {
                  unitBuffer.append(' ');
                }
                unitBuffer.append(tokenBuffer, 0, tokenBufferLength);
              }
              return IDENTIFIER;
            }
            if (readWS && unitBuffer.length() > 0) {
              unitBuffer.append(' ');
            }
            unitBuffer.append(tokenBuffer, 0, tokenBufferLength);
          }
          break;

        case BRACKET_OPEN:
        case BRACE_OPEN:
        case PAREN_OPEN:
          ubl = unitBuffer.length();
          if (wantsBlocks) {
            unitBuffer.append(charMapping[nextToken]);
          }
          parseTillClosed(nextToken);
          if (!wantsBlocks) {
            unitBuffer.setLength(ubl);
          }
          break;

        case BRACE_CLOSE:
          // No need to throw for these two, we return token and
          // caller can do whatever.
        case BRACKET_CLOSE:
        case PAREN_CLOSE:
        case END:
          // Hit the end
          return nextToken;
      }
    }
  }
  /**
   * Output the specified {@link Collection} in proper columns.
   *
   * @param stuff the stuff to print
   */
  public void printColumns(final Collection stuff) throws IOException {
    if ((stuff == null) || (stuff.size() == 0)) {
      return;
    }

    int width = getTermwidth();
    int maxwidth = 0;

    for (Iterator i = stuff.iterator();
        i.hasNext();
        maxwidth = Math.max(maxwidth, i.next().toString().length())) {;
    }

    StringBuffer line = new StringBuffer();

    int showLines;

    if (usePagination) showLines = getTermheight() - 1; // page limit
    else showLines = Integer.MAX_VALUE;

    for (Iterator i = stuff.iterator(); i.hasNext(); ) {
      String cur = (String) i.next();

      if ((line.length() + maxwidth) > width) {
        printString(line.toString().trim());
        printNewline();
        line.setLength(0);
        if (--showLines == 0) { // Overflow
          printString(loc.getString("display-more"));
          flushConsole();
          int c = readVirtualKey();
          if (c == '\r' || c == '\n') showLines = 1; // one step forward
          else if (c != 'q') showLines = getTermheight() - 1; // page forward

          back(loc.getString("display-more").length());
          if (c == 'q') break; // cancel
        }
      }

      pad(cur, maxwidth + 3, line);
    }

    if (line.length() > 0) {
      printString(line.toString().trim());
      printNewline();
      line.setLength(0);
    }
  }
示例#4
0
  /**
   * @return the clipboard content as a String (DataFlavor.stringFlavor) Code snippet adapted from
   *     jEdit (Registers.java), http://www.jedit.org. Returns null if clipboard is empty.
   */
  public static String getClipboardStringContent(Clipboard clipboard) {
    // Clipboard clipboard = Toolkit.getDefaultToolkit().getSystemClipboard();
    try {
      String selection =
          (String) (clipboard.getContents(null).getTransferData(DataFlavor.stringFlavor));
      if (selection == null) return null;

      boolean trailingEOL =
          (selection.endsWith("\n") || selection.endsWith(System.getProperty("line.separator")));

      // Some Java versions return the clipboard contents using the native line separator,
      // so have to convert it here , see jEdit's "registers.java"
      BufferedReader in = new BufferedReader(new StringReader(selection));
      StringBuffer buf = new StringBuffer();
      String line;
      while ((line = in.readLine()) != null) {
        buf.append(line);
        buf.append('\n');
      }
      // remove trailing \n
      if (!trailingEOL) buf.setLength(buf.length() - 1);
      return buf.toString();
    } catch (Exception e) {
      e.printStackTrace();
      return null;
    }
  }
  /** Copies all data that it can read from the given reader to the given writer. */
  protected void copyData(Reader reader, Writer writer) throws IOException {
    StringBuffer word = new StringBuffer();

    while (true) {
      int i = reader.read();
      if (i < 0) {
        break;
      }

      // Is the character part of a word?
      char c = (char) i;
      if (Character.isJavaIdentifierPart(c) || c == '.' || c == '-') {
        // Collect the characters in this word.
        word.append(c);
      } else {
        // Write out the updated word, if any.
        writeUpdatedWord(writer, word.toString());
        word.setLength(0);

        // Write out the character that terminated it.
        writer.write(c);
      }
    }

    // Write out the final word.
    writeUpdatedWord(writer, word.toString());
  }
示例#6
0
  /**
   * Reads an array of strings from the TIFF file.
   *
   * @param count Number of strings to read
   * @param value Offset from which to read
   */
  protected String[] readASCIIArray(long count, long value) throws IOException {
    _raf.seek(value);

    int nstrs = 0;
    List list = new LinkedList();
    byte[] buf = new byte[(int) count];
    _raf.read(buf);
    StringBuffer strbuf = new StringBuffer();
    for (int i = 0; i < count; i++) {
      int b = buf[i];
      if (b == 0) {
        list.add(strbuf.toString());
        strbuf.setLength(0);
      } else {
        strbuf.append((char) b);
      }
    }
    /* We can't use ArrayList.toArray because that returns an
    Object[], not a String[] ... sigh. */
    String[] strs = new String[nstrs];
    ListIterator iter = list.listIterator();
    for (int i = 0; i < nstrs; i++) {
      strs[i] = (String) iter.next();
    }
    return strs;
  }
示例#7
0
    public void run() {
      StringBuffer data = new StringBuffer();
      Print.logDebug("Client:InputThread started");

      while (true) {
        data.setLength(0);
        boolean timeout = false;
        try {
          if (this.readTimeout > 0L) {
            this.socket.setSoTimeout((int) this.readTimeout);
          }
          ClientSocketThread.socketReadLine(this.socket, -1, data);
        } catch (InterruptedIOException ee) { // SocketTimeoutException ee) {
          // error("Read interrupted (timeout) ...");
          if (getRunStatus() != THREAD_RUNNING) {
            break;
          }
          timeout = true;
          // continue;
        } catch (Throwable t) {
          Print.logError("Client:InputThread - " + t);
          t.printStackTrace();
          break;
        }
        if (!timeout || (data.length() > 0)) {
          ClientSocketThread.this.handleMessage(data.toString());
        }
      }

      synchronized (this.threadLock) {
        this.isRunning = false;
        Print.logDebug("Client:InputThread stopped");
        this.threadLock.notify();
      }
    }
示例#8
0
  /**
   * Gets the next statement, returning false if the end is reached. A statement is either an @rule,
   * or a ruleset.
   */
  private boolean getNextStatement() throws IOException {
    unitBuffer.setLength(0);

    int token = nextToken((char) 0);

    switch (token) {
      case IDENTIFIER:
        if (tokenBufferLength > 0) {
          if (tokenBuffer[0] == '@') {
            parseAtRule();
          } else {
            encounteredRuleSet = true;
            parseRuleSet();
          }
        }
        return true;
      case BRACKET_OPEN:
      case BRACE_OPEN:
      case PAREN_OPEN:
        parseTillClosed(token);
        return true;

      case BRACKET_CLOSE:
      case BRACE_CLOSE:
      case PAREN_CLOSE:
        // Shouldn't happen...
        throw new RuntimeException("Unexpected top level block close");

      case END:
        return false;
    }
    return true;
  }
示例#9
0
  private int printThreadGroup(OutputSink out, ThreadGroupReference tg, int iThread) {
    out.println("Group " + tg.name() + ":");
    List<ThreadReference> tlist = tg.threads();
    int maxId = 0;
    int maxName = 0;
    for (int i = 0; i < tlist.size(); i++) {
      ThreadReference thr = tlist.get(i);
      int len = Utils.description(thr).length();
      if (len > maxId) {
        maxId = len;
      }
      String name = thr.name();
      int iDot = name.lastIndexOf('.');
      if (iDot >= 0 && name.length() > iDot) {
        name = name.substring(iDot + 1);
      }
      if (name.length() > maxName) {
        maxName = name.length();
      }
    }
    String maxNumString = String.valueOf(iThread + tlist.size());
    int maxNumDigits = maxNumString.length();
    for (int i = 0; i < tlist.size(); i++) {
      ThreadReference thr = tlist.get(i);
      char buf[] = new char[80];
      for (int j = 0; j < 79; j++) {
        buf[j] = ' ';
      }
      buf[79] = '\0';
      StringBuffer sbOut = new StringBuffer();
      sbOut.append(buf);

      // Right-justify the thread number at start of output string
      String numString = String.valueOf(iThread + i + 1);
      sbOut.insert(maxNumDigits - numString.length(), numString);
      sbOut.insert(maxNumDigits, ".");

      int iBuf = maxNumDigits + 2;
      sbOut.insert(iBuf, Utils.description(thr));
      iBuf += maxId + 1;
      String name = thr.name();
      int iDot = name.lastIndexOf('.');
      if (iDot >= 0 && name.length() > iDot) {
        name = name.substring(iDot + 1);
      }
      sbOut.insert(iBuf, name);
      iBuf += maxName + 1;
      sbOut.insert(iBuf, Utils.getStatus(thr));
      sbOut.setLength(79);
      out.println(sbOut.toString());
    }
    for (ThreadGroupReference tg0 : tg.threadGroups()) {
      if (!tg.equals(tg0)) { // TODO ref mgt
        iThread += printThreadGroup(out, tg0, iThread + tlist.size());
      }
    }
    return tlist.size();
  }
示例#10
0
  public static void BeginRedirect(
      int target, StringBuffer buffer, int buffersize, RD_Flusher flush) {
    if (0 == target || null == buffer || 0 == buffersize || null == flush) return;

    rd_target = target;
    rd_buffer = buffer;
    rd_buffersize = buffersize;
    rd_flusher = flush;

    rd_buffer.setLength(0);
  }
  /**
   * Creates a new <code>DictionaryNameFactory</code>.
   *
   * @param file the file from which the names can be read.
   * @param nameFactory the name factory from which names will be retrieved if the list of read
   *     names has been exhausted.
   */
  public DictionaryNameFactory(File file, NameFactory nameFactory) throws IOException {
    this.names = new ArrayList();
    this.nameFactory = nameFactory;

    Reader reader = new FileReader(file);

    try {
      StringBuffer buffer = new StringBuffer();

      while (true) {
        // Read the next character.
        int c = reader.read();

        // Is it a valid identifier character?
        if (c != -1
            && (buffer.length() == 0
                ? Character.isJavaIdentifierStart((char) c)
                : Character.isJavaIdentifierPart((char) c))) {
          // Append it to the current identifier.
          buffer.append((char) c);
        } else {
          // Did we collect a new identifier?
          if (buffer.length() > 0) {
            // Add the completed name to the list of names, if it's
            // not in it yet.
            String name = buffer.toString();
            if (!names.contains(name)) {
              names.add(name);
            }

            // Clear the buffer.
            buffer.setLength(0);
          }

          // Is this the beginning of a comment line?
          if (c == COMMENT_CHARACTER) {
            // Skip all characters till the end of the line.
            do {
              c = reader.read();
            } while (c != -1 && c != '\n' && c != '\r');
          }

          // Is this the end of the file?
          if (c == -1) {
            // Just return.
            return;
          }
        }
      }
    } finally {
      reader.close();
    }
  }
示例#12
0
 private static void setupType(
     JaxType currentType,
     StringBuffer currentTypeName,
     ClassDoc containingClass,
     JAXDoclet<?> doclet)
     throws InvalidJaxTypeException {
   if (currentTypeName.length() == 0) {
     throw new InvalidJaxTypeException();
   }
   currentType.typeName = currentTypeName.toString();
   currentType.type = resolveType(currentType.typeName, containingClass, doclet);
   currentTypeName.setLength(0);
 }
 /**
  * @param cssFilePath - path of the cssFile
  * @param contextPath - web app context path or custom context path
  * @param inputStream - input stream
  * @param outputStream - output stream
  * @throws java.io.IOException - throws exception in case something woes wrong (IO read/write)
  */
 private void processCSS(
     String contextPath, String cssFilePath, InputStream inputStream, OutputStream outputStream)
     throws IOException {
   BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
   String line;
   StringBuffer buffer = new StringBuffer();
   while ((line = bufferedReader.readLine()) != null) {
     buffer.setLength(0);
     buffer.append(line);
     line = this.processCSSLine(context, contextPath, cssFilePath, buffer);
     outputStream.write((line + "\n").getBytes());
   }
 }
示例#14
0
  /** Parses a set of selectors, returning false if the end of the stream is reached. */
  private boolean parseSelectors() throws IOException {
    // Parse the selectors
    int nextToken;

    if (tokenBufferLength > 0) {
      callback.handleSelector(new String(tokenBuffer, 0, tokenBufferLength));
    }

    unitBuffer.setLength(0);
    for (; ; ) {
      while ((nextToken = nextToken((char) 0)) == IDENTIFIER) {
        if (tokenBufferLength > 0) {
          callback.handleSelector(new String(tokenBuffer, 0, tokenBufferLength));
        }
      }
      switch (nextToken) {
        case BRACE_OPEN:
          return true;

        case BRACKET_OPEN:
        case PAREN_OPEN:
          parseTillClosed(nextToken);
          // Not too sure about this, how we handle this isn't very
          // well spec'd.
          unitBuffer.setLength(0);
          break;

        case BRACKET_CLOSE:
        case BRACE_CLOSE:
        case PAREN_CLOSE:
          throw new RuntimeException("Unexpected block close in selector");

        case END:
          // Prematurely hit end.
          return false;
      }
    }
  }
示例#15
0
  /** Prints out messages, which can also be redirected to a remote client. */
  public static void Printf(String fmt, Vargs vargs) {
    String msg = sprintf(_debugContext + fmt, vargs);
    if (rd_target != 0) {
      if ((msg.length() + rd_buffer.length()) > (rd_buffersize - 1)) {
        rd_flusher.rd_flush(rd_target, rd_buffer);
        rd_buffer.setLength(0);
      }
      rd_buffer.append(msg);
      return;
    }

    Console.Print(msg);

    // also echo to debugging console
    Sys.ConsoleOutput(msg);

    // logfile
    if (Globals.logfile_active != null && Globals.logfile_active.value != 0) {
      String name;

      if (Globals.logfile == null) {
        name = QuakeFileSystem.Gamedir() + "/qconsole.log";
        if (Globals.logfile_active.value > 2)
          try {
            Globals.logfile = new RandomAccessFile(name, "rw");
            Globals.logfile.seek(Globals.logfile.length());
          } catch (Exception e) {
            // TODO: do quake2 error handling!
            e.printStackTrace();
          }
        else
          try {
            Globals.logfile = new RandomAccessFile(name, "rw");
          } catch (FileNotFoundException e1) {
            // TODO: do quake2 error handling!
            e1.printStackTrace();
          }
      }
      if (Globals.logfile != null)
        try {
          Globals.logfile.writeChars(msg);
        } catch (IOException e) {
          // TODO: do quake2 error handling!
          e.printStackTrace();
        }
      if (Globals.logfile_active.value > 1) ; // do nothing
      // fflush (logfile);		// force it to save every time
    }
  }
示例#16
0
 public static StringBuffer buildSaveBuffer(XMLElement auctionsData, XMLElement deletedData) {
   synchronized (_saveBuf) {
     _saveBuf.setLength(0);
     _saveBuf.append("<?xml version=\"1.0\"?>\n\n");
     _saveBuf.append(Constants.XML_SAVE_DOCTYPE);
     _saveBuf.append('\n');
     _saveBuf.append("<jbidwatcher format=\"0101\">\n");
     auctionsData.toStringBuffer(_saveBuf, 1);
     if (deletedData != null) {
       deletedData.toStringBuffer(_saveBuf, 1);
     }
     _saveBuf.append("</jbidwatcher>");
   }
   return _saveBuf;
 }
示例#17
0
  /**
   * Update the SQL string and replace the ? markers with parameter names eg @P0, @P1 etc.
   *
   * @param sql the SQL containing markers to substitute
   * @param list the parameter list
   * @return the modified SQL as a <code>String</code>
   */
  static String substituteParamMarkers(String sql, ParamInfo[] list) {
    // A parameter can have at most 8 characters: " @P" plus at most 4
    // digits plus " ". We subtract the "?" placeholder, that's at most
    // 7 extra characters needed for each parameter.
    char[] buf = new char[sql.length() + list.length * 7];
    int bufferPtr = 0; // Output buffer pointer
    int start = 0; // Input string pointer
    StringBuffer number = new StringBuffer(4);

    for (int i = 0; i < list.length; i++) {
      int pos = list[i].markerPos;

      if (pos > 0) {
        sql.getChars(start, pos, buf, bufferPtr);
        bufferPtr += (pos - start);
        start = pos + 1;

        // Append " @P"
        buf[bufferPtr++] = ' ';
        buf[bufferPtr++] = '@';
        buf[bufferPtr++] = 'P';

        // Append parameter number
        // Rather complicated, but it's the only way in which no
        // unnecessary objects are created
        number.setLength(0);
        number.append(i);
        number.getChars(0, number.length(), buf, bufferPtr);
        bufferPtr += number.length();

        // Append " "
        buf[bufferPtr++] = ' ';
      }
    }

    if (start < sql.length()) {
      sql.getChars(start, sql.length(), buf, bufferPtr);
      bufferPtr += (sql.length() - start);
    }

    return new String(buf, 0, bufferPtr);
  }
示例#18
0
  /**
   * parse: break the input String into fields
   *
   * @return java.util.Iterator containing each field from the original as a String, in order.
   */
  public List parse(String line) {
    StringBuffer sb = new StringBuffer();
    list.clear(); // recycle to initial state
    int i = 0;

    if (line.length() == 0) {
      list.add(line);
      return list;
    }

    do {
      sb.setLength(0);
      if (i < line.length() && line.charAt(i) == '"') i = advQuoted(line, sb, ++i); // skip quote
      else i = advPlain(line, sb, i);
      list.add(sb.toString());
      i++;
    } while (i < line.length());

    return list;
  }
  /**
   * Parse end of element from document. Collects character data to the end tag and returns it with
   * whitespace stripped. Throws an exception if a start tag is seen before an end tag, or if the
   * end tag seen does not match the expected name.
   *
   * @param tag element name expected
   * @return content text with whitespace stripped
   * @throws IOException if error reading document
   * @throws XmlPullParserException if expected element not found, or if other parse error
   */
  protected String parseEndTag(String tag) throws IOException, XmlPullParserException {
    m_buffer.setLength(0);
    while (true) {
      switch (m_parser.next()) {
        case XmlPullParser.CONTENT:
          m_buffer.append(m_parser.readContent());
          break;

        case XmlPullParser.END_TAG:
          m_parser.readEndTag(m_endTag);
          if (m_endTag.getLocalName().equals(tag)) {
            return m_buffer.toString().trim();
          }
          // fall through for error handling

        case XmlPullParser.START_TAG:
        case XmlPullParser.END_DOCUMENT:
          throw new XmlPullParserException("Missing expected end tag " + tag);
      }
    }
  }
示例#20
0
  private void checkBodyEnd(String endTag, Map replaces) {
    int index = test.length() - endTag.length();
    if (index >= 0) {
      test = test.substring(index);
      if (test.equalsIgnoreCase(endTag)) {
        if (index > 0) {

          buf.setLength(index);
          String temp = buf.toString();
          if (replaces != null) temp = StringUtils.replaceStrings(temp, replaces);

          unitList.add(new ParseUnit(temp, mode));
        }
        mode = NORMAL;
      } else {
        test = null;
      }
    } else {
      test = null;
    }
  }
示例#21
0
  private byte[] readMultiPartChunk(ServletInputStream requestStream, String contentType)
      throws IOException {
    // fast forward stream past multi-part header
    int boundaryOff = contentType.indexOf("boundary="); // $NON-NLS-1$
    String boundary = contentType.substring(boundaryOff + 9);
    BufferedReader reader =
        new BufferedReader(new InputStreamReader(requestStream, "ISO-8859-1")); // $NON-NLS-1$
    StringBuffer out = new StringBuffer();
    // skip headers up to the first blank line
    String line = reader.readLine();
    while (line != null && line.length() > 0) line = reader.readLine();
    // now process the file

    char[] buf = new char[1000];
    int read;
    while ((read = reader.read(buf)) > 0) {
      out.append(buf, 0, read);
    }
    // remove the boundary from the output (end of input is \r\n--<boundary>--\r\n)
    out.setLength(out.length() - (boundary.length() + 8));
    return out.toString().getBytes("ISO-8859-1"); // $NON-NLS-1$
  }
  /** EdiDialog constructor comment. */
  public void keyReleased(java.awt.event.KeyEvent e) {
    if (e.getSource() == list) {
      switch (e.getKeyCode()) {
        case KeyEvent.VK_DELETE:
        case KeyEvent.VK_BACK_SPACE:
          if (keys.length() > 0) keys.setLength(keys.length() - 1);
          break;
        case KeyEvent.VK_ESCAPE:
          dispose();
          break;
        case KeyEvent.VK_ENTER:
          dispose();
          actionOK();
          break;
        case KeyEvent.VK_SPACE:
          actionAdd();
          break;
        default:
          // keys.append((char) e.getKeyChar());
          list.ensureIndexIsVisible(list.getSelectedIndex());
      }

      //		if (debug)
      //			System.out.println("keys: " + keys);
      return;
    }

    switch (e.getKeyCode()) {
      case KeyEvent.VK_ENTER:
      case KeyEvent.VK_ESCAPE:
        dispose();
        break;
      default:
        break;
    }

    super.keyReleased(e);
  }
  public void startElement(String namespaceURI, String localName, String qName, Attributes atts)
      throws SAXException {
    try {
      // we output the remaining text
      if (!counting) {
        writer.write(getText());
        accumulator.setLength(0);
      }
      if (!counting) {
        writer.write("<" + qName);

        int length = atts.getLength();

        // Process each attribute
        for (int i = 0; i < length; i++) {
          // Get names and values for each attribute
          String name = atts.getQName(i);
          String value = atts.getValue(i);

          if ((name != null) && (value != null)) {
            writer.write(" " + name + "=\"" + value + "\"");
          }
        }

        writer.write(">");
      }

      if (qName.equals("description")) {
        offset = 0;
        counting = true;
      } else if (qName.equals("patent-document")) {
        counting = false;
      }
    } catch (Exception e) {
      //		    e.printStackTrace();
      throw new GrobidException("An exception occured while running Grobid.", e);
    }
  }
示例#24
0
 private static String FilterID(String id) {
   if (id == null) {
     return null;
   } else {
     StringBuffer newID = new StringBuffer();
     int st = 0;
     for (int i = 0; i < id.length(); i++) {
       char ch = Character.toLowerCase(id.charAt(i));
       if (Character.isLetterOrDigit(ch)) {
         newID.append(ch);
         st = 1;
       } else if (st == 1) {
         newID.append("_");
         st = 0;
       } else {
         // ignore char
       }
     }
     while ((newID.length() > 0) && (newID.charAt(newID.length() - 1) == '_')) {
       newID.setLength(newID.length() - 1);
     }
     return newID.toString();
   }
 }
示例#25
0
  private void parseStream(BufferedReader in, Map replaces) throws IOException {
    int c;
    ParseUnit unit;

    unitList.clear();
    mode = NORMAL;
    buf = new StringBuffer();

    c = in.read();
    while (c >= 0) {
      switch (c) {
        case '<':
          if (mode == NORMAL && buf.length() > 0) {
            test = buf.toString();
            if (test.startsWith("<!--")) {
              mode = COMMENT;
            } else if (test.startsWith("<%--")) {
              mode = JSP_COMMENT;
            } else if (test.startsWith("<%")) {
              mode = JSP;
            } else {
              if (replaces != null) test = StringUtils.replaceStrings(test, replaces);
              // test = StringUtils.stripEnterSymbol(test);
              unitList.add(new ParseUnit(test, mode));
              buf.setLength(0);
            }
          }
          buf.append((char) c);
          break;
        case '>':
          buf.append((char) c);
          test = buf.toString();
          if (mode == NORMAL) {
            if (test.startsWith("<!--")) {
              mode = COMMENT;
            } else if (test.startsWith("<%--")) {
              mode = JSP_COMMENT;
            } else if (test.startsWith("<%")) {
              mode = JSP;
            }
          }
          switch (mode) {
            case COMMENT:
              if (!test.endsWith("-->")) {
                test = null;
              }
              break;
            case JSP_COMMENT:
              if (!test.endsWith("--%>")) {
                test = null;
              }
              break;
            case JSP:
              if (!test.endsWith("%>")) {
                test = null;
              }
              break;
            case SCRIPT_BODY:
              checkBodyEnd("</SCRIPT>", replaces);
              break;
            case STYLE_BODY:
              checkBodyEnd("</STYLE>", replaces);
              break;
          }

          if (test != null) {
            //
            if (replaces != null) test = StringUtils.replaceStrings(test, replaces);

            unit = new ParseUnit(test, mode);
            unitList.add(unit);
            buf.setLength(0);
            mode = NORMAL;
            if (unit.isStartTag()) {
              test = unit.getTagName();
              if (test.equals("script")) {
                mode = SCRIPT_BODY;
              } else if (test.equals("style")) {
                mode = STYLE_BODY;
              }
            }
          }
          break;
        default:
          buf.append((char) c);
      }
      c = in.read();
    }
    if (buf.length() > 0) {
      String temp = buf.toString();
      if (replaces != null) temp = StringUtils.replaceStrings(temp, replaces);

      unitList.add(new ParseUnit(temp, mode));
    }
    units = new ParseUnit[unitList.size()];
    unitList.toArray(units);
  }
示例#26
0
  /**
   * Resumes scanning until the next regular expression is matched, the end of input is encountered
   * or an I/O-Error occurs.
   *
   * @return the next token
   * @exception java.io.IOException if any I/O-Error occurs
   */
  public XiSymbol next_token() throws java.io.IOException {
    int zzInput;
    int zzAction;

    // cached fields:
    int zzCurrentPosL;
    int zzMarkedPosL;
    int zzEndReadL = zzEndRead;
    char[] zzBufferL = zzBuffer;
    char[] zzCMapL = ZZ_CMAP;

    int[] zzTransL = ZZ_TRANS;
    int[] zzRowMapL = ZZ_ROWMAP;
    int[] zzAttrL = ZZ_ATTRIBUTE;

    while (true) {
      zzMarkedPosL = zzMarkedPos;

      yychar += zzMarkedPosL - zzStartRead;

      boolean zzR = false;
      for (zzCurrentPosL = zzStartRead; zzCurrentPosL < zzMarkedPosL; zzCurrentPosL++) {
        switch (zzBufferL[zzCurrentPosL]) {
          case '\u000B':
          case '\u000C':
          case '\u0085':
          case '\u2028':
          case '\u2029':
            yyline++;
            yycolumn = 0;
            zzR = false;
            break;
          case '\r':
            yyline++;
            yycolumn = 0;
            zzR = true;
            break;
          case '\n':
            if (zzR) zzR = false;
            else {
              yyline++;
              yycolumn = 0;
            }
            break;
          default:
            zzR = false;
            yycolumn++;
        }
      }

      if (zzR) {
        // peek one character ahead if it is \n (if we have counted one line too much)
        boolean zzPeek;
        if (zzMarkedPosL < zzEndReadL) zzPeek = zzBufferL[zzMarkedPosL] == '\n';
        else if (zzAtEOF) zzPeek = false;
        else {
          boolean eof = zzRefill();
          zzEndReadL = zzEndRead;
          zzMarkedPosL = zzMarkedPos;
          zzBufferL = zzBuffer;
          if (eof) zzPeek = false;
          else zzPeek = zzBufferL[zzMarkedPosL] == '\n';
        }
        if (zzPeek) yyline--;
      }
      zzAction = -1;

      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

      zzState = ZZ_LEXSTATE[zzLexicalState];

      zzForAction:
      {
        while (true) {

          if (zzCurrentPosL < zzEndReadL) zzInput = zzBufferL[zzCurrentPosL++];
          else if (zzAtEOF) {
            zzInput = YYEOF;
            break zzForAction;
          } else {
            // store back cached positions
            zzCurrentPos = zzCurrentPosL;
            zzMarkedPos = zzMarkedPosL;
            boolean eof = zzRefill();
            // get translated positions and possibly new buffer
            zzCurrentPosL = zzCurrentPos;
            zzMarkedPosL = zzMarkedPos;
            zzBufferL = zzBuffer;
            zzEndReadL = zzEndRead;
            if (eof) {
              zzInput = YYEOF;
              break zzForAction;
            } else {
              zzInput = zzBufferL[zzCurrentPosL++];
            }
          }
          int zzNext = zzTransL[zzRowMapL[zzState] + zzCMapL[zzInput]];
          if (zzNext == -1) break zzForAction;
          zzState = zzNext;

          int zzAttributes = zzAttrL[zzState];
          if ((zzAttributes & 1) == 1) {
            zzAction = zzState;
            zzMarkedPosL = zzCurrentPosL;
            if ((zzAttributes & 8) == 8) break zzForAction;
          }
        }
      }

      // store back cached position
      zzMarkedPos = zzMarkedPosL;

      switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
        case 1:
          {
            /* ignore */
          }
        case 72:
          break;
        case 19:
          {
            return xiSymbol(Sym.OPEN_BRACKET, yytext());
          }
        case 73:
          break;
        case 29:
          {
            throw new RuntimeException("Unterminated string at end of line:" + yyline);
          }
        case 74:
          break;
        case 68:
          {
            return xiSymbol(Sym.LENGTH, yytext());
          }
        case 75:
          break;
        case 16:
          {
            return xiSymbol(Sym.GT, yytext());
          }
        case 76:
          break;
        case 41:
          {
            string.append('\f');
          }
        case 77:
          break;
        case 49:
          {
            return xiSymbol(Sym.NEW, yytext());
          }
        case 78:
          break;
        case 65:
          {
            return xiSymbol(Sym.WHILE, yytext());
          }
        case 79:
          break;
        case 35:
          {
            return xiSymbol(Sym.EQUAL, yytext());
          }
        case 80:
          break;
        case 11:
          {
            return xiSymbol(Sym.DIVIDE, yytext());
          }
        case 81:
          break;
        case 47:
          {
            return xiSymbol(Sym.INT, yytext());
          }
        case 82:
          break;
        case 42:
          {
            string.append('\b');
          }
        case 83:
          break;
        case 60:
          {
            return xiSymbol(Sym.BOOL, yytext());
          }
        case 84:
          break;
        case 15:
          {
            return xiSymbol(Sym.GETS, yytext());
          }
        case 85:
          break;
        case 3:
          {
            return xiSymbol(Sym.INTEGER_LITERAL, Long.parseLong(yytext()));
          }
        case 86:
          break;
        case 46:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character(yytext().charAt(0)), 1);
          }
        case 87:
          break;
        case 2:
          {
            return xiSymbol(Sym.IDENTIFIER, yytext());
          }
        case 88:
          break;
        case 44:
          {
            string.append('\t');
          }
        case 89:
          break;
        case 5:
          {
            yybegin(CHARACTER);
          }
        case 90:
          break;
        case 25:
          {
            return xiSymbol(Sym.COLON, yytext());
          }
        case 91:
          break;
        case 38:
          {
            string.append('\'');
          }
        case 92:
          break;
        case 48:
          {
            return xiSymbol(Sym.USE, yytext());
          }
        case 93:
          break;
        case 20:
          {
            return xiSymbol(Sym.CLOSE_BRACKET, yytext());
          }
        case 94:
          break;
        case 10:
          {
            return xiSymbol(Sym.TIMES, yytext());
          }
        case 95:
          break;
        case 34:
          {
            return xiSymbol(Sym.LEQ, yytext());
          }
        case 96:
          break;
        case 22:
          {
            return xiSymbol(Sym.CLOSE_PAREN, yytext());
          }
        case 97:
          break;
        case 7:
          {
            return xiSymbol(Sym.PERIOD, yytext());
          }
        case 98:
          break;
        case 69:
          {
            return xiSymbol(Sym.RETURN, yytext());
          }
        case 99:
          break;
        case 13:
          {
            return xiSymbol(Sym.NOT, yytext());
          }
        case 100:
          break;
        case 71:
          {
            return xiSymbol(Sym.CONTINUE, yytext());
          }
        case 101:
          break;
        case 33:
          {
            return xiSymbol(Sym.NOT_EQUAL, yytext());
          }
        case 102:
          break;
        case 8:
          {
            return xiSymbol(Sym.PLUS, yytext());
          }
        case 103:
          break;
        case 70:
          {
            return xiSymbol(Sym.EXTENDS, yytext());
          }
        case 104:
          break;
        case 14:
          {
            return xiSymbol(Sym.LT, yytext());
          }
        case 105:
          break;
        case 64:
          {
            return xiSymbol(Sym.FALSE, yytext());
          }
        case 106:
          break;
        case 4:
          {
            return xiSymbol(Sym.UNDERSCORE, yytext());
          }
        case 107:
          break;
        case 9:
          {
            return xiSymbol(Sym.MINUS, yytext());
          }
        case 108:
          break;
        case 18:
          {
            return xiSymbol(Sym.OR, yytext());
          }
        case 109:
          break;
        case 30:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.STRING_LITERAL, string.toString(), string.length());
          }
        case 110:
          break;
        case 21:
          {
            return xiSymbol(Sym.OPEN_PAREN, yytext());
          }
        case 111:
          break;
        case 26:
          {
            return xiSymbol(Sym.COMMA, yytext());
          }
        case 112:
          break;
        case 61:
          {
            return xiSymbol(Sym.THIS, yytext());
          }
        case 113:
          break;
        case 66:
          {
            return xiSymbol(Sym.BREAK, yytext());
          }
        case 114:
          break;
        case 40:
          {
            string.append('\"');
          }
        case 115:
          break;
        case 17:
          {
            return xiSymbol(Sym.AND, yytext());
          }
        case 116:
          break;
        case 24:
          {
            return xiSymbol(Sym.CLOSE_BRACE, yytext());
          }
        case 117:
          break;
        case 12:
          {
            return xiSymbol(Sym.MODULO, yytext());
          }
        case 118:
          break;
        case 31:
          {
            throw new RuntimeException("Unterminated character literal at end of line");
          }
        case 119:
          break;
        case 53:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\"'), 1);
          }
        case 120:
          break;
        case 28:
          {
            string.append(yytext());
          }
        case 121:
          break;
        case 51:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\''), 1);
          }
        case 122:
          break;
        case 36:
          {
            return xiSymbol(Sym.GEQ, yytext());
          }
        case 123:
          break;
        case 63:
          {
            return xiSymbol(Sym.NULL, yytext());
          }
        case 124:
          break;
        case 43:
          {
            string.append('\r');
          }
        case 125:
          break;
        case 62:
          {
            return xiSymbol(Sym.TRUE, yytext());
          }
        case 126:
          break;
        case 39:
          {
            string.append('\\');
          }
        case 127:
          break;
        case 6:
          {
            yybegin(STRING);
            string.setLength(0);
          }
        case 128:
          break;
        case 59:
          {
            return xiSymbol(Sym.ELSE, yytext());
          }
        case 129:
          break;
        case 45:
          {
            string.append('\n');
          }
        case 130:
          break;
        case 52:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\\'), 1);
          }
        case 131:
          break;
        case 23:
          {
            return xiSymbol(Sym.OPEN_BRACE, yytext());
          }
        case 132:
          break;
        case 54:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\f'), 1);
          }
        case 133:
          break;
        case 32:
          {
            return xiSymbol(Sym.IF, yytext());
          }
        case 134:
          break;
        case 67:
          {
            return xiSymbol(Sym.CLASS, yytext());
          }
        case 135:
          break;
        case 55:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\b'), 1);
          }
        case 136:
          break;
        case 50:
          {
            /* IGNORE */
          }
        case 137:
          break;
        case 27:
          {
            return xiSymbol(Sym.SEMICOLON, yytext());
          }
        case 138:
          break;
        case 56:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\r'), 1);
          }
        case 139:
          break;
        case 58:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\n'), 1);
          }
        case 140:
          break;
        case 57:
          {
            yybegin(YYINITIAL);
            return xiSymbol(Sym.CHAR_LITERAL, new Character('\t'), 1);
          }
        case 141:
          break;
        case 37:
          {
            throw new RuntimeException("Illegal escape sequence \"" + yytext() + "\"");
          }
        case 142:
          break;
        default:
          if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
            zzAtEOF = true;
            zzDoEOF();
            {
              return new XiSymbol(Sym.EOF);
            }
          } else {
            zzScanError(ZZ_NO_MATCH);
          }
      }
    }
  }
示例#27
0
文件: Lexer.java 项目: chirino/janet
 protected void newTokenHere() {
   lexbuf.setLength(0);
   pbeg.copyFrom(loc);
 }
示例#28
0
文件: Lexer.java 项目: chirino/janet
 protected void backupChar() throws LexException {
   ibuf.backup();
   lexbuf.setLength(lexbuf.length() - 1);
 }
  public static void main(String[] args) {
    // String inFile = "C:/My Documents/HMM/Chunk/wsj_15_18_train.log";
    String inFile = chunkDir + "wsj_15_18_train.log";
    // String featureFile = "C:/My Documents/HMM/Chunk/chunk features.txt";
    String featureFile = chunkDir + "chunk features.txt";

    try {
      BufferedReader reader = new BufferedReader(new FileReader(inFile));
      PrintStream writer = new PrintStream(new FileOutputStream(featureFile));
      String line;
      String prevToken = "";
      String prevPOS = "";
      String prevTag = "";
      String currentToken = "";
      String currentPOS = "";
      String currentTag = "";
      String nextToken = "";
      String nextPOS = "";
      String nextTag = "";
      String followingToken = "";
      String followingPOS = "";
      String followingTag = "";
      StringBuffer features = new StringBuffer(200);

      boolean inGroup = false;
      boolean firstToken = true;
      while ((line = reader.readLine()) != null) {
        StringTokenizer st = new StringTokenizer(line);
        int count = st.countTokens();
        if (count == 0) {
          // blank line -- end of sentence
          followingToken = "";
          followingPOS = "";
          followingTag = "";
        } else if (count >= 3) {
          // token
          followingToken = st.nextToken();
          followingPOS = st.nextToken();
          followingTag = st.nextToken();
        } else {
          System.out.println("Error:  invalid input line: " + line);
        }
        if (currentToken != "") {
          features.setLength(0);
          features.append("prevPOS=" + prevPOS + " ");
          features.append("currPOS=" + currentPOS + " ");
          features.append("nextPOS=" + nextPOS + " ");
          if (nextToken == "") features.append("POS012=" + currentPOS + ":: ");
          else features.append("POS012=" + currentPOS + ":" + nextPOS + ":" + followingPOS + " ");
          features.append("prevTag=" + prevTag + " ");
          features.append("currWord=" + currentToken + " ");
          features.append("W-1W0=" + prevToken + ":" + currentToken + " ");
          features.append("W0W1=" + currentToken + ":" + nextToken + " ");
          features.append(currentTag);
          writer.println(features);
        }
        prevToken = currentToken;
        prevPOS = currentPOS;
        prevTag = currentTag;
        currentToken = nextToken;
        currentPOS = nextPOS;
        currentTag = nextTag;
        nextToken = followingToken;
        nextPOS = followingPOS;
        nextTag = followingTag;
      }
    } catch (IOException e) {
      System.out.println(e);
    }
  }
示例#30
0
  /** Performs the subsequent ReTrace operations. */
  public void execute() throws IOException {
    // Read the mapping file.
    MappingReader mappingReader = new MappingReader(mappingFile);
    mappingReader.pump(this);

    StringBuffer expressionBuffer = new StringBuffer(regularExpression.length() + 32);
    char[] expressionTypes = new char[32];
    int expressionTypeCount = 0;
    int index = 0;
    while (true) {
      int nextIndex = regularExpression.indexOf('%', index);
      if (nextIndex < 0
          || nextIndex == regularExpression.length() - 1
          || expressionTypeCount == expressionTypes.length) {
        break;
      }

      expressionBuffer.append(regularExpression.substring(index, nextIndex));
      expressionBuffer.append('(');

      char expressionType = regularExpression.charAt(nextIndex + 1);
      switch (expressionType) {
        case 'c':
          expressionBuffer.append(REGEX_CLASS);
          break;

        case 'C':
          expressionBuffer.append(REGEX_CLASS_SLASH);
          break;

        case 'l':
          expressionBuffer.append(REGEX_LINE_NUMBER);
          break;

        case 't':
          expressionBuffer.append(REGEX_TYPE);
          break;

        case 'f':
          expressionBuffer.append(REGEX_MEMBER);
          break;

        case 'm':
          expressionBuffer.append(REGEX_MEMBER);
          break;

        case 'a':
          expressionBuffer.append(REGEX_ARGUMENTS);
          break;
      }

      expressionBuffer.append(')');

      expressionTypes[expressionTypeCount++] = expressionType;

      index = nextIndex + 2;
    }

    expressionBuffer.append(regularExpression.substring(index));

    Pattern pattern = Pattern.compile(expressionBuffer.toString());

    // Read the stack trace file.
    LineNumberReader reader =
        new LineNumberReader(
            stackTraceFile == null
                ? (Reader) new InputStreamReader(System.in)
                : (Reader) new BufferedReader(new FileReader(stackTraceFile)));

    try {
      StringBuffer outLine = new StringBuffer(256);
      List extraOutLines = new ArrayList();

      String className = null;

      // Read the line in the stack trace.
      while (true) {
        String line = reader.readLine();
        if (line == null) {
          break;
        }

        Matcher matcher = pattern.matcher(line);

        if (matcher.matches()) {
          int lineNumber = 0;
          String type = null;
          String arguments = null;

          // Figure out a class name, line number, type, and
          // arguments beforehand.
          for (int expressionTypeIndex = 0;
              expressionTypeIndex < expressionTypeCount;
              expressionTypeIndex++) {
            int startIndex = matcher.start(expressionTypeIndex + 1);
            if (startIndex >= 0) {
              String match = matcher.group(expressionTypeIndex + 1);

              char expressionType = expressionTypes[expressionTypeIndex];
              switch (expressionType) {
                case 'c':
                  className = originalClassName(match);
                  break;

                case 'C':
                  className = originalClassName(ClassUtil.externalClassName(match));
                  break;

                case 'l':
                  lineNumber = Integer.parseInt(match);
                  break;

                case 't':
                  type = originalType(match);
                  break;

                case 'a':
                  arguments = originalArguments(match);
                  break;
              }
            }
          }

          // Actually construct the output line.
          int lineIndex = 0;

          outLine.setLength(0);
          extraOutLines.clear();

          for (int expressionTypeIndex = 0;
              expressionTypeIndex < expressionTypeCount;
              expressionTypeIndex++) {
            int startIndex = matcher.start(expressionTypeIndex + 1);
            if (startIndex >= 0) {
              int endIndex = matcher.end(expressionTypeIndex + 1);
              String match = matcher.group(expressionTypeIndex + 1);

              // Copy a literal piece of input line.
              outLine.append(line.substring(lineIndex, startIndex));

              char expressionType = expressionTypes[expressionTypeIndex];
              switch (expressionType) {
                case 'c':
                  className = originalClassName(match);
                  outLine.append(className);
                  break;

                case 'C':
                  className = originalClassName(ClassUtil.externalClassName(match));
                  outLine.append(ClassUtil.internalClassName(className));
                  break;

                case 'l':
                  lineNumber = Integer.parseInt(match);
                  outLine.append(match);
                  break;

                case 't':
                  type = originalType(match);
                  outLine.append(type);
                  break;

                case 'f':
                  originalFieldName(className, match, type, outLine, extraOutLines);
                  break;

                case 'm':
                  originalMethodName(
                      className, match, lineNumber, type, arguments, outLine, extraOutLines);
                  break;

                case 'a':
                  arguments = originalArguments(match);
                  outLine.append(arguments);
                  break;
              }

              // Skip the original element whose processed version
              // has just been appended.
              lineIndex = endIndex;
            }
          }

          // Copy the last literal piece of input line.
          outLine.append(line.substring(lineIndex));

          // Print out the main line.
          System.out.println(outLine);

          // Print out any additional lines.
          for (int extraLineIndex = 0; extraLineIndex < extraOutLines.size(); extraLineIndex++) {
            System.out.println(extraOutLines.get(extraLineIndex));
          }
        } else {
          // Print out the original line.
          System.out.println(line);
        }
      }
    } catch (IOException ex) {
      throw new IOException("Can't read stack trace (" + ex.getMessage() + ")");
    } finally {
      if (stackTraceFile != null) {
        try {
          reader.close();
        } catch (IOException ex) {
          // This shouldn't happen.
        }
      }
    }
  }