示例#1
0
    public void run() {
      StringBuffer data = new StringBuffer();
      Print.logDebug("Client:InputThread started");

      while (true) {
        data.setLength(0);
        boolean timeout = false;
        try {
          if (this.readTimeout > 0L) {
            this.socket.setSoTimeout((int) this.readTimeout);
          }
          ClientSocketThread.socketReadLine(this.socket, -1, data);
        } catch (InterruptedIOException ee) { // SocketTimeoutException ee) {
          // error("Read interrupted (timeout) ...");
          if (getRunStatus() != THREAD_RUNNING) {
            break;
          }
          timeout = true;
          // continue;
        } catch (Throwable t) {
          Print.logError("Client:InputThread - " + t);
          t.printStackTrace();
          break;
        }
        if (!timeout || (data.length() > 0)) {
          ClientSocketThread.this.handleMessage(data.toString());
        }
      }

      synchronized (this.threadLock) {
        this.isRunning = false;
        Print.logDebug("Client:InputThread stopped");
        this.threadLock.notify();
      }
    }
示例#2
0
  private byte[] readMultiPartChunk(ServletInputStream requestStream, String contentType)
      throws IOException {
    // fast forward stream past multi-part header
    int boundaryOff = contentType.indexOf("boundary="); // $NON-NLS-1$
    String boundary = contentType.substring(boundaryOff + 9);
    BufferedReader reader =
        new BufferedReader(new InputStreamReader(requestStream, "ISO-8859-1")); // $NON-NLS-1$
    StringBuffer out = new StringBuffer();
    // skip headers up to the first blank line
    String line = reader.readLine();
    while (line != null && line.length() > 0) line = reader.readLine();
    // now process the file

    char[] buf = new char[1000];
    int read;
    while ((read = reader.read(buf)) > 0) {
      out.append(buf, 0, read);
    }
    // remove the boundary from the output (end of input is \r\n--<boundary>--\r\n)
    out.setLength(out.length() - (boundary.length() + 8));
    return out.toString().getBytes("ISO-8859-1"); // $NON-NLS-1$
  }
示例#3
0
  // The main procedure
  public static void main(String args[]) {
    String s;

    initGUI();

    while (true) {
      try { // Poll every ~10 ms
        Thread.sleep(10);
      } catch (InterruptedException e) {
      }

      switch (connectionStatus) {
        case BEGIN_CONNECT:
          try {
            // Try to set up a server if host
            if (isHost) {
              hostServer = new ServerSocket(port);
              socket = hostServer.accept();
            }

            // If guest, try to connect to the server
            else {
              socket = new Socket(hostIP, port);
            }

            in = new BufferedReader(new InputStreamReader(socket.getInputStream()));
            out = new PrintWriter(socket.getOutputStream(), true);
            changeStatusTS(CONNECTED, true);
          }
          // If error, clean up and output an error message
          catch (IOException e) {
            cleanUp();
            changeStatusTS(DISCONNECTED, false);
          }
          break;

        case CONNECTED:
          try {
            // Send data
            if (toSend.length() != 0) {
              out.print(toSend);
              out.flush();
              toSend.setLength(0);
              changeStatusTS(NULL, true);
            }

            // Receive data
            if (in.ready()) {
              s = in.readLine();
              if ((s != null) && (s.length() != 0)) {
                // Check if it is the end of a trasmission
                if (s.equals(END_CHAT_SESSION)) {
                  changeStatusTS(DISCONNECTING, true);
                }

                // Otherwise, receive what text
                else {
                  appendToChatBox("INCOMING: " + s + "\n");
                  changeStatusTS(NULL, true);
                }
              }
            }
          } catch (IOException e) {
            cleanUp();
            changeStatusTS(DISCONNECTED, false);
          }
          break;

        case DISCONNECTING:
          // Tell other chatter to disconnect as well
          out.print(END_CHAT_SESSION);
          out.flush();

          // Clean up (close all streams/sockets)
          cleanUp();
          changeStatusTS(DISCONNECTED, true);
          break;

        default:
          break; // do nothing
      }
    }
  }
示例#4
0
  // Checks the current state and sets the enables/disables
  // accordingly
  public void run() {
    switch (connectionStatus) {
      case DISCONNECTED:
        connectButton.setEnabled(true);
        disconnectButton.setEnabled(false);
        ipField.setEnabled(true);
        portField.setEnabled(true);
        hostOption.setEnabled(true);
        guestOption.setEnabled(true);
        chatLine.setText("");
        chatLine.setEnabled(false);
        statusColor.setBackground(Color.red);
        break;

      case DISCONNECTING:
        connectButton.setEnabled(false);
        disconnectButton.setEnabled(false);
        ipField.setEnabled(false);
        portField.setEnabled(false);
        hostOption.setEnabled(false);
        guestOption.setEnabled(false);
        chatLine.setEnabled(false);
        statusColor.setBackground(Color.orange);
        break;

      case CONNECTED:
        connectButton.setEnabled(false);
        disconnectButton.setEnabled(true);
        ipField.setEnabled(false);
        portField.setEnabled(false);
        hostOption.setEnabled(false);
        guestOption.setEnabled(false);
        chatLine.setEnabled(true);
        statusColor.setBackground(Color.green);
        break;

      case BEGIN_CONNECT:
        connectButton.setEnabled(false);
        disconnectButton.setEnabled(false);
        ipField.setEnabled(false);
        portField.setEnabled(false);
        hostOption.setEnabled(false);
        guestOption.setEnabled(false);
        chatLine.setEnabled(false);
        chatLine.grabFocus();
        statusColor.setBackground(Color.orange);
        break;
    }

    // Make sure that the button/text field states are consistent
    // with the internal states
    ipField.setText(hostIP);
    portField.setText((new Integer(port)).toString());
    hostOption.setSelected(isHost);
    guestOption.setSelected(!isHost);
    statusField.setText(statusString);
    chatText.append(toAppend.toString());
    toAppend.setLength(0);

    mainFrame.repaint();
  }
示例#5
0
  // initiate either a server or a user session
  public void run() {
    if (isDaemon) {
      daemon();
      return;
    }
    ;

    boolean loggedIn = false;
    int i, h1;
    String di, str1, user = "******", user_id = "0";
    InetAddress localNode;
    byte dataBuffer[] = new byte[1024];
    String command = null;
    StringBuffer statusMessage = new StringBuffer(40);
    File targetFile = null;

    try {
      // start mysql
      Class.forName("com.mysql.jdbc.Driver").newInstance();
      this.db_conn = DriverManager.getConnection(db_url);
      this.db_stmt = this.db_conn.createStatement();

      this.db_stmt.executeUpdate("INSERT INTO test_table (name) VALUES ('hello world')");

      incoming.setSoTimeout(inactivityTimer); // enforce I/O timeout
      remoteNode = incoming.getInetAddress();
      localNode = InetAddress.getLocalHost();

      BufferedReader in =
          new BufferedReader(new InputStreamReader(incoming.getInputStream(), TELNET));
      PrintWriter out =
          new PrintWriter(new OutputStreamWriter(incoming.getOutputStream(), TELNET), true);
      str1 = "220 Flickr FTP Server Ready";
      out.println(str1);
      if (log) System.out.println(remoteNode.getHostName() + " " + str1);

      boolean done = false;
      char dataType = 0;

      while (!done) {
        statusMessage.setLength(0);

        // obtain and tokenize command
        String str = in.readLine();
        if (str == null) break; // EOS reached
        i = str.indexOf(' ');
        if (i == -1) i = str.length();
        command = str.substring(0, i).toUpperCase().intern();
        if (log)
          System.out.print(
              user
                  + "@"
                  + remoteNode.getHostName()
                  + " "
                  + (String) ((command != "PASS") ? str : "PASS ***"));
        str = str.substring(i).trim();

        try {
          if (command == "USER") {

            user = str;
            statusMessage.append("331 Password");

          } else if (command == "PASS") {

            String pass = str;
            String pass_md5 = md5(pass);

            this.db_rs =
                this.db_stmt.executeQuery(
                    "SELECT * FROM users WHERE email='"
                        + user
                        + "' AND password='******'");
            if (this.db_rs.first()) {
              loggedIn = true;
              user_id = this.db_rs.getString("id");
              System.out.println("Account id is " + user_id);
            }

            statusMessage.append(loggedIn ? "230 logged in User" : "530 Login Incorrect");

          } else if (!loggedIn) {

            statusMessage.append("530 Not logged in");

          } else if (command == "RETR") {

            statusMessage.append("999 Not likely");

          } else if (command == "STOR") {

            out.println(BINARY_XFER);

            // trim a leading slash off the filename if there is one
            if (str.substring(0, 1).equals("/")) str = str.substring(1);
            String filename = user_id + "_" + str;
            // TODO: sanitise filename
            targetFile = new File(upload_root + "/" + filename);

            RandomAccessFile dataFile = null;
            InputStream inStream = null;
            OutputStream outStream = null;
            BufferedReader br = null;
            PrintWriter pw = null;

            try {
              int amount;
              dataSocket = setupDataLink();

              // ensure timeout on reads.
              dataSocket.setSoTimeout(inactivityTimer);

              dataFile = new RandomAccessFile(targetFile, "rw");

              inStream = dataSocket.getInputStream();
              while ((amount = inStream.read(dataBuffer)) != -1)
                dataFile.write(dataBuffer, 0, amount);

              statusMessage.append(XFER_COMPLETE);

              shell_exec(ingest_path + " " + user_id + " " + filename);
            } finally {
              try {
                if (inStream != null) inStream.close();
              } catch (Exception e1) {
              }
              ;
              try {
                if (outStream != null) outStream.close();
              } catch (Exception e1) {
              }
              ;
              try {
                if (dataFile != null) dataFile.close();
              } catch (Exception e1) {
              }
              ;
              try {
                if (dataSocket != null) dataSocket.close();
              } catch (Exception e1) {
              }
              ;
              dataSocket = null;
            }

          } else if (command == "REST") {

            statusMessage.append("502 Sorry, no resuming");

          } else if (command == "TYPE") {

            if (Character.toUpperCase(str.charAt(0)) == 'I') {
              statusMessage.append(COMMAND_OK);
            } else {
              statusMessage.append("504 Only binary baybee");
            }

          } else if (command == "DELE"
              || command == "RMD"
              || command == "XRMD"
              || command == "MKD"
              || command == "XMKD"
              || command == "RNFR"
              || command == "RNTO"
              || command == "CDUP"
              || command == "XCDUP"
              || command == "CWD"
              || command == "SIZE"
              || command == "MDTM") {

            statusMessage.append("502 None of that malarky!");

          } else if (command == "QUIT") {

            statusMessage.append(COMMAND_OK).append("GOOD BYE");
            done = true;

          } else if (command == "PWD" | command == "XPWD") {

            statusMessage.append("257 \"/\" is current directory");

          } else if (command == "PORT") {

            int lng, lng1, lng2, ip2;
            String a1 = "", a2 = "";
            lng = str.length() - 1;
            lng2 = str.lastIndexOf(",");
            lng1 = str.lastIndexOf(",", lng2 - 1);

            for (i = lng1 + 1; i < lng2; i++) {
              a1 = a1 + str.charAt(i);
            }

            for (i = lng2 + 1; i <= lng; i++) {
              a2 = a2 + str.charAt(i);
            }

            remotePort = Integer.parseInt(a1);
            ip2 = Integer.parseInt(a2);
            remotePort = (remotePort << 8) + ip2;
            statusMessage.append(COMMAND_OK).append(remotePort);

          } else if (command == "LIST" | command == "NLST") {

            try {

              out.println("150 ASCII data");
              dataSocket = setupDataLink();

              PrintWriter out2 = new PrintWriter(dataSocket.getOutputStream(), true);

              if ((command == "NLST")) {
                out2.println(".");
                out2.println("..");
              } else {
                out2.println("total 8.0k");
                out2.println("dr--r--r-- 1 owner group           213 Aug 26 16:31 .");
                out2.println("dr--r--r-- 1 owner group           213 Aug 26 16:31 ..");
              }

              // socket MUST be closed before signalling EOD
              dataSocket.close();
              dataSocket = null;
              statusMessage.setLength(0);
              statusMessage.append(XFER_COMPLETE);
            } finally {
              try {
                if (dataSocket != null) dataSocket.close();
              } catch (Exception e) {
              }
              ;
              dataSocket = null;
            }

          } else if (command == "NOOP") {

            statusMessage.append(COMMAND_OK);

          } else if (command == "SYST") {

            statusMessage.append("215 UNIX"); // allows NS to do long dir

          } else if (command == "MODE") {

            if (Character.toUpperCase(str.charAt(0)) == 'S') {
              statusMessage.append(COMMAND_OK);
            } else {
              statusMessage.append("504");
            }

          } else if (command == "STRU") {

            if (str.equals("F")) {
              statusMessage.append(COMMAND_OK);
            } else {
              statusMessage.append("504");
            }

          } else if (command == "PASV") {

            try {

              int num = 0, j = 0;
              if (passiveSocket != null)
                try {
                  passiveSocket.close();
                } catch (Exception e) {
                }
              ;
              passiveSocket = new ServerSocket(0); // any port

              // ensure timeout on reads.
              passiveSocket.setSoTimeout(inactivityTimer);

              statusMessage.append("227 Entering Passive Mode (");
              String s = localNode.getHostAddress().replace('.', ','); // get host #
              statusMessage.append(s).append(',');
              num = passiveSocket.getLocalPort(); // get port #
              j = (num >> 8) & 0xff;
              statusMessage.append(j);
              statusMessage.append(',');
              j = num & 0xff;
              statusMessage.append(j);
              statusMessage.append(')');
            } catch (Exception e) {
              try {
                if (passiveSocket != null) passiveSocket.close();
              } catch (Exception e1) {
              }
              ;
              passiveSocket = null;
              throw e;
            }

          } else {
            statusMessage.append("502 unimplemented ").append(command);
          }
        }

        // shutdown causes an interruption to be thrown
        catch (InterruptedException e) {
          throw (e);
        } catch (Exception e) // catch all for any errors (including files)
        {
          statusMessage.append(FAULT).append(e.getMessage());
          if (debug) {
            System.out.println("\nFAULT - lastfile " + targetFile);
            e.printStackTrace();
          }
          ;
        }

        // send result status to remote
        out.println(statusMessage);
        if (log) System.out.println("\t" + statusMessage);
      }
    } catch (Exception e) // usually network errors (including timeout)
    {
      if (log) System.out.println("forced instance exit " + e);
      if (debug) e.printStackTrace();
    } finally // exiting server instance
    {
      // tear down mysql
      if (this.db_rs != null) {
        try {
          this.db_rs.close();
        } catch (SQLException SQLE) {;
        }
      }
      if (this.db_stmt != null) {
        try {
          this.db_stmt.close();
        } catch (SQLException SQLE) {;
        }
      }
      if (this.db_pstmt != null) {
        try {
          this.db_pstmt.close();
        } catch (SQLException SQLE) {;
        }
      }
      if (this.db_conn != null) {
        try {
          this.db_conn.close();
        } catch (SQLException SQLE) {;
        }
      }

      forceClose();
    }
  }
  // This uses the key to get the entire record from the web page
  // For now the fields that compose the record will be hardcoded
  // but these fields need to be checked with the advertisement
  // in future releases. It then uses the parameter passed to it
  // to decide which fields to return
  // Also put's the field values in a Hashtable that can be later accessed
  private synchronized String getRecordFields(String key, String param) {

    fieldsTable = null;

    String city = new String(key);
    displayMessage("DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " + "city = " + city);

    // Add the URL extension corresponding to the city.
    StringBuffer sbuf = new StringBuffer();
    for (int i = 0; i < city.length(); i++) {
      char ch = city.charAt(i);
      if (ch != ' ' && ch != '\t' && ch != '\n') sbuf.append(ch);
    }
    String formURLString = siteURLString + new String(sbuf) + ".html";

    URL infoURL = null;
    try {
      infoURL = new URL(formURLString);
    } catch (MalformedURLException e) {
      displayMessage(
          "ERROR",
          "WCNForecastWeatherQueryFn::getRecordFields " + "Malformed URL " + formURLString);
      return null;
    }
    // displayMessage("DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " +
    //	   "Got URL for site: " + formURLString);
    Object content = null;
    try {
      content = infoURL.getContent();
    } catch (IOException e) {
      displayMessage(
          "ERROR",
          "WCNForecastWeatherQueryFn::getRecordFields "
              + "Can't load content from "
              + formURLString);
      return null;
    }
    InputStreamReader input = new InputStreamReader((InputStream) content);
    BufferedReader reader = new BufferedReader(input);
    sbuf.setLength(0);
    boolean stillReading = true;
    char[] cbuf = new char[100000];
    int ccount = 0;
    long startTime = System.currentTimeMillis();
    while (stillReading) {
      try {
        ccount = reader.read(cbuf);
      } catch (IOException e) {
        displayMessage(
            "ERROR",
            "WCNForecastWeatherQueryFn::getRecordFields "
                + "IO error getting content from "
                + formURLString);
        return null;
      }
      if (ccount > 0) sbuf.append(cbuf, 0, ccount);
      if (ccount < 0) stillReading = false;
    }
    // displayMessage("DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " +
    //	   "Time to read content: " + ((System.currentTimeMillis() - startTime) / 1000.0) +
    //            " seconds");
    String page = sbuf.toString();
    StringBuffer day = new StringBuffer();
    StringBuffer condition = new StringBuffer();

    int pos = page.indexOf("<!-- forecast include");
    int delimit;
    pos = page.indexOf("<TR>", pos);
    pos = page.indexOf("<TR>", pos + 1);
    pos = page.indexOf("<TR>", pos + 1);
    for (int n = 0; n < 10; n++) {
      pos = page.indexOf("<TR>", pos + 1);
      int daypos = page.indexOf("<BR>", pos) + 4;
      delimit = page.indexOf("</B>", daypos);
      day.append("(" + page.substring(daypos, delimit) + (n == 9 ? ")" : ") "));
      pos = page.indexOf("<FONT", daypos);
      int condpos = page.indexOf(">", pos) + 1;
      delimit = page.indexOf("</FONT>", condpos);
      condition.append("(" + page.substring(condpos, delimit) + (n == 9 ? ")" : ") "));
    }

    // Get the parser corresponding to the initial URL
    /*
       Parser infoParser = null;
       try {
         infoParser = new Parser(infoURL,true,display);
       } catch (Exception ex) {
         infoParser = null;
       }

       if (infoParser == null)     {
         displayMessage("ERROR",  "WCNForecastWeatherQueryFn::getRecordFields " +
    	     "Cannot initialize parser for page " + formURLString );
         return null ;
       }


       if (FILE_WRITE) {
         displayMessage("DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " +
    	     "Site URL: " + formURLString +
    	     " Calling writeContent()." + FILE_WRITE_EXT_NUM);
         writeContent(infoParser.getContent());
       }


       displayMessage("DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " +
    	   "ready to return record fields: " + param);

       displayMessage("DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " +
    	   "city is: " + key);



       String ct = null;
       StringBuffer day = new StringBuffer();
       StringBuffer condition = new StringBuffer();
       String tag = null;

       while ((tag = infoParser.nextTag("NONE")) != null) {
         ct = infoParser.getContentBetween ();
         if ((ct == null) || (ct.length() == 0)) continue;
         ct = ct.replace('\n', ' ');
         ct = ct.replace('\r', ' ');
         if (ct.toLowerCase().indexOf("sunrise:") != -1) {
           tag = infoParser.nextTag("NONE");
    while (tag != null && !tag.equalsIgnoreCase("table"))
      tag = infoParser.nextTag("NONE");
    while (tag != null && !tag.equalsIgnoreCase("span"))
      tag = infoParser.nextTag("NONE");
           for ( int n = 0; n < 5; n++ ) {
             if (tag != null) {
               infoParser.removeContentBetween();
               tag = infoParser.nextTag("NONE");
               while (tag != null && !tag.equalsIgnoreCase("/span"))
                 tag = infoParser.nextTag("NONE");
               if (tag != null) {
                 day.append(infoParser.getContentBetween() + (n == 4 ? "" : " "));
                 tag = infoParser.nextTag("IMG");
                 while (tag != null && !tag.equalsIgnoreCase("img"))
                   tag = infoParser.nextTag("IMG");
                 if (tag != null)
                   condition.append(infoParser.getParameterValue("ALT") + (n == 4 ? "" : " "));
               }
             }
             tag = infoParser.nextTag("NONE");
             while (tag != null && !tag.equalsIgnoreCase("img"))
               tag = infoParser.nextTag("NONE");
             while (tag != null && !tag.equalsIgnoreCase("span"))
               tag = infoParser.nextTag("NONE");
             System.out.println("Content after condition = " + infoParser.getContentBetween());
           }
         }
         infoParser.removeContentBetween();
       }
       */

    // If we couldn't get at least a temperature from the page, it must
    // have been an invalid URL, ie.e, a city not in CNN database or
    // incorrectly spelled or designated.
    if (day == null) {
      displayMessage(
          "ERROR",
          "WCNForecastWeatherQueryFn::getRecordFields "
              + "URL "
              + formURLString
              + " is not a valid URL. "
              + "The \"CityCountry\" or \"CityState\" key may be wrong.");
      return null;
    }

    String weather =
        WEATHER_PERF_STRING
            + " "
            + ":"
            + DAY_STRING
            + " ("
            + day.toString()
            + ") "
            + ":"
            + CONDITION_STRING
            + " ("
            + condition.toString()
            + ")";

    if (param.equalsIgnoreCase(ALL_FIELDS)) {

      fieldsTable = new Hashtable();

      if (key != null) {
        fieldsTable.put(CITY_STRING, key);
      }
      if (day != null) {
        fieldsTable.put(DAY_STRING, day);
      }
      if (condition != null) {
        fieldsTable.put(CONDITION_STRING, condition);
      }
      if (formURLString != null) {
        fieldsTable.put(WEATHER_URL_STRING, formURLString);
      }

      if (key == null) {
        key = " ";
      }

      if (formURLString == null) {
        formURLString = " ";
      }

      String reply =
          "(reply :"
              + CITY_STRING
              + " ("
              + key
              + ") "
              + ":"
              + WEATHER_STRING
              + " ("
              + weather
              + ") "
              + ":"
              + WEATHER_URL_STRING
              + " ("
              + formURLString
              + "))";
      displayMessage(
          "DEBUG", "WCNForecastWeatherQueryFn::getRecordFields " + "Weather is: " + reply);
      return reply;
    } else if (param.equalsIgnoreCase(CITY_STRING)) {
      fieldsTable = new Hashtable();
      fieldsTable.put(CITY_STRING, key);
      return key;
    } else if (param.equalsIgnoreCase(WEATHER_STRING)) {
      fieldsTable = new Hashtable();
      fieldsTable.put(WEATHER_STRING, weather);
      return weather;
    } else if (param.equalsIgnoreCase(WEATHER_URL_STRING)) {
      fieldsTable = new Hashtable();
      fieldsTable.put(WEATHER_URL_STRING, formURLString);
      return formURLString;
    } else {
      return null;
    }
  }
示例#7
0
  private void readwrite(long s1[], long s2[], int pass) {
    // oids[] should correspond to s1[] going in, s2[] going out
    String[] names = null;
    time_retrieve = 0;
    store_bytes = 0;
    time_store = 0;

    String algorithm_id = algorithms[randIndex(algorithms.length)];
    String content_type = content_types[randIndex(content_types.length)];
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < s1.length; i++) {
      String retrieved = null;
      boolean error1 = false;
      long t1 = System.currentTimeMillis();
      while (true) {
        boolean error = false;
        try {
          retrieved = retrieve(oids[i], times, 0);
          time_retrieve += times[0];
          total_retrieve_time += times[0];
        } catch (HoneycombTestException e) {
          flex("retrv " + oids[i], e);
          String s = "#e retrv " + oids[i] + ": " + e.getMessage();
          log.log(s);
          // flog(s + "\n");
          // e.printStackTrace();
          error = true;
        } catch (Throwable t) {
          flex("retrv " + oids[i], t);
          log.log("retrieve got throwable");
          t.printStackTrace();
          error = true;
        }
        if (error) {
          error1 = true;
          retrieve_retries++;
          if (System.currentTimeMillis() - t1 > GIVEUP) {
            retrieve_failures++;
            retrieved = null;
            break;
          }
          sleep(SLEEP);
          continue;
        }
        break;
      }
      if (retrieved == null) {
        failed_rtrv.add(oids[i]);
      } else {
        if (error1) flog("#E ok\n");
        dot("r");
        // compare retrieved w/ original
        File f = new File(retrieved);
        if (f.length() != s1[i]) {
          bad_file_errors++;
          String s =
              "#e size mismatch ["
                  + oids[i]
                  + "] store/retrieve: "
                  + s1[i]
                  + " / "
                  + f.length()
                  + "\n";
          flog(s);
        } else {
          String sha1sum = null;
          try {
            sha1sum = shell.sha1sum(retrieved);
          } catch (Exception e) {
            e.printStackTrace();
          }
          if (sha1sum != null && !sha1sum.equals(shas[i])) {
            bad_file_errors++;
            String s =
                "#e sha1 mismatch ["
                    + oids[i]
                    + "] store/retrieve: "
                    + shas[i]
                    + " / "
                    + sha1sum
                    + "\n";
            flog(s);
          }
        }
        deleteFile(retrieved);
      }
      // compare mdata
      t1 = System.currentTimeMillis();
      NameValueRecord nvr = null;
      error1 = false;
      while (true) {
        boolean error = false;
        try {
          nvr = retrieveMetadata(oids[i], times, 0);
          md_retrieve_time += times[0];
          md_retrieves++;
        } catch (HoneycombTestException e) {
          flex("md retrv " + oids[i], e);
          String s = "#e md retrv " + oids[i] + ": " + e.getMessage();
          log.log(s);
          // flog(s + "\n");
          // e.printStackTrace();
          error = true;
        } catch (Throwable t) {
          flex("md retrv " + oids[i], t);
          log.log("md retrieve got throwable");
          t.printStackTrace();
          error = true;
        }
        if (error) {
          error1 = true;
          md_retrieve_retries++;
          if (System.currentTimeMillis() - t1 > GIVEUP) {
            md_retrieve_failures++;
            nvr = null;
            flog("#E " + oids[i] + ": giving up\n");
            break;
          }
          sleep(SLEEP);
          continue;
        }
        break;
      }
      if (nvr == null) {
        failed_md_rtrv.add(oids[i]);
      } else {
        if (error1) flog("#E ok\n");
        SystemRecord sr = nvr.getSystemRecord();
        String stored_oid = sr.getObjectIdentifier().toString();
        if (!oids[i].equals(stored_oid)) {
          // one oid looked up another
          sb.append("#e  oid mismatch! ").append(oids[i]);
          sb.append(" / ").append(stored_oid).append("\n");
        }
        String sha2 = nvr.getString("sha1");
        if (sha2 == null) {
          sb.append("#e  no sha1 in metadata\n");
        } else if (!sha2.equals(shas[i])) {
          sb.append("#e  original/md sha:  ").append(shas[i]);
          sb.append(" / ").append(sha2).append("\n");
        }

        checkMD(sb, nvr, "cedars_subject_id", subject_id);
        checkMD(sb, nvr, "cedars_study_id", study_id);
        checkMD(sb, nvr, "cedars_species_id", species_id);
        checkMD(sb, nvr, "cedars_sample_id", sample_id);
        checkMD(sb, nvr, "cedars_collection_date", collection_date);
        checkMD(sb, nvr, "cedars_processing_date", processing_date);
        checkMD(sb, nvr, "cedars_collected_by", collected_by);
        checkMD(sb, nvr, "cedars_processed_by", processed_by);
        checkMD(sb, nvr, "cedars_sample_type", sample_type);
        checkMD(sb, nvr, "cedars_experiment_date", experiment_date);
        checkMD(sb, nvr, "cedars_experiment_number", experiment_number);

        checkMD(sb, nvr, "cedars_scan_number", Integer.toString(i));

        if (pass > 2) {
          checkMD(sb, nvr, "cedars_algorithm_id", prev_algorithm_id);
          checkMD(sb, nvr, "cedars_content_type", prev_content_type);
        }

        if (sb.length() > 0) {
          bad_md_errors++;
          flog("#e md error " + oids[i] + "\n");
          flog(sb.toString());
          sb.setLength(0);
        }
      }

      if (s2 != null) {
        // store next smaller file and save oid
        nvr = createRecord();
        nvr.put("cedars_subject_id", subject_id);
        nvr.put("cedars_study_id", study_id);
        nvr.put("cedars_species_id", species_id);

        nvr.put("cedars_sample_id", sample_id);
        nvr.put("cedars_collection_date", collection_date);
        nvr.put("cedars_processing_date", processing_date);
        nvr.put("cedars_collected_by", collected_by);
        nvr.put("cedars_processed_by", processed_by);
        nvr.put("cedars_sample_type", sample_type);

        nvr.put("cedars_experiment_date", experiment_date);
        nvr.put("cedars_experiment_number", experiment_number);

        nvr.put("cedars_pass", Long.toString(pass));
        nvr.put("cedars_scan_number", Long.toString(i));

        nvr.put("cedars_algorithm_id", algorithm_id);
        nvr.put("cedars_content_type", content_type);

        names = store(names, s2[i], i, nvr);
        oids[i] = names[1];
        dot("s");
      }
    }
    prev_algorithm_id = algorithm_id;
    prev_content_type = content_type;

    if (names != null && names[0] != null) deleteFile(names[0]);
  }
  /**
   * Gets the next token from a tokenizer.
   *
   * @param wantWhitespace If true, leading whitespace will be returned as a token.
   * @param wantComment If true, comments are returned as tokens.
   * @return The next token in the stream.
   * @throws TextParseException The input was invalid.
   * @throws IOException An I/O error occurred.
   */
  public Token get(boolean wantWhitespace, boolean wantComment) throws IOException {
    int type;
    int c;

    if (ungottenToken) {
      ungottenToken = false;
      if (current.type == WHITESPACE) {
        if (wantWhitespace) return current;
      } else if (current.type == COMMENT) {
        if (wantComment) return current;
      } else {
        if (current.type == EOL) line++;
        return current;
      }
    }
    int skipped = skipWhitespace();
    if (skipped > 0 && wantWhitespace) return current.set(WHITESPACE, null);
    type = IDENTIFIER;
    sb.setLength(0);
    while (true) {
      c = getChar();
      if (c == -1 || delimiters.indexOf(c) != -1) {
        if (c == -1) {
          if (quoting) throw exception("EOF in " + "quoted string");
          else if (sb.length() == 0) return current.set(EOF, null);
          else return current.set(type, sb);
        }
        if (sb.length() == 0 && type != QUOTED_STRING) {
          if (c == '(') {
            multiline++;
            skipWhitespace();
            continue;
          } else if (c == ')') {
            if (multiline <= 0) throw exception("invalid " + "close " + "parenthesis");
            multiline--;
            skipWhitespace();
            continue;
          } else if (c == '"') {
            if (!quoting) {
              quoting = true;
              delimiters = quotes;
              type = QUOTED_STRING;
            } else {
              quoting = false;
              delimiters = delim;
              skipWhitespace();
            }
            continue;
          } else if (c == '\n') {
            return current.set(EOL, null);
          } else if (c == ';') {
            while (true) {
              c = getChar();
              if (c == '\n' || c == -1) break;
              sb.append((char) c);
            }
            if (wantComment) {
              ungetChar(c);
              return current.set(COMMENT, sb);
            } else if (c == -1 && type != QUOTED_STRING) {
              checkUnbalancedParens();
              return current.set(EOF, null);
            } else if (multiline > 0) {
              skipWhitespace();
              sb.setLength(0);
              continue;
            } else return current.set(EOL, null);
          } else throw new IllegalStateException();
        } else ungetChar(c);
        break;
      } else if (c == '\\') {
        c = getChar();
        if (c == -1) throw exception("unterminated escape sequence");
        sb.append('\\');
      } else if (quoting && c == '\n') {
        throw exception("newline in quoted string");
      }
      sb.append((char) c);
    }
    if (sb.length() == 0 && type != QUOTED_STRING) {
      checkUnbalancedParens();
      return current.set(EOF, null);
    }
    return current.set(type, sb);
  }