Example #1
0
  public boolean load(String abspath) {
    abspath = abspath.replace('\\', '/');
    File rcFile = new File(abspath);
    if (!rcFile.exists() || !rcFile.canRead()) {
      return false;
    }
    if (showlog) log.debug("Loading rc file: " + abspath);
    try (BufferedReader rdr =
        new BufferedReader(new InputStreamReader(new FileInputStream(rcFile), CDM.UTF8))) {
      for (int lineno = 1; ; lineno++) {
        URL url = null;
        String line = rdr.readLine();
        if (line == null) break;
        // trim leading blanks
        line = line.trim();
        if (line.length() == 0) continue; // empty line
        if (line.charAt(0) == '#') continue; // check for comment
        // parse the line
        if (line.charAt(0) == LTAG) {
          int rindex = line.indexOf(RTAG);
          if (rindex < 0) return false;
          if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
          String surl = line.substring(1, rindex);
          try {
            url = new URL(surl);
          } catch (MalformedURLException mue) {
            if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
          }
          line = line.substring(rindex + 1);
          // trim again
          line = line.trim();
        }
        // Get the key,value part
        String[] pieces = line.split("\\s*=\\s*");
        assert (pieces.length == 1 || pieces.length == 2);
        // Create the triple
        String value = "1";
        if (pieces.length == 2) value = pieces[1].trim();
        Triple triple = new Triple(pieces[0].trim(), value, url);
        List<Triple> list = triplestore.get(triple.key);
        if (list == null) list = new ArrayList<Triple>();
        Triple prev = addtriple(list, triple);
        triplestore.put(triple.key, list);
      }

    } catch (FileNotFoundException fe) {
      if (showlog) log.debug("Loading rc file: " + abspath);
      return false;

    } catch (IOException ioe) {
      if (showlog) log.error("File " + abspath + ": IO exception: " + ioe.getMessage());
      return false;
    }
    return true;
  }
Example #2
0
  // any time the server needs access to the dataset, it gets a "GuardedDataset" which allows us to
  // add caching
  // optionally, a session may be established, which allows us to reserve the dataset for that
  // session.
  protected GuardedDataset getDataset(ReqState preq) throws Exception {
    HttpServletRequest req = preq.getRequest();
    String reqPath = preq.getDataSet();

    // see if the client wants sessions
    boolean acceptSession = false;
    String s = req.getHeader("X-Accept-Session");
    if (s != null && s.equalsIgnoreCase("true") && allowSessions) acceptSession = true;

    HttpSession session = null;
    if (acceptSession) {
      // see if theres already a session established, create one if not
      session = req.getSession();
      if (!session.isNew()) {
        GuardedDataset gdataset = (GuardedDataset) session.getAttribute(reqPath);
        if (null != gdataset) {
          if (debugSession)
            System.out.printf(" found gdataset %s in session %s %n", reqPath, session.getId());
          if (log.isDebugEnabled())
            log.debug(" found gdataset " + gdataset + " in session " + session.getId());
          return gdataset;
        }
      }
    }

    NetcdfFile ncd = DatasetHandler.getNetcdfFile(req, preq.getResponse(), reqPath);
    if (null == ncd) return null;

    GuardedDataset gdataset = new GuardedDatasetCacheAndClone(reqPath, ncd, acceptSession);
    // GuardedDataset gdataset = new GuardedDatasetImpl(reqPath, ncd, acceptSession);

    if (acceptSession) {
      String cookiePath = req.getRequestURI();
      String suffix = "." + preq.getRequestSuffix();
      if (cookiePath.endsWith(suffix)) // snip off the suffix
      cookiePath = cookiePath.substring(0, cookiePath.length() - suffix.length());
      session.setAttribute(reqPath, gdataset);
      session.setAttribute(CookieFilter.SESSION_PATH, cookiePath);
      // session.setAttribute("dataset", ncd.getLocation());  // for UsageValve
      // session.setMaxInactiveInterval(30); // 30 second timeout !!
      if (debugSession)
        System.out.printf(
            " added gdataset %s in session %s cookiePath %s %n",
            reqPath, session.getId(), cookiePath);
      if (log.isDebugEnabled())
        log.debug(" added gdataset " + gdataset + " in session " + session.getId());
    } /* else {
        session = req.getSession();
        session.setAttribute("dataset", ncd.getLocation()); // for UsageValve
      } */

    return gdataset;
  }
  public void createSelf() {
    Project project = new Project();
    project.setAlias("闪电狗");
    project.setName("flash_dog");
    setMongoInfoByLog4j(project);

    if (projectService.findProject(project.getName()) == null) {
      logger.debug("try create a monitor project for flash-dog {}", project);
      project.setTasks(initTasks);
      create(project);

    } else {
      logger.debug("projectName={} has exist ,skip create ", project.getName());
    }
  }
Example #4
0
  public static boolean saveFile(
      HttpServlet servlet,
      String contentPath,
      String path,
      HttpServletRequest req,
      HttpServletResponse res) {

    // @todo Need to use logServerAccess() below here.
    boolean debugRequest = Debug.isSet("SaveFile");
    if (debugRequest) log.debug(" saveFile(): path= " + path);

    String filename = contentPath + path; // absolute path
    File want = new File(filename);

    // backup current version if it exists
    int version = getBackupVersion(want.getParent(), want.getName());
    String fileSave = filename + "~" + version;
    File file = new File(filename);
    if (file.exists()) {
      try {
        IO.copyFile(filename, fileSave);
      } catch (IOException e) {
        log.error(
            "saveFile(): Unable to save copy of file "
                + filename
                + " to "
                + fileSave
                + "\n"
                + e.getMessage());
        return false;
      }
    }

    // save new file
    try {
      OutputStream out = new BufferedOutputStream(new FileOutputStream(filename));
      IO.copy(req.getInputStream(), out);
      out.close();
      if (debugRequest) log.debug("saveFile(): ok= " + filename);
      res.setStatus(HttpServletResponse.SC_CREATED);
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_CREATED, -1));
      return true;
    } catch (IOException e) {
      log.error(
          "saveFile(): Unable to PUT file " + filename + " to " + fileSave + "\n" + e.getMessage());
      return false;
    }
  }
 private boolean verifySignature(
     Principal principal,
     byte[] dataToSign,
     String signature,
     ContainerRequestContext requestContext) {
   try {
     final byte[] signatureData = StringUtils.base64Decode(signature);
     if (logger.isDebugEnabled()) {
       logger.debug(
           "Verifying REST request - principal: "
               + principal
               + " data: "
               + fingerprint(dataToSign)
               + " signature: "
               + fingerprint(signatureData));
     }
     SignatureVerificationKey key = findVerificationKey(principal);
     if (key == null) {
       return false;
     }
     try {
       cryptoEngine.verifySignature(key, digestAlgorithm, dataToSign, signatureData);
       return true;
     } catch (InvalidKeyException e) {
       logServerError(
           "Invalid key found while verifying signature: " + e.getMessage(), e, requestContext);
       throw new WebApplicationException(INTERNAL_SERVER_ERROR);
     } catch (SignatureException e) {
       return false;
     }
   } catch (BackendAccessException e) {
     logServerError("Unexpected BackendAccessException: " + e.getMessage(), e, requestContext);
     throw new WebApplicationException(INTERNAL_SERVER_ERROR);
   }
 }
  public void create(Project project) {
    Assert.isNull(
        projectService.findProject(project.getName()),
        "project  [" + project.getName() + "] has exist");
    MongoTemplate template = project.fetchMongoTemplate();
    Assert.notNull(template, "mongo uri is not access");
    Assert.notNull(template.getDb(), "mongo uri is not access");
    Assert.isTrue(
        project.fetchMongoTemplate().collectionExists(project.getLogCollection()),
        " [" + project.getLogCollection() + "] 日志表不存在");
    try {
      List<Task> taskList = Lists.newArrayList();
      logger.debug("init task count:{}", project.getTasks().size());
      for (Task _task : project.getTasks()) {

        Task task = getTemplateTask(_task);
        Task tempTask = renderTemplateTask(task, project);
        if (tempTask != null) taskList.add(tempTask);
      }
      project.setTasks(taskList);
    } catch (Exception e) {
      logger.error("", e);
      throw new IllegalArgumentException("自动添加监控脚本错误:" + e.getMessage());
    }
    projectService.saveProject(project);
  }
 /**
  * _more_
  *
  * @param aTableList _more_
  * @param aTables _more_
  * @return Was read successful
  * @throws IOException On badness
  */
 private static boolean readTableEntries(String aTableList, ArrayList<GribPDSParamTable> aTables)
     throws IOException {
   InputStream inputStream = GribResourceReader.getInputStream(aTableList);
   if (inputStream == null) {
     logger.debug("Could not open table file:" + aTableList);
     return false;
   }
   return readTableEntries(inputStream, aTableList, aTables);
 }
Example #8
0
 private void checkSize(ServerDDS dds, boolean isAscii) throws Exception {
   long size = computeSize(dds, isAscii);
   // System.err.printf("total (constrained) size=%s\n", size);
   log.debug("total (constrained) size={}", size);
   double dsize = size / (1000 * 1000);
   double maxSize = isAscii ? ascLimit : binLimit; // Mbytes
   if (dsize > maxSize) {
     log.info("Reject request size = {} Mbytes", dsize);
     throw new UnsupportedOperationException(
         "Request too big=" + dsize + " Mbytes, max=" + maxSize);
   }
 }
 public RESTAuthenticationFilter(
     CryptoEngine cryptoEngine,
     Long contentMaxSize,
     DigestAlgorithm digestAlgorithm,
     long expiry,
     ReplayAttackValidator replayAttackValidator) {
   this.cryptoEngine = cryptoEngine;
   this.contentMaxSize = contentMaxSize;
   this.digestAlgorithm = digestAlgorithm;
   this.expiry = expiry;
   this.replayAttackValidator = replayAttackValidator;
   logger.debug(
       "REST Authentication filter using crypto engine: " + cryptoEngine.getClass().getName());
 }
Example #10
0
  /**
   * Write a file to the response stream.
   *
   * @param servlet called from here
   * @param contentPath file root path
   * @param path file path reletive to the root
   * @param req the request
   * @param res the response
   * @param contentType content type, or null
   * @throws IOException on write error
   */
  public static void returnFile(
      HttpServlet servlet,
      String contentPath,
      String path,
      HttpServletRequest req,
      HttpServletResponse res,
      String contentType)
      throws IOException {

    String filename = ServletUtil.formFilename(contentPath, path);

    log.debug("returnFile(): returning file <" + filename + ">.");
    // No file, nothing to view
    if (filename == null) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      res.sendError(HttpServletResponse.SC_NOT_FOUND);
      return;
    }

    // dontallow ..
    if (filename.indexOf("..") != -1) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
      res.sendError(HttpServletResponse.SC_FORBIDDEN);
      return;
    }

    // dont allow access to WEB-INF or META-INF
    String upper = filename.toUpperCase();
    if (upper.indexOf("WEB-INF") != -1 || upper.indexOf("META-INF") != -1) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
      res.sendError(HttpServletResponse.SC_FORBIDDEN);
      return;
    }

    returnFile(servlet, req, res, new File(filename), contentType);
  }
Example #11
0
  static void loadDefaults() {
    RC rc0 = new RC();
    String[] locations =
        new String[] {
          System.getProperty("user.home"), System.getProperty("user.dir"),
        };

    boolean found1 = false;
    for (String loc : locations) {
      if (loc == null) continue;
      String dir = loc.replace('\\', '/');
      if (dir.endsWith("/")) dir = dir.substring(0, dir.length() - 1);
      for (String rcpath : rcfilelist) {
        String filepath = loc + "/" + rcpath;
        if (rc0.load(filepath)) found1 = true;
      }
    }
    if (!found1) if (showlog) log.debug("No .rc file found");
    dfaltRC = rc0;
  }
Example #12
0
  public void doGetASC(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (ds == null) return;

      response.setHeader("XDODS-Server", getServerVersion());
      response.setContentType("text/plain");
      response.setHeader("Content-Description", "dods-ascii");

      log.debug(
          "Sending OPeNDAP ASCII Data For: " + rs + "  CE: '" + rs.getConstraintExpression() + "'");

      ServerDDS dds = ds.getDDS();
      CEEvaluator ce = new CEEvaluator(dds);
      ce.parseConstraint(rs);
      checkSize(dds, true);

      PrintWriter pw = new PrintWriter(response.getOutputStream());
      dds.printConstrained(pw);
      pw.println("---------------------------------------------");

      AsciiWriter writer = new AsciiWriter(); // could be static
      writer.toASCII(pw, dds, ds);

      // the way that getDAP2Data works
      // DataOutputStream sink = new DataOutputStream(bOut);
      // ce.send(myDDS.getName(), sink, ds);

      pw.flush();

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }
Example #13
0
  public void doGet(HttpServletRequest request, HttpServletResponse response) {
    log.info("doGet(): " + UsageLog.setupRequestContext(request));
    // System.out.printf("opendap doGet: req=%s%n%s%n", ServletUtil.getRequest(request),
    // ServletUtil.showRequestDetail(this, request));

    String path = null;

    ReqState rs = getRequestState(request, response);

    try {
      path = request.getPathInfo();
      log.debug("doGet path={}", path);

      if (thredds.servlet.Debug.isSet("showRequestDetail"))
        log.debug(ServletUtil.showRequestDetail(this, request));

      if (path == null) {
        log.info(
            "doGet(): "
                + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
        response.sendError(HttpServletResponse.SC_NOT_FOUND);
        return;
      }

      if (baseURI == null) { // first time, set baseURI
        URI reqURI = ServletUtil.getRequestURI(request);
        // Build base URI from request (rather than hard-coding "/thredds/dodsC/").
        String baseUriString = request.getContextPath() + request.getServletPath() + "/";
        baseURI = reqURI.resolve(baseUriString);
        log.debug("doGet(): baseURI was set = {}", baseURI);
      }

      if (path.endsWith("latest.xml")) {
        DataRootHandler.getInstance().processReqForLatestDataset(this, request, response);
        return;
      }

      // Redirect all catalog requests at the root level.
      if (path.equals("/") || path.equals("/catalog.html") || path.equals("/catalog.xml")) {
        ServletUtil.sendPermanentRedirect(ServletUtil.getContextPath() + path, request, response);
        return;
      }

      // Make sure catalog requests match a dataRoot before trying to handle.
      if (path.endsWith("/") || path.endsWith("/catalog.html") || path.endsWith("/catalog.xml")) {
        if (!DataRootHandler.getInstance().hasDataRootMatch(path)) {
          log.info(
              "doGet(): "
                  + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
          response.sendError(HttpServletResponse.SC_NOT_FOUND);
          return;
        }

        if (!DataRootHandler.getInstance().processReqForCatalog(request, response))
          log.error(
              "doGet(): "
                  + UsageLog.closingMessageForRequestContext(
                      ServletUtil.STATUS_FORWARD_FAILURE, -1));

        return;
      }

      if (rs != null) {
        String dataSet = rs.getDataSet();
        String requestSuffix = rs.getRequestSuffix();

        if ((dataSet == null) || dataSet.equals("/") || dataSet.equals("")) {
          doGetDIR(rs);
        } else if (requestSuffix.equalsIgnoreCase("blob")) {
          doGetBLOB(rs);
        } else if (requestSuffix.equalsIgnoreCase("close")) {
          doClose(rs);
        } else if (requestSuffix.equalsIgnoreCase("dds")) {
          doGetDDS(rs);
        } else if (requestSuffix.equalsIgnoreCase("das")) {
          doGetDAS(rs);
        } else if (requestSuffix.equalsIgnoreCase("ddx")) {
          doGetDDX(rs);
        } else if (requestSuffix.equalsIgnoreCase("dods")) {
          doGetDAP2Data(rs);
        } else if (requestSuffix.equalsIgnoreCase("asc")
            || requestSuffix.equalsIgnoreCase("ascii")) {
          doGetASC(rs);
        } else if (requestSuffix.equalsIgnoreCase("info")) {
          doGetINFO(rs);
        } else if (requestSuffix.equalsIgnoreCase("html")
            || requestSuffix.equalsIgnoreCase("htm")) {
          doGetHTML(rs);
        } else if (requestSuffix.equalsIgnoreCase("ver")
            || requestSuffix.equalsIgnoreCase("version")
            || dataSet.equalsIgnoreCase("/version")
            || dataSet.equalsIgnoreCase("/version/")) {
          doGetVER(rs);
        } else if (dataSet.equalsIgnoreCase("/help")
            || dataSet.equalsIgnoreCase("/help/")
            || dataSet.equalsIgnoreCase("/" + requestSuffix)
            || requestSuffix.equalsIgnoreCase("help")) {
          doGetHELP(rs);
        } else {
          sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
          return;
        }

      } else {
        sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
        return;
      }

      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, -1));

      // plain ol' 404
    } catch (FileNotFoundException e) {
      sendErrorResponse(response, HttpServletResponse.SC_NOT_FOUND, e.getMessage());

      // DAP2Exception bad url
    } catch (BadURLException e) {
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      dap2ExceptionHandler(e, rs);

      // all other DAP2Exception
    } catch (DAP2Exception de) {
      int status =
          (de.getErrorCode() == DAP2Exception.NO_SUCH_FILE)
              ? HttpServletResponse.SC_NOT_FOUND
              : HttpServletResponse.SC_BAD_REQUEST;
      if ((de.getErrorCode() != DAP2Exception.NO_SUCH_FILE) && (de.getErrorMessage() != null))
        log.debug(de.getErrorMessage());
      log.info(UsageLog.closingMessageForRequestContext(status, -1));
      response.setStatus(status);
      dap2ExceptionHandler(de, rs);

      // parsing, usually the CE
    } catch (ParseException pe) {
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      parseExceptionHandler(pe, response);

      // 403 - request too big
    } catch (UnsupportedOperationException e) {
      sendErrorResponse(response, HttpServletResponse.SC_FORBIDDEN, e.getMessage());

    } catch (java.net.SocketException e) {
      log.info("SocketException: " + e.getMessage(), e);
      log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));

    } catch (IOException e) {
      String eName =
          e.getClass().getName(); // dont want compile time dependency on ClientAbortException
      if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
        log.debug("ClientAbortException: " + e.getMessage());
        log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));
        return;
      }

      log.error("path= " + path, e);
      sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());

      // everything else
    } catch (Throwable t) {
      log.error("path= " + path, t);
      t.printStackTrace();
      sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, t.getMessage());
    }
  }
Example #14
0
  /**
   * Write a file to the response stream. Handles Range requests.
   *
   * @param req request
   * @param res response
   * @param file must exists and not be a directory
   * @param contentType must not be null
   * @throws IOException or error
   */
  public static void returnFile(
      HttpServletRequest req, HttpServletResponse res, File file, String contentType)
      throws IOException {
    res.setContentType(contentType);

    // see if its a Range Request
    boolean isRangeRequest = false;
    long startPos = 0, endPos = Long.MAX_VALUE;
    String rangeRequest = req.getHeader("Range");
    if (rangeRequest != null) { // bytes=12-34 or bytes=12-
      int pos = rangeRequest.indexOf("=");
      if (pos > 0) {
        int pos2 = rangeRequest.indexOf("-");
        if (pos2 > 0) {
          String startString = rangeRequest.substring(pos + 1, pos2);
          String endString = rangeRequest.substring(pos2 + 1);
          startPos = Long.parseLong(startString);
          if (endString.length() > 0) endPos = Long.parseLong(endString) + 1;
          isRangeRequest = true;
        }
      }
    }

    // set content length
    long fileSize = file.length();
    long contentLength = fileSize;
    if (isRangeRequest) {
      endPos = Math.min(endPos, fileSize);
      contentLength = endPos - startPos;
    }

    if (contentLength > Integer.MAX_VALUE)
      res.addHeader(
          "Content-Length", Long.toString(contentLength)); // allow content length > MAX_INT
    else res.setContentLength((int) contentLength); // note HEAD only allows this

    String filename = file.getPath();
    boolean debugRequest = Debug.isSet("returnFile");
    if (debugRequest)
      log.debug(
          "returnFile(): filename = "
              + filename
              + " contentType = "
              + contentType
              + " contentLength = "
              + contentLength);

    // indicate we allow Range Requests
    res.addHeader("Accept-Ranges", "bytes");

    if (req.getMethod().equals("HEAD")) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, 0));
      return;
    }

    try {

      if (isRangeRequest) {
        // set before content is sent
        res.addHeader("Content-Range", "bytes " + startPos + "-" + (endPos - 1) + "/" + fileSize);
        res.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);

        FileCacheRaf.Raf craf = null;
        try {
          craf = fileCacheRaf.acquire(filename);
          IO.copyRafB(
              craf.getRaf(), startPos, contentLength, res.getOutputStream(), new byte[60000]);
          log.info(
              "returnFile(): "
                  + UsageLog.closingMessageForRequestContext(
                      HttpServletResponse.SC_PARTIAL_CONTENT, contentLength));
          return;
        } finally {
          if (craf != null) fileCacheRaf.release(craf);
        }
      }

      // Return the file
      ServletOutputStream out = res.getOutputStream();
      IO.copyFileB(file, out, 60000);
      res.flushBuffer();
      out.close();
      if (debugRequest) log.debug("returnFile(): returnFile ok = " + filename);
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, contentLength));
    }

    // @todo Split up this exception handling: those from file access vs those from dealing with
    // response
    //       File access: catch and res.sendError()
    //       response: don't catch (let bubble up out of doGet() etc)
    catch (FileNotFoundException e) {
      log.error("returnFile(): FileNotFoundException= " + filename);
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_NOT_FOUND);
    } catch (java.net.SocketException e) {
      log.info("returnFile(): SocketException sending file: " + filename + " " + e.getMessage());
      log.info("returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
    } catch (IOException e) {
      String eName =
          e.getClass().getName(); // dont want compile time dependency on ClientAbortException
      if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
        log.info(
            "returnFile(): ClientAbortException while sending file: "
                + filename
                + " "
                + e.getMessage());
        log.info(
            "returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
        return;
      }

      log.error("returnFile(): IOException (" + e.getClass().getName() + ") sending file ", e);
      log.error(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      if (!res.isCommitted())
        res.sendError(HttpServletResponse.SC_NOT_FOUND, "Problem sending file: " + e.getMessage());
    }
  }
  /**
   * Write equivilent uncompressed version of the file.
   *
   * @param inputRaf file to uncompress
   * @param ufilename write to this file
   * @return raf of uncompressed file
   * @throws IOException on read error
   */
  private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
      throws IOException {
    RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
    FileLock lock = null;

    while (true) { // loop waiting for the lock
      try {
        lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
        break;

      } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
        try {
          Thread.sleep(100); // msecs
        } catch (InterruptedException e1) {
        }
      }
    }

    try {
      inputRaf.seek(0);
      byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
      inputRaf.read(header);
      outputRaf.write(header);

      boolean eof = false;
      int numCompBytes;
      byte[] ubuff = new byte[40000];
      byte[] obuff = new byte[40000];
      try {
        CBZip2InputStream cbzip2 = new CBZip2InputStream();
        while (!eof) {
          try {
            numCompBytes = inputRaf.readInt();
            if (numCompBytes == -1) {
              if (log.isDebugEnabled()) log.debug("  done: numCompBytes=-1 ");
              break;
            }
          } catch (EOFException ee) {
            log.warn("  got EOFException ");
            break; // assume this is ok
          }

          if (log.isDebugEnabled()) {
            log.debug(
                "reading compressed bytes "
                    + numCompBytes
                    + " input starts at "
                    + inputRaf.getFilePointer()
                    + "; output starts at "
                    + outputRaf.getFilePointer());
          }
          /*
           * For some stupid reason, the last block seems to
           * have the number of bytes negated.  So, we just
           * assume that any negative number (other than -1)
           * is the last block and go on our merry little way.
           */
          if (numCompBytes < 0) {
            if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
            numCompBytes = -numCompBytes;
            eof = true;
          }
          byte[] buf = new byte[numCompBytes];
          inputRaf.readFully(buf);
          ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);

          // CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
          cbzip2.setStream(bis);
          int total = 0;
          int nread;
          /*
          while ((nread = cbzip2.read(ubuff)) != -1) {
            dout2.write(ubuff, 0, nread);
            total += nread;
          }
          */
          try {
            while ((nread = cbzip2.read(ubuff)) != -1) {
              if (total + nread > obuff.length) {
                byte[] temp = obuff;
                obuff = new byte[temp.length * 2];
                System.arraycopy(temp, 0, obuff, 0, temp.length);
              }
              System.arraycopy(ubuff, 0, obuff, total, nread);
              total += nread;
            }
            if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
          } catch (BZip2ReadException ioe) {
            log.warn("Nexrad2IOSP.uncompress ", ioe);
          }
          float nrecords = (float) (total / 2432.0);
          if (log.isDebugEnabled())
            log.debug(
                "  unpacked "
                    + total
                    + " num bytes "
                    + nrecords
                    + " records; ouput ends at "
                    + outputRaf.getFilePointer());
        }

      } catch (Exception e) {
        if (outputRaf != null) outputRaf.close();
        outputRaf = null;

        // dont leave bad files around
        File ufile = new File(ufilename);
        if (ufile.exists()) {
          if (!ufile.delete())
            log.warn("failed to delete uncompressed file (IOException)" + ufilename);
        }

        if (e instanceof IOException) throw (IOException) e;
        else throw new RuntimeException(e);
      }

    } finally {
      if (null != outputRaf) outputRaf.flush();
      if (lock != null) lock.release();
    }

    return outputRaf;
  }
  Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
    this.raf = orgRaf;

    if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());

    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);

    // volume scan header
    dataFormat = raf.readString(8);
    raf.skipBytes(1);
    String volumeNo = raf.readString(3);
    title_julianDay = raf.readInt(); // since 1/1/70
    title_msecs = raf.readInt();
    stationId = raf.readString(4).trim(); // only in AR2V0001
    if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);

    if (stationId.length() == 0) {
      // try to get it from the filename LOOK

      stationId = null;
    }

    // try to find the station
    if (stationId != null) {
      if (!stationId.startsWith("K") && stationId.length() == 4) {
        String _stationId = "K" + stationId;
        station = NexradStationDB.get(_stationId);
      } else station = NexradStationDB.get(stationId);
    }

    // see if we have to uncompress
    if (dataFormat.equals(AR2V0001)
        || dataFormat.equals(AR2V0003)
        || dataFormat.equals(AR2V0004)
        || dataFormat.equals(AR2V0006)) {
      raf.skipBytes(4);
      String BZ = raf.readString(2);
      if (BZ.equals("BZ")) {
        RandomAccessFile uraf;
        File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");

        if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
          // see if its locked - another thread is writing it
          FileInputStream fstream = null;
          FileLock lock = null;
          try {
            fstream = new FileInputStream(uncompressedFile);
            // lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked

            while (true) { // loop waiting for the lock
              try {
                lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
                break;

              } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
                try {
                  Thread.sleep(100); // msecs
                } catch (InterruptedException e1) {
                  break;
                }
              }
            }

          } finally {
            if (lock != null) lock.release();
            if (fstream != null) fstream.close();
          }
          uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");

        } else {
          // nope, gotta uncompress it
          uraf = uncompress(raf, uncompressedFile.getPath());
          if (log.isDebugEnabled())
            log.debug("made uncompressed file= " + uncompressedFile.getPath());
        }

        // switch to uncompressed file
        raf.close();
        raf = uraf;
        raf.order(RandomAccessFile.BIG_ENDIAN);
      }

      raf.seek(Level2Record.FILE_HEADER_SIZE);
    }

    List<Level2Record> reflectivity = new ArrayList<Level2Record>();
    List<Level2Record> doppler = new ArrayList<Level2Record>();
    List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highVelocity = new ArrayList<Level2Record>();
    List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
    List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
    List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();

    long message_offset31 = 0;
    int recno = 0;
    while (true) {

      Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
      if (r == null) break;
      if (showData) r.dump2(System.out);
      // skip non-data messages
      if (r.message_type == 31) {
        message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
      }

      if (r.message_type != 1 && r.message_type != 31) {
        if (showMessages) r.dumpMessage(System.out);
        continue;
      }

      //  if (showData) r.dump2(System.out);

      /* skip bad
      if (!r.checkOk()) {
        r.dump(System.out);
        continue;
      }   */

      // some global params
      if (vcp == 0) vcp = r.vcp;
      if (first == null) first = r;
      last = r;

      if (runCheck && !r.checkOk()) {
        continue;
      }

      if (r.hasReflectData) reflectivity.add(r);
      if (r.hasDopplerData) doppler.add(r);

      if (r.message_type == 31) {
        if (r.hasHighResREFData) highReflectivity.add(r);
        if (r.hasHighResVELData) highVelocity.add(r);
        if (r.hasHighResSWData) highSpectrum.add(r);
        if (r.hasHighResZDRData) highDiffReflectivity.add(r);
        if (r.hasHighResPHIData) highDiffPhase.add(r);
        if (r.hasHighResRHOData) highCorreCoefficient.add(r);
      }

      if ((cancelTask != null) && cancelTask.isCancel()) return;
    }
    if (debugRadials)
      System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
    if (highReflectivity.size() == 0) {
      reflectivityGroups = sortScans("reflect", reflectivity, 600);
      dopplerGroups = sortScans("doppler", doppler, 600);
    }
    if (highReflectivity.size() > 0)
      reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
    if (highVelocity.size() > 0)
      velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
    if (highSpectrum.size() > 0)
      spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
    if (highDiffReflectivity.size() > 0)
      diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
    if (highDiffPhase.size() > 0)
      diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
    if (highCorreCoefficient.size() > 0)
      coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
  }