public boolean load(String abspath) {
abspath = abspath.replace('\\', '/');
File rcFile = new File(abspath);
if (!rcFile.exists() || !rcFile.canRead()) {
return false;
}
if (showlog) log.debug("Loading rc file: " + abspath);
try (BufferedReader rdr =
new BufferedReader(new InputStreamReader(new FileInputStream(rcFile), CDM.UTF8))) {
for (int lineno = 1; ; lineno++) {
URL url = null;
String line = rdr.readLine();
if (line == null) break;
// trim leading blanks
line = line.trim();
if (line.length() == 0) continue; // empty line
if (line.charAt(0) == '#') continue; // check for comment
// parse the line
if (line.charAt(0) == LTAG) {
int rindex = line.indexOf(RTAG);
if (rindex < 0) return false;
if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
String surl = line.substring(1, rindex);
try {
url = new URL(surl);
} catch (MalformedURLException mue) {
if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
}
line = line.substring(rindex + 1);
// trim again
line = line.trim();
}
// Get the key,value part
String[] pieces = line.split("\\s*=\\s*");
assert (pieces.length == 1 || pieces.length == 2);
// Create the triple
String value = "1";
if (pieces.length == 2) value = pieces[1].trim();
Triple triple = new Triple(pieces[0].trim(), value, url);
List<Triple> list = triplestore.get(triple.key);
if (list == null) list = new ArrayList<Triple>();
Triple prev = addtriple(list, triple);
triplestore.put(triple.key, list);
}
} catch (FileNotFoundException fe) {
if (showlog) log.debug("Loading rc file: " + abspath);
return false;
} catch (IOException ioe) {
if (showlog) log.error("File " + abspath + ": IO exception: " + ioe.getMessage());
return false;
}
return true;
}
// any time the server needs access to the dataset, it gets a "GuardedDataset" which allows us to
// add caching
// optionally, a session may be established, which allows us to reserve the dataset for that
// session.
protected GuardedDataset getDataset(ReqState preq) throws Exception {
HttpServletRequest req = preq.getRequest();
String reqPath = preq.getDataSet();
// see if the client wants sessions
boolean acceptSession = false;
String s = req.getHeader("X-Accept-Session");
if (s != null && s.equalsIgnoreCase("true") && allowSessions) acceptSession = true;
HttpSession session = null;
if (acceptSession) {
// see if theres already a session established, create one if not
session = req.getSession();
if (!session.isNew()) {
GuardedDataset gdataset = (GuardedDataset) session.getAttribute(reqPath);
if (null != gdataset) {
if (debugSession)
System.out.printf(" found gdataset %s in session %s %n", reqPath, session.getId());
if (log.isDebugEnabled())
log.debug(" found gdataset " + gdataset + " in session " + session.getId());
return gdataset;
}
}
}
NetcdfFile ncd = DatasetHandler.getNetcdfFile(req, preq.getResponse(), reqPath);
if (null == ncd) return null;
GuardedDataset gdataset = new GuardedDatasetCacheAndClone(reqPath, ncd, acceptSession);
// GuardedDataset gdataset = new GuardedDatasetImpl(reqPath, ncd, acceptSession);
if (acceptSession) {
String cookiePath = req.getRequestURI();
String suffix = "." + preq.getRequestSuffix();
if (cookiePath.endsWith(suffix)) // snip off the suffix
cookiePath = cookiePath.substring(0, cookiePath.length() - suffix.length());
session.setAttribute(reqPath, gdataset);
session.setAttribute(CookieFilter.SESSION_PATH, cookiePath);
// session.setAttribute("dataset", ncd.getLocation()); // for UsageValve
// session.setMaxInactiveInterval(30); // 30 second timeout !!
if (debugSession)
System.out.printf(
" added gdataset %s in session %s cookiePath %s %n",
reqPath, session.getId(), cookiePath);
if (log.isDebugEnabled())
log.debug(" added gdataset " + gdataset + " in session " + session.getId());
} /* else {
session = req.getSession();
session.setAttribute("dataset", ncd.getLocation()); // for UsageValve
} */
return gdataset;
}
public void createSelf() {
Project project = new Project();
project.setAlias("闪电狗");
project.setName("flash_dog");
setMongoInfoByLog4j(project);
if (projectService.findProject(project.getName()) == null) {
logger.debug("try create a monitor project for flash-dog {}", project);
project.setTasks(initTasks);
create(project);
} else {
logger.debug("projectName={} has exist ,skip create ", project.getName());
}
}
public static boolean saveFile(
HttpServlet servlet,
String contentPath,
String path,
HttpServletRequest req,
HttpServletResponse res) {
// @todo Need to use logServerAccess() below here.
boolean debugRequest = Debug.isSet("SaveFile");
if (debugRequest) log.debug(" saveFile(): path= " + path);
String filename = contentPath + path; // absolute path
File want = new File(filename);
// backup current version if it exists
int version = getBackupVersion(want.getParent(), want.getName());
String fileSave = filename + "~" + version;
File file = new File(filename);
if (file.exists()) {
try {
IO.copyFile(filename, fileSave);
} catch (IOException e) {
log.error(
"saveFile(): Unable to save copy of file "
+ filename
+ " to "
+ fileSave
+ "\n"
+ e.getMessage());
return false;
}
}
// save new file
try {
OutputStream out = new BufferedOutputStream(new FileOutputStream(filename));
IO.copy(req.getInputStream(), out);
out.close();
if (debugRequest) log.debug("saveFile(): ok= " + filename);
res.setStatus(HttpServletResponse.SC_CREATED);
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_CREATED, -1));
return true;
} catch (IOException e) {
log.error(
"saveFile(): Unable to PUT file " + filename + " to " + fileSave + "\n" + e.getMessage());
return false;
}
}
private boolean verifySignature(
Principal principal,
byte[] dataToSign,
String signature,
ContainerRequestContext requestContext) {
try {
final byte[] signatureData = StringUtils.base64Decode(signature);
if (logger.isDebugEnabled()) {
logger.debug(
"Verifying REST request - principal: "
+ principal
+ " data: "
+ fingerprint(dataToSign)
+ " signature: "
+ fingerprint(signatureData));
}
SignatureVerificationKey key = findVerificationKey(principal);
if (key == null) {
return false;
}
try {
cryptoEngine.verifySignature(key, digestAlgorithm, dataToSign, signatureData);
return true;
} catch (InvalidKeyException e) {
logServerError(
"Invalid key found while verifying signature: " + e.getMessage(), e, requestContext);
throw new WebApplicationException(INTERNAL_SERVER_ERROR);
} catch (SignatureException e) {
return false;
}
} catch (BackendAccessException e) {
logServerError("Unexpected BackendAccessException: " + e.getMessage(), e, requestContext);
throw new WebApplicationException(INTERNAL_SERVER_ERROR);
}
}
public void create(Project project) {
Assert.isNull(
projectService.findProject(project.getName()),
"project [" + project.getName() + "] has exist");
MongoTemplate template = project.fetchMongoTemplate();
Assert.notNull(template, "mongo uri is not access");
Assert.notNull(template.getDb(), "mongo uri is not access");
Assert.isTrue(
project.fetchMongoTemplate().collectionExists(project.getLogCollection()),
" [" + project.getLogCollection() + "] 日志表不存在");
try {
List<Task> taskList = Lists.newArrayList();
logger.debug("init task count:{}", project.getTasks().size());
for (Task _task : project.getTasks()) {
Task task = getTemplateTask(_task);
Task tempTask = renderTemplateTask(task, project);
if (tempTask != null) taskList.add(tempTask);
}
project.setTasks(taskList);
} catch (Exception e) {
logger.error("", e);
throw new IllegalArgumentException("自动添加监控脚本错误:" + e.getMessage());
}
projectService.saveProject(project);
}
/**
* _more_
*
* @param aTableList _more_
* @param aTables _more_
* @return Was read successful
* @throws IOException On badness
*/
private static boolean readTableEntries(String aTableList, ArrayList<GribPDSParamTable> aTables)
throws IOException {
InputStream inputStream = GribResourceReader.getInputStream(aTableList);
if (inputStream == null) {
logger.debug("Could not open table file:" + aTableList);
return false;
}
return readTableEntries(inputStream, aTableList, aTables);
}
private void checkSize(ServerDDS dds, boolean isAscii) throws Exception {
long size = computeSize(dds, isAscii);
// System.err.printf("total (constrained) size=%s\n", size);
log.debug("total (constrained) size={}", size);
double dsize = size / (1000 * 1000);
double maxSize = isAscii ? ascLimit : binLimit; // Mbytes
if (dsize > maxSize) {
log.info("Reject request size = {} Mbytes", dsize);
throw new UnsupportedOperationException(
"Request too big=" + dsize + " Mbytes, max=" + maxSize);
}
}
public RESTAuthenticationFilter(
CryptoEngine cryptoEngine,
Long contentMaxSize,
DigestAlgorithm digestAlgorithm,
long expiry,
ReplayAttackValidator replayAttackValidator) {
this.cryptoEngine = cryptoEngine;
this.contentMaxSize = contentMaxSize;
this.digestAlgorithm = digestAlgorithm;
this.expiry = expiry;
this.replayAttackValidator = replayAttackValidator;
logger.debug(
"REST Authentication filter using crypto engine: " + cryptoEngine.getClass().getName());
}
/**
* Write a file to the response stream.
*
* @param servlet called from here
* @param contentPath file root path
* @param path file path reletive to the root
* @param req the request
* @param res the response
* @param contentType content type, or null
* @throws IOException on write error
*/
public static void returnFile(
HttpServlet servlet,
String contentPath,
String path,
HttpServletRequest req,
HttpServletResponse res,
String contentType)
throws IOException {
String filename = ServletUtil.formFilename(contentPath, path);
log.debug("returnFile(): returning file <" + filename + ">.");
// No file, nothing to view
if (filename == null) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
res.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
// dontallow ..
if (filename.indexOf("..") != -1) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
res.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
// dont allow access to WEB-INF or META-INF
String upper = filename.toUpperCase();
if (upper.indexOf("WEB-INF") != -1 || upper.indexOf("META-INF") != -1) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
res.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
returnFile(servlet, req, res, new File(filename), contentType);
}
static void loadDefaults() {
RC rc0 = new RC();
String[] locations =
new String[] {
System.getProperty("user.home"), System.getProperty("user.dir"),
};
boolean found1 = false;
for (String loc : locations) {
if (loc == null) continue;
String dir = loc.replace('\\', '/');
if (dir.endsWith("/")) dir = dir.substring(0, dir.length() - 1);
for (String rcpath : rcfilelist) {
String filepath = loc + "/" + rcpath;
if (rc0.load(filepath)) found1 = true;
}
}
if (!found1) if (showlog) log.debug("No .rc file found");
dfaltRC = rc0;
}
public void doGetASC(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (ds == null) return;
response.setHeader("XDODS-Server", getServerVersion());
response.setContentType("text/plain");
response.setHeader("Content-Description", "dods-ascii");
log.debug(
"Sending OPeNDAP ASCII Data For: " + rs + " CE: '" + rs.getConstraintExpression() + "'");
ServerDDS dds = ds.getDDS();
CEEvaluator ce = new CEEvaluator(dds);
ce.parseConstraint(rs);
checkSize(dds, true);
PrintWriter pw = new PrintWriter(response.getOutputStream());
dds.printConstrained(pw);
pw.println("---------------------------------------------");
AsciiWriter writer = new AsciiWriter(); // could be static
writer.toASCII(pw, dds, ds);
// the way that getDAP2Data works
// DataOutputStream sink = new DataOutputStream(bOut);
// ce.send(myDDS.getName(), sink, ds);
pw.flush();
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGet(HttpServletRequest request, HttpServletResponse response) {
log.info("doGet(): " + UsageLog.setupRequestContext(request));
// System.out.printf("opendap doGet: req=%s%n%s%n", ServletUtil.getRequest(request),
// ServletUtil.showRequestDetail(this, request));
String path = null;
ReqState rs = getRequestState(request, response);
try {
path = request.getPathInfo();
log.debug("doGet path={}", path);
if (thredds.servlet.Debug.isSet("showRequestDetail"))
log.debug(ServletUtil.showRequestDetail(this, request));
if (path == null) {
log.info(
"doGet(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
response.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (baseURI == null) { // first time, set baseURI
URI reqURI = ServletUtil.getRequestURI(request);
// Build base URI from request (rather than hard-coding "/thredds/dodsC/").
String baseUriString = request.getContextPath() + request.getServletPath() + "/";
baseURI = reqURI.resolve(baseUriString);
log.debug("doGet(): baseURI was set = {}", baseURI);
}
if (path.endsWith("latest.xml")) {
DataRootHandler.getInstance().processReqForLatestDataset(this, request, response);
return;
}
// Redirect all catalog requests at the root level.
if (path.equals("/") || path.equals("/catalog.html") || path.equals("/catalog.xml")) {
ServletUtil.sendPermanentRedirect(ServletUtil.getContextPath() + path, request, response);
return;
}
// Make sure catalog requests match a dataRoot before trying to handle.
if (path.endsWith("/") || path.endsWith("/catalog.html") || path.endsWith("/catalog.xml")) {
if (!DataRootHandler.getInstance().hasDataRootMatch(path)) {
log.info(
"doGet(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
response.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (!DataRootHandler.getInstance().processReqForCatalog(request, response))
log.error(
"doGet(): "
+ UsageLog.closingMessageForRequestContext(
ServletUtil.STATUS_FORWARD_FAILURE, -1));
return;
}
if (rs != null) {
String dataSet = rs.getDataSet();
String requestSuffix = rs.getRequestSuffix();
if ((dataSet == null) || dataSet.equals("/") || dataSet.equals("")) {
doGetDIR(rs);
} else if (requestSuffix.equalsIgnoreCase("blob")) {
doGetBLOB(rs);
} else if (requestSuffix.equalsIgnoreCase("close")) {
doClose(rs);
} else if (requestSuffix.equalsIgnoreCase("dds")) {
doGetDDS(rs);
} else if (requestSuffix.equalsIgnoreCase("das")) {
doGetDAS(rs);
} else if (requestSuffix.equalsIgnoreCase("ddx")) {
doGetDDX(rs);
} else if (requestSuffix.equalsIgnoreCase("dods")) {
doGetDAP2Data(rs);
} else if (requestSuffix.equalsIgnoreCase("asc")
|| requestSuffix.equalsIgnoreCase("ascii")) {
doGetASC(rs);
} else if (requestSuffix.equalsIgnoreCase("info")) {
doGetINFO(rs);
} else if (requestSuffix.equalsIgnoreCase("html")
|| requestSuffix.equalsIgnoreCase("htm")) {
doGetHTML(rs);
} else if (requestSuffix.equalsIgnoreCase("ver")
|| requestSuffix.equalsIgnoreCase("version")
|| dataSet.equalsIgnoreCase("/version")
|| dataSet.equalsIgnoreCase("/version/")) {
doGetVER(rs);
} else if (dataSet.equalsIgnoreCase("/help")
|| dataSet.equalsIgnoreCase("/help/")
|| dataSet.equalsIgnoreCase("/" + requestSuffix)
|| requestSuffix.equalsIgnoreCase("help")) {
doGetHELP(rs);
} else {
sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
return;
}
} else {
sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
return;
}
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, -1));
// plain ol' 404
} catch (FileNotFoundException e) {
sendErrorResponse(response, HttpServletResponse.SC_NOT_FOUND, e.getMessage());
// DAP2Exception bad url
} catch (BadURLException e) {
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
dap2ExceptionHandler(e, rs);
// all other DAP2Exception
} catch (DAP2Exception de) {
int status =
(de.getErrorCode() == DAP2Exception.NO_SUCH_FILE)
? HttpServletResponse.SC_NOT_FOUND
: HttpServletResponse.SC_BAD_REQUEST;
if ((de.getErrorCode() != DAP2Exception.NO_SUCH_FILE) && (de.getErrorMessage() != null))
log.debug(de.getErrorMessage());
log.info(UsageLog.closingMessageForRequestContext(status, -1));
response.setStatus(status);
dap2ExceptionHandler(de, rs);
// parsing, usually the CE
} catch (ParseException pe) {
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
parseExceptionHandler(pe, response);
// 403 - request too big
} catch (UnsupportedOperationException e) {
sendErrorResponse(response, HttpServletResponse.SC_FORBIDDEN, e.getMessage());
} catch (java.net.SocketException e) {
log.info("SocketException: " + e.getMessage(), e);
log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));
} catch (IOException e) {
String eName =
e.getClass().getName(); // dont want compile time dependency on ClientAbortException
if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
log.debug("ClientAbortException: " + e.getMessage());
log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));
return;
}
log.error("path= " + path, e);
sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
// everything else
} catch (Throwable t) {
log.error("path= " + path, t);
t.printStackTrace();
sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, t.getMessage());
}
}
/**
* Write a file to the response stream. Handles Range requests.
*
* @param req request
* @param res response
* @param file must exists and not be a directory
* @param contentType must not be null
* @throws IOException or error
*/
public static void returnFile(
HttpServletRequest req, HttpServletResponse res, File file, String contentType)
throws IOException {
res.setContentType(contentType);
// see if its a Range Request
boolean isRangeRequest = false;
long startPos = 0, endPos = Long.MAX_VALUE;
String rangeRequest = req.getHeader("Range");
if (rangeRequest != null) { // bytes=12-34 or bytes=12-
int pos = rangeRequest.indexOf("=");
if (pos > 0) {
int pos2 = rangeRequest.indexOf("-");
if (pos2 > 0) {
String startString = rangeRequest.substring(pos + 1, pos2);
String endString = rangeRequest.substring(pos2 + 1);
startPos = Long.parseLong(startString);
if (endString.length() > 0) endPos = Long.parseLong(endString) + 1;
isRangeRequest = true;
}
}
}
// set content length
long fileSize = file.length();
long contentLength = fileSize;
if (isRangeRequest) {
endPos = Math.min(endPos, fileSize);
contentLength = endPos - startPos;
}
if (contentLength > Integer.MAX_VALUE)
res.addHeader(
"Content-Length", Long.toString(contentLength)); // allow content length > MAX_INT
else res.setContentLength((int) contentLength); // note HEAD only allows this
String filename = file.getPath();
boolean debugRequest = Debug.isSet("returnFile");
if (debugRequest)
log.debug(
"returnFile(): filename = "
+ filename
+ " contentType = "
+ contentType
+ " contentLength = "
+ contentLength);
// indicate we allow Range Requests
res.addHeader("Accept-Ranges", "bytes");
if (req.getMethod().equals("HEAD")) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, 0));
return;
}
try {
if (isRangeRequest) {
// set before content is sent
res.addHeader("Content-Range", "bytes " + startPos + "-" + (endPos - 1) + "/" + fileSize);
res.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
FileCacheRaf.Raf craf = null;
try {
craf = fileCacheRaf.acquire(filename);
IO.copyRafB(
craf.getRaf(), startPos, contentLength, res.getOutputStream(), new byte[60000]);
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(
HttpServletResponse.SC_PARTIAL_CONTENT, contentLength));
return;
} finally {
if (craf != null) fileCacheRaf.release(craf);
}
}
// Return the file
ServletOutputStream out = res.getOutputStream();
IO.copyFileB(file, out, 60000);
res.flushBuffer();
out.close();
if (debugRequest) log.debug("returnFile(): returnFile ok = " + filename);
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, contentLength));
}
// @todo Split up this exception handling: those from file access vs those from dealing with
// response
// File access: catch and res.sendError()
// response: don't catch (let bubble up out of doGet() etc)
catch (FileNotFoundException e) {
log.error("returnFile(): FileNotFoundException= " + filename);
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_NOT_FOUND);
} catch (java.net.SocketException e) {
log.info("returnFile(): SocketException sending file: " + filename + " " + e.getMessage());
log.info("returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
} catch (IOException e) {
String eName =
e.getClass().getName(); // dont want compile time dependency on ClientAbortException
if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
log.info(
"returnFile(): ClientAbortException while sending file: "
+ filename
+ " "
+ e.getMessage());
log.info(
"returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
return;
}
log.error("returnFile(): IOException (" + e.getClass().getName() + ") sending file ", e);
log.error(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
if (!res.isCommitted())
res.sendError(HttpServletResponse.SC_NOT_FOUND, "Problem sending file: " + e.getMessage());
}
}
/**
* Write equivilent uncompressed version of the file.
*
* @param inputRaf file to uncompress
* @param ufilename write to this file
* @return raf of uncompressed file
* @throws IOException on read error
*/
private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
throws IOException {
RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
FileLock lock = null;
while (true) { // loop waiting for the lock
try {
lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
}
}
}
try {
inputRaf.seek(0);
byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
inputRaf.read(header);
outputRaf.write(header);
boolean eof = false;
int numCompBytes;
byte[] ubuff = new byte[40000];
byte[] obuff = new byte[40000];
try {
CBZip2InputStream cbzip2 = new CBZip2InputStream();
while (!eof) {
try {
numCompBytes = inputRaf.readInt();
if (numCompBytes == -1) {
if (log.isDebugEnabled()) log.debug(" done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
log.warn(" got EOFException ");
break; // assume this is ok
}
if (log.isDebugEnabled()) {
log.debug(
"reading compressed bytes "
+ numCompBytes
+ " input starts at "
+ inputRaf.getFilePointer()
+ "; output starts at "
+ outputRaf.getFilePointer());
}
/*
* For some stupid reason, the last block seems to
* have the number of bytes negated. So, we just
* assume that any negative number (other than -1)
* is the last block and go on our merry little way.
*/
if (numCompBytes < 0) {
if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
numCompBytes = -numCompBytes;
eof = true;
}
byte[] buf = new byte[numCompBytes];
inputRaf.readFully(buf);
ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);
// CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
cbzip2.setStream(bis);
int total = 0;
int nread;
/*
while ((nread = cbzip2.read(ubuff)) != -1) {
dout2.write(ubuff, 0, nread);
total += nread;
}
*/
try {
while ((nread = cbzip2.read(ubuff)) != -1) {
if (total + nread > obuff.length) {
byte[] temp = obuff;
obuff = new byte[temp.length * 2];
System.arraycopy(temp, 0, obuff, 0, temp.length);
}
System.arraycopy(ubuff, 0, obuff, total, nread);
total += nread;
}
if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
} catch (BZip2ReadException ioe) {
log.warn("Nexrad2IOSP.uncompress ", ioe);
}
float nrecords = (float) (total / 2432.0);
if (log.isDebugEnabled())
log.debug(
" unpacked "
+ total
+ " num bytes "
+ nrecords
+ " records; ouput ends at "
+ outputRaf.getFilePointer());
}
} catch (Exception e) {
if (outputRaf != null) outputRaf.close();
outputRaf = null;
// dont leave bad files around
File ufile = new File(ufilename);
if (ufile.exists()) {
if (!ufile.delete())
log.warn("failed to delete uncompressed file (IOException)" + ufilename);
}
if (e instanceof IOException) throw (IOException) e;
else throw new RuntimeException(e);
}
} finally {
if (null != outputRaf) outputRaf.flush();
if (lock != null) lock.release();
}
return outputRaf;
}
Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
this.raf = orgRaf;
if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());
raf.seek(0);
raf.order(RandomAccessFile.BIG_ENDIAN);
// volume scan header
dataFormat = raf.readString(8);
raf.skipBytes(1);
String volumeNo = raf.readString(3);
title_julianDay = raf.readInt(); // since 1/1/70
title_msecs = raf.readInt();
stationId = raf.readString(4).trim(); // only in AR2V0001
if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);
if (stationId.length() == 0) {
// try to get it from the filename LOOK
stationId = null;
}
// try to find the station
if (stationId != null) {
if (!stationId.startsWith("K") && stationId.length() == 4) {
String _stationId = "K" + stationId;
station = NexradStationDB.get(_stationId);
} else station = NexradStationDB.get(stationId);
}
// see if we have to uncompress
if (dataFormat.equals(AR2V0001)
|| dataFormat.equals(AR2V0003)
|| dataFormat.equals(AR2V0004)
|| dataFormat.equals(AR2V0006)) {
raf.skipBytes(4);
String BZ = raf.readString(2);
if (BZ.equals("BZ")) {
RandomAccessFile uraf;
File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");
if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
// see if its locked - another thread is writing it
FileInputStream fstream = null;
FileLock lock = null;
try {
fstream = new FileInputStream(uncompressedFile);
// lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
while (true) { // loop waiting for the lock
try {
lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
break;
}
}
}
} finally {
if (lock != null) lock.release();
if (fstream != null) fstream.close();
}
uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");
} else {
// nope, gotta uncompress it
uraf = uncompress(raf, uncompressedFile.getPath());
if (log.isDebugEnabled())
log.debug("made uncompressed file= " + uncompressedFile.getPath());
}
// switch to uncompressed file
raf.close();
raf = uraf;
raf.order(RandomAccessFile.BIG_ENDIAN);
}
raf.seek(Level2Record.FILE_HEADER_SIZE);
}
List<Level2Record> reflectivity = new ArrayList<Level2Record>();
List<Level2Record> doppler = new ArrayList<Level2Record>();
List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highVelocity = new ArrayList<Level2Record>();
List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();
long message_offset31 = 0;
int recno = 0;
while (true) {
Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
if (r == null) break;
if (showData) r.dump2(System.out);
// skip non-data messages
if (r.message_type == 31) {
message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
}
if (r.message_type != 1 && r.message_type != 31) {
if (showMessages) r.dumpMessage(System.out);
continue;
}
// if (showData) r.dump2(System.out);
/* skip bad
if (!r.checkOk()) {
r.dump(System.out);
continue;
} */
// some global params
if (vcp == 0) vcp = r.vcp;
if (first == null) first = r;
last = r;
if (runCheck && !r.checkOk()) {
continue;
}
if (r.hasReflectData) reflectivity.add(r);
if (r.hasDopplerData) doppler.add(r);
if (r.message_type == 31) {
if (r.hasHighResREFData) highReflectivity.add(r);
if (r.hasHighResVELData) highVelocity.add(r);
if (r.hasHighResSWData) highSpectrum.add(r);
if (r.hasHighResZDRData) highDiffReflectivity.add(r);
if (r.hasHighResPHIData) highDiffPhase.add(r);
if (r.hasHighResRHOData) highCorreCoefficient.add(r);
}
if ((cancelTask != null) && cancelTask.isCancel()) return;
}
if (debugRadials)
System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
if (highReflectivity.size() == 0) {
reflectivityGroups = sortScans("reflect", reflectivity, 600);
dopplerGroups = sortScans("doppler", doppler, 600);
}
if (highReflectivity.size() > 0)
reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
if (highVelocity.size() > 0)
velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
if (highSpectrum.size() > 0)
spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
if (highDiffReflectivity.size() > 0)
diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
if (highDiffPhase.size() > 0)
diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
if (highCorreCoefficient.size() > 0)
coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
}