/**
* Read the grid
*
* @param gr the grid record
* @return the data
*/
public float[] readGrid(McIDASGridRecord gr) {
float[] data = null;
try {
int te = (gr.getOffsetToHeader() + 64) * 4;
int rows = gr.getRows();
int cols = gr.getColumns();
rf.seek(te);
float scale = (float) gr.getParamScale();
data = new float[rows * cols];
rf.order(needToSwap ? rf.LITTLE_ENDIAN : rf.BIG_ENDIAN);
int n = 0;
// store such that 0,0 is in lower left corner...
for (int nc = 0; nc < cols; nc++) {
for (int nr = 0; nr < rows; nr++) {
int temp = rf.readInt(); // check for missing value
data[(rows - nr - 1) * cols + nc] =
(temp == McIDASUtil.MCMISSING) ? Float.NaN : ((float) temp) / scale;
}
}
rf.order(rf.BIG_ENDIAN);
} catch (Exception esc) {
System.out.println(esc);
}
return data;
}
/**
* Read an integer
*
* @param word word in file (0 based) to read
* @return int read
* @throws IOException problem reading file
*/
public int readInt(int word) throws IOException {
if (rf == null) {
throw new IOException("no file to read from");
}
rf.seek(word * 4);
// set the order
if (needToSwap) {
rf.order(RandomAccessFile.LITTLE_ENDIAN); // swap
} else {
rf.order(RandomAccessFile.BIG_ENDIAN);
}
int idata = rf.readInt();
rf.order(RandomAccessFile.BIG_ENDIAN);
return idata;
}
/**
* Read some global data from SIGMET file. The SIGMET file consists of records with fixed
* length=6144 bytes.
*/
public static java.util.Map<String, Number> readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) {
java.util.Map<String, Number> recHdr1 = new java.util.HashMap<String, Number>();
try {
int nparams = 0;
// -- Read from <product_end> of the 1st record -- 12+320+120
// -- Calculate Nyquist velocity --------------------
raf.seek(452);
int prf = raf.readInt();
raf.seek(480);
int wave = raf.readInt();
float vNyq = calcNyquist(prf, wave);
recHdr1.put("vNyq", vNyq);
// -- Read from the 2nd record----------- 6144+12(strucr_hdr)+168(from ingest_config)
raf.seek(6324);
int radar_lat = raf.readInt();
int radar_lon = raf.readInt(); // 6328
short ground_height = raf.readShort(); // 6332
short radar_height = raf.readShort(); // 6334
raf.skipBytes(4);
short num_rays = raf.readShort(); // 6340
raf.skipBytes(2);
int radar_alt = raf.readInt(); // 6344
raf.seek(6648);
int time_beg = raf.readInt();
raf.seek(6652);
int time_end = raf.readInt();
raf.seek(6772);
int data_mask = raf.readInt();
for (int j = 0; j < 32; j++) {
nparams += ((data_mask >> j) & (0x1));
}
raf.seek(6912);
short multiprf = raf.readShort();
raf.seek(7408);
int range_first = raf.readInt(); // cm 7408
int range_last = raf.readInt(); // cm 7412
raf.skipBytes(2);
short bins = raf.readShort(); // 7418
if (bins % 2 != 0) bins = (short) (bins + 1);
raf.skipBytes(4);
int step = raf.readInt(); // cm 7424
raf.seek(7574);
short number_sweeps = raf.readShort(); // 7574
raf.seek(12312);
int base_time = raf.readInt(); // <ingest_data_header> 3d rec
raf.skipBytes(2);
short year = raf.readShort();
short month = raf.readShort();
short day = raf.readShort();
recHdr1.put("radar_lat", calcAngle(radar_lat));
recHdr1.put("radar_lon", calcAngle(radar_lon));
recHdr1.put("range_first", range_first);
recHdr1.put("range_last", range_last);
recHdr1.put("ground_height", ground_height);
recHdr1.put("radar_height", radar_height);
recHdr1.put("radar_alt", radar_alt);
recHdr1.put("step", step);
recHdr1.put("bins", bins); // System.out.println(" bins="+bins);
recHdr1.put("num_rays", num_rays); // System.out.println(" rays="+num_rays);
recHdr1.put("nparams", nparams); // System.out.println(" nparams="+nparams);
recHdr1.put("multiprf", multiprf);
recHdr1.put(
"number_sweeps",
number_sweeps); // System.out.println("IN HDR: number_sweeps="+number_sweeps);
recHdr1.put("year", year);
recHdr1.put("month", month);
recHdr1.put("day", day);
recHdr1.put("base_time", base_time);
} catch (Exception e) {
System.out.println(e.toString());
e.printStackTrace();
}
return recHdr1;
}
public boolean readIndex(RandomAccessFile raf) {
gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
try {
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
//// header message
if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
logger.error("GribCollection {}: invalid index", gc.getName());
return false;
}
int v = raf.readInt();
if (v != getVersion()) {
logger.warn(
"GribCollection {}: index found version={}, want version= {} on file {}",
new Object[] {gc.getName(), v, version, raf.getLocation()});
return false;
}
long skip = raf.readLong();
raf.skipBytes(skip);
int size = NcStream.readVInt(raf);
if ((size < 0) || (size > 100 * 1000 * 1000)) {
logger.warn("GribCollection {}: invalid index ", gc.getName());
return false;
}
byte[] m = new byte[size];
raf.readFully(m);
GribCollectionProto.GribCollectionIndex proto =
GribCollectionProto.GribCollectionIndex.parseFrom(m);
gc.center = proto.getCenter();
gc.subcenter = proto.getSubcenter();
gc.master = proto.getMaster();
gc.local = proto.getLocal();
gc.genProcessType = proto.getGenProcessType();
gc.genProcessId = proto.getGenProcessId();
gc.backProcessId = proto.getBackProcessId();
gc.local = proto.getLocal();
// gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);
gc.filenames = new ArrayList<String>(proto.getFilesCount());
for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));
// error condition on a GribCollection Index
if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
return false;
}
gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
for (int i = 0; i < proto.getGroupsCount(); i++)
gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
Collections.sort(gc.groups);
gc.params = new ArrayList<Parameter>(proto.getParamsCount());
for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));
if (!readPartitions(proto)) {
logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
return false;
}
return true;
} catch (Throwable t) {
logger.error("Error reading index " + raf.getLocation(), t);
return false;
}
}
// check if compressed file seems ok
public static long testValid(String ufilename) throws IOException {
boolean lookForHeader = false;
// gotta make it
RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
byte[] b = new byte[8];
raf.read(b);
String test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println("--Good header= " + test);
raf.seek(24);
} else {
System.out.println("--No header ");
lookForHeader = true;
raf.seek(0);
}
boolean eof = false;
int numCompBytes;
try {
while (!eof) {
if (lookForHeader) {
raf.read(b);
test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println(" found header= " + test);
raf.skipBytes(16);
lookForHeader = false;
} else {
raf.skipBytes(-8);
}
}
try {
numCompBytes = raf.readInt();
if (numCompBytes == -1) {
System.out.println("\n--done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
System.out.println("\n--got EOFException ");
break; // assume this is ok
}
System.out.print(" " + numCompBytes + ",");
if (numCompBytes < 0) {
System.out.println("\n--last block " + numCompBytes);
numCompBytes = -numCompBytes;
if (!lookForHeader) eof = true;
}
raf.skipBytes(numCompBytes);
}
} catch (EOFException e) {
e.printStackTrace();
}
return raf.getFilePointer();
}
/**
* Write equivilent uncompressed version of the file.
*
* @param inputRaf file to uncompress
* @param ufilename write to this file
* @return raf of uncompressed file
* @throws IOException on read error
*/
private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
throws IOException {
RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
FileLock lock = null;
while (true) { // loop waiting for the lock
try {
lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
}
}
}
try {
inputRaf.seek(0);
byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
inputRaf.read(header);
outputRaf.write(header);
boolean eof = false;
int numCompBytes;
byte[] ubuff = new byte[40000];
byte[] obuff = new byte[40000];
try {
CBZip2InputStream cbzip2 = new CBZip2InputStream();
while (!eof) {
try {
numCompBytes = inputRaf.readInt();
if (numCompBytes == -1) {
if (log.isDebugEnabled()) log.debug(" done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
log.warn(" got EOFException ");
break; // assume this is ok
}
if (log.isDebugEnabled()) {
log.debug(
"reading compressed bytes "
+ numCompBytes
+ " input starts at "
+ inputRaf.getFilePointer()
+ "; output starts at "
+ outputRaf.getFilePointer());
}
/*
* For some stupid reason, the last block seems to
* have the number of bytes negated. So, we just
* assume that any negative number (other than -1)
* is the last block and go on our merry little way.
*/
if (numCompBytes < 0) {
if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
numCompBytes = -numCompBytes;
eof = true;
}
byte[] buf = new byte[numCompBytes];
inputRaf.readFully(buf);
ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);
// CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
cbzip2.setStream(bis);
int total = 0;
int nread;
/*
while ((nread = cbzip2.read(ubuff)) != -1) {
dout2.write(ubuff, 0, nread);
total += nread;
}
*/
try {
while ((nread = cbzip2.read(ubuff)) != -1) {
if (total + nread > obuff.length) {
byte[] temp = obuff;
obuff = new byte[temp.length * 2];
System.arraycopy(temp, 0, obuff, 0, temp.length);
}
System.arraycopy(ubuff, 0, obuff, total, nread);
total += nread;
}
if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
} catch (BZip2ReadException ioe) {
log.warn("Nexrad2IOSP.uncompress ", ioe);
}
float nrecords = (float) (total / 2432.0);
if (log.isDebugEnabled())
log.debug(
" unpacked "
+ total
+ " num bytes "
+ nrecords
+ " records; ouput ends at "
+ outputRaf.getFilePointer());
}
} catch (Exception e) {
if (outputRaf != null) outputRaf.close();
outputRaf = null;
// dont leave bad files around
File ufile = new File(ufilename);
if (ufile.exists()) {
if (!ufile.delete())
log.warn("failed to delete uncompressed file (IOException)" + ufilename);
}
if (e instanceof IOException) throw (IOException) e;
else throw new RuntimeException(e);
}
} finally {
if (null != outputRaf) outputRaf.flush();
if (lock != null) lock.release();
}
return outputRaf;
}
Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
this.raf = orgRaf;
if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());
raf.seek(0);
raf.order(RandomAccessFile.BIG_ENDIAN);
// volume scan header
dataFormat = raf.readString(8);
raf.skipBytes(1);
String volumeNo = raf.readString(3);
title_julianDay = raf.readInt(); // since 1/1/70
title_msecs = raf.readInt();
stationId = raf.readString(4).trim(); // only in AR2V0001
if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);
if (stationId.length() == 0) {
// try to get it from the filename LOOK
stationId = null;
}
// try to find the station
if (stationId != null) {
if (!stationId.startsWith("K") && stationId.length() == 4) {
String _stationId = "K" + stationId;
station = NexradStationDB.get(_stationId);
} else station = NexradStationDB.get(stationId);
}
// see if we have to uncompress
if (dataFormat.equals(AR2V0001)
|| dataFormat.equals(AR2V0003)
|| dataFormat.equals(AR2V0004)
|| dataFormat.equals(AR2V0006)) {
raf.skipBytes(4);
String BZ = raf.readString(2);
if (BZ.equals("BZ")) {
RandomAccessFile uraf;
File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");
if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
// see if its locked - another thread is writing it
FileInputStream fstream = null;
FileLock lock = null;
try {
fstream = new FileInputStream(uncompressedFile);
// lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
while (true) { // loop waiting for the lock
try {
lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
break;
}
}
}
} finally {
if (lock != null) lock.release();
if (fstream != null) fstream.close();
}
uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");
} else {
// nope, gotta uncompress it
uraf = uncompress(raf, uncompressedFile.getPath());
if (log.isDebugEnabled())
log.debug("made uncompressed file= " + uncompressedFile.getPath());
}
// switch to uncompressed file
raf.close();
raf = uraf;
raf.order(RandomAccessFile.BIG_ENDIAN);
}
raf.seek(Level2Record.FILE_HEADER_SIZE);
}
List<Level2Record> reflectivity = new ArrayList<Level2Record>();
List<Level2Record> doppler = new ArrayList<Level2Record>();
List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highVelocity = new ArrayList<Level2Record>();
List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();
long message_offset31 = 0;
int recno = 0;
while (true) {
Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
if (r == null) break;
if (showData) r.dump2(System.out);
// skip non-data messages
if (r.message_type == 31) {
message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
}
if (r.message_type != 1 && r.message_type != 31) {
if (showMessages) r.dumpMessage(System.out);
continue;
}
// if (showData) r.dump2(System.out);
/* skip bad
if (!r.checkOk()) {
r.dump(System.out);
continue;
} */
// some global params
if (vcp == 0) vcp = r.vcp;
if (first == null) first = r;
last = r;
if (runCheck && !r.checkOk()) {
continue;
}
if (r.hasReflectData) reflectivity.add(r);
if (r.hasDopplerData) doppler.add(r);
if (r.message_type == 31) {
if (r.hasHighResREFData) highReflectivity.add(r);
if (r.hasHighResVELData) highVelocity.add(r);
if (r.hasHighResSWData) highSpectrum.add(r);
if (r.hasHighResZDRData) highDiffReflectivity.add(r);
if (r.hasHighResPHIData) highDiffPhase.add(r);
if (r.hasHighResRHOData) highCorreCoefficient.add(r);
}
if ((cancelTask != null) && cancelTask.isCancel()) return;
}
if (debugRadials)
System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
if (highReflectivity.size() == 0) {
reflectivityGroups = sortScans("reflect", reflectivity, 600);
dopplerGroups = sortScans("doppler", doppler, 600);
}
if (highReflectivity.size() > 0)
reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
if (highVelocity.size() > 0)
velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
if (highSpectrum.size() > 0)
spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
if (highDiffReflectivity.size() > 0)
diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
if (highDiffPhase.size() > 0)
diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
if (highCorreCoefficient.size() > 0)
coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
}
/**
* Initialize this reader. Get the Grid specific info
*
* @param fullCheck for a full check reading grids
* @return true if successful
* @throws IOException problem reading the data
*/
protected boolean init(boolean fullCheck) throws IOException {
if (rf == null) {
logError("File is null");
return false;
}
gridIndex = new GridIndex(rf.getLocation());
rf.order(RandomAccessFile.BIG_ENDIAN);
int numEntries = Math.abs(readInt(10));
if (numEntries > 1000000) {
needToSwap = true;
numEntries = Math.abs(McIDASUtil.swbyt4(numEntries));
}
if (numEntries > MAX_GRIDS) {
return false;
}
// System.out.println("need to Swap = " + needToSwap);
// System.out.println("number entries="+numEntries);
// go back to the beginning
rf.seek(0);
// read the fileheader
String label = rf.readString(32);
// GEMPAK too closely like McIDAS
if (label.indexOf("GEMPAK DATA MANAGEMENT FILE") >= 0) {
logError("label indicates this is a GEMPAK grid");
return false;
} else { // check that they are all printable ASCII chars
for (int i = 0; i < label.length(); i++) {
String s0 = label.substring(i, i + 1);
if (!(0 <= s0.compareTo(" ") && s0.compareTo("~") <= 0)) {
logError("bad label, not a McIDAS grid");
return false;
}
}
}
// System.out.println("label = " + label);
int project = readInt(8);
// System.out.println("Project = " + project);
int date = readInt(9);
// dates are supposed to be yyyddd, but account for ccyyddd up to year 4000
if ((date < 10000) || (date > 400000)) {
logError("date wrong, not a McIDAS grid");
return false;
}
// System.out.println("date = " + date);
int[] entries = new int[numEntries];
for (int i = 0; i < numEntries; i++) {
entries[i] = readInt(i + 11);
// sanity check that this is indeed a McIDAS Grid file
if (entries[i] < -1) {
logError("bad grid offset " + i + ": " + entries[i]);
return false;
}
}
if (!fullCheck) {
return true;
}
// Don't swap:
rf.order(RandomAccessFile.BIG_ENDIAN);
for (int i = 0; i < numEntries; i++) {
if (entries[i] == -1) {
continue;
}
int[] header = new int[64];
rf.seek(entries[i] * 4);
rf.readInt(header, 0, 64);
if (needToSwap) {
swapGridHeader(header);
}
try {
McIDASGridRecord gr = new McIDASGridRecord(entries[i], header);
// if (gr.getGridDefRecordId().equals("CONF X:93 Y:65")) {
// if (gr.getGridDefRecordId().equals("CONF X:54 Y:47")) {
// figure out how to handle Mercator projections
// if ( !(gr.getGridDefRecordId().startsWith("MERC"))) {
gridIndex.addGridRecord(gr);
if (gdsMap.get(gr.getGridDefRecordId()) == null) {
McGridDefRecord mcdef = gr.getGridDefRecord();
// System.out.println("new nav " + mcdef.toString());
gdsMap.put(mcdef.toString(), mcdef);
gridIndex.addHorizCoordSys(mcdef);
}
// }
} catch (McIDASException me) {
logError("problem creating grid dir");
return false;
}
}
// check to see if there are any grids that we can handle
if (gridIndex.getGridRecords().isEmpty()) {
logError("no grids found");
return false;
}
return true;
}
