public boolean isValidFile(RandomAccessFile raf) {
int data_msecs = 0;
short data_julian_date = 0;
try {
raf.order(RandomAccessFile.LITTLE_ENDIAN);
raf.seek(0);
raf.skipBytes(14);
short message_type = raf.readShort();
if (message_type != 1) return false;
raf.skipBytes(12);
// data header
byte[] b4 = raf.readBytes(4);
data_msecs = bytesToInt(b4, true);
byte[] b2 = raf.readBytes(2);
data_julian_date = (short) bytesToShort(b2, true);
java.util.Date dd = Cinrad2Record.getDate(data_julian_date, data_msecs);
Calendar cal = new GregorianCalendar(new SimpleTimeZone(0, "GMT"));
cal.clear();
cal.setTime(dd);
int year = cal.get(Calendar.YEAR);
cal.setTime(new Date());
int cyear = cal.get(Calendar.YEAR);
if (year < 1990 || year > cyear) return false;
return true;
} catch (IOException ioe) {
return false;
}
}
public static boolean isValidFile(RandomAccessFile raf) {
try {
raf.seek(0);
while (raf.getFilePointer() < maxScan) {
boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
if (!found) return false;
raf.skipBytes(7); // will be positioned on byte 0 of indicator section
int edition = raf.read(); // read at byte 8
if (edition == 2) return true;
}
} catch (IOException e) {
return false;
}
return false;
}
public boolean hasNext() throws IOException {
if (lastPos >= raf.length()) return false;
if (repeatPos > 0) {
if (nextRepeating()) // this has created a new repeatRecord
return true;
} else {
repeatRecord = null;
repeatBms = null;
// fall through to new record
}
boolean more;
long stop = 0;
while (true) { // scan until we get a GRIB-2 or more == false
raf.seek(lastPos);
more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
if (!more) break;
stop = raf.getFilePointer();
// see if its GRIB-2
raf.skipBytes(7);
int edition = raf.read();
if (edition == 2) break;
lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
}
if (more) {
int sizeHeader =
(int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
// if (sizeHeader > 30) sizeHeader = 30;
header = new byte[sizeHeader];
startPos = stop - sizeHeader;
raf.seek(startPos);
raf.read(header);
}
if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
return more;
}
/**
* Read some global data from SIGMET file. The SIGMET file consists of records with fixed
* length=6144 bytes.
*/
public static java.util.Map<String, Number> readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) {
java.util.Map<String, Number> recHdr1 = new java.util.HashMap<String, Number>();
try {
int nparams = 0;
// -- Read from <product_end> of the 1st record -- 12+320+120
// -- Calculate Nyquist velocity --------------------
raf.seek(452);
int prf = raf.readInt();
raf.seek(480);
int wave = raf.readInt();
float vNyq = calcNyquist(prf, wave);
recHdr1.put("vNyq", vNyq);
// -- Read from the 2nd record----------- 6144+12(strucr_hdr)+168(from ingest_config)
raf.seek(6324);
int radar_lat = raf.readInt();
int radar_lon = raf.readInt(); // 6328
short ground_height = raf.readShort(); // 6332
short radar_height = raf.readShort(); // 6334
raf.skipBytes(4);
short num_rays = raf.readShort(); // 6340
raf.skipBytes(2);
int radar_alt = raf.readInt(); // 6344
raf.seek(6648);
int time_beg = raf.readInt();
raf.seek(6652);
int time_end = raf.readInt();
raf.seek(6772);
int data_mask = raf.readInt();
for (int j = 0; j < 32; j++) {
nparams += ((data_mask >> j) & (0x1));
}
raf.seek(6912);
short multiprf = raf.readShort();
raf.seek(7408);
int range_first = raf.readInt(); // cm 7408
int range_last = raf.readInt(); // cm 7412
raf.skipBytes(2);
short bins = raf.readShort(); // 7418
if (bins % 2 != 0) bins = (short) (bins + 1);
raf.skipBytes(4);
int step = raf.readInt(); // cm 7424
raf.seek(7574);
short number_sweeps = raf.readShort(); // 7574
raf.seek(12312);
int base_time = raf.readInt(); // <ingest_data_header> 3d rec
raf.skipBytes(2);
short year = raf.readShort();
short month = raf.readShort();
short day = raf.readShort();
recHdr1.put("radar_lat", calcAngle(radar_lat));
recHdr1.put("radar_lon", calcAngle(radar_lon));
recHdr1.put("range_first", range_first);
recHdr1.put("range_last", range_last);
recHdr1.put("ground_height", ground_height);
recHdr1.put("radar_height", radar_height);
recHdr1.put("radar_alt", radar_alt);
recHdr1.put("step", step);
recHdr1.put("bins", bins); // System.out.println(" bins="+bins);
recHdr1.put("num_rays", num_rays); // System.out.println(" rays="+num_rays);
recHdr1.put("nparams", nparams); // System.out.println(" nparams="+nparams);
recHdr1.put("multiprf", multiprf);
recHdr1.put(
"number_sweeps",
number_sweeps); // System.out.println("IN HDR: number_sweeps="+number_sweeps);
recHdr1.put("year", year);
recHdr1.put("month", month);
recHdr1.put("day", day);
recHdr1.put("base_time", base_time);
} catch (Exception e) {
System.out.println(e.toString());
e.printStackTrace();
}
return recHdr1;
}
public boolean readIndex(RandomAccessFile raf) {
gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
try {
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
//// header message
if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
logger.error("GribCollection {}: invalid index", gc.getName());
return false;
}
int v = raf.readInt();
if (v != getVersion()) {
logger.warn(
"GribCollection {}: index found version={}, want version= {} on file {}",
new Object[] {gc.getName(), v, version, raf.getLocation()});
return false;
}
long skip = raf.readLong();
raf.skipBytes(skip);
int size = NcStream.readVInt(raf);
if ((size < 0) || (size > 100 * 1000 * 1000)) {
logger.warn("GribCollection {}: invalid index ", gc.getName());
return false;
}
byte[] m = new byte[size];
raf.readFully(m);
GribCollectionProto.GribCollectionIndex proto =
GribCollectionProto.GribCollectionIndex.parseFrom(m);
gc.center = proto.getCenter();
gc.subcenter = proto.getSubcenter();
gc.master = proto.getMaster();
gc.local = proto.getLocal();
gc.genProcessType = proto.getGenProcessType();
gc.genProcessId = proto.getGenProcessId();
gc.backProcessId = proto.getBackProcessId();
gc.local = proto.getLocal();
// gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);
gc.filenames = new ArrayList<String>(proto.getFilesCount());
for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));
// error condition on a GribCollection Index
if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
return false;
}
gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
for (int i = 0; i < proto.getGroupsCount(); i++)
gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
Collections.sort(gc.groups);
gc.params = new ArrayList<Parameter>(proto.getParamsCount());
for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));
if (!readPartitions(proto)) {
logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
return false;
}
return true;
} catch (Throwable t) {
logger.error("Error reading index " + raf.getLocation(), t);
return false;
}
}
// check if compressed file seems ok
public static long testValid(String ufilename) throws IOException {
boolean lookForHeader = false;
// gotta make it
RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
byte[] b = new byte[8];
raf.read(b);
String test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println("--Good header= " + test);
raf.seek(24);
} else {
System.out.println("--No header ");
lookForHeader = true;
raf.seek(0);
}
boolean eof = false;
int numCompBytes;
try {
while (!eof) {
if (lookForHeader) {
raf.read(b);
test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println(" found header= " + test);
raf.skipBytes(16);
lookForHeader = false;
} else {
raf.skipBytes(-8);
}
}
try {
numCompBytes = raf.readInt();
if (numCompBytes == -1) {
System.out.println("\n--done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
System.out.println("\n--got EOFException ");
break; // assume this is ok
}
System.out.print(" " + numCompBytes + ",");
if (numCompBytes < 0) {
System.out.println("\n--last block " + numCompBytes);
numCompBytes = -numCompBytes;
if (!lookForHeader) eof = true;
}
raf.skipBytes(numCompBytes);
}
} catch (EOFException e) {
e.printStackTrace();
}
return raf.getFilePointer();
}
Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
this.raf = orgRaf;
if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());
raf.seek(0);
raf.order(RandomAccessFile.BIG_ENDIAN);
// volume scan header
dataFormat = raf.readString(8);
raf.skipBytes(1);
String volumeNo = raf.readString(3);
title_julianDay = raf.readInt(); // since 1/1/70
title_msecs = raf.readInt();
stationId = raf.readString(4).trim(); // only in AR2V0001
if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);
if (stationId.length() == 0) {
// try to get it from the filename LOOK
stationId = null;
}
// try to find the station
if (stationId != null) {
if (!stationId.startsWith("K") && stationId.length() == 4) {
String _stationId = "K" + stationId;
station = NexradStationDB.get(_stationId);
} else station = NexradStationDB.get(stationId);
}
// see if we have to uncompress
if (dataFormat.equals(AR2V0001)
|| dataFormat.equals(AR2V0003)
|| dataFormat.equals(AR2V0004)
|| dataFormat.equals(AR2V0006)) {
raf.skipBytes(4);
String BZ = raf.readString(2);
if (BZ.equals("BZ")) {
RandomAccessFile uraf;
File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");
if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
// see if its locked - another thread is writing it
FileInputStream fstream = null;
FileLock lock = null;
try {
fstream = new FileInputStream(uncompressedFile);
// lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
while (true) { // loop waiting for the lock
try {
lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
break;
}
}
}
} finally {
if (lock != null) lock.release();
if (fstream != null) fstream.close();
}
uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");
} else {
// nope, gotta uncompress it
uraf = uncompress(raf, uncompressedFile.getPath());
if (log.isDebugEnabled())
log.debug("made uncompressed file= " + uncompressedFile.getPath());
}
// switch to uncompressed file
raf.close();
raf = uraf;
raf.order(RandomAccessFile.BIG_ENDIAN);
}
raf.seek(Level2Record.FILE_HEADER_SIZE);
}
List<Level2Record> reflectivity = new ArrayList<Level2Record>();
List<Level2Record> doppler = new ArrayList<Level2Record>();
List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highVelocity = new ArrayList<Level2Record>();
List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();
long message_offset31 = 0;
int recno = 0;
while (true) {
Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
if (r == null) break;
if (showData) r.dump2(System.out);
// skip non-data messages
if (r.message_type == 31) {
message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
}
if (r.message_type != 1 && r.message_type != 31) {
if (showMessages) r.dumpMessage(System.out);
continue;
}
// if (showData) r.dump2(System.out);
/* skip bad
if (!r.checkOk()) {
r.dump(System.out);
continue;
} */
// some global params
if (vcp == 0) vcp = r.vcp;
if (first == null) first = r;
last = r;
if (runCheck && !r.checkOk()) {
continue;
}
if (r.hasReflectData) reflectivity.add(r);
if (r.hasDopplerData) doppler.add(r);
if (r.message_type == 31) {
if (r.hasHighResREFData) highReflectivity.add(r);
if (r.hasHighResVELData) highVelocity.add(r);
if (r.hasHighResSWData) highSpectrum.add(r);
if (r.hasHighResZDRData) highDiffReflectivity.add(r);
if (r.hasHighResPHIData) highDiffPhase.add(r);
if (r.hasHighResRHOData) highCorreCoefficient.add(r);
}
if ((cancelTask != null) && cancelTask.isCancel()) return;
}
if (debugRadials)
System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
if (highReflectivity.size() == 0) {
reflectivityGroups = sortScans("reflect", reflectivity, 600);
dopplerGroups = sortScans("doppler", doppler, 600);
}
if (highReflectivity.size() > 0)
reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
if (highVelocity.size() > 0)
velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
if (highSpectrum.size() > 0)
spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
if (highDiffReflectivity.size() > 0)
diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
if (highDiffPhase.size() > 0)
diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
if (highCorreCoefficient.size() > 0)
coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
}
