private ArrayList sortScans(String name, List<Level2Record> scans, int siz) {
// now group by elevation_num
Map<Short, List<Level2Record>> groupHash = new HashMap<Short, List<Level2Record>>(siz);
for (Level2Record record : scans) {
List<Level2Record> group = groupHash.get(record.elevation_num);
if (null == group) {
group = new ArrayList<Level2Record>();
groupHash.put(record.elevation_num, group);
}
group.add(record);
}
// sort the groups by elevation_num
ArrayList groups = new ArrayList(groupHash.values());
Collections.sort(groups, new GroupComparator());
// use the maximum radials
for (int i = 0; i < groups.size(); i++) {
ArrayList group = (ArrayList) groups.get(i);
Level2Record r = (Level2Record) group.get(0);
if (runCheck) testScan(name, group);
if (r.getGateCount(REFLECTIVITY_HIGH) > 500 || r.getGateCount(VELOCITY_HIGH) > 1000) {
max_radials_hr = Math.max(max_radials_hr, group.size());
min_radials_hr = Math.min(min_radials_hr, group.size());
} else {
max_radials = Math.max(max_radials, group.size());
min_radials = Math.min(min_radials, group.size());
}
}
if (debugRadials) {
System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials);
for (int i = 0; i < groups.size(); i++) {
ArrayList group = (ArrayList) groups.get(i);
Level2Record lastr = (Level2Record) group.get(0);
for (int j = 1; j < group.size(); j++) {
Level2Record r = (Level2Record) group.get(j);
if (r.data_msecs < lastr.data_msecs) System.out.println(" out of order " + j);
lastr = r;
}
}
}
testVariable(name, groups);
if (debugScans) System.out.println("-----------------------------");
return groups;
}
public Date getEndDate() {
return last.getDate();
}
public Date getStartDate() {
return first.getDate();
}
// do we have same characteristics for all groups in a variable?
private boolean testVariable(String name, List scans) {
int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
if (scans.size() == 0) {
log.warn(" No data for = " + name);
return false;
}
boolean ok = true;
List firstScan = (List) scans.get(0);
Level2Record firstRecord = (Level2Record) firstScan.get(0);
dopplarResolution = firstRecord.resolution;
if (debugGroups2)
System.out.println(
"Group "
+ Level2Record.getDatatypeName(datatype)
+ " ngates = "
+ firstRecord.getGateCount(datatype)
+ " start = "
+ firstRecord.getGateStart(datatype)
+ " size = "
+ firstRecord.getGateSize(datatype));
for (int i = 1; i < scans.size(); i++) {
List scan = (List) scans.get(i);
Level2Record record = (Level2Record) scan.get(0);
if ((datatype == Level2Record.VELOCITY_HI)
&& (record.resolution
!= firstRecord.resolution)) { // do all velocity resolutions match ??
log.warn(
name
+ " scan "
+ i
+ " diff resolutions = "
+ record.resolution
+ ", "
+ firstRecord.resolution
+ " elev= "
+ record.elevation_num
+ " "
+ record.getElevation());
ok = false;
hasDifferentDopplarResolutions = true;
}
if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) {
log.warn(
name
+ " scan "
+ i
+ " diff gates size = "
+ record.getGateSize(datatype)
+ " "
+ firstRecord.getGateSize(datatype)
+ " elev= "
+ record.elevation_num
+ " "
+ record.getElevation());
ok = false;
} else if (debugGroups2)
System.out.println(
" ok gates size elev= " + record.elevation_num + " " + record.getElevation());
if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) {
log.warn(
name
+ " scan "
+ i
+ " diff gates start = "
+ record.getGateStart(datatype)
+ " "
+ firstRecord.getGateStart(datatype)
+ " elev= "
+ record.elevation_num
+ " "
+ record.getElevation());
ok = false;
} else if (debugGroups2)
System.out.println(
" ok gates start elev= " + record.elevation_num + " " + record.getElevation());
if (record.message_type == 31) {
hasHighResolutionData = true;
// each data type
if (record.hasHighResREFData) hasHighResolutionREF = true;
if (record.hasHighResVELData) hasHighResolutionVEL = true;
if (record.hasHighResSWData) hasHighResolutionSW = true;
if (record.hasHighResZDRData) hasHighResolutionZDR = true;
if (record.hasHighResPHIData) hasHighResolutionPHI = true;
if (record.hasHighResRHOData) hasHighResolutionRHO = true;
}
}
return ok;
}
private boolean testScan(String name, ArrayList group) {
int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
Level2Record first = (Level2Record) group.get(0);
int n = group.size();
if (debugScans) {
boolean hasBoth = first.hasDopplerData && first.hasReflectData;
System.out.println(
name
+ " "
+ first
+ " has "
+ n
+ " radials resolution= "
+ first.resolution
+ " has both = "
+ hasBoth);
}
boolean ok = true;
double sum = 0.0;
double sum2 = 0.0;
for (int i = 0; i < MAX_RADIAL; i++) radial[i] = 0;
for (int i = 0; i < group.size(); i++) {
Level2Record r = (Level2Record) group.get(i);
/* this appears to be common - seems to be ok, we put missing values in
if (r.getGateCount(datatype) != first.getGateCount(datatype)) {
log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+
"!="+first.getGateCount(datatype)+") in record "+name+ " "+r);
ok = false;
} */
if (r.getGateSize(datatype) != first.getGateSize(datatype)) {
log.warn(
raf.getLocation()
+ " different gate size ("
+ r.getGateSize(datatype)
+ ") in record "
+ name
+ " "
+ r);
ok = false;
}
if (r.getGateStart(datatype) != first.getGateStart(datatype)) {
log.warn(
raf.getLocation()
+ " different gate start ("
+ r.getGateStart(datatype)
+ ") in record "
+ name
+ " "
+ r);
ok = false;
}
if (r.resolution != first.resolution) {
log.warn(
raf.getLocation()
+ " different resolution ("
+ r.resolution
+ ") in record "
+ name
+ " "
+ r);
ok = false;
}
if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) {
log.info(
raf.getLocation()
+ " radial out of range= "
+ r.radial_num
+ " in record "
+ name
+ " "
+ r);
continue;
}
if (radial[r.radial_num] > 0) {
log.warn(
raf.getLocation()
+ " duplicate radial = "
+ r.radial_num
+ " in record "
+ name
+ " "
+ r);
ok = false;
}
radial[r.radial_num] = r.recno + 1;
sum += r.getElevation();
sum2 += r.getElevation() * r.getElevation();
// System.out.println(" elev="+r.getElevation()+" azi="+r.getAzimuth());
}
for (int i = 1; i < radial.length; i++) {
if (0 == radial[i]) {
if (n != (i - 1)) {
log.warn(" missing radial(s)");
ok = false;
}
break;
}
}
double avg = sum / n;
double sd = Math.sqrt((n * sum2 - sum * sum) / (n * (n - 1)));
// System.out.println(" avg elev="+avg+" std.dev="+sd);
return ok;
}
Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
this.raf = orgRaf;
if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());
raf.seek(0);
raf.order(RandomAccessFile.BIG_ENDIAN);
// volume scan header
dataFormat = raf.readString(8);
raf.skipBytes(1);
String volumeNo = raf.readString(3);
title_julianDay = raf.readInt(); // since 1/1/70
title_msecs = raf.readInt();
stationId = raf.readString(4).trim(); // only in AR2V0001
if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);
if (stationId.length() == 0) {
// try to get it from the filename LOOK
stationId = null;
}
// try to find the station
if (stationId != null) {
if (!stationId.startsWith("K") && stationId.length() == 4) {
String _stationId = "K" + stationId;
station = NexradStationDB.get(_stationId);
} else station = NexradStationDB.get(stationId);
}
// see if we have to uncompress
if (dataFormat.equals(AR2V0001)
|| dataFormat.equals(AR2V0003)
|| dataFormat.equals(AR2V0004)
|| dataFormat.equals(AR2V0006)) {
raf.skipBytes(4);
String BZ = raf.readString(2);
if (BZ.equals("BZ")) {
RandomAccessFile uraf;
File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");
if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
// see if its locked - another thread is writing it
FileInputStream fstream = null;
FileLock lock = null;
try {
fstream = new FileInputStream(uncompressedFile);
// lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
while (true) { // loop waiting for the lock
try {
lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
break;
}
}
}
} finally {
if (lock != null) lock.release();
if (fstream != null) fstream.close();
}
uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");
} else {
// nope, gotta uncompress it
uraf = uncompress(raf, uncompressedFile.getPath());
if (log.isDebugEnabled())
log.debug("made uncompressed file= " + uncompressedFile.getPath());
}
// switch to uncompressed file
raf.close();
raf = uraf;
raf.order(RandomAccessFile.BIG_ENDIAN);
}
raf.seek(Level2Record.FILE_HEADER_SIZE);
}
List<Level2Record> reflectivity = new ArrayList<Level2Record>();
List<Level2Record> doppler = new ArrayList<Level2Record>();
List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highVelocity = new ArrayList<Level2Record>();
List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();
long message_offset31 = 0;
int recno = 0;
while (true) {
Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
if (r == null) break;
if (showData) r.dump2(System.out);
// skip non-data messages
if (r.message_type == 31) {
message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
}
if (r.message_type != 1 && r.message_type != 31) {
if (showMessages) r.dumpMessage(System.out);
continue;
}
// if (showData) r.dump2(System.out);
/* skip bad
if (!r.checkOk()) {
r.dump(System.out);
continue;
} */
// some global params
if (vcp == 0) vcp = r.vcp;
if (first == null) first = r;
last = r;
if (runCheck && !r.checkOk()) {
continue;
}
if (r.hasReflectData) reflectivity.add(r);
if (r.hasDopplerData) doppler.add(r);
if (r.message_type == 31) {
if (r.hasHighResREFData) highReflectivity.add(r);
if (r.hasHighResVELData) highVelocity.add(r);
if (r.hasHighResSWData) highSpectrum.add(r);
if (r.hasHighResZDRData) highDiffReflectivity.add(r);
if (r.hasHighResPHIData) highDiffPhase.add(r);
if (r.hasHighResRHOData) highCorreCoefficient.add(r);
}
if ((cancelTask != null) && cancelTask.isCancel()) return;
}
if (debugRadials)
System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
if (highReflectivity.size() == 0) {
reflectivityGroups = sortScans("reflect", reflectivity, 600);
dopplerGroups = sortScans("doppler", doppler, 600);
}
if (highReflectivity.size() > 0)
reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
if (highVelocity.size() > 0)
velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
if (highSpectrum.size() > 0)
spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
if (highDiffReflectivity.size() > 0)
diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
if (highDiffPhase.size() > 0)
diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
if (highCorreCoefficient.size() > 0)
coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
}