private ArrayList sortScans(String name, List<Level2Record> scans, int siz) { // now group by elevation_num Map<Short, List<Level2Record>> groupHash = new HashMap<Short, List<Level2Record>>(siz); for (Level2Record record : scans) { List<Level2Record> group = groupHash.get(record.elevation_num); if (null == group) { group = new ArrayList<Level2Record>(); groupHash.put(record.elevation_num, group); } group.add(record); } // sort the groups by elevation_num ArrayList groups = new ArrayList(groupHash.values()); Collections.sort(groups, new GroupComparator()); // use the maximum radials for (int i = 0; i < groups.size(); i++) { ArrayList group = (ArrayList) groups.get(i); Level2Record r = (Level2Record) group.get(0); if (runCheck) testScan(name, group); if (r.getGateCount(REFLECTIVITY_HIGH) > 500 || r.getGateCount(VELOCITY_HIGH) > 1000) { max_radials_hr = Math.max(max_radials_hr, group.size()); min_radials_hr = Math.min(min_radials_hr, group.size()); } else { max_radials = Math.max(max_radials, group.size()); min_radials = Math.min(min_radials, group.size()); } } if (debugRadials) { System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials); for (int i = 0; i < groups.size(); i++) { ArrayList group = (ArrayList) groups.get(i); Level2Record lastr = (Level2Record) group.get(0); for (int j = 1; j < group.size(); j++) { Level2Record r = (Level2Record) group.get(j); if (r.data_msecs < lastr.data_msecs) System.out.println(" out of order " + j); lastr = r; } } } testVariable(name, groups); if (debugScans) System.out.println("-----------------------------"); return groups; }
public Date getEndDate() { return last.getDate(); }
public Date getStartDate() { return first.getDate(); }
// do we have same characteristics for all groups in a variable? private boolean testVariable(String name, List scans) { int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI; if (scans.size() == 0) { log.warn(" No data for = " + name); return false; } boolean ok = true; List firstScan = (List) scans.get(0); Level2Record firstRecord = (Level2Record) firstScan.get(0); dopplarResolution = firstRecord.resolution; if (debugGroups2) System.out.println( "Group " + Level2Record.getDatatypeName(datatype) + " ngates = " + firstRecord.getGateCount(datatype) + " start = " + firstRecord.getGateStart(datatype) + " size = " + firstRecord.getGateSize(datatype)); for (int i = 1; i < scans.size(); i++) { List scan = (List) scans.get(i); Level2Record record = (Level2Record) scan.get(0); if ((datatype == Level2Record.VELOCITY_HI) && (record.resolution != firstRecord.resolution)) { // do all velocity resolutions match ?? log.warn( name + " scan " + i + " diff resolutions = " + record.resolution + ", " + firstRecord.resolution + " elev= " + record.elevation_num + " " + record.getElevation()); ok = false; hasDifferentDopplarResolutions = true; } if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) { log.warn( name + " scan " + i + " diff gates size = " + record.getGateSize(datatype) + " " + firstRecord.getGateSize(datatype) + " elev= " + record.elevation_num + " " + record.getElevation()); ok = false; } else if (debugGroups2) System.out.println( " ok gates size elev= " + record.elevation_num + " " + record.getElevation()); if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) { log.warn( name + " scan " + i + " diff gates start = " + record.getGateStart(datatype) + " " + firstRecord.getGateStart(datatype) + " elev= " + record.elevation_num + " " + record.getElevation()); ok = false; } else if (debugGroups2) System.out.println( " ok gates start elev= " + record.elevation_num + " " + record.getElevation()); if (record.message_type == 31) { hasHighResolutionData = true; // each data type if (record.hasHighResREFData) hasHighResolutionREF = true; if (record.hasHighResVELData) hasHighResolutionVEL = true; if (record.hasHighResSWData) hasHighResolutionSW = true; if (record.hasHighResZDRData) hasHighResolutionZDR = true; if (record.hasHighResPHIData) hasHighResolutionPHI = true; if (record.hasHighResRHOData) hasHighResolutionRHO = true; } } return ok; }
private boolean testScan(String name, ArrayList group) { int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI; Level2Record first = (Level2Record) group.get(0); int n = group.size(); if (debugScans) { boolean hasBoth = first.hasDopplerData && first.hasReflectData; System.out.println( name + " " + first + " has " + n + " radials resolution= " + first.resolution + " has both = " + hasBoth); } boolean ok = true; double sum = 0.0; double sum2 = 0.0; for (int i = 0; i < MAX_RADIAL; i++) radial[i] = 0; for (int i = 0; i < group.size(); i++) { Level2Record r = (Level2Record) group.get(i); /* this appears to be common - seems to be ok, we put missing values in if (r.getGateCount(datatype) != first.getGateCount(datatype)) { log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+ "!="+first.getGateCount(datatype)+") in record "+name+ " "+r); ok = false; } */ if (r.getGateSize(datatype) != first.getGateSize(datatype)) { log.warn( raf.getLocation() + " different gate size (" + r.getGateSize(datatype) + ") in record " + name + " " + r); ok = false; } if (r.getGateStart(datatype) != first.getGateStart(datatype)) { log.warn( raf.getLocation() + " different gate start (" + r.getGateStart(datatype) + ") in record " + name + " " + r); ok = false; } if (r.resolution != first.resolution) { log.warn( raf.getLocation() + " different resolution (" + r.resolution + ") in record " + name + " " + r); ok = false; } if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) { log.info( raf.getLocation() + " radial out of range= " + r.radial_num + " in record " + name + " " + r); continue; } if (radial[r.radial_num] > 0) { log.warn( raf.getLocation() + " duplicate radial = " + r.radial_num + " in record " + name + " " + r); ok = false; } radial[r.radial_num] = r.recno + 1; sum += r.getElevation(); sum2 += r.getElevation() * r.getElevation(); // System.out.println(" elev="+r.getElevation()+" azi="+r.getAzimuth()); } for (int i = 1; i < radial.length; i++) { if (0 == radial[i]) { if (n != (i - 1)) { log.warn(" missing radial(s)"); ok = false; } break; } } double avg = sum / n; double sd = Math.sqrt((n * sum2 - sum * sum) / (n * (n - 1))); // System.out.println(" avg elev="+avg+" std.dev="+sd); return ok; }
Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException { this.raf = orgRaf; if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation()); raf.seek(0); raf.order(RandomAccessFile.BIG_ENDIAN); // volume scan header dataFormat = raf.readString(8); raf.skipBytes(1); String volumeNo = raf.readString(3); title_julianDay = raf.readInt(); // since 1/1/70 title_msecs = raf.readInt(); stationId = raf.readString(4).trim(); // only in AR2V0001 if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId); if (stationId.length() == 0) { // try to get it from the filename LOOK stationId = null; } // try to find the station if (stationId != null) { if (!stationId.startsWith("K") && stationId.length() == 4) { String _stationId = "K" + stationId; station = NexradStationDB.get(_stationId); } else station = NexradStationDB.get(stationId); } // see if we have to uncompress if (dataFormat.equals(AR2V0001) || dataFormat.equals(AR2V0003) || dataFormat.equals(AR2V0004) || dataFormat.equals(AR2V0006)) { raf.skipBytes(4); String BZ = raf.readString(2); if (BZ.equals("BZ")) { RandomAccessFile uraf; File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress"); if (uncompressedFile.exists() && uncompressedFile.length() > 0) { // see if its locked - another thread is writing it FileInputStream fstream = null; FileLock lock = null; try { fstream = new FileInputStream(uncompressedFile); // lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked while (true) { // loop waiting for the lock try { lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked break; } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block try { Thread.sleep(100); // msecs } catch (InterruptedException e1) { break; } } } } finally { if (lock != null) lock.release(); if (fstream != null) fstream.close(); } uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r"); } else { // nope, gotta uncompress it uraf = uncompress(raf, uncompressedFile.getPath()); if (log.isDebugEnabled()) log.debug("made uncompressed file= " + uncompressedFile.getPath()); } // switch to uncompressed file raf.close(); raf = uraf; raf.order(RandomAccessFile.BIG_ENDIAN); } raf.seek(Level2Record.FILE_HEADER_SIZE); } List<Level2Record> reflectivity = new ArrayList<Level2Record>(); List<Level2Record> doppler = new ArrayList<Level2Record>(); List<Level2Record> highReflectivity = new ArrayList<Level2Record>(); List<Level2Record> highVelocity = new ArrayList<Level2Record>(); List<Level2Record> highSpectrum = new ArrayList<Level2Record>(); List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>(); List<Level2Record> highDiffPhase = new ArrayList<Level2Record>(); List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>(); long message_offset31 = 0; int recno = 0; while (true) { Level2Record r = Level2Record.factory(raf, recno++, message_offset31); if (r == null) break; if (showData) r.dump2(System.out); // skip non-data messages if (r.message_type == 31) { message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432); } if (r.message_type != 1 && r.message_type != 31) { if (showMessages) r.dumpMessage(System.out); continue; } // if (showData) r.dump2(System.out); /* skip bad if (!r.checkOk()) { r.dump(System.out); continue; } */ // some global params if (vcp == 0) vcp = r.vcp; if (first == null) first = r; last = r; if (runCheck && !r.checkOk()) { continue; } if (r.hasReflectData) reflectivity.add(r); if (r.hasDopplerData) doppler.add(r); if (r.message_type == 31) { if (r.hasHighResREFData) highReflectivity.add(r); if (r.hasHighResVELData) highVelocity.add(r); if (r.hasHighResSWData) highSpectrum.add(r); if (r.hasHighResZDRData) highDiffReflectivity.add(r); if (r.hasHighResPHIData) highDiffPhase.add(r); if (r.hasHighResRHOData) highCorreCoefficient.add(r); } if ((cancelTask != null) && cancelTask.isCancel()) return; } if (debugRadials) System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size()); if (highReflectivity.size() == 0) { reflectivityGroups = sortScans("reflect", reflectivity, 600); dopplerGroups = sortScans("doppler", doppler, 600); } if (highReflectivity.size() > 0) reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720); if (highVelocity.size() > 0) velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720); if (highSpectrum.size() > 0) spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720); if (highDiffReflectivity.size() > 0) diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720); if (highDiffPhase.size() > 0) diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720); if (highCorreCoefficient.size() > 0) coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720); }