private ArrayList sortScans(String name, List<Level2Record> scans, int siz) {

    // now group by elevation_num
    Map<Short, List<Level2Record>> groupHash = new HashMap<Short, List<Level2Record>>(siz);
    for (Level2Record record : scans) {
      List<Level2Record> group = groupHash.get(record.elevation_num);
      if (null == group) {
        group = new ArrayList<Level2Record>();
        groupHash.put(record.elevation_num, group);
      }
      group.add(record);
    }

    // sort the groups by elevation_num
    ArrayList groups = new ArrayList(groupHash.values());
    Collections.sort(groups, new GroupComparator());

    // use the maximum radials
    for (int i = 0; i < groups.size(); i++) {
      ArrayList group = (ArrayList) groups.get(i);
      Level2Record r = (Level2Record) group.get(0);
      if (runCheck) testScan(name, group);
      if (r.getGateCount(REFLECTIVITY_HIGH) > 500 || r.getGateCount(VELOCITY_HIGH) > 1000) {
        max_radials_hr = Math.max(max_radials_hr, group.size());
        min_radials_hr = Math.min(min_radials_hr, group.size());
      } else {
        max_radials = Math.max(max_radials, group.size());
        min_radials = Math.min(min_radials, group.size());
      }
    }

    if (debugRadials) {
      System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials);
      for (int i = 0; i < groups.size(); i++) {
        ArrayList group = (ArrayList) groups.get(i);
        Level2Record lastr = (Level2Record) group.get(0);
        for (int j = 1; j < group.size(); j++) {
          Level2Record r = (Level2Record) group.get(j);
          if (r.data_msecs < lastr.data_msecs) System.out.println(" out of order " + j);
          lastr = r;
        }
      }
    }

    testVariable(name, groups);
    if (debugScans) System.out.println("-----------------------------");

    return groups;
  }
 public Date getEndDate() {
   return last.getDate();
 }
 public Date getStartDate() {
   return first.getDate();
 }
  // do we have same characteristics for all groups in a variable?
  private boolean testVariable(String name, List scans) {
    int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
    if (scans.size() == 0) {
      log.warn(" No data for = " + name);
      return false;
    }

    boolean ok = true;
    List firstScan = (List) scans.get(0);
    Level2Record firstRecord = (Level2Record) firstScan.get(0);
    dopplarResolution = firstRecord.resolution;

    if (debugGroups2)
      System.out.println(
          "Group "
              + Level2Record.getDatatypeName(datatype)
              + " ngates = "
              + firstRecord.getGateCount(datatype)
              + " start = "
              + firstRecord.getGateStart(datatype)
              + " size = "
              + firstRecord.getGateSize(datatype));

    for (int i = 1; i < scans.size(); i++) {
      List scan = (List) scans.get(i);
      Level2Record record = (Level2Record) scan.get(0);

      if ((datatype == Level2Record.VELOCITY_HI)
          && (record.resolution
              != firstRecord.resolution)) { // do all velocity resolutions match ??
        log.warn(
            name
                + " scan "
                + i
                + " diff resolutions = "
                + record.resolution
                + ", "
                + firstRecord.resolution
                + " elev= "
                + record.elevation_num
                + " "
                + record.getElevation());
        ok = false;
        hasDifferentDopplarResolutions = true;
      }

      if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) {
        log.warn(
            name
                + " scan "
                + i
                + " diff gates size = "
                + record.getGateSize(datatype)
                + " "
                + firstRecord.getGateSize(datatype)
                + " elev= "
                + record.elevation_num
                + " "
                + record.getElevation());
        ok = false;

      } else if (debugGroups2)
        System.out.println(
            " ok gates size elev= " + record.elevation_num + " " + record.getElevation());

      if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) {
        log.warn(
            name
                + " scan "
                + i
                + " diff gates start = "
                + record.getGateStart(datatype)
                + " "
                + firstRecord.getGateStart(datatype)
                + " elev= "
                + record.elevation_num
                + " "
                + record.getElevation());
        ok = false;

      } else if (debugGroups2)
        System.out.println(
            " ok gates start elev= " + record.elevation_num + " " + record.getElevation());

      if (record.message_type == 31) {
        hasHighResolutionData = true;
        // each data type
        if (record.hasHighResREFData) hasHighResolutionREF = true;
        if (record.hasHighResVELData) hasHighResolutionVEL = true;
        if (record.hasHighResSWData) hasHighResolutionSW = true;
        if (record.hasHighResZDRData) hasHighResolutionZDR = true;
        if (record.hasHighResPHIData) hasHighResolutionPHI = true;
        if (record.hasHighResRHOData) hasHighResolutionRHO = true;
      }
    }

    return ok;
  }
  private boolean testScan(String name, ArrayList group) {
    int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
    Level2Record first = (Level2Record) group.get(0);

    int n = group.size();
    if (debugScans) {
      boolean hasBoth = first.hasDopplerData && first.hasReflectData;
      System.out.println(
          name
              + " "
              + first
              + " has "
              + n
              + " radials resolution= "
              + first.resolution
              + " has both = "
              + hasBoth);
    }

    boolean ok = true;
    double sum = 0.0;
    double sum2 = 0.0;

    for (int i = 0; i < MAX_RADIAL; i++) radial[i] = 0;

    for (int i = 0; i < group.size(); i++) {
      Level2Record r = (Level2Record) group.get(i);

      /* this appears to be common - seems to be ok, we put missing values in
      if (r.getGateCount(datatype) != first.getGateCount(datatype)) {
        log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+
                "!="+first.getGateCount(datatype)+") in record "+name+ " "+r);
        ok = false;
      } */

      if (r.getGateSize(datatype) != first.getGateSize(datatype)) {
        log.warn(
            raf.getLocation()
                + " different gate size ("
                + r.getGateSize(datatype)
                + ") in record "
                + name
                + " "
                + r);
        ok = false;
      }
      if (r.getGateStart(datatype) != first.getGateStart(datatype)) {
        log.warn(
            raf.getLocation()
                + " different gate start ("
                + r.getGateStart(datatype)
                + ") in record "
                + name
                + " "
                + r);
        ok = false;
      }
      if (r.resolution != first.resolution) {
        log.warn(
            raf.getLocation()
                + " different resolution ("
                + r.resolution
                + ") in record "
                + name
                + " "
                + r);
        ok = false;
      }

      if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) {
        log.info(
            raf.getLocation()
                + " radial out of range= "
                + r.radial_num
                + " in record "
                + name
                + " "
                + r);
        continue;
      }
      if (radial[r.radial_num] > 0) {
        log.warn(
            raf.getLocation()
                + " duplicate radial = "
                + r.radial_num
                + " in record "
                + name
                + " "
                + r);
        ok = false;
      }
      radial[r.radial_num] = r.recno + 1;

      sum += r.getElevation();
      sum2 += r.getElevation() * r.getElevation();
      // System.out.println("  elev="+r.getElevation()+" azi="+r.getAzimuth());
    }

    for (int i = 1; i < radial.length; i++) {
      if (0 == radial[i]) {
        if (n != (i - 1)) {
          log.warn(" missing radial(s)");
          ok = false;
        }
        break;
      }
    }

    double avg = sum / n;
    double sd = Math.sqrt((n * sum2 - sum * sum) / (n * (n - 1)));
    // System.out.println(" avg elev="+avg+" std.dev="+sd);

    return ok;
  }
  Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
    this.raf = orgRaf;

    if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());

    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);

    // volume scan header
    dataFormat = raf.readString(8);
    raf.skipBytes(1);
    String volumeNo = raf.readString(3);
    title_julianDay = raf.readInt(); // since 1/1/70
    title_msecs = raf.readInt();
    stationId = raf.readString(4).trim(); // only in AR2V0001
    if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);

    if (stationId.length() == 0) {
      // try to get it from the filename LOOK

      stationId = null;
    }

    // try to find the station
    if (stationId != null) {
      if (!stationId.startsWith("K") && stationId.length() == 4) {
        String _stationId = "K" + stationId;
        station = NexradStationDB.get(_stationId);
      } else station = NexradStationDB.get(stationId);
    }

    // see if we have to uncompress
    if (dataFormat.equals(AR2V0001)
        || dataFormat.equals(AR2V0003)
        || dataFormat.equals(AR2V0004)
        || dataFormat.equals(AR2V0006)) {
      raf.skipBytes(4);
      String BZ = raf.readString(2);
      if (BZ.equals("BZ")) {
        RandomAccessFile uraf;
        File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");

        if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
          // see if its locked - another thread is writing it
          FileInputStream fstream = null;
          FileLock lock = null;
          try {
            fstream = new FileInputStream(uncompressedFile);
            // lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked

            while (true) { // loop waiting for the lock
              try {
                lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
                break;

              } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
                try {
                  Thread.sleep(100); // msecs
                } catch (InterruptedException e1) {
                  break;
                }
              }
            }

          } finally {
            if (lock != null) lock.release();
            if (fstream != null) fstream.close();
          }
          uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");

        } else {
          // nope, gotta uncompress it
          uraf = uncompress(raf, uncompressedFile.getPath());
          if (log.isDebugEnabled())
            log.debug("made uncompressed file= " + uncompressedFile.getPath());
        }

        // switch to uncompressed file
        raf.close();
        raf = uraf;
        raf.order(RandomAccessFile.BIG_ENDIAN);
      }

      raf.seek(Level2Record.FILE_HEADER_SIZE);
    }

    List<Level2Record> reflectivity = new ArrayList<Level2Record>();
    List<Level2Record> doppler = new ArrayList<Level2Record>();
    List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highVelocity = new ArrayList<Level2Record>();
    List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
    List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
    List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();

    long message_offset31 = 0;
    int recno = 0;
    while (true) {

      Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
      if (r == null) break;
      if (showData) r.dump2(System.out);
      // skip non-data messages
      if (r.message_type == 31) {
        message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
      }

      if (r.message_type != 1 && r.message_type != 31) {
        if (showMessages) r.dumpMessage(System.out);
        continue;
      }

      //  if (showData) r.dump2(System.out);

      /* skip bad
      if (!r.checkOk()) {
        r.dump(System.out);
        continue;
      }   */

      // some global params
      if (vcp == 0) vcp = r.vcp;
      if (first == null) first = r;
      last = r;

      if (runCheck && !r.checkOk()) {
        continue;
      }

      if (r.hasReflectData) reflectivity.add(r);
      if (r.hasDopplerData) doppler.add(r);

      if (r.message_type == 31) {
        if (r.hasHighResREFData) highReflectivity.add(r);
        if (r.hasHighResVELData) highVelocity.add(r);
        if (r.hasHighResSWData) highSpectrum.add(r);
        if (r.hasHighResZDRData) highDiffReflectivity.add(r);
        if (r.hasHighResPHIData) highDiffPhase.add(r);
        if (r.hasHighResRHOData) highCorreCoefficient.add(r);
      }

      if ((cancelTask != null) && cancelTask.isCancel()) return;
    }
    if (debugRadials)
      System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
    if (highReflectivity.size() == 0) {
      reflectivityGroups = sortScans("reflect", reflectivity, 600);
      dopplerGroups = sortScans("doppler", doppler, 600);
    }
    if (highReflectivity.size() > 0)
      reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
    if (highVelocity.size() > 0)
      velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
    if (highSpectrum.size() > 0)
      spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
    if (highDiffReflectivity.size() > 0)
      diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
    if (highDiffPhase.size() > 0)
      diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
    if (highCorreCoefficient.size() > 0)
      coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
  }