public boolean isValidFile(RandomAccessFile raf) {
    int data_msecs = 0;
    short data_julian_date = 0;

    try {
      raf.order(RandomAccessFile.LITTLE_ENDIAN);
      raf.seek(0);
      raf.skipBytes(14);
      short message_type = raf.readShort();
      if (message_type != 1) return false;

      raf.skipBytes(12);
      // data header
      byte[] b4 = raf.readBytes(4);
      data_msecs = bytesToInt(b4, true);
      byte[] b2 = raf.readBytes(2);
      data_julian_date = (short) bytesToShort(b2, true);
      java.util.Date dd = Cinrad2Record.getDate(data_julian_date, data_msecs);

      Calendar cal = new GregorianCalendar(new SimpleTimeZone(0, "GMT"));
      cal.clear();
      cal.setTime(dd);
      int year = cal.get(Calendar.YEAR);
      cal.setTime(new Date());
      int cyear = cal.get(Calendar.YEAR);
      if (year < 1990 || year > cyear) return false;
      return true;
    } catch (IOException ioe) {
      return false;
    }
  }
  public static boolean isValidFile(RandomAccessFile raf) {
    try {
      raf.seek(0);
      while (raf.getFilePointer() < maxScan) {
        boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
        if (!found) return false;
        raf.skipBytes(7); // will be positioned on byte 0 of indicator section
        int edition = raf.read(); // read at byte 8
        if (edition == 2) return true;
      }

    } catch (IOException e) {
      return false;
    }

    return false;
  }
  public boolean hasNext() throws IOException {
    if (lastPos >= raf.length()) return false;
    if (repeatPos > 0) {
      if (nextRepeating()) // this has created a new repeatRecord
      return true;
    } else {
      repeatRecord = null;
      repeatBms = null;
      // fall through to new record
    }

    boolean more;
    long stop = 0;

    while (true) { // scan until we get a GRIB-2 or more == false
      raf.seek(lastPos);
      more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
      if (!more) break;

      stop = raf.getFilePointer();
      // see if its GRIB-2
      raf.skipBytes(7);
      int edition = raf.read();
      if (edition == 2) break;
      lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
    }

    if (more) {
      int sizeHeader =
          (int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
      // if (sizeHeader > 30) sizeHeader = 30;
      header = new byte[sizeHeader];
      startPos = stop - sizeHeader;
      raf.seek(startPos);
      raf.read(header);
    }
    if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
    return more;
  }
  /**
   * Read some global data from SIGMET file. The SIGMET file consists of records with fixed
   * length=6144 bytes.
   */
  public static java.util.Map<String, Number> readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) {
    java.util.Map<String, Number> recHdr1 = new java.util.HashMap<String, Number>();
    try {
      int nparams = 0;
      //      -- Read from <product_end> of the 1st record -- 12+320+120
      //      -- Calculate Nyquist velocity --------------------
      raf.seek(452);
      int prf = raf.readInt();
      raf.seek(480);
      int wave = raf.readInt();
      float vNyq = calcNyquist(prf, wave);
      recHdr1.put("vNyq", vNyq);

      //      -- Read from the 2nd record----------- 6144+12(strucr_hdr)+168(from ingest_config)
      raf.seek(6324);
      int radar_lat = raf.readInt();
      int radar_lon = raf.readInt(); // 6328
      short ground_height = raf.readShort(); // 6332
      short radar_height = raf.readShort(); // 6334
      raf.skipBytes(4);
      short num_rays = raf.readShort(); // 6340
      raf.skipBytes(2);
      int radar_alt = raf.readInt(); // 6344
      raf.seek(6648);
      int time_beg = raf.readInt();
      raf.seek(6652);
      int time_end = raf.readInt();
      raf.seek(6772);
      int data_mask = raf.readInt();
      for (int j = 0; j < 32; j++) {
        nparams += ((data_mask >> j) & (0x1));
      }
      raf.seek(6912);
      short multiprf = raf.readShort();
      raf.seek(7408);
      int range_first = raf.readInt(); //  cm    7408
      int range_last = raf.readInt(); //  cm  7412
      raf.skipBytes(2);
      short bins = raf.readShort(); // 7418
      if (bins % 2 != 0) bins = (short) (bins + 1);
      raf.skipBytes(4);
      int step = raf.readInt(); //  cm    7424
      raf.seek(7574);
      short number_sweeps = raf.readShort(); // 7574
      raf.seek(12312);
      int base_time = raf.readInt(); // <ingest_data_header> 3d rec
      raf.skipBytes(2);
      short year = raf.readShort();
      short month = raf.readShort();
      short day = raf.readShort();
      recHdr1.put("radar_lat", calcAngle(radar_lat));
      recHdr1.put("radar_lon", calcAngle(radar_lon));
      recHdr1.put("range_first", range_first);
      recHdr1.put("range_last", range_last);
      recHdr1.put("ground_height", ground_height);
      recHdr1.put("radar_height", radar_height);
      recHdr1.put("radar_alt", radar_alt);
      recHdr1.put("step", step);
      recHdr1.put("bins", bins); // System.out.println("  bins="+bins);
      recHdr1.put("num_rays", num_rays); // System.out.println("  rays="+num_rays);
      recHdr1.put("nparams", nparams); // System.out.println("  nparams="+nparams);
      recHdr1.put("multiprf", multiprf);
      recHdr1.put(
          "number_sweeps",
          number_sweeps); // System.out.println("IN HDR:  number_sweeps="+number_sweeps);
      recHdr1.put("year", year);
      recHdr1.put("month", month);
      recHdr1.put("day", day);
      recHdr1.put("base_time", base_time);
    } catch (Exception e) {
      System.out.println(e.toString());
      e.printStackTrace();
    }
    return recHdr1;
  }
  public boolean readIndex(RandomAccessFile raf) {
    gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
    try {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      //// header message
      if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
        logger.error("GribCollection {}: invalid index", gc.getName());
        return false;
      }

      int v = raf.readInt();
      if (v != getVersion()) {
        logger.warn(
            "GribCollection {}: index found version={}, want version= {} on file {}",
            new Object[] {gc.getName(), v, version, raf.getLocation()});
        return false;
      }

      long skip = raf.readLong();
      raf.skipBytes(skip);

      int size = NcStream.readVInt(raf);
      if ((size < 0) || (size > 100 * 1000 * 1000)) {
        logger.warn("GribCollection {}: invalid index ", gc.getName());
        return false;
      }

      byte[] m = new byte[size];
      raf.readFully(m);

      GribCollectionProto.GribCollectionIndex proto =
          GribCollectionProto.GribCollectionIndex.parseFrom(m);

      gc.center = proto.getCenter();
      gc.subcenter = proto.getSubcenter();
      gc.master = proto.getMaster();
      gc.local = proto.getLocal();
      gc.genProcessType = proto.getGenProcessType();
      gc.genProcessId = proto.getGenProcessId();
      gc.backProcessId = proto.getBackProcessId();
      gc.local = proto.getLocal();
      // gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);

      gc.filenames = new ArrayList<String>(proto.getFilesCount());
      for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));

      // error condition on a GribCollection Index
      if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
        logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
        return false;
      }

      gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
      for (int i = 0; i < proto.getGroupsCount(); i++)
        gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
      Collections.sort(gc.groups);

      gc.params = new ArrayList<Parameter>(proto.getParamsCount());
      for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));

      if (!readPartitions(proto)) {
        logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
        return false;
      }

      return true;

    } catch (Throwable t) {
      logger.error("Error reading index " + raf.getLocation(), t);
      return false;
    }
  }
  // check if compressed file seems ok
  public static long testValid(String ufilename) throws IOException {
    boolean lookForHeader = false;

    // gotta make it
    RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    raf.seek(0);
    byte[] b = new byte[8];
    raf.read(b);
    String test = new String(b);
    if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
      System.out.println("--Good header= " + test);
      raf.seek(24);
    } else {
      System.out.println("--No header ");
      lookForHeader = true;
      raf.seek(0);
    }

    boolean eof = false;
    int numCompBytes;
    try {

      while (!eof) {

        if (lookForHeader) {
          raf.read(b);
          test = new String(b);
          if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
            System.out.println("  found header= " + test);
            raf.skipBytes(16);
            lookForHeader = false;
          } else {
            raf.skipBytes(-8);
          }
        }

        try {
          numCompBytes = raf.readInt();
          if (numCompBytes == -1) {
            System.out.println("\n--done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          System.out.println("\n--got EOFException ");
          break; // assume this is ok
        }

        System.out.print(" " + numCompBytes + ",");
        if (numCompBytes < 0) {
          System.out.println("\n--last block " + numCompBytes);
          numCompBytes = -numCompBytes;
          if (!lookForHeader) eof = true;
        }

        raf.skipBytes(numCompBytes);
      }
    } catch (EOFException e) {
      e.printStackTrace();
    }

    return raf.getFilePointer();
  }
  Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
    this.raf = orgRaf;

    if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());

    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);

    // volume scan header
    dataFormat = raf.readString(8);
    raf.skipBytes(1);
    String volumeNo = raf.readString(3);
    title_julianDay = raf.readInt(); // since 1/1/70
    title_msecs = raf.readInt();
    stationId = raf.readString(4).trim(); // only in AR2V0001
    if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);

    if (stationId.length() == 0) {
      // try to get it from the filename LOOK

      stationId = null;
    }

    // try to find the station
    if (stationId != null) {
      if (!stationId.startsWith("K") && stationId.length() == 4) {
        String _stationId = "K" + stationId;
        station = NexradStationDB.get(_stationId);
      } else station = NexradStationDB.get(stationId);
    }

    // see if we have to uncompress
    if (dataFormat.equals(AR2V0001)
        || dataFormat.equals(AR2V0003)
        || dataFormat.equals(AR2V0004)
        || dataFormat.equals(AR2V0006)) {
      raf.skipBytes(4);
      String BZ = raf.readString(2);
      if (BZ.equals("BZ")) {
        RandomAccessFile uraf;
        File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");

        if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
          // see if its locked - another thread is writing it
          FileInputStream fstream = null;
          FileLock lock = null;
          try {
            fstream = new FileInputStream(uncompressedFile);
            // lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked

            while (true) { // loop waiting for the lock
              try {
                lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
                break;

              } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
                try {
                  Thread.sleep(100); // msecs
                } catch (InterruptedException e1) {
                  break;
                }
              }
            }

          } finally {
            if (lock != null) lock.release();
            if (fstream != null) fstream.close();
          }
          uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");

        } else {
          // nope, gotta uncompress it
          uraf = uncompress(raf, uncompressedFile.getPath());
          if (log.isDebugEnabled())
            log.debug("made uncompressed file= " + uncompressedFile.getPath());
        }

        // switch to uncompressed file
        raf.close();
        raf = uraf;
        raf.order(RandomAccessFile.BIG_ENDIAN);
      }

      raf.seek(Level2Record.FILE_HEADER_SIZE);
    }

    List<Level2Record> reflectivity = new ArrayList<Level2Record>();
    List<Level2Record> doppler = new ArrayList<Level2Record>();
    List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highVelocity = new ArrayList<Level2Record>();
    List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
    List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
    List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();

    long message_offset31 = 0;
    int recno = 0;
    while (true) {

      Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
      if (r == null) break;
      if (showData) r.dump2(System.out);
      // skip non-data messages
      if (r.message_type == 31) {
        message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
      }

      if (r.message_type != 1 && r.message_type != 31) {
        if (showMessages) r.dumpMessage(System.out);
        continue;
      }

      //  if (showData) r.dump2(System.out);

      /* skip bad
      if (!r.checkOk()) {
        r.dump(System.out);
        continue;
      }   */

      // some global params
      if (vcp == 0) vcp = r.vcp;
      if (first == null) first = r;
      last = r;

      if (runCheck && !r.checkOk()) {
        continue;
      }

      if (r.hasReflectData) reflectivity.add(r);
      if (r.hasDopplerData) doppler.add(r);

      if (r.message_type == 31) {
        if (r.hasHighResREFData) highReflectivity.add(r);
        if (r.hasHighResVELData) highVelocity.add(r);
        if (r.hasHighResSWData) highSpectrum.add(r);
        if (r.hasHighResZDRData) highDiffReflectivity.add(r);
        if (r.hasHighResPHIData) highDiffPhase.add(r);
        if (r.hasHighResRHOData) highCorreCoefficient.add(r);
      }

      if ((cancelTask != null) && cancelTask.isCancel()) return;
    }
    if (debugRadials)
      System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
    if (highReflectivity.size() == 0) {
      reflectivityGroups = sortScans("reflect", reflectivity, 600);
      dopplerGroups = sortScans("doppler", doppler, 600);
    }
    if (highReflectivity.size() > 0)
      reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
    if (highVelocity.size() > 0)
      velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
    if (highSpectrum.size() > 0)
      spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
    if (highDiffReflectivity.size() > 0)
      diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
    if (highDiffPhase.size() > 0)
      diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
    if (highCorreCoefficient.size() > 0)
      coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
  }