Esempio n. 1
0
  /**
   * Read the grid
   *
   * @param gr the grid record
   * @return the data
   */
  public float[] readGrid(McIDASGridRecord gr) {

    float[] data = null;
    try {
      int te = (gr.getOffsetToHeader() + 64) * 4;
      int rows = gr.getRows();
      int cols = gr.getColumns();
      rf.seek(te);

      float scale = (float) gr.getParamScale();

      data = new float[rows * cols];
      rf.order(needToSwap ? rf.LITTLE_ENDIAN : rf.BIG_ENDIAN);
      int n = 0;
      // store such that 0,0 is in lower left corner...
      for (int nc = 0; nc < cols; nc++) {
        for (int nr = 0; nr < rows; nr++) {
          int temp = rf.readInt(); // check for missing value
          data[(rows - nr - 1) * cols + nc] =
              (temp == McIDASUtil.MCMISSING) ? Float.NaN : ((float) temp) / scale;
        }
      }
      rf.order(rf.BIG_ENDIAN);
    } catch (Exception esc) {
      System.out.println(esc);
    }
    return data;
  }
Esempio n. 2
0
 /**
  * Read an integer
  *
  * @param word word in file (0 based) to read
  * @return int read
  * @throws IOException problem reading file
  */
 public int readInt(int word) throws IOException {
   if (rf == null) {
     throw new IOException("no file to read from");
   }
   rf.seek(word * 4);
   // set the order
   if (needToSwap) {
     rf.order(RandomAccessFile.LITTLE_ENDIAN); // swap
   } else {
     rf.order(RandomAccessFile.BIG_ENDIAN);
   }
   int idata = rf.readInt();
   rf.order(RandomAccessFile.BIG_ENDIAN);
   return idata;
 }
  /**
   * Read some global data from SIGMET file. The SIGMET file consists of records with fixed
   * length=6144 bytes.
   */
  public static java.util.Map<String, Number> readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) {
    java.util.Map<String, Number> recHdr1 = new java.util.HashMap<String, Number>();
    try {
      int nparams = 0;
      //      -- Read from <product_end> of the 1st record -- 12+320+120
      //      -- Calculate Nyquist velocity --------------------
      raf.seek(452);
      int prf = raf.readInt();
      raf.seek(480);
      int wave = raf.readInt();
      float vNyq = calcNyquist(prf, wave);
      recHdr1.put("vNyq", vNyq);

      //      -- Read from the 2nd record----------- 6144+12(strucr_hdr)+168(from ingest_config)
      raf.seek(6324);
      int radar_lat = raf.readInt();
      int radar_lon = raf.readInt(); // 6328
      short ground_height = raf.readShort(); // 6332
      short radar_height = raf.readShort(); // 6334
      raf.skipBytes(4);
      short num_rays = raf.readShort(); // 6340
      raf.skipBytes(2);
      int radar_alt = raf.readInt(); // 6344
      raf.seek(6648);
      int time_beg = raf.readInt();
      raf.seek(6652);
      int time_end = raf.readInt();
      raf.seek(6772);
      int data_mask = raf.readInt();
      for (int j = 0; j < 32; j++) {
        nparams += ((data_mask >> j) & (0x1));
      }
      raf.seek(6912);
      short multiprf = raf.readShort();
      raf.seek(7408);
      int range_first = raf.readInt(); //  cm    7408
      int range_last = raf.readInt(); //  cm  7412
      raf.skipBytes(2);
      short bins = raf.readShort(); // 7418
      if (bins % 2 != 0) bins = (short) (bins + 1);
      raf.skipBytes(4);
      int step = raf.readInt(); //  cm    7424
      raf.seek(7574);
      short number_sweeps = raf.readShort(); // 7574
      raf.seek(12312);
      int base_time = raf.readInt(); // <ingest_data_header> 3d rec
      raf.skipBytes(2);
      short year = raf.readShort();
      short month = raf.readShort();
      short day = raf.readShort();
      recHdr1.put("radar_lat", calcAngle(radar_lat));
      recHdr1.put("radar_lon", calcAngle(radar_lon));
      recHdr1.put("range_first", range_first);
      recHdr1.put("range_last", range_last);
      recHdr1.put("ground_height", ground_height);
      recHdr1.put("radar_height", radar_height);
      recHdr1.put("radar_alt", radar_alt);
      recHdr1.put("step", step);
      recHdr1.put("bins", bins); // System.out.println("  bins="+bins);
      recHdr1.put("num_rays", num_rays); // System.out.println("  rays="+num_rays);
      recHdr1.put("nparams", nparams); // System.out.println("  nparams="+nparams);
      recHdr1.put("multiprf", multiprf);
      recHdr1.put(
          "number_sweeps",
          number_sweeps); // System.out.println("IN HDR:  number_sweeps="+number_sweeps);
      recHdr1.put("year", year);
      recHdr1.put("month", month);
      recHdr1.put("day", day);
      recHdr1.put("base_time", base_time);
    } catch (Exception e) {
      System.out.println(e.toString());
      e.printStackTrace();
    }
    return recHdr1;
  }
  public boolean readIndex(RandomAccessFile raf) {
    gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
    try {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      //// header message
      if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
        logger.error("GribCollection {}: invalid index", gc.getName());
        return false;
      }

      int v = raf.readInt();
      if (v != getVersion()) {
        logger.warn(
            "GribCollection {}: index found version={}, want version= {} on file {}",
            new Object[] {gc.getName(), v, version, raf.getLocation()});
        return false;
      }

      long skip = raf.readLong();
      raf.skipBytes(skip);

      int size = NcStream.readVInt(raf);
      if ((size < 0) || (size > 100 * 1000 * 1000)) {
        logger.warn("GribCollection {}: invalid index ", gc.getName());
        return false;
      }

      byte[] m = new byte[size];
      raf.readFully(m);

      GribCollectionProto.GribCollectionIndex proto =
          GribCollectionProto.GribCollectionIndex.parseFrom(m);

      gc.center = proto.getCenter();
      gc.subcenter = proto.getSubcenter();
      gc.master = proto.getMaster();
      gc.local = proto.getLocal();
      gc.genProcessType = proto.getGenProcessType();
      gc.genProcessId = proto.getGenProcessId();
      gc.backProcessId = proto.getBackProcessId();
      gc.local = proto.getLocal();
      // gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);

      gc.filenames = new ArrayList<String>(proto.getFilesCount());
      for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));

      // error condition on a GribCollection Index
      if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
        logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
        return false;
      }

      gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
      for (int i = 0; i < proto.getGroupsCount(); i++)
        gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
      Collections.sort(gc.groups);

      gc.params = new ArrayList<Parameter>(proto.getParamsCount());
      for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));

      if (!readPartitions(proto)) {
        logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
        return false;
      }

      return true;

    } catch (Throwable t) {
      logger.error("Error reading index " + raf.getLocation(), t);
      return false;
    }
  }
  // check if compressed file seems ok
  public static long testValid(String ufilename) throws IOException {
    boolean lookForHeader = false;

    // gotta make it
    RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    raf.seek(0);
    byte[] b = new byte[8];
    raf.read(b);
    String test = new String(b);
    if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
      System.out.println("--Good header= " + test);
      raf.seek(24);
    } else {
      System.out.println("--No header ");
      lookForHeader = true;
      raf.seek(0);
    }

    boolean eof = false;
    int numCompBytes;
    try {

      while (!eof) {

        if (lookForHeader) {
          raf.read(b);
          test = new String(b);
          if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
            System.out.println("  found header= " + test);
            raf.skipBytes(16);
            lookForHeader = false;
          } else {
            raf.skipBytes(-8);
          }
        }

        try {
          numCompBytes = raf.readInt();
          if (numCompBytes == -1) {
            System.out.println("\n--done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          System.out.println("\n--got EOFException ");
          break; // assume this is ok
        }

        System.out.print(" " + numCompBytes + ",");
        if (numCompBytes < 0) {
          System.out.println("\n--last block " + numCompBytes);
          numCompBytes = -numCompBytes;
          if (!lookForHeader) eof = true;
        }

        raf.skipBytes(numCompBytes);
      }
    } catch (EOFException e) {
      e.printStackTrace();
    }

    return raf.getFilePointer();
  }
  /**
   * Write equivilent uncompressed version of the file.
   *
   * @param inputRaf file to uncompress
   * @param ufilename write to this file
   * @return raf of uncompressed file
   * @throws IOException on read error
   */
  private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
      throws IOException {
    RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
    FileLock lock = null;

    while (true) { // loop waiting for the lock
      try {
        lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
        break;

      } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
        try {
          Thread.sleep(100); // msecs
        } catch (InterruptedException e1) {
        }
      }
    }

    try {
      inputRaf.seek(0);
      byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
      inputRaf.read(header);
      outputRaf.write(header);

      boolean eof = false;
      int numCompBytes;
      byte[] ubuff = new byte[40000];
      byte[] obuff = new byte[40000];
      try {
        CBZip2InputStream cbzip2 = new CBZip2InputStream();
        while (!eof) {
          try {
            numCompBytes = inputRaf.readInt();
            if (numCompBytes == -1) {
              if (log.isDebugEnabled()) log.debug("  done: numCompBytes=-1 ");
              break;
            }
          } catch (EOFException ee) {
            log.warn("  got EOFException ");
            break; // assume this is ok
          }

          if (log.isDebugEnabled()) {
            log.debug(
                "reading compressed bytes "
                    + numCompBytes
                    + " input starts at "
                    + inputRaf.getFilePointer()
                    + "; output starts at "
                    + outputRaf.getFilePointer());
          }
          /*
           * For some stupid reason, the last block seems to
           * have the number of bytes negated.  So, we just
           * assume that any negative number (other than -1)
           * is the last block and go on our merry little way.
           */
          if (numCompBytes < 0) {
            if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
            numCompBytes = -numCompBytes;
            eof = true;
          }
          byte[] buf = new byte[numCompBytes];
          inputRaf.readFully(buf);
          ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);

          // CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
          cbzip2.setStream(bis);
          int total = 0;
          int nread;
          /*
          while ((nread = cbzip2.read(ubuff)) != -1) {
            dout2.write(ubuff, 0, nread);
            total += nread;
          }
          */
          try {
            while ((nread = cbzip2.read(ubuff)) != -1) {
              if (total + nread > obuff.length) {
                byte[] temp = obuff;
                obuff = new byte[temp.length * 2];
                System.arraycopy(temp, 0, obuff, 0, temp.length);
              }
              System.arraycopy(ubuff, 0, obuff, total, nread);
              total += nread;
            }
            if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
          } catch (BZip2ReadException ioe) {
            log.warn("Nexrad2IOSP.uncompress ", ioe);
          }
          float nrecords = (float) (total / 2432.0);
          if (log.isDebugEnabled())
            log.debug(
                "  unpacked "
                    + total
                    + " num bytes "
                    + nrecords
                    + " records; ouput ends at "
                    + outputRaf.getFilePointer());
        }

      } catch (Exception e) {
        if (outputRaf != null) outputRaf.close();
        outputRaf = null;

        // dont leave bad files around
        File ufile = new File(ufilename);
        if (ufile.exists()) {
          if (!ufile.delete())
            log.warn("failed to delete uncompressed file (IOException)" + ufilename);
        }

        if (e instanceof IOException) throw (IOException) e;
        else throw new RuntimeException(e);
      }

    } finally {
      if (null != outputRaf) outputRaf.flush();
      if (lock != null) lock.release();
    }

    return outputRaf;
  }
  Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
    this.raf = orgRaf;

    if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());

    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);

    // volume scan header
    dataFormat = raf.readString(8);
    raf.skipBytes(1);
    String volumeNo = raf.readString(3);
    title_julianDay = raf.readInt(); // since 1/1/70
    title_msecs = raf.readInt();
    stationId = raf.readString(4).trim(); // only in AR2V0001
    if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);

    if (stationId.length() == 0) {
      // try to get it from the filename LOOK

      stationId = null;
    }

    // try to find the station
    if (stationId != null) {
      if (!stationId.startsWith("K") && stationId.length() == 4) {
        String _stationId = "K" + stationId;
        station = NexradStationDB.get(_stationId);
      } else station = NexradStationDB.get(stationId);
    }

    // see if we have to uncompress
    if (dataFormat.equals(AR2V0001)
        || dataFormat.equals(AR2V0003)
        || dataFormat.equals(AR2V0004)
        || dataFormat.equals(AR2V0006)) {
      raf.skipBytes(4);
      String BZ = raf.readString(2);
      if (BZ.equals("BZ")) {
        RandomAccessFile uraf;
        File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");

        if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
          // see if its locked - another thread is writing it
          FileInputStream fstream = null;
          FileLock lock = null;
          try {
            fstream = new FileInputStream(uncompressedFile);
            // lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked

            while (true) { // loop waiting for the lock
              try {
                lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
                break;

              } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
                try {
                  Thread.sleep(100); // msecs
                } catch (InterruptedException e1) {
                  break;
                }
              }
            }

          } finally {
            if (lock != null) lock.release();
            if (fstream != null) fstream.close();
          }
          uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");

        } else {
          // nope, gotta uncompress it
          uraf = uncompress(raf, uncompressedFile.getPath());
          if (log.isDebugEnabled())
            log.debug("made uncompressed file= " + uncompressedFile.getPath());
        }

        // switch to uncompressed file
        raf.close();
        raf = uraf;
        raf.order(RandomAccessFile.BIG_ENDIAN);
      }

      raf.seek(Level2Record.FILE_HEADER_SIZE);
    }

    List<Level2Record> reflectivity = new ArrayList<Level2Record>();
    List<Level2Record> doppler = new ArrayList<Level2Record>();
    List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highVelocity = new ArrayList<Level2Record>();
    List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
    List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
    List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();

    long message_offset31 = 0;
    int recno = 0;
    while (true) {

      Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
      if (r == null) break;
      if (showData) r.dump2(System.out);
      // skip non-data messages
      if (r.message_type == 31) {
        message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
      }

      if (r.message_type != 1 && r.message_type != 31) {
        if (showMessages) r.dumpMessage(System.out);
        continue;
      }

      //  if (showData) r.dump2(System.out);

      /* skip bad
      if (!r.checkOk()) {
        r.dump(System.out);
        continue;
      }   */

      // some global params
      if (vcp == 0) vcp = r.vcp;
      if (first == null) first = r;
      last = r;

      if (runCheck && !r.checkOk()) {
        continue;
      }

      if (r.hasReflectData) reflectivity.add(r);
      if (r.hasDopplerData) doppler.add(r);

      if (r.message_type == 31) {
        if (r.hasHighResREFData) highReflectivity.add(r);
        if (r.hasHighResVELData) highVelocity.add(r);
        if (r.hasHighResSWData) highSpectrum.add(r);
        if (r.hasHighResZDRData) highDiffReflectivity.add(r);
        if (r.hasHighResPHIData) highDiffPhase.add(r);
        if (r.hasHighResRHOData) highCorreCoefficient.add(r);
      }

      if ((cancelTask != null) && cancelTask.isCancel()) return;
    }
    if (debugRadials)
      System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
    if (highReflectivity.size() == 0) {
      reflectivityGroups = sortScans("reflect", reflectivity, 600);
      dopplerGroups = sortScans("doppler", doppler, 600);
    }
    if (highReflectivity.size() > 0)
      reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
    if (highVelocity.size() > 0)
      velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
    if (highSpectrum.size() > 0)
      spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
    if (highDiffReflectivity.size() > 0)
      diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
    if (highDiffPhase.size() > 0)
      diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
    if (highCorreCoefficient.size() > 0)
      coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
  }
Esempio n. 8
0
  /**
   * Initialize this reader. Get the Grid specific info
   *
   * @param fullCheck for a full check reading grids
   * @return true if successful
   * @throws IOException problem reading the data
   */
  protected boolean init(boolean fullCheck) throws IOException {
    if (rf == null) {
      logError("File is null");
      return false;
    }
    gridIndex = new GridIndex(rf.getLocation());

    rf.order(RandomAccessFile.BIG_ENDIAN);
    int numEntries = Math.abs(readInt(10));
    if (numEntries > 1000000) {
      needToSwap = true;
      numEntries = Math.abs(McIDASUtil.swbyt4(numEntries));
    }
    if (numEntries > MAX_GRIDS) {
      return false;
    }
    // System.out.println("need to Swap = " + needToSwap);
    // System.out.println("number entries="+numEntries);

    // go back to the beginning
    rf.seek(0);
    // read the fileheader
    String label = rf.readString(32);
    // GEMPAK too closely like McIDAS
    if (label.indexOf("GEMPAK DATA MANAGEMENT FILE") >= 0) {
      logError("label indicates this is a GEMPAK grid");
      return false;
    } else { // check that they are all printable ASCII chars
      for (int i = 0; i < label.length(); i++) {
        String s0 = label.substring(i, i + 1);
        if (!(0 <= s0.compareTo(" ") && s0.compareTo("~") <= 0)) {
          logError("bad label, not a McIDAS grid");
          return false;
        }
      }
    }

    // System.out.println("label = " + label);

    int project = readInt(8);
    // System.out.println("Project = " + project);

    int date = readInt(9);
    // dates are supposed to be yyyddd, but account for ccyyddd up to year 4000
    if ((date < 10000) || (date > 400000)) {
      logError("date wrong, not a McIDAS grid");
      return false;
    }
    // System.out.println("date = " + date);

    int[] entries = new int[numEntries];
    for (int i = 0; i < numEntries; i++) {
      entries[i] = readInt(i + 11);
      // sanity check that this is indeed a McIDAS Grid file
      if (entries[i] < -1) {
        logError("bad grid offset " + i + ": " + entries[i]);
        return false;
      }
    }
    if (!fullCheck) {
      return true;
    }

    // Don't swap:
    rf.order(RandomAccessFile.BIG_ENDIAN);
    for (int i = 0; i < numEntries; i++) {
      if (entries[i] == -1) {
        continue;
      }
      int[] header = new int[64];
      rf.seek(entries[i] * 4);
      rf.readInt(header, 0, 64);
      if (needToSwap) {
        swapGridHeader(header);
      }
      try {

        McIDASGridRecord gr = new McIDASGridRecord(entries[i], header);
        // if (gr.getGridDefRecordId().equals("CONF X:93 Y:65")) {
        // if (gr.getGridDefRecordId().equals("CONF X:54 Y:47")) {
        // figure out how to handle Mercator projections
        // if ( !(gr.getGridDefRecordId().startsWith("MERC"))) {
        gridIndex.addGridRecord(gr);
        if (gdsMap.get(gr.getGridDefRecordId()) == null) {
          McGridDefRecord mcdef = gr.getGridDefRecord();
          // System.out.println("new nav " + mcdef.toString());
          gdsMap.put(mcdef.toString(), mcdef);
          gridIndex.addHorizCoordSys(mcdef);
        }
        // }
      } catch (McIDASException me) {
        logError("problem creating grid dir");
        return false;
      }
    }
    // check to see if there are any grids that we can handle
    if (gridIndex.getGridRecords().isEmpty()) {
      logError("no grids found");
      return false;
    }
    return true;
  }