Exemplo n.º 1
1
  public boolean isValidFile(RandomAccessFile raf) {
    int data_msecs = 0;
    short data_julian_date = 0;

    try {
      raf.order(RandomAccessFile.LITTLE_ENDIAN);
      raf.seek(0);
      raf.skipBytes(14);
      short message_type = raf.readShort();
      if (message_type != 1) return false;

      raf.skipBytes(12);
      // data header
      byte[] b4 = raf.readBytes(4);
      data_msecs = bytesToInt(b4, true);
      byte[] b2 = raf.readBytes(2);
      data_julian_date = (short) bytesToShort(b2, true);
      java.util.Date dd = Cinrad2Record.getDate(data_julian_date, data_msecs);

      Calendar cal = new GregorianCalendar(new SimpleTimeZone(0, "GMT"));
      cal.clear();
      cal.setTime(dd);
      int year = cal.get(Calendar.YEAR);
      cal.setTime(new Date());
      int cyear = cal.get(Calendar.YEAR);
      if (year < 1990 || year > cyear) return false;
      return true;
    } catch (IOException ioe) {
      return false;
    }
  }
Exemplo n.º 2
0
 public static void main(String[] args) throws IOException {
   RandomAccessFile raf =
       new RandomAccessFile("Q:/cdmUnitTest/formats/grib2/sfc_d01_20080430_1200_f00000.grb2", "r");
   Grib2RecordScanner scan = new Grib2RecordScanner(raf);
   while (scan.hasNext()) scan.next();
   raf.close();
 }
  /**
   * _more_
   *
   * @param raf _more_
   * @param stop _more_
   * @return _more_
   * @throws IOException _more_
   */
  private boolean seekHeader(RandomAccessFile raf, long stop) throws IOException {
    // seek header
    StringBuffer hdr = new StringBuffer();
    int match = 0;

    while (raf.getFilePointer() < stop) {
      // code must be "G" "R" "I" "B"
      char c = (char) raf.read();

      hdr.append((char) c);
      if (c == 'G') {
        match = 1;
      } else if ((c == 'R') && (match == 1)) {
        match = 2;
      } else if ((c == 'I') && (match == 2)) {
        match = 3;
      } else if ((c == 'B') && (match == 3)) {
        match = 4;

        Matcher m = productID.matcher(hdr.toString());
        if (m.find()) {
          header = m.group(1);
        } else {
          // header = hdr.toString();
          header = "GRIB1";
        }
        // System.out.println( "header =" + header.toString() );
        return true;
      } else {
        match = 0; /* Needed to protect against "GaRaIaB" case. */
      }
    }
    return false;
  } // end seekHeader
  @Test
  public void testDBZ() throws IOException {
    TaskConfiguration taskConfiguration = new TaskConfiguration();
    taskConfiguration.builder(dbzFile, 160);
    System.out.println(dbzFile.getFilePointer());
    System.out.println(taskConfiguration);
    assertTrue(taskConfiguration.getCutNumber() == 8);
    for (int i = 0; i < taskConfiguration.getCutNumber(); i++) {
      CutConfiguration cutConfiguration = new CutConfiguration();
      cutConfiguration.builder(dbzFile, -1);
      System.out.println(cutConfiguration);

      System.out.println(dbzFile.getFilePointer());
    }
    ProductHeader productHeader = new ProductHeader();
    productHeader.builder(dbzFile, -1);
    System.out.println(productHeader);
    assertTrue(productHeader.getProductNumber() == 1);

    ProductDependentParameter productDependentParameter =
        new ProductDependentParameter(
            CinradXUtils.getProductType(productHeader.getProductNumber()), dbzFile.readBytes(64));
    assertTrue(productDependentParameter.getProductType().name().equalsIgnoreCase("PPI"));
    // productDependentParameter.buildProductParameter(dbzFile.readBytes(64));
    System.out.println(productDependentParameter.getProductType());
    System.out.println(productDependentParameter);
  }
Exemplo n.º 5
0
  /**
   * Sync the file
   *
   * @return true if needed to sync
   * @throws IOException problem synching the file
   */
  public boolean sync() throws IOException {
    // printStack("***************************** sync ************************", 100);
    // System.out.printf("check sync on %s (%s) %n", raf.getLocation(),
    // Calendar.getInstance().getTime());

    if (gemreader.getInitFileSize() < raf.length()) {
      long start = System.currentTimeMillis();
      Trace.msg(
          "GEMPAKStationIOSP.sync: file "
              + raf.getLocation()
              + " is bigger: "
              + raf.length()
              + " > "
              + gemreader.getInitFileSize());
      Trace.call1("GEMPAKStationIOSP.sync: reader.init");
      gemreader.init(raf, true);
      Trace.call2("GEMPAKStationIOSP.sync: reader.init");
      Trace.call1("GEMPAKStationIOSP.sync: buildNCFile");
      // reconstruct the ncfile objects
      buildNCFile();
      Trace.call2("GEMPAKSIOSP.sync: buildNCFile");
      // System.out.printf("sync on %s took %d msecs%n", raf.getLocation(),
      // (System.currentTimeMillis()-start));
      return true;
    }
    return false;
  }
Exemplo n.º 6
0
  /**
   * Read the grid
   *
   * @param gr the grid record
   * @return the data
   */
  public float[] readGrid(McIDASGridRecord gr) {

    float[] data = null;
    try {
      int te = (gr.getOffsetToHeader() + 64) * 4;
      int rows = gr.getRows();
      int cols = gr.getColumns();
      rf.seek(te);

      float scale = (float) gr.getParamScale();

      data = new float[rows * cols];
      rf.order(needToSwap ? rf.LITTLE_ENDIAN : rf.BIG_ENDIAN);
      int n = 0;
      // store such that 0,0 is in lower left corner...
      for (int nc = 0; nc < cols; nc++) {
        for (int nr = 0; nr < rows; nr++) {
          int temp = rf.readInt(); // check for missing value
          data[(rows - nr - 1) * cols + nc] =
              (temp == McIDASUtil.MCMISSING) ? Float.NaN : ((float) temp) / scale;
        }
      }
      rf.order(rf.BIG_ENDIAN);
    } catch (Exception esc) {
      System.out.println(esc);
    }
    return data;
  }
  @After
  public void tearDown() throws Exception {
    dbzFile.flush();
    dbzFile.close();

    cappiFile.flush();
    cappiFile.close();
  }
Exemplo n.º 8
0
  public Grib2RecordScanner(RandomAccessFile raf) throws IOException {
    startPos = 0;
    this.raf = raf;
    raf.seek(startPos);
    raf.order(RandomAccessFile.BIG_ENDIAN);
    lastPos = startPos;

    if (debugRepeat) System.out.printf(" Grib2RecordScanner %s%n", raf.getLocation());
  }
 /**
  * Grib edition number 1, 2 or 0 not a Grib file.
  *
  * @return int 0 not a Grib file, 1 Grib1, 2 Grib2
  * @throws NotSupportedException
  * @throws IOException
  */
 public final int getEdition() throws IOException, NotSupportedException {
   int check = 0; // Not a valid Grib file
   long length = (raf.length() < 4000L) ? raf.length() : 4000L;
   if (!seekHeader(raf, length)) {
     return 0; // not valid Grib file
   }
   //  Read Section 0 Indicator Section to get Edition number
   Grib1IndicatorSection is = new Grib1IndicatorSection(raf); // section 0
   return is.getGribEdition();
 } // end getEdition
Exemplo n.º 10
0
  // LOOK what about extending an index ??
  public boolean makeIndex(String filename, RandomAccessFile dataRaf, Formatter f)
      throws IOException {
    File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
    FileOutputStream fout = new FileOutputStream(idxFile);
    RandomAccessFile raf = null;
    try {
      //// header message
      fout.write(MAGIC_START.getBytes("UTF-8"));
      NcStream.writeVInt(fout, version);

      Map<Long, Integer> gdsMap = new HashMap<Long, Integer>();
      gdsList = new ArrayList<Grib1SectionGridDefinition>();
      records = new ArrayList<Grib1Record>(200);

      Grib1IndexProto.Grib1Index.Builder rootBuilder = Grib1IndexProto.Grib1Index.newBuilder();
      rootBuilder.setFilename(filename);

      if (dataRaf == null) {
        raf = new RandomAccessFile(filename, "r");
        dataRaf = raf;
      }

      Grib1RecordScanner scan = new Grib1RecordScanner(dataRaf);
      while (scan.hasNext()) {
        Grib1Record r = scan.next();
        if (r == null) break; // done
        records.add(r);

        Grib1SectionGridDefinition gds = r.getGDSsection();
        Integer index = gdsMap.get(gds.calcCRC());
        if (gds.getPredefinedGridDefinition()
            >= 0) // skip predefined gds - they dont have raw bytes
        index = 0;
        else if (index == null) {
          gdsList.add(gds);
          index = gdsList.size() - 1;
          gdsMap.put(gds.calcCRC(), index);
          rootBuilder.addGdsList(makeGdsProto(gds));
        }
        rootBuilder.addRecords(makeRecordProto(r, index));
      }

      ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Index index = rootBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(fout, b.length); // message size
      fout.write(b); // message  - all in one gulp
      f.format("  made gbx9 index for %s size=%d%n", filename, b.length);
      return true;

    } finally {
      fout.close();
      if (raf != null) raf.close();
    }
  }
Exemplo n.º 11
0
 /**
  * tricky bit of business. recapture the entire record based on drs position. for validation.
  *
  * @param raf from this RandomAccessFile
  * @param drsPos Grib2SectionDataRepresentation starts here
  */
 public static Grib2Record findRecordByDrspos(RandomAccessFile raf, long drsPos)
     throws IOException {
   Grib2Record result = null;
   Grib2RecordScanner scanner = new Grib2RecordScanner(raf);
   long pos = Math.max(0, drsPos - 10000); // go back 10000 bytes
   raf.seek(pos);
   while (scanner.hasNext()) { // find GRIB header
     result = scanner.next();
     if (result.getDataRepresentationSection().getStartingPosition() == drsPos) return result;
     if (raf.getFilePointer() > drsPos) break;
   }
   return null;
 }
Exemplo n.º 12
0
 /**
  * Read an integer
  *
  * @param word word in file (0 based) to read
  * @return int read
  * @throws IOException problem reading file
  */
 public int readInt(int word) throws IOException {
   if (rf == null) {
     throw new IOException("no file to read from");
   }
   rf.seek(word * 4);
   // set the order
   if (needToSwap) {
     rf.order(RandomAccessFile.LITTLE_ENDIAN); // swap
   } else {
     rf.order(RandomAccessFile.BIG_ENDIAN);
   }
   int idata = rf.readInt();
   rf.order(RandomAccessFile.BIG_ENDIAN);
   return idata;
 }
 /** Read StationName strings */
 public java.util.Map<String, String> readStnNames(ucar.unidata.io.RandomAccessFile raf) {
   java.util.Map<String, String> hdrNames = new java.util.HashMap<String, String>();
   try {
     raf.seek(6288);
     String stnName = raf.readString(16); // System.out.println(" stnName="+stnName.trim());
     raf.seek(6306);
     String stnName_util = raf.readString(16);
     hdrNames.put("StationName", stnName.trim());
     hdrNames.put("StationName_SetupUtility", stnName_util.trim());
   } catch (Exception e) {
     System.out.println(e.toString());
     e.printStackTrace();
   }
   return hdrNames;
 }
Exemplo n.º 14
0
  /**
   * Open the service provider for reading.
   *
   * @param raf file to read from
   * @param ncfile netCDF file we are writing to (memory)
   * @param cancelTask task for cancelling
   * @throws IOException problem reading file
   */
  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask)
      throws IOException {
    super.open(raf, ncfile, cancelTask);

    if (areaReader == null) areaReader = new AreaReader();

    try {
      areaReader.init(raf.getLocation(), ncfile);

    } catch (Throwable e) {
      close(); // try not to leak files
      throw new IOException(e);

    } finally {
      raf.close(); // avoid leaks
    }
  }
Exemplo n.º 15
0
  public static boolean isValidFile(RandomAccessFile raf) {
    try {
      raf.seek(0);
      while (raf.getFilePointer() < maxScan) {
        boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
        if (!found) return false;
        raf.skipBytes(7); // will be positioned on byte 0 of indicator section
        int edition = raf.read(); // read at byte 8
        if (edition == 2) return true;
      }

    } catch (IOException e) {
      return false;
    }

    return false;
  }
Exemplo n.º 16
0
 /**
  * Initialize the file, read in all the metadata (ala DM_OPEN)
  *
  * @param fullCheck for a full check reading grids
  * @param raf RandomAccessFile to read.
  * @throws IOException problem reading file
  */
 public final void init(RandomAccessFile raf, boolean fullCheck) throws IOException {
   rf = raf;
   raf.order(RandomAccessFile.BIG_ENDIAN);
   boolean ok = init(fullCheck);
   if (!ok) {
     throw new IOException("Unable to open McIDAS Grid file: " + errorMessage);
   }
 }
 /** Check if this is a valid SIGMET-IRIS file for this IOServiceProvider. */
 public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) {
   try {
     raf.order(RandomAccessFile.LITTLE_ENDIAN);
     // The first struct in the file is the product_hdr, which will have the
     // standard structure_header, followed by other embedded structures.
     // Each of these structures also have a structure header. To validate
     // the file we check for a product_hdr (by looking for type 27 in the
     // structure_header), then a product_configuration structure (by looking
     // for type 26 in its structure_header), then checking that that
     // the product_configuration does indicate a type of RAW data (type 15)
     raf.seek(0);
     short[] data = new short[13];
     raf.readShort(data, 0, 13);
     return (data[0] == (short) 27 && data[6] == (short) 26 && data[12] == (short) 15);
   } catch (IOException ioe) {
     System.out.println("In isValidFile(): " + ioe.toString());
     return false;
   }
 }
Exemplo n.º 18
0
 public boolean isValidFileOld(RandomAccessFile raf) {
   try {
     String loc = raf.getLocation();
     int posFirst = loc.lastIndexOf('/') + 1;
     if (posFirst < 0) posFirst = 0;
     String stationId = loc.substring(posFirst, posFirst + 4);
     NexradStationDB.init();
     NexradStationDB.Station station = NexradStationDB.get("K" + stationId);
     if (station != null) return true;
     else return false;
   } catch (IOException ioe) {
     return false;
   }
 }
Exemplo n.º 19
0
  /*
  MAGIC_START
  version
  sizeRecords
  VariableRecords (sizeRecords bytes)
  sizeIndex
  GribCollectionIndex (sizeIndex bytes)
  */
  private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
    File file = tp.getIndexFile();
    if (file.exists()) {
      if (!file.delete()) logger.error("Cant delete " + file.getPath());
    }

    RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      raf.writeLong(0); // no record section

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(tp.getName());

      GribCollection canonGc = canon.makeGribCollection(f);
      for (GribCollection.GroupHcs g : canonGc.getGroups())
        indexBuilder.addGroups(writeGroupProto(g));

      indexBuilder.setCenter(canonGc.getCenter());
      indexBuilder.setSubcenter(canonGc.getSubcenter());
      indexBuilder.setMaster(canonGc.getMaster());
      indexBuilder.setLocal(canonGc.getLocal());

      for (TimePartition.Partition p : tp.getPartitions()) {
        indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
      }

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);

    } finally {
      f.format("file size =  %d bytes%n", raf.length());
      raf.close();
    }

    return true;
  }
Exemplo n.º 20
0
  public boolean hasNext() throws IOException {
    if (lastPos >= raf.length()) return false;
    if (repeatPos > 0) {
      if (nextRepeating()) // this has created a new repeatRecord
      return true;
    } else {
      repeatRecord = null;
      repeatBms = null;
      // fall through to new record
    }

    boolean more;
    long stop = 0;

    while (true) { // scan until we get a GRIB-2 or more == false
      raf.seek(lastPos);
      more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
      if (!more) break;

      stop = raf.getFilePointer();
      // see if its GRIB-2
      raf.skipBytes(7);
      int edition = raf.read();
      if (edition == 2) break;
      lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
    }

    if (more) {
      int sizeHeader =
          (int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
      // if (sizeHeader > 30) sizeHeader = 30;
      header = new byte[sizeHeader];
      startPos = stop - sizeHeader;
      raf.seek(startPos);
      raf.read(header);
    }
    if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
    return more;
  }
Exemplo n.º 21
0
  public float[] getData(RandomAccessFile raf, byte[] bitmap, Grib2Drs gdrs) throws IOException {
    this.bitmap = bitmap;

    if (bitmap != null) { // is bitmap ok ?
      if (bitmap.length * 8 < totalNPoints) { // gdsNumberPoints == nx * ny ??
        System.out.printf(
            "Bitmap section length = %d != grid length %d (%d,%d)",
            bitmap.length, totalNPoints, nx, totalNPoints / nx);
        throw new IllegalStateException("Bitmap section length!= grid length %");
      }
    }

    raf.seek(startPos + 5); // skip past first 5 bytes in data section, now ready to read

    float[] data;
    switch (dataTemplate) {
      case 0:
        data = getData0(raf, (Grib2Drs.Type0) gdrs);
        break;
      case 2:
        data = getData2(raf, (Grib2Drs.Type2) gdrs);
        break;
      case 3:
        data = getData3(raf, (Grib2Drs.Type3) gdrs);
        break;
      case 40:
        data = getData40(raf, (Grib2Drs.Type40) gdrs);
        break;
      default:
        throw new UnsupportedOperationException("Unsupported DRS type = " + dataTemplate);
    }

    // int scanMode = gds.getGds().getScanMode();
    // int nx = gds.getGds().getNx();  // needs some smarts for different type Grids
    scanningModeCheck(data, scanMode, nx);

    return data;
  }
  /**
   * Read some global data from SIGMET file. The SIGMET file consists of records with fixed
   * length=6144 bytes.
   */
  public static java.util.Map<String, Number> readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) {
    java.util.Map<String, Number> recHdr1 = new java.util.HashMap<String, Number>();
    try {
      int nparams = 0;
      //      -- Read from <product_end> of the 1st record -- 12+320+120
      //      -- Calculate Nyquist velocity --------------------
      raf.seek(452);
      int prf = raf.readInt();
      raf.seek(480);
      int wave = raf.readInt();
      float vNyq = calcNyquist(prf, wave);
      recHdr1.put("vNyq", vNyq);

      //      -- Read from the 2nd record----------- 6144+12(strucr_hdr)+168(from ingest_config)
      raf.seek(6324);
      int radar_lat = raf.readInt();
      int radar_lon = raf.readInt(); // 6328
      short ground_height = raf.readShort(); // 6332
      short radar_height = raf.readShort(); // 6334
      raf.skipBytes(4);
      short num_rays = raf.readShort(); // 6340
      raf.skipBytes(2);
      int radar_alt = raf.readInt(); // 6344
      raf.seek(6648);
      int time_beg = raf.readInt();
      raf.seek(6652);
      int time_end = raf.readInt();
      raf.seek(6772);
      int data_mask = raf.readInt();
      for (int j = 0; j < 32; j++) {
        nparams += ((data_mask >> j) & (0x1));
      }
      raf.seek(6912);
      short multiprf = raf.readShort();
      raf.seek(7408);
      int range_first = raf.readInt(); //  cm    7408
      int range_last = raf.readInt(); //  cm  7412
      raf.skipBytes(2);
      short bins = raf.readShort(); // 7418
      if (bins % 2 != 0) bins = (short) (bins + 1);
      raf.skipBytes(4);
      int step = raf.readInt(); //  cm    7424
      raf.seek(7574);
      short number_sweeps = raf.readShort(); // 7574
      raf.seek(12312);
      int base_time = raf.readInt(); // <ingest_data_header> 3d rec
      raf.skipBytes(2);
      short year = raf.readShort();
      short month = raf.readShort();
      short day = raf.readShort();
      recHdr1.put("radar_lat", calcAngle(radar_lat));
      recHdr1.put("radar_lon", calcAngle(radar_lon));
      recHdr1.put("range_first", range_first);
      recHdr1.put("range_last", range_last);
      recHdr1.put("ground_height", ground_height);
      recHdr1.put("radar_height", radar_height);
      recHdr1.put("radar_alt", radar_alt);
      recHdr1.put("step", step);
      recHdr1.put("bins", bins); // System.out.println("  bins="+bins);
      recHdr1.put("num_rays", num_rays); // System.out.println("  rays="+num_rays);
      recHdr1.put("nparams", nparams); // System.out.println("  nparams="+nparams);
      recHdr1.put("multiprf", multiprf);
      recHdr1.put(
          "number_sweeps",
          number_sweeps); // System.out.println("IN HDR:  number_sweeps="+number_sweeps);
      recHdr1.put("year", year);
      recHdr1.put("month", month);
      recHdr1.put("day", day);
      recHdr1.put("base_time", base_time);
    } catch (Exception e) {
      System.out.println(e.toString());
      e.printStackTrace();
    }
    return recHdr1;
  }
  // check if compressed file seems ok
  public static long testValid(String ufilename) throws IOException {
    boolean lookForHeader = false;

    // gotta make it
    RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    raf.seek(0);
    byte[] b = new byte[8];
    raf.read(b);
    String test = new String(b);
    if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
      System.out.println("--Good header= " + test);
      raf.seek(24);
    } else {
      System.out.println("--No header ");
      lookForHeader = true;
      raf.seek(0);
    }

    boolean eof = false;
    int numCompBytes;
    try {

      while (!eof) {

        if (lookForHeader) {
          raf.read(b);
          test = new String(b);
          if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
            System.out.println("  found header= " + test);
            raf.skipBytes(16);
            lookForHeader = false;
          } else {
            raf.skipBytes(-8);
          }
        }

        try {
          numCompBytes = raf.readInt();
          if (numCompBytes == -1) {
            System.out.println("\n--done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          System.out.println("\n--got EOFException ");
          break; // assume this is ok
        }

        System.out.print(" " + numCompBytes + ",");
        if (numCompBytes < 0) {
          System.out.println("\n--last block " + numCompBytes);
          numCompBytes = -numCompBytes;
          if (!lookForHeader) eof = true;
        }

        raf.skipBytes(numCompBytes);
      }
    } catch (EOFException e) {
      e.printStackTrace();
    }

    return raf.getFilePointer();
  }
Exemplo n.º 24
0
 /**
  * Close this IOSP
  *
  * @throws IOException problem closing file
  */
 public void close() throws IOException {
   raf.close();
 }
  /**
   * scans a Grib file to gather information that could be used to create an index or dump the
   * metadata contents.
   *
   * @param parm
   * @throws NotSupportedException
   * @throws IOException if raf does not contain a valid GRIB record
   */
  public final boolean scan(int[] parm) throws NotSupportedException, IOException {
    long start = System.currentTimeMillis();
    // stores the number of times a particular GDS is used
    HashMap gdsCounter = new HashMap();
    Grib1ProductDefinitionSection pds = null;
    Grib1GridDefinitionSection gds = null;
    long gdsOffset = 0;
    DataOutputStream dos = new DataOutputStream(System.out);

    // System.out.println("file position =" + raf.getFilePointer());
    long SOR = raf.getFilePointer();
    while (raf.getFilePointer() < raf.length()) {
      SOR = raf.getFilePointer();
      if (seekHeader(raf, raf.length())) {
        // Read Section 0 Indicator Section
        Grib1IndicatorSection is = new Grib1IndicatorSection(raf);
        // System.out.println( "Grib record length=" + is.getGribLength());
        // EOR (EndOfRecord) calculated so skipping data sections is faster
        long EOR = raf.getFilePointer() + is.getGribLength() - is.getLength();
        // long SOR = raf.getFilePointer() - is.getLength();

        // skip Grib 2 records in a Grib 1 file
        if (is.getGribEdition() == 2) {
          // System.out.println( "Error Grib 2 record in Grib1 file" ) ;
          raf.seek(EOR);
          continue;
        }

        if (parm[0] == -1) { // extract only 1st record
          raf.seek(SOR);
          byte[] oneRecord = new byte[(int) is.getGribLength()];
          raf.read(oneRecord);
          dos.write(oneRecord, 0, oneRecord.length);
          dos.flush();
          break;
        }
        long dataOffset = 0;
        try { // catch all exceptions and seek to EOR

          // Read Section 1 Product Definition Section PDS
          pds = new Grib1ProductDefinitionSection(raf);
          if (pds.getLengthErr()) {
            raf.seek(EOR);
            continue;
          }
          Grib1PDSVariables pdsv = pds.getPdsVars();
          if (getParameter && parm[0] == pdsv.getParameterNumber()) {
            raf.seek(SOR);
            byte[] oneRecord = new byte[(int) is.getGribLength()];
            raf.read(oneRecord);
            dos.write(oneRecord, 0, oneRecord.length);
            dos.flush();
          } else if (parm[0] <= pdsv.getForecastTime() && parm[1] >= pdsv.getForecastTime()) {
            raf.seek(SOR);
            byte[] oneRecord = new byte[(int) is.getGribLength()];
            raf.read(oneRecord);
            dos.write(oneRecord, 0, oneRecord.length);
            dos.flush();
          }

          raf.seek(EOR);
          continue;

        } catch (Exception e) {
          // .println( "Caught Exception scannning record" );
          e.printStackTrace();
          raf.seek(EOR);
          continue;
        }
      } // end if seekHeader
      // System.out.println( "raf.getFilePointer()=" + raf.getFilePointer());
      // System.out.println( "raf.length()=" + raf.length() );
    } // end while raf.getFilePointer() < raf.length()
    // System.out.println("GribInput: processed in " +
    //   (System.currentTimeMillis()- start) + " milliseconds");
    dos.close();
    return true;
  } // end scan
Exemplo n.º 26
0
  public boolean readIndex(RandomAccessFile raf) {
    gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
    try {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      //// header message
      if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
        logger.error("GribCollection {}: invalid index", gc.getName());
        return false;
      }

      int v = raf.readInt();
      if (v != getVersion()) {
        logger.warn(
            "GribCollection {}: index found version={}, want version= {} on file {}",
            new Object[] {gc.getName(), v, version, raf.getLocation()});
        return false;
      }

      long skip = raf.readLong();
      raf.skipBytes(skip);

      int size = NcStream.readVInt(raf);
      if ((size < 0) || (size > 100 * 1000 * 1000)) {
        logger.warn("GribCollection {}: invalid index ", gc.getName());
        return false;
      }

      byte[] m = new byte[size];
      raf.readFully(m);

      GribCollectionProto.GribCollectionIndex proto =
          GribCollectionProto.GribCollectionIndex.parseFrom(m);

      gc.center = proto.getCenter();
      gc.subcenter = proto.getSubcenter();
      gc.master = proto.getMaster();
      gc.local = proto.getLocal();
      gc.genProcessType = proto.getGenProcessType();
      gc.genProcessId = proto.getGenProcessId();
      gc.backProcessId = proto.getBackProcessId();
      gc.local = proto.getLocal();
      // gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);

      gc.filenames = new ArrayList<String>(proto.getFilesCount());
      for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));

      // error condition on a GribCollection Index
      if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
        logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
        return false;
      }

      gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
      for (int i = 0; i < proto.getGroupsCount(); i++)
        gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
      Collections.sort(gc.groups);

      gc.params = new ArrayList<Parameter>(proto.getParamsCount());
      for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));

      if (!readPartitions(proto)) {
        logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
        return false;
      }

      return true;

    } catch (Throwable t) {
      logger.error("Error reading index " + raf.getLocation(), t);
      return false;
    }
  }
Exemplo n.º 27
0
  private void createIndex(
      File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
      throws IOException {
    Grib2Record first = null; // take global metadata from here

    if (indexFile.exists()) indexFile.delete(); // replace it
    f.format(" createIndex for %s%n", indexFile.getPath());

    RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;
      for (Group g : groups) {
        g.fileSet = new HashSet<Integer>();
        for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
          if (first == null) first = vb.first;
          GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.recordMap.length;
        }
      }
      long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
      f.format(
          "  write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
          countBytes, countRecords, bytesPerRecord);

      if (first == null) {
        logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
        throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(gc.getName());

      for (String fn : filenames) indexBuilder.addFiles(fn);

      for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));

      /* int count = 0;
      for (DatasetCollectionManager dcm : collections) {
        indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
        count++;
      } */

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("  write GribCollectionIndex= %d bytes%n", b.length);

    } finally {
      f.format("  file size =  %d bytes%n", raf.length());
      raf.close();
      if (raf != null) raf.close();
    }
  }
Exemplo n.º 28
0
  // Grid point data - JPEG 2000 code stream format
  public float[] getData40(RandomAccessFile raf, Grib2Drs.Type40 gdrs) throws IOException {
    // 6-xx  jpeg2000 data block to decode

    // dataPoints are number of points encoded, it could be less than the
    // totalNPoints in the grid record if bitMap is used, otherwise equal
    // int dataPoints = drs.getDataPoints();

    int nb = gdrs.numberOfBits;
    int D = gdrs.decimalScaleFactor;
    float DD = (float) java.lang.Math.pow((double) 10, (double) D);
    float R = gdrs.referenceValue;
    int E = gdrs.binaryScaleFactor;
    float EE = (float) java.lang.Math.pow((double) 2.0, (double) E);

    Grib2JpegDecoder g2j = null;
    try {
      if (nb != 0) { // there's data to decode
        String[] argv = new String[6];
        argv[0] = "-rate";
        argv[1] = Integer.toString(nb);
        argv[2] = "-verbose";
        argv[3] = "off";
        argv[4] = "-debug";
        argv[5] = "on";
        // argv[ 2 ] = "-nocolorspace" ;
        // argv[ 3 ] = "-Rno_roi" ;
        // argv[ 4 ] = "-cdstr_info" ;
        // argv[ 5 ] = "-verbose" ;
        g2j = new Grib2JpegDecoder(argv);
        // how jpeg2000.jar use to decode, used raf
        // g2j.decode(raf, length - 5);
        // jpeg-1.0.jar added method to have the data read first
        byte[] buf = new byte[dataLength - 5];
        raf.read(buf);
        g2j.decode(buf);
        gdrs.hasSignedProblem = g2j.hasSignedProblem();
      }

      float[] data = new float[totalNPoints];

      if (nb == 0) { // no data to decoded, set to reference or  MissingValue
        for (int i = 0; i < dataNPoints; i++) data[i] = R;
      } else if (bitmap == null) {
        if (g2j.data.length != dataNPoints) {
          data = null;
          return data;
        }
        for (int i = 0; i < dataNPoints; i++) {
          // Y * 10^D = R + (X1 + X2) * 2^E ; // regulation 92.9.4
          // Y = (R + ( 0 + X2) * EE)/DD ;
          data[i] = (R + g2j.data[i] * EE) / DD;
        }
      } else { // use bitmap
        for (int i = 0, j = 0; i < totalNPoints; i++) {
          if ((bitmap[i / 8] & GribNumbers.bitmask[i % 8]) != 0) {
            data[i] = (R + g2j.data[j++] * EE) / DD;
          } else {
            data[i] = staticMissingValue; // LOOK ??
          }
        }
      }
      return data;

    } catch (NullPointerException npe) {

      log.error(
          "Grib2DataSection.jpeg2000Unpacking: bit rate too small nb ="
              + nb
              + " for file"
              + raf.getLocation());
      float[] data = new float[dataNPoints];
      for (int i = 0; i < dataNPoints; i++) {
        data[i] = staticMissingValue; // LOOK ??
      }
      return data;
    }
  }
Exemplo n.º 29
0
  public Grib2Record next() throws IOException {
    if (repeatRecord != null) { // serve current repeatRecord if it exists
      return new Grib2Record(repeatRecord);
    }

    Grib2SectionIndicator is = null;
    try {
      is = new Grib2SectionIndicator(raf);
      Grib2SectionIdentification ids = new Grib2SectionIdentification(raf);
      Grib2SectionLocalUse lus = new Grib2SectionLocalUse(raf);
      Grib2SectionGridDefinition gds = new Grib2SectionGridDefinition(raf);
      Grib2SectionProductDefinition pds = new Grib2SectionProductDefinition(raf);
      Grib2SectionDataRepresentation drs = new Grib2SectionDataRepresentation(raf);
      Grib2SectionBitMap bms = new Grib2SectionBitMap(raf);
      Grib2SectionData dataSection = new Grib2SectionData(raf);
      if (dataSection.getMsgLength() > is.getMessageLength()) { // presumably corrupt
        raf.seek(drs.getStartingPosition()); // go back to before the dataSection
        throw new IllegalStateException("Illegal Grib2SectionData Message Length");
      }

      // look for duplicate gds
      long crc = gds.calcCRC();
      Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
      if (gdsCached != null) gds = gdsCached;
      else gdsMap.put(crc, gds);

      // look for duplicate pds
      /* crc = pds.calcCRC();
      Grib2SectionProductDefinition pdsCached = pdsMap.get(crc);
      if (pdsCached != null)
        pds = pdsCached;
      else
        pdsMap.put(crc, pds); */

      // check to see if we have a repeating record
      long pos = raf.getFilePointer();
      long ending = is.getEndPos();
      if (pos + 34 < ending) { // give it 30 bytes of slop
        if (debugRepeat) System.out.printf(" REPEAT AT %d != %d%n", pos + 4, ending);
        repeatPos = pos;
        repeatRecord =
            new Grib2Record(
                header,
                is,
                ids,
                lus,
                gds,
                pds,
                drs,
                bms,
                dataSection,
                false); // this assumes immutable sections
        // track bms in case its a repeat
        if (bms.getBitMapIndicator() == 0) repeatBms = bms;
        return new Grib2Record(
            repeatRecord); // GribRecord isnt immutable; still may not be necessary
      }

      if (debug)
        System.out.printf(
            " read until %d grib ending at %d header ='%s'%n",
            raf.getFilePointer(), ending, StringUtil2.cleanup(header));

      // check that end section is correct
      raf.seek(ending - 4);
      for (int i = 0; i < 4; i++) {
        if (raf.read() != 55) {
          log.warn(
              "Missing End of GRIB message at pos="
                  + ending
                  + " header= "
                  + StringUtil2.cleanup(header)
                  + " for="
                  + raf.getLocation());
          break;
        }
      }
      lastPos = raf.getFilePointer();

      return new Grib2Record(header, is, ids, lus, gds, pds, drs, bms, dataSection, false);

    } catch (Throwable t) {
      long pos = (is == null) ? -1 : is.getStartPos();
      log.warn("Bad GRIB2 record in file {}, skipping pos={}", raf.getLocation(), pos);
      lastPos = raf.getFilePointer();
      if (hasNext()) // skip forward
      return next();
    }

    return null; // last record was incomplete
  }
Exemplo n.º 30
0
  // return true if got another repeat out of this record
  // side effect is that the new record is in repeatRecord
  private boolean nextRepeating() throws IOException {
    raf.seek(repeatPos);

    // octets 1-4 (Length of GDS)
    int length = GribNumbers.int4(raf);
    int section = raf.read();
    raf.seek(repeatPos);

    if (section == 2) {
      repeatRecord.setLus(new Grib2SectionLocalUse(raf));
      repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
      repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
      repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
      repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
      repeatRecord.setDataSection(new Grib2SectionData(raf));
      repeatRecord.repeat = section;

    } else if (section == 3) {
      repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
      repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
      repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
      repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
      repeatRecord.setDataSection(new Grib2SectionData(raf));
      repeatRecord.repeat = section;

    } else if (section == 4) {
      repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
      repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
      repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
      repeatRecord.setDataSection(new Grib2SectionData(raf));
      repeatRecord.repeat = section;

    } else {
      if (debugRepeat) System.out.printf(" REPEAT Terminate %d%n", section);
      repeatPos = -1;
      repeatRecord = null;
      repeatBms = null;
      return false;
    }

    // look for repeating bms
    Grib2SectionBitMap bms = repeatRecord.getBitmapSection();
    if (bms.getBitMapIndicator() == 254) {
      // replace BMS with last good one
      if (repeatBms == null) throw new IllegalStateException("No bms in repeating section");
      repeatRecord.setBms(repeatBms, true);
      // debug
      if (debugRepeat) System.out.printf("replaced bms %d%n", section);
      repeatRecord.repeat += 1000;

    } else if (bms.getBitMapIndicator() == 0) {
      // track last good bms
      repeatBms = repeatRecord.getBitmapSection();
    }

    // keep only unique gds
    if ((section == 2) || (section == 3)) {
      // look for duplicate gds
      Grib2SectionGridDefinition gds = repeatRecord.getGDSsection();
      long crc = gds.calcCRC();
      Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
      if (gdsCached != null) repeatRecord.setGdss(gdsCached);
      else gdsMap.put(crc, gds);
    }

    // check to see if we are at the end
    long pos = raf.getFilePointer();
    long ending = repeatRecord.getIs().getEndPos();
    if (pos + 34 < ending) { // give it 30 bytes of slop
      if (debugRepeat) System.out.printf(" REPEAT AGAIN %d != %d%n", pos + 4, ending);
      repeatPos = pos;
      return true;
    }

    if (debug)
      System.out.printf(
          " REPEAT read until %d grib ending at %d header ='%s'%n",
          raf.getFilePointer(), ending, StringUtil2.cleanup(header));

    // check that end section is correct
    raf.seek(ending - 4);
    for (int i = 0; i < 4; i++) {
      if (raf.read() != 55) {
        log.warn(
            "  REPEAT Missing End of GRIB message at pos="
                + ending
                + " header= "
                + StringUtil2.cleanup(header)
                + " for="
                + raf.getLocation());
        break;
      }
    }
    lastPos = raf.getFilePointer();

    if (debugRepeat) System.out.printf(" REPEAT DONE%n");
    repeatPos = -1; // no more repeats in this record
    return true;
  }