Ejemplo n.º 1
0
  /**
   * Sync the file
   *
   * @return true if needed to sync
   * @throws IOException problem synching the file
   */
  public boolean sync() throws IOException {
    // printStack("***************************** sync ************************", 100);
    // System.out.printf("check sync on %s (%s) %n", raf.getLocation(),
    // Calendar.getInstance().getTime());

    if (gemreader.getInitFileSize() < raf.length()) {
      long start = System.currentTimeMillis();
      Trace.msg(
          "GEMPAKStationIOSP.sync: file "
              + raf.getLocation()
              + " is bigger: "
              + raf.length()
              + " > "
              + gemreader.getInitFileSize());
      Trace.call1("GEMPAKStationIOSP.sync: reader.init");
      gemreader.init(raf, true);
      Trace.call2("GEMPAKStationIOSP.sync: reader.init");
      Trace.call1("GEMPAKStationIOSP.sync: buildNCFile");
      // reconstruct the ncfile objects
      buildNCFile();
      Trace.call2("GEMPAKSIOSP.sync: buildNCFile");
      // System.out.printf("sync on %s took %d msecs%n", raf.getLocation(),
      // (System.currentTimeMillis()-start));
      return true;
    }
    return false;
  }
 /**
  * Grib edition number 1, 2 or 0 not a Grib file.
  *
  * @return int 0 not a Grib file, 1 Grib1, 2 Grib2
  * @throws NotSupportedException
  * @throws IOException
  */
 public final int getEdition() throws IOException, NotSupportedException {
   int check = 0; // Not a valid Grib file
   long length = (raf.length() < 4000L) ? raf.length() : 4000L;
   if (!seekHeader(raf, length)) {
     return 0; // not valid Grib file
   }
   //  Read Section 0 Indicator Section to get Edition number
   Grib1IndicatorSection is = new Grib1IndicatorSection(raf); // section 0
   return is.getGribEdition();
 } // end getEdition
Ejemplo n.º 3
0
  /*
  MAGIC_START
  version
  sizeRecords
  VariableRecords (sizeRecords bytes)
  sizeIndex
  GribCollectionIndex (sizeIndex bytes)
  */
  private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
    File file = tp.getIndexFile();
    if (file.exists()) {
      if (!file.delete()) logger.error("Cant delete " + file.getPath());
    }

    RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      raf.writeLong(0); // no record section

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(tp.getName());

      GribCollection canonGc = canon.makeGribCollection(f);
      for (GribCollection.GroupHcs g : canonGc.getGroups())
        indexBuilder.addGroups(writeGroupProto(g));

      indexBuilder.setCenter(canonGc.getCenter());
      indexBuilder.setSubcenter(canonGc.getSubcenter());
      indexBuilder.setMaster(canonGc.getMaster());
      indexBuilder.setLocal(canonGc.getLocal());

      for (TimePartition.Partition p : tp.getPartitions()) {
        indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
      }

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);

    } finally {
      f.format("file size =  %d bytes%n", raf.length());
      raf.close();
    }

    return true;
  }
Ejemplo n.º 4
0
  public boolean hasNext() throws IOException {
    if (lastPos >= raf.length()) return false;
    if (repeatPos > 0) {
      if (nextRepeating()) // this has created a new repeatRecord
      return true;
    } else {
      repeatRecord = null;
      repeatBms = null;
      // fall through to new record
    }

    boolean more;
    long stop = 0;

    while (true) { // scan until we get a GRIB-2 or more == false
      raf.seek(lastPos);
      more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
      if (!more) break;

      stop = raf.getFilePointer();
      // see if its GRIB-2
      raf.skipBytes(7);
      int edition = raf.read();
      if (edition == 2) break;
      lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
    }

    if (more) {
      int sizeHeader =
          (int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
      // if (sizeHeader > 30) sizeHeader = 30;
      header = new byte[sizeHeader];
      startPos = stop - sizeHeader;
      raf.seek(startPos);
      raf.read(header);
    }
    if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
    return more;
  }
  /**
   * scans a Grib file to gather information that could be used to create an index or dump the
   * metadata contents.
   *
   * @param parm
   * @throws NotSupportedException
   * @throws IOException if raf does not contain a valid GRIB record
   */
  public final boolean scan(int[] parm) throws NotSupportedException, IOException {
    long start = System.currentTimeMillis();
    // stores the number of times a particular GDS is used
    HashMap gdsCounter = new HashMap();
    Grib1ProductDefinitionSection pds = null;
    Grib1GridDefinitionSection gds = null;
    long gdsOffset = 0;
    DataOutputStream dos = new DataOutputStream(System.out);

    // System.out.println("file position =" + raf.getFilePointer());
    long SOR = raf.getFilePointer();
    while (raf.getFilePointer() < raf.length()) {
      SOR = raf.getFilePointer();
      if (seekHeader(raf, raf.length())) {
        // Read Section 0 Indicator Section
        Grib1IndicatorSection is = new Grib1IndicatorSection(raf);
        // System.out.println( "Grib record length=" + is.getGribLength());
        // EOR (EndOfRecord) calculated so skipping data sections is faster
        long EOR = raf.getFilePointer() + is.getGribLength() - is.getLength();
        // long SOR = raf.getFilePointer() - is.getLength();

        // skip Grib 2 records in a Grib 1 file
        if (is.getGribEdition() == 2) {
          // System.out.println( "Error Grib 2 record in Grib1 file" ) ;
          raf.seek(EOR);
          continue;
        }

        if (parm[0] == -1) { // extract only 1st record
          raf.seek(SOR);
          byte[] oneRecord = new byte[(int) is.getGribLength()];
          raf.read(oneRecord);
          dos.write(oneRecord, 0, oneRecord.length);
          dos.flush();
          break;
        }
        long dataOffset = 0;
        try { // catch all exceptions and seek to EOR

          // Read Section 1 Product Definition Section PDS
          pds = new Grib1ProductDefinitionSection(raf);
          if (pds.getLengthErr()) {
            raf.seek(EOR);
            continue;
          }
          Grib1PDSVariables pdsv = pds.getPdsVars();
          if (getParameter && parm[0] == pdsv.getParameterNumber()) {
            raf.seek(SOR);
            byte[] oneRecord = new byte[(int) is.getGribLength()];
            raf.read(oneRecord);
            dos.write(oneRecord, 0, oneRecord.length);
            dos.flush();
          } else if (parm[0] <= pdsv.getForecastTime() && parm[1] >= pdsv.getForecastTime()) {
            raf.seek(SOR);
            byte[] oneRecord = new byte[(int) is.getGribLength()];
            raf.read(oneRecord);
            dos.write(oneRecord, 0, oneRecord.length);
            dos.flush();
          }

          raf.seek(EOR);
          continue;

        } catch (Exception e) {
          // .println( "Caught Exception scannning record" );
          e.printStackTrace();
          raf.seek(EOR);
          continue;
        }
      } // end if seekHeader
      // System.out.println( "raf.getFilePointer()=" + raf.getFilePointer());
      // System.out.println( "raf.length()=" + raf.length() );
    } // end while raf.getFilePointer() < raf.length()
    // System.out.println("GribInput: processed in " +
    //   (System.currentTimeMillis()- start) + " milliseconds");
    dos.close();
    return true;
  } // end scan
Ejemplo n.º 6
0
  private void createIndex(
      File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
      throws IOException {
    Grib2Record first = null; // take global metadata from here

    if (indexFile.exists()) indexFile.delete(); // replace it
    f.format(" createIndex for %s%n", indexFile.getPath());

    RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;
      for (Group g : groups) {
        g.fileSet = new HashSet<Integer>();
        for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
          if (first == null) first = vb.first;
          GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.recordMap.length;
        }
      }
      long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
      f.format(
          "  write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
          countBytes, countRecords, bytesPerRecord);

      if (first == null) {
        logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
        throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(gc.getName());

      for (String fn : filenames) indexBuilder.addFiles(fn);

      for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));

      /* int count = 0;
      for (DatasetCollectionManager dcm : collections) {
        indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
        count++;
      } */

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("  write GribCollectionIndex= %d bytes%n", b.length);

    } finally {
      f.format("  file size =  %d bytes%n", raf.length());
      raf.close();
      if (raf != null) raf.close();
    }
  }