@Test
  public void testDBZ() throws IOException {
    TaskConfiguration taskConfiguration = new TaskConfiguration();
    taskConfiguration.builder(dbzFile, 160);
    System.out.println(dbzFile.getFilePointer());
    System.out.println(taskConfiguration);
    assertTrue(taskConfiguration.getCutNumber() == 8);
    for (int i = 0; i < taskConfiguration.getCutNumber(); i++) {
      CutConfiguration cutConfiguration = new CutConfiguration();
      cutConfiguration.builder(dbzFile, -1);
      System.out.println(cutConfiguration);

      System.out.println(dbzFile.getFilePointer());
    }
    ProductHeader productHeader = new ProductHeader();
    productHeader.builder(dbzFile, -1);
    System.out.println(productHeader);
    assertTrue(productHeader.getProductNumber() == 1);

    ProductDependentParameter productDependentParameter =
        new ProductDependentParameter(
            CinradXUtils.getProductType(productHeader.getProductNumber()), dbzFile.readBytes(64));
    assertTrue(productDependentParameter.getProductType().name().equalsIgnoreCase("PPI"));
    // productDependentParameter.buildProductParameter(dbzFile.readBytes(64));
    System.out.println(productDependentParameter.getProductType());
    System.out.println(productDependentParameter);
  }
  /**
   * _more_
   *
   * @param raf _more_
   * @param stop _more_
   * @return _more_
   * @throws IOException _more_
   */
  private boolean seekHeader(RandomAccessFile raf, long stop) throws IOException {
    // seek header
    StringBuffer hdr = new StringBuffer();
    int match = 0;

    while (raf.getFilePointer() < stop) {
      // code must be "G" "R" "I" "B"
      char c = (char) raf.read();

      hdr.append((char) c);
      if (c == 'G') {
        match = 1;
      } else if ((c == 'R') && (match == 1)) {
        match = 2;
      } else if ((c == 'I') && (match == 2)) {
        match = 3;
      } else if ((c == 'B') && (match == 3)) {
        match = 4;

        Matcher m = productID.matcher(hdr.toString());
        if (m.find()) {
          header = m.group(1);
        } else {
          // header = hdr.toString();
          header = "GRIB1";
        }
        // System.out.println( "header =" + header.toString() );
        return true;
      } else {
        match = 0; /* Needed to protect against "GaRaIaB" case. */
      }
    }
    return false;
  } // end seekHeader
Ejemplo n.º 3
0
  public boolean hasNext() throws IOException {
    if (lastPos >= raf.length()) return false;
    if (repeatPos > 0) {
      if (nextRepeating()) // this has created a new repeatRecord
      return true;
    } else {
      repeatRecord = null;
      repeatBms = null;
      // fall through to new record
    }

    boolean more;
    long stop = 0;

    while (true) { // scan until we get a GRIB-2 or more == false
      raf.seek(lastPos);
      more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
      if (!more) break;

      stop = raf.getFilePointer();
      // see if its GRIB-2
      raf.skipBytes(7);
      int edition = raf.read();
      if (edition == 2) break;
      lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
    }

    if (more) {
      int sizeHeader =
          (int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
      // if (sizeHeader > 30) sizeHeader = 30;
      header = new byte[sizeHeader];
      startPos = stop - sizeHeader;
      raf.seek(startPos);
      raf.read(header);
    }
    if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
    return more;
  }
Ejemplo n.º 4
0
 /**
  * tricky bit of business. recapture the entire record based on drs position. for validation.
  *
  * @param raf from this RandomAccessFile
  * @param drsPos Grib2SectionDataRepresentation starts here
  */
 public static Grib2Record findRecordByDrspos(RandomAccessFile raf, long drsPos)
     throws IOException {
   Grib2Record result = null;
   Grib2RecordScanner scanner = new Grib2RecordScanner(raf);
   long pos = Math.max(0, drsPos - 10000); // go back 10000 bytes
   raf.seek(pos);
   while (scanner.hasNext()) { // find GRIB header
     result = scanner.next();
     if (result.getDataRepresentationSection().getStartingPosition() == drsPos) return result;
     if (raf.getFilePointer() > drsPos) break;
   }
   return null;
 }
Ejemplo n.º 5
0
  public static boolean isValidFile(RandomAccessFile raf) {
    try {
      raf.seek(0);
      while (raf.getFilePointer() < maxScan) {
        boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
        if (!found) return false;
        raf.skipBytes(7); // will be positioned on byte 0 of indicator section
        int edition = raf.read(); // read at byte 8
        if (edition == 2) return true;
      }

    } catch (IOException e) {
      return false;
    }

    return false;
  }
  /**
   * scans a Grib file to gather information that could be used to create an index or dump the
   * metadata contents.
   *
   * @param parm
   * @throws NotSupportedException
   * @throws IOException if raf does not contain a valid GRIB record
   */
  public final boolean scan(int[] parm) throws NotSupportedException, IOException {
    long start = System.currentTimeMillis();
    // stores the number of times a particular GDS is used
    HashMap gdsCounter = new HashMap();
    Grib1ProductDefinitionSection pds = null;
    Grib1GridDefinitionSection gds = null;
    long gdsOffset = 0;
    DataOutputStream dos = new DataOutputStream(System.out);

    // System.out.println("file position =" + raf.getFilePointer());
    long SOR = raf.getFilePointer();
    while (raf.getFilePointer() < raf.length()) {
      SOR = raf.getFilePointer();
      if (seekHeader(raf, raf.length())) {
        // Read Section 0 Indicator Section
        Grib1IndicatorSection is = new Grib1IndicatorSection(raf);
        // System.out.println( "Grib record length=" + is.getGribLength());
        // EOR (EndOfRecord) calculated so skipping data sections is faster
        long EOR = raf.getFilePointer() + is.getGribLength() - is.getLength();
        // long SOR = raf.getFilePointer() - is.getLength();

        // skip Grib 2 records in a Grib 1 file
        if (is.getGribEdition() == 2) {
          // System.out.println( "Error Grib 2 record in Grib1 file" ) ;
          raf.seek(EOR);
          continue;
        }

        if (parm[0] == -1) { // extract only 1st record
          raf.seek(SOR);
          byte[] oneRecord = new byte[(int) is.getGribLength()];
          raf.read(oneRecord);
          dos.write(oneRecord, 0, oneRecord.length);
          dos.flush();
          break;
        }
        long dataOffset = 0;
        try { // catch all exceptions and seek to EOR

          // Read Section 1 Product Definition Section PDS
          pds = new Grib1ProductDefinitionSection(raf);
          if (pds.getLengthErr()) {
            raf.seek(EOR);
            continue;
          }
          Grib1PDSVariables pdsv = pds.getPdsVars();
          if (getParameter && parm[0] == pdsv.getParameterNumber()) {
            raf.seek(SOR);
            byte[] oneRecord = new byte[(int) is.getGribLength()];
            raf.read(oneRecord);
            dos.write(oneRecord, 0, oneRecord.length);
            dos.flush();
          } else if (parm[0] <= pdsv.getForecastTime() && parm[1] >= pdsv.getForecastTime()) {
            raf.seek(SOR);
            byte[] oneRecord = new byte[(int) is.getGribLength()];
            raf.read(oneRecord);
            dos.write(oneRecord, 0, oneRecord.length);
            dos.flush();
          }

          raf.seek(EOR);
          continue;

        } catch (Exception e) {
          // .println( "Caught Exception scannning record" );
          e.printStackTrace();
          raf.seek(EOR);
          continue;
        }
      } // end if seekHeader
      // System.out.println( "raf.getFilePointer()=" + raf.getFilePointer());
      // System.out.println( "raf.length()=" + raf.length() );
    } // end while raf.getFilePointer() < raf.length()
    // System.out.println("GribInput: processed in " +
    //   (System.currentTimeMillis()- start) + " milliseconds");
    dos.close();
    return true;
  } // end scan
Ejemplo n.º 7
0
  // return true if got another repeat out of this record
  // side effect is that the new record is in repeatRecord
  private boolean nextRepeating() throws IOException {
    raf.seek(repeatPos);

    // octets 1-4 (Length of GDS)
    int length = GribNumbers.int4(raf);
    int section = raf.read();
    raf.seek(repeatPos);

    if (section == 2) {
      repeatRecord.setLus(new Grib2SectionLocalUse(raf));
      repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
      repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
      repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
      repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
      repeatRecord.setDataSection(new Grib2SectionData(raf));
      repeatRecord.repeat = section;

    } else if (section == 3) {
      repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
      repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
      repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
      repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
      repeatRecord.setDataSection(new Grib2SectionData(raf));
      repeatRecord.repeat = section;

    } else if (section == 4) {
      repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
      repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
      repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
      repeatRecord.setDataSection(new Grib2SectionData(raf));
      repeatRecord.repeat = section;

    } else {
      if (debugRepeat) System.out.printf(" REPEAT Terminate %d%n", section);
      repeatPos = -1;
      repeatRecord = null;
      repeatBms = null;
      return false;
    }

    // look for repeating bms
    Grib2SectionBitMap bms = repeatRecord.getBitmapSection();
    if (bms.getBitMapIndicator() == 254) {
      // replace BMS with last good one
      if (repeatBms == null) throw new IllegalStateException("No bms in repeating section");
      repeatRecord.setBms(repeatBms, true);
      // debug
      if (debugRepeat) System.out.printf("replaced bms %d%n", section);
      repeatRecord.repeat += 1000;

    } else if (bms.getBitMapIndicator() == 0) {
      // track last good bms
      repeatBms = repeatRecord.getBitmapSection();
    }

    // keep only unique gds
    if ((section == 2) || (section == 3)) {
      // look for duplicate gds
      Grib2SectionGridDefinition gds = repeatRecord.getGDSsection();
      long crc = gds.calcCRC();
      Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
      if (gdsCached != null) repeatRecord.setGdss(gdsCached);
      else gdsMap.put(crc, gds);
    }

    // check to see if we are at the end
    long pos = raf.getFilePointer();
    long ending = repeatRecord.getIs().getEndPos();
    if (pos + 34 < ending) { // give it 30 bytes of slop
      if (debugRepeat) System.out.printf(" REPEAT AGAIN %d != %d%n", pos + 4, ending);
      repeatPos = pos;
      return true;
    }

    if (debug)
      System.out.printf(
          " REPEAT read until %d grib ending at %d header ='%s'%n",
          raf.getFilePointer(), ending, StringUtil2.cleanup(header));

    // check that end section is correct
    raf.seek(ending - 4);
    for (int i = 0; i < 4; i++) {
      if (raf.read() != 55) {
        log.warn(
            "  REPEAT Missing End of GRIB message at pos="
                + ending
                + " header= "
                + StringUtil2.cleanup(header)
                + " for="
                + raf.getLocation());
        break;
      }
    }
    lastPos = raf.getFilePointer();

    if (debugRepeat) System.out.printf(" REPEAT DONE%n");
    repeatPos = -1; // no more repeats in this record
    return true;
  }
Ejemplo n.º 8
0
  public Grib2Record next() throws IOException {
    if (repeatRecord != null) { // serve current repeatRecord if it exists
      return new Grib2Record(repeatRecord);
    }

    Grib2SectionIndicator is = null;
    try {
      is = new Grib2SectionIndicator(raf);
      Grib2SectionIdentification ids = new Grib2SectionIdentification(raf);
      Grib2SectionLocalUse lus = new Grib2SectionLocalUse(raf);
      Grib2SectionGridDefinition gds = new Grib2SectionGridDefinition(raf);
      Grib2SectionProductDefinition pds = new Grib2SectionProductDefinition(raf);
      Grib2SectionDataRepresentation drs = new Grib2SectionDataRepresentation(raf);
      Grib2SectionBitMap bms = new Grib2SectionBitMap(raf);
      Grib2SectionData dataSection = new Grib2SectionData(raf);
      if (dataSection.getMsgLength() > is.getMessageLength()) { // presumably corrupt
        raf.seek(drs.getStartingPosition()); // go back to before the dataSection
        throw new IllegalStateException("Illegal Grib2SectionData Message Length");
      }

      // look for duplicate gds
      long crc = gds.calcCRC();
      Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
      if (gdsCached != null) gds = gdsCached;
      else gdsMap.put(crc, gds);

      // look for duplicate pds
      /* crc = pds.calcCRC();
      Grib2SectionProductDefinition pdsCached = pdsMap.get(crc);
      if (pdsCached != null)
        pds = pdsCached;
      else
        pdsMap.put(crc, pds); */

      // check to see if we have a repeating record
      long pos = raf.getFilePointer();
      long ending = is.getEndPos();
      if (pos + 34 < ending) { // give it 30 bytes of slop
        if (debugRepeat) System.out.printf(" REPEAT AT %d != %d%n", pos + 4, ending);
        repeatPos = pos;
        repeatRecord =
            new Grib2Record(
                header,
                is,
                ids,
                lus,
                gds,
                pds,
                drs,
                bms,
                dataSection,
                false); // this assumes immutable sections
        // track bms in case its a repeat
        if (bms.getBitMapIndicator() == 0) repeatBms = bms;
        return new Grib2Record(
            repeatRecord); // GribRecord isnt immutable; still may not be necessary
      }

      if (debug)
        System.out.printf(
            " read until %d grib ending at %d header ='%s'%n",
            raf.getFilePointer(), ending, StringUtil2.cleanup(header));

      // check that end section is correct
      raf.seek(ending - 4);
      for (int i = 0; i < 4; i++) {
        if (raf.read() != 55) {
          log.warn(
              "Missing End of GRIB message at pos="
                  + ending
                  + " header= "
                  + StringUtil2.cleanup(header)
                  + " for="
                  + raf.getLocation());
          break;
        }
      }
      lastPos = raf.getFilePointer();

      return new Grib2Record(header, is, ids, lus, gds, pds, drs, bms, dataSection, false);

    } catch (Throwable t) {
      long pos = (is == null) ? -1 : is.getStartPos();
      log.warn("Bad GRIB2 record in file {}, skipping pos={}", raf.getLocation(), pos);
      lastPos = raf.getFilePointer();
      if (hasNext()) // skip forward
      return next();
    }

    return null; // last record was incomplete
  }
Ejemplo n.º 9
0
  private void createIndex(
      File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
      throws IOException {
    Grib2Record first = null; // take global metadata from here

    if (indexFile.exists()) indexFile.delete(); // replace it
    f.format(" createIndex for %s%n", indexFile.getPath());

    RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;
      for (Group g : groups) {
        g.fileSet = new HashSet<Integer>();
        for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
          if (first == null) first = vb.first;
          GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.recordMap.length;
        }
      }
      long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
      f.format(
          "  write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
          countBytes, countRecords, bytesPerRecord);

      if (first == null) {
        logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
        throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(gc.getName());

      for (String fn : filenames) indexBuilder.addFiles(fn);

      for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));

      /* int count = 0;
      for (DatasetCollectionManager dcm : collections) {
        indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
        count++;
      } */

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("  write GribCollectionIndex= %d bytes%n", b.length);

    } finally {
      f.format("  file size =  %d bytes%n", raf.length());
      raf.close();
      if (raf != null) raf.close();
    }
  }
  // check if compressed file seems ok
  public static long testValid(String ufilename) throws IOException {
    boolean lookForHeader = false;

    // gotta make it
    RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    raf.seek(0);
    byte[] b = new byte[8];
    raf.read(b);
    String test = new String(b);
    if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
      System.out.println("--Good header= " + test);
      raf.seek(24);
    } else {
      System.out.println("--No header ");
      lookForHeader = true;
      raf.seek(0);
    }

    boolean eof = false;
    int numCompBytes;
    try {

      while (!eof) {

        if (lookForHeader) {
          raf.read(b);
          test = new String(b);
          if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
            System.out.println("  found header= " + test);
            raf.skipBytes(16);
            lookForHeader = false;
          } else {
            raf.skipBytes(-8);
          }
        }

        try {
          numCompBytes = raf.readInt();
          if (numCompBytes == -1) {
            System.out.println("\n--done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          System.out.println("\n--got EOFException ");
          break; // assume this is ok
        }

        System.out.print(" " + numCompBytes + ",");
        if (numCompBytes < 0) {
          System.out.println("\n--last block " + numCompBytes);
          numCompBytes = -numCompBytes;
          if (!lookForHeader) eof = true;
        }

        raf.skipBytes(numCompBytes);
      }
    } catch (EOFException e) {
      e.printStackTrace();
    }

    return raf.getFilePointer();
  }
  /**
   * Write equivilent uncompressed version of the file.
   *
   * @param inputRaf file to uncompress
   * @param ufilename write to this file
   * @return raf of uncompressed file
   * @throws IOException on read error
   */
  private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
      throws IOException {
    RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
    FileLock lock = null;

    while (true) { // loop waiting for the lock
      try {
        lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
        break;

      } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
        try {
          Thread.sleep(100); // msecs
        } catch (InterruptedException e1) {
        }
      }
    }

    try {
      inputRaf.seek(0);
      byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
      inputRaf.read(header);
      outputRaf.write(header);

      boolean eof = false;
      int numCompBytes;
      byte[] ubuff = new byte[40000];
      byte[] obuff = new byte[40000];
      try {
        CBZip2InputStream cbzip2 = new CBZip2InputStream();
        while (!eof) {
          try {
            numCompBytes = inputRaf.readInt();
            if (numCompBytes == -1) {
              if (log.isDebugEnabled()) log.debug("  done: numCompBytes=-1 ");
              break;
            }
          } catch (EOFException ee) {
            log.warn("  got EOFException ");
            break; // assume this is ok
          }

          if (log.isDebugEnabled()) {
            log.debug(
                "reading compressed bytes "
                    + numCompBytes
                    + " input starts at "
                    + inputRaf.getFilePointer()
                    + "; output starts at "
                    + outputRaf.getFilePointer());
          }
          /*
           * For some stupid reason, the last block seems to
           * have the number of bytes negated.  So, we just
           * assume that any negative number (other than -1)
           * is the last block and go on our merry little way.
           */
          if (numCompBytes < 0) {
            if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
            numCompBytes = -numCompBytes;
            eof = true;
          }
          byte[] buf = new byte[numCompBytes];
          inputRaf.readFully(buf);
          ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);

          // CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
          cbzip2.setStream(bis);
          int total = 0;
          int nread;
          /*
          while ((nread = cbzip2.read(ubuff)) != -1) {
            dout2.write(ubuff, 0, nread);
            total += nread;
          }
          */
          try {
            while ((nread = cbzip2.read(ubuff)) != -1) {
              if (total + nread > obuff.length) {
                byte[] temp = obuff;
                obuff = new byte[temp.length * 2];
                System.arraycopy(temp, 0, obuff, 0, temp.length);
              }
              System.arraycopy(ubuff, 0, obuff, total, nread);
              total += nread;
            }
            if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
          } catch (BZip2ReadException ioe) {
            log.warn("Nexrad2IOSP.uncompress ", ioe);
          }
          float nrecords = (float) (total / 2432.0);
          if (log.isDebugEnabled())
            log.debug(
                "  unpacked "
                    + total
                    + " num bytes "
                    + nrecords
                    + " records; ouput ends at "
                    + outputRaf.getFilePointer());
        }

      } catch (Exception e) {
        if (outputRaf != null) outputRaf.close();
        outputRaf = null;

        // dont leave bad files around
        File ufile = new File(ufilename);
        if (ufile.exists()) {
          if (!ufile.delete())
            log.warn("failed to delete uncompressed file (IOException)" + ufilename);
        }

        if (e instanceof IOException) throw (IOException) e;
        else throw new RuntimeException(e);
      }

    } finally {
      if (null != outputRaf) outputRaf.flush();
      if (lock != null) lock.release();
    }

    return outputRaf;
  }