Exemple #1
0
  // LOOK what about extending an index ??
  public boolean makeIndex(String filename, RandomAccessFile dataRaf, Formatter f)
      throws IOException {
    File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
    FileOutputStream fout = new FileOutputStream(idxFile);
    RandomAccessFile raf = null;
    try {
      //// header message
      fout.write(MAGIC_START.getBytes("UTF-8"));
      NcStream.writeVInt(fout, version);

      Map<Long, Integer> gdsMap = new HashMap<Long, Integer>();
      gdsList = new ArrayList<Grib1SectionGridDefinition>();
      records = new ArrayList<Grib1Record>(200);

      Grib1IndexProto.Grib1Index.Builder rootBuilder = Grib1IndexProto.Grib1Index.newBuilder();
      rootBuilder.setFilename(filename);

      if (dataRaf == null) {
        raf = new RandomAccessFile(filename, "r");
        dataRaf = raf;
      }

      Grib1RecordScanner scan = new Grib1RecordScanner(dataRaf);
      while (scan.hasNext()) {
        Grib1Record r = scan.next();
        if (r == null) break; // done
        records.add(r);

        Grib1SectionGridDefinition gds = r.getGDSsection();
        Integer index = gdsMap.get(gds.calcCRC());
        if (gds.getPredefinedGridDefinition()
            >= 0) // skip predefined gds - they dont have raw bytes
        index = 0;
        else if (index == null) {
          gdsList.add(gds);
          index = gdsList.size() - 1;
          gdsMap.put(gds.calcCRC(), index);
          rootBuilder.addGdsList(makeGdsProto(gds));
        }
        rootBuilder.addRecords(makeRecordProto(r, index));
      }

      ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Index index = rootBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(fout, b.length); // message size
      fout.write(b); // message  - all in one gulp
      f.format("  made gbx9 index for %s size=%d%n", filename, b.length);
      return true;

    } finally {
      fout.close();
      if (raf != null) raf.close();
    }
  }
 private boolean needsUpdate(long collectionLastModified, Formatter f) throws IOException {
   CollectionManager.ChangeChecker cc = Grib1Index.getChangeChecker();
   for (CollectionManager dcm :
       tpc.makePartitions()) { // LOOK not really right, since we dont know if these files are the
     // same as in the index
     File idxFile = GribCollection.getIndexFile(dcm);
     if (!idxFile.exists()) return true;
     if (collectionLastModified < idxFile.lastModified()) return true;
     for (MFile mfile : dcm.getFiles()) {
       if (cc.hasChangedSince(mfile, idxFile.lastModified())) return true;
     }
   }
   return false;
 }
  // read or create index
  private void readOrCreateIndex(CollectionManager.Force ff, Formatter f) throws IOException {

    // force new index or test for new index needed
    boolean force =
        ((ff == CollectionManager.Force.always)
            || (ff == CollectionManager.Force.test && needsUpdate()));

    // otherwise, we're good as long as the index file exists
    File idx = gc.getIndexFile();
    if (force || !idx.exists() || !readIndex(idx.getPath())) {
      logger.info("GribCollection {}: createIndex {}", gc.getName(), idx.getPath());
      createIndex(idx, ff, f); // write out index
      gc.rafLocation = idx.getPath();
      gc.setRaf(new RandomAccessFile(idx.getPath(), "r"));
      readIndex(gc.getRaf()); // read back in index
    }
  }
Exemple #4
0
  public boolean readIndex(String filename, long gribLastModified, CollectionManager.Force force)
      throws IOException {
    File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
    if (!idxFile.exists()) return false;
    long idxModified = idxFile.lastModified();
    if ((force != CollectionManager.Force.nocheck) && (idxModified < gribLastModified))
      return false;
    // force new index if file was updated

    FileInputStream fin =
        new FileInputStream(idxFile); // LOOK need DiskCache for non-writeable directories

    try {
      //// check header is ok
      if (!NcStream.readAndTest(fin, MAGIC_START.getBytes())) {
        log.info("Bad magic number of grib index, on file" + idxFile);
        return false;
      }

      int v = NcStream.readVInt(fin);
      if (v != version) {
        if ((v == 0) || (v > version))
          throw new IOException(
              "Grib1Index found version " + v + ", want version " + version + " on " + filename);
        if (log.isDebugEnabled())
          log.debug(
              "Grib1Index found version " + v + ", want version " + version + " on " + filename);
        return false;
      }

      int size = NcStream.readVInt(fin);
      if (size <= 0 || size > 100 * 1000 * 1000) { // try to catch garbage
        log.warn("Grib1Index bad size = {} for {} ", size, filename);
        return false;
      }

      byte[] m = new byte[size];
      NcStream.readFully(fin, m);

      Grib1IndexProto.Grib1Index proto = Grib1IndexProto.Grib1Index.parseFrom(m);
      String fname = proto.getFilename();
      if (debug) System.out.printf("%s for %s%n", fname, filename);

      gdsList = new ArrayList<Grib1SectionGridDefinition>(proto.getGdsListCount());
      for (Grib1IndexProto.Grib1GdsSection pgds : proto.getGdsListList()) {
        Grib1SectionGridDefinition gds = readGds(pgds);
        gdsList.add(gds);
      }
      if (debug) System.out.printf(" read %d gds%n", gdsList.size());

      records = new ArrayList<Grib1Record>(proto.getRecordsCount());
      for (Grib1IndexProto.Grib1Record precord : proto.getRecordsList()) {
        records.add(readRecord(precord));
      }
      if (debug) System.out.printf(" read %d records%n", records.size());

    } catch (java.lang.NegativeArraySizeException e) {
      log.error("GribIndex failed on " + filename, e);
      return false;

    } catch (IOException e) {
      log.error("GribIndex failed on " + filename, e);
      return false;

    } finally {
      fin.close();
    }

    return true;
  }
 public boolean needsUpdate() {
   File idx = gc.getIndexFile();
   return !idx.exists() || needsUpdate(idx.lastModified());
 }