private Grib2CollectionBuilder(CollectionManager dcm) {
   this.collections.add(dcm);
   FeatureCollectionConfig.GribConfig config =
       (FeatureCollectionConfig.GribConfig)
           dcm.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG);
   this.gc = new Grib2Collection(dcm.getCollectionName(), new File(dcm.getRoot()), config);
 }
Exemplo n.º 2
0
 private boolean needsUpdate(long idxLastModified) {
   CollectionManager.ChangeChecker cc = Grib2Index.getChangeChecker();
   for (CollectionManager dcm : collections) {
     for (MFile mfile : dcm.getFiles()) {
       if (cc.hasChangedSince(mfile, idxLastModified)) return true;
     }
   }
   return false;
 }
Exemplo n.º 3
0
  // scan has been done, create FmrcInv
  private FmrcInv makeFmrcInv(Formatter debug) throws IOException {
    try {
      Map<CalendarDate, FmrInv> fmrMap =
          new HashMap<CalendarDate, FmrInv>(); // all files are grouped by run date in an FmrInv
      List<FmrInv> fmrList = new ArrayList<FmrInv>(); // an fmrc is a collection of fmr

      // get the inventory, sorted by path
      for (MFile f : manager.getFiles()) {
        if (logger.isDebugEnabled()) logger.debug("Fmrc: " + config.spec + ": file=" + f.getPath());

        GridDatasetInv inv = null;
        try {
          inv =
              GridDatasetInv.open(
                  manager, f, config.innerNcml); // inventory is discovered for each GDS
        } catch (IOException ioe) {
          logger.warn("Error opening " + f.getPath() + "(skipped)", ioe);
          continue; // skip
        }

        CalendarDate runDate = inv.getRunDate();
        if (debug != null)
          debug.format("  opened %s rundate = %s%n", f.getPath(), inv.getRunDateString());

        // add to fmr for that rundate
        FmrInv fmr = fmrMap.get(runDate);
        if (fmr == null) {
          fmr = new FmrInv(runDate);
          fmrMap.put(runDate, fmr);
          fmrList.add(fmr);
        }
        fmr.addDataset(inv, debug);
      }
      if (debug != null) debug.format("%n");

      // finish the FmrInv
      Collections.sort(fmrList);
      for (FmrInv fmr : fmrList) {
        fmr.finish();
        if (logger.isDebugEnabled())
          logger.debug(
              "Fmrc: spec="
                  + config.spec
                  + ": fmr rundate="
                  + fmr.getRunDate()
                  + " nfiles= "
                  + fmr.getFiles().size());
      }

      return new FmrcInv(
          "fmrc:" + manager.getCollectionName(), fmrList, config.fmrcConfig.regularize);

    } catch (Throwable t) {
      logger.error("makeFmrcInv", t);
      throw new RuntimeException(t);
    }
  }
 private boolean needsUpdate(long collectionLastModified, Formatter f) throws IOException {
   CollectionManager.ChangeChecker cc = Grib1Index.getChangeChecker();
   for (CollectionManager dcm :
       tpc.makePartitions()) { // LOOK not really right, since we dont know if these files are the
     // same as in the index
     File idxFile = GribCollection.getIndexFile(dcm);
     if (!idxFile.exists()) return true;
     if (collectionLastModified < idxFile.lastModified()) return true;
     for (MFile mfile : dcm.getFiles()) {
       if (cc.hasChangedSince(mfile, idxFile.lastModified())) return true;
     }
   }
   return false;
 }
Exemplo n.º 5
0
  // single file
  private Grib2CollectionBuilder(File file, Formatter f) throws IOException {
    try {
      // String spec = StringUtil2.substitute(file.getPath(), "\\", "/");
      CollectionManager dcm = new DatasetCollectionSingleFile(file);
      this.collections.add(dcm);
      this.gc = new Grib2Collection(file.getName(), new File(dcm.getRoot()));

    } catch (Exception e) {
      ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
      e.printStackTrace(new PrintStream(bos));
      f.format("%s", bos.toString());
      throw new IOException(e);
    }
  }
  // single file
  private Grib2CollectionBuilder(MFile file, FeatureCollectionConfig.GribConfig config, Formatter f)
      throws IOException {
    this.isSingleFile = true;
    try {
      // String spec = StringUtil2.substitute(file.getPath(), "\\", "/");
      CollectionManager dcm = new CollectionManagerSingleFile(file);
      if (config != null) dcm.putAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG, config);
      this.collections.add(dcm);
      this.gc = new Grib2Collection(file.getName(), new File(dcm.getRoot()), config);

    } catch (Exception e) {
      ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
      e.printStackTrace(new PrintStream(bos));
      f.format("%s", bos.toString());
      throw new IOException(e);
    }
  }
Exemplo n.º 7
0
  public void checkNeeded(boolean force) {
    synchronized (lock) {
      if (fmrcDataset == null) {
        try {
          manager.scan(true);
          update();
          return;
        } catch (Throwable t) {
          logger.error(config.spec + ": rescan failed");
          throw new RuntimeException(t);
        }
      }

      if (!force && !manager.isScanNeeded()) return;
      try {
        if (!manager.scan(true)) return;
        update();
      } catch (Throwable t) {
        logger.error(config.spec + ": rescan failed");
        throw new RuntimeException(t);
      }
    }
  }
Exemplo n.º 8
0
  // read all records in all files,
  // divide into groups based on GDS hash
  // each group has an arraylist of all records that belong to it.
  // for each group, run rectlizer to derive the coordinates and variables
  public List<Group> makeAggregatedGroups(
      ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
    Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();

    f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
    int total = 0;
    int fileno = 0;
    for (CollectionManager dcm : collections) {
      // dcm.scanIfNeeded(); // LOOK ??
      f.format(" dcm= %s%n", dcm);
      Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash");

      for (MFile mfile : dcm.getFiles()) {
        // f.format("%3d: %s%n", fileno, mfile.getPath());
        filenames.add(mfile.getPath());

        Grib2Index index = new Grib2Index();
        try {
          if (!index.readIndex(
              mfile.getPath(),
              mfile.getLastModified(),
              force)) { // heres where the index date is checked against the data file
            index.makeIndex(mfile.getPath(), f);
            f.format(
                "  Index written: %s == %d records %n",
                mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
          } else if (debug) {
            f.format(
                "  Index read: %s == %d records %n",
                mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
          }
        } catch (IOException ioe) {
          f.format(
              "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n  skipping %s%n",
              ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT);
          continue;
        }

        for (Grib2Record gr : index.getRecords()) {
          gr.setFile(fileno); // each record tracks which file it belongs to
          int gdsHash =
              gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
          if (gdsConvert != null
              && gdsConvert.get(gdsHash)
                  != null) { // allow external config to muck with gdsHash. Why? because of error in
            // encoding
            gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
          }

          Group g = gdsMap.get(gdsHash);
          if (g == null) {
            g = new Group(gr.getGDSsection(), gdsHash);
            gdsMap.put(gdsHash, g);
          }
          g.records.add(gr);
          total++;
        }
        fileno++;
      }
    }
    f.format(" total grib records= %d%n", total);

    Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter();
    List<Group> result = new ArrayList<Group>(gdsMap.values());
    for (Group g : result) {
      g.rect = new Grib2Rectilyser(g.records, g.gdsHash);
      f.format(" GDS hash %d == ", g.gdsHash);
      g.rect.make(f, c);
    }
    f.format(
        " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
        c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);

    return result;
  }
Exemplo n.º 9
0
 private Grib2CollectionBuilder(CollectionManager dcm) {
   this.collections.add(dcm);
   this.gc = new Grib2Collection(dcm.getCollectionName(), new File(dcm.getRoot()));
 }
  // read all records in all files,
  // divide into groups based on GDS hash
  // each group has an arraylist of all records that belong to it.
  // for each group, run rectlizer to derive the coordinates and variables
  public List<Group> makeAggregatedGroups(
      List<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
    Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();
    boolean intvMerge = mergeIntvDefault;

    f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
    int total = 0;
    int fileno = 0;

    for (CollectionManager dcm : collections) {
      f.format(" dcm= %s%n", dcm);
      FeatureCollectionConfig.GribConfig config =
          (FeatureCollectionConfig.GribConfig)
              dcm.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG);
      Map<Integer, Integer> gdsConvert = (config != null) ? config.gdsHash : null;
      FeatureCollectionConfig.GribIntvFilter intvMap = (config != null) ? config.intvFilter : null;
      intvMerge =
          (config == null) || (config.intvMerge == null) ? mergeIntvDefault : config.intvMerge;

      for (MFile mfile : dcm.getFiles()) {
        // f.format("%3d: %s%n", fileno, mfile.getPath());
        filenames.add(mfile.getPath());

        Grib2Index index = null;
        try {
          index =
              (Grib2Index)
                  GribIndex.readOrCreateIndexFromSingleFile(
                      false, !isSingleFile, mfile, config, force, f);

        } catch (IOException ioe) {
          logger.warn(
              "GribCollectionBuilder {}: reading/Creating gbx9 index failed err={}",
              gc.getName(),
              ioe.getMessage());
          f.format(
              "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n  skipping %s%n",
              ioe.getMessage(), mfile.getPath() + GribIndex.IDX_EXT);
          continue;
        }

        for (Grib2Record gr : index.getRecords()) {
          if (this.tables == null) {
            Grib2SectionIdentification ids =
                gr.getId(); // so all records must use the same table (!)
            this.tables =
                Grib2Customizer.factory(
                    ids.getCenter_id(),
                    ids.getSubcenter_id(),
                    ids.getMaster_table_version(),
                    ids.getLocal_table_version());
            if (config != null)
              tables.setTimeUnitConverter(
                  config
                      .getTimeUnitConverter()); // LOOK doesnt really work with multiple collections
          }
          if (intvMap != null && filterTinv(gr, intvMap, f)) continue; // skip

          gr.setFile(fileno); // each record tracks which file it belongs to
          int gdsHash =
              gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
          if (gdsConvert != null
              && gdsConvert.get(gdsHash)
                  != null) // allow external config to muck with gdsHash. Why? because of error in
            // encoding
            gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching

          Group g = gdsMap.get(gdsHash);
          if (g == null) {
            g = new Group(gr.getGDSsection(), gdsHash);
            gdsMap.put(gdsHash, g);
          }
          g.records.add(gr);
          total++;
        }
        fileno++;
      }
    }
    f.format(" total grib records= %d%n", total);

    Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter(); // debugging
    List<Group> result = new ArrayList<Group>(gdsMap.values());
    for (Group g : result) {
      g.rect = new Grib2Rectilyser(tables, g.records, g.gdsHash, intvMerge);
      f.format(" GDS hash %d == ", g.gdsHash);
      g.rect.make(f, c, filenames);
    }
    f.format(
        " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
        c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);

    return result;
  }
Exemplo n.º 11
0
 public void close() {
   manager.close();
 }