Exemple #1
0
  public void testEnhanceDefer() throws IOException {
    NetcdfDataset ncd =
        NetcdfDataset.openDataset(
            filename, EnumSet.of(NetcdfDataset.Enhance.ScaleMissing), -1, null, null);
    VariableDS enhancedVar = (VariableDS) ncd.findVariable("t1");

    NetcdfDataset ncdefer =
        NetcdfDataset.openDataset(
            filename, EnumSet.of(NetcdfDataset.Enhance.ScaleMissingDefer), -1, null, null);
    VariableDS deferVar = (VariableDS) ncdefer.findVariable("t1");

    Array data = enhancedVar.read();
    Array dataDefer = deferVar.read();

    System.out.printf("Enhanced=");
    NCdumpW.printArray(data);
    System.out.printf("%nDeferred=");
    NCdumpW.printArray(dataDefer);
    System.out.printf("%nProcessed=");

    CompareNetcdf2 nc = new CompareNetcdf2(new Formatter(System.out), false, false, true);
    assert !nc.compareData(enhancedVar.getShortName(), data, dataDefer, false);

    IndexIterator ii = dataDefer.getIndexIterator();
    while (ii.hasNext()) {
      double val = deferVar.convertScaleOffsetMissing(ii.getDoubleNext());
      ii.setDoubleCurrent(val);
    }
    NCdumpW.printArray(dataDefer);

    assert nc.compareData(enhancedVar.getShortName(), data, dataDefer, false);

    ncd.close();
    ncdefer.close();
  }
Exemple #2
0
  public void testReadStandardVar() throws Exception {
    ncfileRead = NetcdfFile.open(filename);
    dsRead = NetcdfDataset.openDataset(filename);

    readDouble();
    readByte2Short();
    readByte();
    readShortMissing();
    readShort2FloatMissing();

    readDoubleMissing();

    ncfileRead.close();
    dsRead.close();
  }
 public List<CoordinateSystemBean> getCoordinateSystemBeans(NetcdfDataset ds) {
   List<CoordinateSystemBean> vlist = new ArrayList<>();
   for (CoordinateSystem elem : ds.getCoordinateSystems()) {
     vlist.add(new CoordinateSystemBean(elem));
   }
   return vlist;
 }
  public void showAtts() {
    if (ds == null) return;
    if (attTable == null) {
      // global attributes
      attTable =
          new BeanTable(AttributeBean.class, (PreferencesExt) prefs.node("AttributeBeans"), false);
      PopupMenu varPopup = new ucar.nc2.ui.widget.PopupMenu(attTable.getJTable(), "Options");
      varPopup.addAction(
          "Show Attribute",
          new AbstractAction() {
            public void actionPerformed(ActionEvent e) {
              AttributeBean bean = (AttributeBean) attTable.getSelectedBean();
              if (bean != null) {
                infoTA.setText(bean.att.toString());
                infoTA.gotoTop();
                infoWindow.show();
              }
            }
          });
      attWindow =
          new IndependentWindow("Global Attributes", BAMutil.getImage("netcdfUI"), attTable);
      attWindow.setBounds(
          (Rectangle) prefs.getBean("AttWindowBounds", new Rectangle(300, 100, 500, 800)));
    }

    List<AttributeBean> attlist = new ArrayList<>();
    for (Attribute att : ds.getGlobalAttributes()) {
      attlist.add(new AttributeBean(att));
    }
    attTable.setBeans(attlist);
    attWindow.show();
  }
Exemple #5
0
  public void readByte2Short() throws Exception {
    Variable t2 = null;
    assert (null != (t2 = ncfileRead.findVariable("t2")));
    assert (t2.getDataType() == DataType.BYTE);

    Attribute att = t2.findAttribute(CDM.SCALE_FACTOR);
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    assert (2 == att.getNumericValue().doubleValue());
    assert (DataType.SHORT == att.getDataType());

    assert (null != (t2 = dsRead.findVariable("t2")));
    assert t2 instanceof VariableEnhanced;
    VariableDS vs = (VariableDS) t2;
    assert (vs.getDataType() == DataType.SHORT) : vs.getDataType();
    assert (!vs.hasMissing());

    Array A = vs.read();
    assert (A.getElementType() == short.class) : A.getElementType();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;
    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getShort(ima.set(i, j)) == (2 * (i * 10 + j) + 77));
      }
    }
    System.out.println("**************TestStandardVar readByte2Short");
  }
  /**
   * Wrap the given Variable, making it into a VariableDS. Delegate data reading to the original
   * variable. Take all metadata from original variable. Does not share cache, iosp.
   *
   * @param g logical container, if null use orgVar's group
   * @param orgVar the original Variable to wrap. The original Variable is not modified. Must not be
   *     a Structure, use StructureDS instead.
   * @param enhance if true, use NetcdfDataset.defaultEnhanceMode to define what enhancements are
   *     made. Note that this can change DataType and data values. You can also call enhance()
   *     later. If orgVar is VariableDS, then enhance is inherited from there, and this parameter is
   *     ignored.
   */
  public VariableDS(Group g, Variable orgVar, boolean enhance) {
    super(orgVar);
    if (g != null)
      this.group =
          g; // otherwise super() sets group; this affects the long name and the dimensions.
    setDimensions(getDimensionsString()); // reset the dimensions

    if (orgVar instanceof Structure)
      throw new IllegalArgumentException(
          "VariableDS must not wrap a Structure; name=" + orgVar.getName());

    // dont share cache, iosp : all IO is delegated
    this.ncfile = null;
    this.spiObject = null;
    createNewCache();

    this.orgVar = orgVar;
    this.orgDataType = orgVar.getDataType();

    if (orgVar instanceof VariableDS) {
      VariableDS ncVarDS = (VariableDS) orgVar;
      this.enhanceProxy = ncVarDS.enhanceProxy;
      this.scaleMissingProxy = ncVarDS.scaleMissingProxy;
      this.enhanceMode = ncVarDS.enhanceMode;

    } else {
      this.enhanceProxy = new EnhancementsImpl(this);
      if (enhance) {
        enhance(NetcdfDataset.getDefaultEnhanceMode());
      } else {
        this.scaleMissingProxy = new EnhanceScaleMissingImpl();
      }
    }
  }
  public void setDataset(NetcdfDataset ds) {
    this.ds = ds;
    parseInfo = new Formatter();

    List<VariableBean> beanList = new ArrayList<>();
    List<AxisBean> axisList = new ArrayList<>();
    setVariables(ds.getVariables(), axisList, beanList);

    varTable.setBeans(beanList);
    axisTable.setBeans(axisList);
    csTable.setBeans(getCoordinateSystemBeans(ds));
  }
  public void summaryInfo(Formatter f) {
    if (ds == null) return;
    f.format("%s%n", ds.getLocation());
    int ngrids = 0;

    for (Object varo : varTable.getBeans()) {
      VariableBean varBean = (VariableBean) varo;
      if (varBean.getDataType().trim().equalsIgnoreCase("grid")) ngrids++;
    }
    int ncoordSys = csTable.getBeans().size();
    int ncoords = axisTable.getBeans().size();

    f.format(" ngrids=%d, ncoords=%d, ncoordSys=%d%n", ngrids, ncoords, ncoordSys);
  }
Exemple #9
0
  public void readShortMissing() throws Exception {
    Variable v = null;
    assert (null != (v = ncfileRead.findVariable("t4")));
    assert (v.getDataType() == DataType.SHORT);

    // default use of missing_value
    assert (null != (v = dsRead.findVariable("t4")));
    assert v instanceof VariableEnhanced;
    assert v instanceof VariableDS;
    VariableDS vs = (VariableDS) v;
    assert (vs.getDataType() == DataType.SHORT);

    Attribute att = vs.findAttribute(CDM.MISSING_VALUE);
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    System.out.println("missing_value = " + att.getNumericValue().shortValue());
    assert (((short) -9999) == att.getNumericValue().shortValue());
    assert (DataType.SHORT == att.getDataType());

    assert (vs.hasMissing());
    assert (vs.hasMissingValue());
    assert (vs.isMissing((double) ((short) -9999)));
    assert (vs.isMissingValue((double) ((short) -9999)));

    Array A = vs.read();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;
    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
      }
    }

    // turn off missing data
    vs.setMissingDataIsMissing(false);
    assert (vs.getDataType() == DataType.SHORT);

    assert (!vs.hasMissing());
    assert (vs.hasMissingValue());
    assert (!vs.isMissing((double) ((short) -9999)));
    assert (vs.isMissingValue((double) ((short) -9999)));

    vs.setMissingDataIsMissing(true);
    assert (vs.hasMissing());
    assert (vs.isMissing((double) ((short) -9999)));

    System.out.println("**************TestStandardVar Read readShortMissing");
  }
Exemple #10
0
  public void readDouble() throws Exception {

    Variable t1 = null;
    assert (null != (t1 = ncfileRead.findVariable("t1")));
    assert (t1.getDataType() == DataType.DOUBLE);

    Attribute att = t1.findAttribute(CDM.SCALE_FACTOR);
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    assert (2.0 == att.getNumericValue().doubleValue());
    assert (DataType.DOUBLE == att.getDataType());

    // read
    Array A = t1.read();
    int i, j;
    Index ima = A.getIndex();
    int[] shape = A.getShape();

    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getDouble(ima.set(i, j)) == (double) (i * 10.0 + j));
      }
    }

    assert (null != (t1 = dsRead.findVariable("t1")));
    assert t1 instanceof VariableEnhanced;
    VariableEnhanced dsVar = (VariableEnhanced) t1;
    assert (dsVar.getDataType() == DataType.DOUBLE);

    A = dsVar.read();
    ima = A.getIndex();
    shape = A.getShape();

    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getDouble(ima.set(i, j)) == (2.0 * (i * 10.0 + j) + 77.0));
      }
    }

    assert (null == t1.findAttribute(CDM.SCALE_FACTOR));
    assert (null == t1.findAttribute("add_offset"));

    System.out.println("**************TestStandardVar ReadDouble");
  }
Exemple #11
0
  public void readDoubleMissing() throws Exception {
    VariableDS v = null;
    assert (null != (v = (VariableDS) dsRead.findVariable("m1")));
    assert (v.getDataType() == DataType.DOUBLE);

    Array A = v.read();
    Index ima = A.getIndex();

    double val = A.getFloat(ima.set(1, 1));
    assert Double.isNaN(val);
    assert v.isMissing(val);

    // reread with useNans off
    v.setUseNaNs(false);
    v.createNewCache();
    A = v.read();
    ima = A.getIndex();

    val = A.getFloat(ima.set(1, 1));
    assert TestAll.closeEnough(val, -999.99) : val;
    assert v.isMissing(val);
  }
Exemple #12
0
  public void readByte() throws Exception {
    Variable v = null;
    assert (null != (v = ncfileRead.findVariable("t3")));
    assert (v.getDataType() == DataType.BYTE);

    assert (null != (v = dsRead.findVariable("t3")));
    assert v instanceof VariableEnhanced;
    assert v instanceof VariableDS;
    VariableDS vs = (VariableDS) v;
    assert (vs.getDataType() == DataType.BYTE);

    Attribute att = vs.findAttribute("_FillValue");
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    System.out.println("_FillValue = " + att.getNumericValue().byteValue());
    assert (((byte) 255) == att.getNumericValue().byteValue());
    assert (DataType.BYTE == att.getDataType());

    assert (vs.hasMissing());
    assert (vs.hasFillValue());
    assert (vs.isMissing((double) ((byte) 255)));
    assert (vs.isFillValue((double) ((byte) 255)));

    Array A = vs.read();
    assert (A.getElementType() == byte.class) : A.getElementType();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;
    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
      }
    }
    System.out.println("**************TestStandardVar ReadByte");
  }
Exemple #13
0
  public void readShort2FloatMissing() throws Exception {
    Variable v = null;
    assert (null != (v = ncfileRead.findVariable("t5")));
    assert (v.getDataType() == DataType.SHORT);

    // standard convert with missing data
    assert (null != (v = dsRead.findVariable("t5")));
    assert v instanceof VariableEnhanced;
    assert v instanceof VariableDS;
    VariableDS vs = (VariableDS) v;
    assert (vs.getDataType() == DataType.FLOAT);

    assert (vs.hasMissing());
    assert (vs.hasMissingValue());
    double mv = 2 * (-9999) + 77;
    assert (vs.isMissing((double) mv));
    assert (vs.isMissingValue((double) mv));

    Array A = vs.read();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;

    assert (vs.isMissing(A.getFloat(ima.set(0, 0))));

    for (i = 0; i < shape[0]; i++) {
      for (j = 1; j < shape[1]; j++) {
        float val = A.getFloat(ima.set(i, j));
        float want = 2 * (i * 10 + j) + 77;
        if (val != want) System.out.println(i + " " + j + " " + val + " " + want);
        assert (val == want);
      }
    }

    // useNaNs
    vs.setUseNaNs(true);
    assert (vs.getDataType() == DataType.FLOAT);

    assert (vs.hasMissing());
    assert (vs.hasMissingValue());
    double mv2 = 2 * (-9999) + 77;
    assert (vs.isMissing((double) mv2));
    assert (vs.isMissingValue((double) mv2));

    Array A2 = vs.read();
    Index ima2 = A2.getIndex();
    int[] shape2 = A2.getShape();

    double mval = A2.getFloat(ima2.set(0, 0));
    assert vs.isMissing(mval);
    assert Double.isNaN(mval);

    for (i = 0; i < shape2[0]; i++) {
      for (j = 1; j < shape2[1]; j++) {
        float val = A2.getFloat(ima2.set(i, j));
        float want = 2 * (i * 10 + j) + 77;
        if (val != want) System.out.println(i + " " + j + " " + val + " " + want);
        assert (val == want) : val + " != " + want;
      }
    }

    assert (null == vs.findAttribute(CDM.SCALE_FACTOR));
    assert (null == vs.findAttribute("add_offset"));
    assert (null == vs.findAttribute(CDM.MISSING_VALUE));

    System.out.println("**************TestStandardVar Read readShort2FloatMissing");
  }
/**
 * Implement NcML Forecast Model Run Collection Aggregation with files that are complete runs (have
 * all forecast times in the same file)
 *
 * @author caron
 */
public class AggregationFmrc extends AggregationOuterDimension {
  protected static Set<NetcdfDataset.Enhance> fmrcEnhanceMode =
      NetcdfDataset.getDefaultEnhanceMode();

  private Fmrc fmrc;
  private String runMatcher; // , forecastMatcher, offsetMatcher; // scanFmrc

  public AggregationFmrc(NetcdfDataset ncd, String dimName, String recheckS) {
    super(ncd, dimName, Type.forecastModelRunCollection, recheckS);
  }

  public void addDirectoryScanFmrc(
      String dirName,
      String suffix,
      String regexpPatternString,
      String subdirs,
      String olderThan,
      String runMatcher,
      String forecastMatcher,
      String offsetMatcher)
      throws IOException {

    // only one
    this.runMatcher = runMatcher;
    // this.forecastMatcher = forecastMatcher;
    // this.offsetMatcher = offsetMatcher;

    // this.enhance = NetcdfDataset.getDefaultEnhanceMode();
    isDate = true;

    // DatasetScanner d = new DatasetScanner(null, dirName, suffix, regexpPatternString, subdirs,
    // olderThan);
    // datasetManager.addDirectoryScan(d);
    datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, null);
    if (runMatcher != null) {
      DateExtractor dateExtractor = new DateExtractorFromName(runMatcher, false);
      datasetManager.setDateExtractor(dateExtractor);
    }
  }

  @Override
  protected void makeDatasets(CancelTask cancelTask) throws IOException {
    super.makeDatasets(cancelTask);
    for (Dataset ds : datasets) ds.enhance = fmrcEnhanceMode;
  }

  @Override
  public void getDetailInfo(Formatter f) {
    super.getDetailInfo(f);

    if (runMatcher != null) f.format("  runMatcher=%s%n", runMatcher);
    /*  if (forecastMatcher != null)
      f.format("  forecastMatcher=%s%n", forecastMatcher);
    if (offsetMatcher != null)
      f.format("  offsetMatcher=%s%n", offsetMatcher); */
  }

  @Override
  protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
    DateExtractor dateExtractor = null;
    if (runMatcher != null)
      dateExtractor = new DateExtractorFromName(runMatcher, false); // uses path
    if (dateExtractor == null && dateFormatMark != null)
      dateExtractor = new DateExtractorFromName(dateFormatMark, true);
    fmrc = new Fmrc(datasetManager, new FeatureCollectionConfig());

    // fill in the ncDataset
    fmrc.getDataset2D(ncDataset);
  }

  // we assume the variables are complete, but the time dimensions and values have to be recomputed
  @Override
  protected void rebuildDataset() throws IOException {
    throw new UnsupportedOperationException();
    // ncDataset.empty();
    // fmrc.getDataset2D(false, true, ncDataset);
  }
}