public void testEnhanceDefer() throws IOException {
NetcdfDataset ncd =
NetcdfDataset.openDataset(
filename, EnumSet.of(NetcdfDataset.Enhance.ScaleMissing), -1, null, null);
VariableDS enhancedVar = (VariableDS) ncd.findVariable("t1");
NetcdfDataset ncdefer =
NetcdfDataset.openDataset(
filename, EnumSet.of(NetcdfDataset.Enhance.ScaleMissingDefer), -1, null, null);
VariableDS deferVar = (VariableDS) ncdefer.findVariable("t1");
Array data = enhancedVar.read();
Array dataDefer = deferVar.read();
System.out.printf("Enhanced=");
NCdumpW.printArray(data);
System.out.printf("%nDeferred=");
NCdumpW.printArray(dataDefer);
System.out.printf("%nProcessed=");
CompareNetcdf2 nc = new CompareNetcdf2(new Formatter(System.out), false, false, true);
assert !nc.compareData(enhancedVar.getShortName(), data, dataDefer, false);
IndexIterator ii = dataDefer.getIndexIterator();
while (ii.hasNext()) {
double val = deferVar.convertScaleOffsetMissing(ii.getDoubleNext());
ii.setDoubleCurrent(val);
}
NCdumpW.printArray(dataDefer);
assert nc.compareData(enhancedVar.getShortName(), data, dataDefer, false);
ncd.close();
ncdefer.close();
}
public void testReadStandardVar() throws Exception {
ncfileRead = NetcdfFile.open(filename);
dsRead = NetcdfDataset.openDataset(filename);
readDouble();
readByte2Short();
readByte();
readShortMissing();
readShort2FloatMissing();
readDoubleMissing();
ncfileRead.close();
dsRead.close();
}
public List<CoordinateSystemBean> getCoordinateSystemBeans(NetcdfDataset ds) {
List<CoordinateSystemBean> vlist = new ArrayList<>();
for (CoordinateSystem elem : ds.getCoordinateSystems()) {
vlist.add(new CoordinateSystemBean(elem));
}
return vlist;
}
public void showAtts() {
if (ds == null) return;
if (attTable == null) {
// global attributes
attTable =
new BeanTable(AttributeBean.class, (PreferencesExt) prefs.node("AttributeBeans"), false);
PopupMenu varPopup = new ucar.nc2.ui.widget.PopupMenu(attTable.getJTable(), "Options");
varPopup.addAction(
"Show Attribute",
new AbstractAction() {
public void actionPerformed(ActionEvent e) {
AttributeBean bean = (AttributeBean) attTable.getSelectedBean();
if (bean != null) {
infoTA.setText(bean.att.toString());
infoTA.gotoTop();
infoWindow.show();
}
}
});
attWindow =
new IndependentWindow("Global Attributes", BAMutil.getImage("netcdfUI"), attTable);
attWindow.setBounds(
(Rectangle) prefs.getBean("AttWindowBounds", new Rectangle(300, 100, 500, 800)));
}
List<AttributeBean> attlist = new ArrayList<>();
for (Attribute att : ds.getGlobalAttributes()) {
attlist.add(new AttributeBean(att));
}
attTable.setBeans(attlist);
attWindow.show();
}
public void readByte2Short() throws Exception {
Variable t2 = null;
assert (null != (t2 = ncfileRead.findVariable("t2")));
assert (t2.getDataType() == DataType.BYTE);
Attribute att = t2.findAttribute(CDM.SCALE_FACTOR);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
assert (2 == att.getNumericValue().doubleValue());
assert (DataType.SHORT == att.getDataType());
assert (null != (t2 = dsRead.findVariable("t2")));
assert t2 instanceof VariableEnhanced;
VariableDS vs = (VariableDS) t2;
assert (vs.getDataType() == DataType.SHORT) : vs.getDataType();
assert (!vs.hasMissing());
Array A = vs.read();
assert (A.getElementType() == short.class) : A.getElementType();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getShort(ima.set(i, j)) == (2 * (i * 10 + j) + 77));
}
}
System.out.println("**************TestStandardVar readByte2Short");
}
/**
* Wrap the given Variable, making it into a VariableDS. Delegate data reading to the original
* variable. Take all metadata from original variable. Does not share cache, iosp.
*
* @param g logical container, if null use orgVar's group
* @param orgVar the original Variable to wrap. The original Variable is not modified. Must not be
* a Structure, use StructureDS instead.
* @param enhance if true, use NetcdfDataset.defaultEnhanceMode to define what enhancements are
* made. Note that this can change DataType and data values. You can also call enhance()
* later. If orgVar is VariableDS, then enhance is inherited from there, and this parameter is
* ignored.
*/
public VariableDS(Group g, Variable orgVar, boolean enhance) {
super(orgVar);
if (g != null)
this.group =
g; // otherwise super() sets group; this affects the long name and the dimensions.
setDimensions(getDimensionsString()); // reset the dimensions
if (orgVar instanceof Structure)
throw new IllegalArgumentException(
"VariableDS must not wrap a Structure; name=" + orgVar.getName());
// dont share cache, iosp : all IO is delegated
this.ncfile = null;
this.spiObject = null;
createNewCache();
this.orgVar = orgVar;
this.orgDataType = orgVar.getDataType();
if (orgVar instanceof VariableDS) {
VariableDS ncVarDS = (VariableDS) orgVar;
this.enhanceProxy = ncVarDS.enhanceProxy;
this.scaleMissingProxy = ncVarDS.scaleMissingProxy;
this.enhanceMode = ncVarDS.enhanceMode;
} else {
this.enhanceProxy = new EnhancementsImpl(this);
if (enhance) {
enhance(NetcdfDataset.getDefaultEnhanceMode());
} else {
this.scaleMissingProxy = new EnhanceScaleMissingImpl();
}
}
}
public void setDataset(NetcdfDataset ds) {
this.ds = ds;
parseInfo = new Formatter();
List<VariableBean> beanList = new ArrayList<>();
List<AxisBean> axisList = new ArrayList<>();
setVariables(ds.getVariables(), axisList, beanList);
varTable.setBeans(beanList);
axisTable.setBeans(axisList);
csTable.setBeans(getCoordinateSystemBeans(ds));
}
public void summaryInfo(Formatter f) {
if (ds == null) return;
f.format("%s%n", ds.getLocation());
int ngrids = 0;
for (Object varo : varTable.getBeans()) {
VariableBean varBean = (VariableBean) varo;
if (varBean.getDataType().trim().equalsIgnoreCase("grid")) ngrids++;
}
int ncoordSys = csTable.getBeans().size();
int ncoords = axisTable.getBeans().size();
f.format(" ngrids=%d, ncoords=%d, ncoordSys=%d%n", ngrids, ncoords, ncoordSys);
}
public void readShortMissing() throws Exception {
Variable v = null;
assert (null != (v = ncfileRead.findVariable("t4")));
assert (v.getDataType() == DataType.SHORT);
// default use of missing_value
assert (null != (v = dsRead.findVariable("t4")));
assert v instanceof VariableEnhanced;
assert v instanceof VariableDS;
VariableDS vs = (VariableDS) v;
assert (vs.getDataType() == DataType.SHORT);
Attribute att = vs.findAttribute(CDM.MISSING_VALUE);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
System.out.println("missing_value = " + att.getNumericValue().shortValue());
assert (((short) -9999) == att.getNumericValue().shortValue());
assert (DataType.SHORT == att.getDataType());
assert (vs.hasMissing());
assert (vs.hasMissingValue());
assert (vs.isMissing((double) ((short) -9999)));
assert (vs.isMissingValue((double) ((short) -9999)));
Array A = vs.read();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
}
}
// turn off missing data
vs.setMissingDataIsMissing(false);
assert (vs.getDataType() == DataType.SHORT);
assert (!vs.hasMissing());
assert (vs.hasMissingValue());
assert (!vs.isMissing((double) ((short) -9999)));
assert (vs.isMissingValue((double) ((short) -9999)));
vs.setMissingDataIsMissing(true);
assert (vs.hasMissing());
assert (vs.isMissing((double) ((short) -9999)));
System.out.println("**************TestStandardVar Read readShortMissing");
}
public void readDouble() throws Exception {
Variable t1 = null;
assert (null != (t1 = ncfileRead.findVariable("t1")));
assert (t1.getDataType() == DataType.DOUBLE);
Attribute att = t1.findAttribute(CDM.SCALE_FACTOR);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
assert (2.0 == att.getNumericValue().doubleValue());
assert (DataType.DOUBLE == att.getDataType());
// read
Array A = t1.read();
int i, j;
Index ima = A.getIndex();
int[] shape = A.getShape();
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getDouble(ima.set(i, j)) == (double) (i * 10.0 + j));
}
}
assert (null != (t1 = dsRead.findVariable("t1")));
assert t1 instanceof VariableEnhanced;
VariableEnhanced dsVar = (VariableEnhanced) t1;
assert (dsVar.getDataType() == DataType.DOUBLE);
A = dsVar.read();
ima = A.getIndex();
shape = A.getShape();
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getDouble(ima.set(i, j)) == (2.0 * (i * 10.0 + j) + 77.0));
}
}
assert (null == t1.findAttribute(CDM.SCALE_FACTOR));
assert (null == t1.findAttribute("add_offset"));
System.out.println("**************TestStandardVar ReadDouble");
}
public void readDoubleMissing() throws Exception {
VariableDS v = null;
assert (null != (v = (VariableDS) dsRead.findVariable("m1")));
assert (v.getDataType() == DataType.DOUBLE);
Array A = v.read();
Index ima = A.getIndex();
double val = A.getFloat(ima.set(1, 1));
assert Double.isNaN(val);
assert v.isMissing(val);
// reread with useNans off
v.setUseNaNs(false);
v.createNewCache();
A = v.read();
ima = A.getIndex();
val = A.getFloat(ima.set(1, 1));
assert TestAll.closeEnough(val, -999.99) : val;
assert v.isMissing(val);
}
public void augmentDataset(NetcdfDataset ds, CancelTask cancelTask) throws IOException {
for (Variable v : ds.getVariables()) checkIfAxis(v);
int year = ds.readAttributeInteger(null, "YEAR", -1);
int doy = ds.readAttributeInteger(null, "DAY", -1);
double time = ds.readAttributeDouble(null, "TIME", Double.NaN);
if ((year > 0) && (doy > 0) && !Double.isNaN(time)) {
Calendar cal = new GregorianCalendar(TimeZone.getTimeZone("UTC"));
cal.clear();
cal.set(Calendar.YEAR, year);
cal.set(Calendar.DAY_OF_YEAR, doy);
int hour = (int) time;
cal.set(Calendar.HOUR_OF_DAY, hour);
time -= hour;
time *= 60;
int minute = (int) time;
cal.set(Calendar.MINUTE, minute);
time -= minute;
time *= 60;
cal.set(Calendar.SECOND, (int) time);
VariableDS var =
new VariableDS(
ds,
null,
null,
"timeFromAtts",
DataType.LONG,
"",
"seconds since 1970-01-01 00:00",
"time generated from global attributes");
// LOOK : cant handle scalar coordinates yet
// var.addAttribute( new Attribute(_Coordinate.AxisType, AxisType.Time.toString()));
ds.addVariable(null, var);
ArrayLong.D0 data = new ArrayLong.D0();
data.set(cal.getTime().getTime() / 1000);
var.setCachedData(data, true);
}
ds.finish();
}
public void readByte() throws Exception {
Variable v = null;
assert (null != (v = ncfileRead.findVariable("t3")));
assert (v.getDataType() == DataType.BYTE);
assert (null != (v = dsRead.findVariable("t3")));
assert v instanceof VariableEnhanced;
assert v instanceof VariableDS;
VariableDS vs = (VariableDS) v;
assert (vs.getDataType() == DataType.BYTE);
Attribute att = vs.findAttribute("_FillValue");
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
System.out.println("_FillValue = " + att.getNumericValue().byteValue());
assert (((byte) 255) == att.getNumericValue().byteValue());
assert (DataType.BYTE == att.getDataType());
assert (vs.hasMissing());
assert (vs.hasFillValue());
assert (vs.isMissing((double) ((byte) 255)));
assert (vs.isFillValue((double) ((byte) 255)));
Array A = vs.read();
assert (A.getElementType() == byte.class) : A.getElementType();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
}
}
System.out.println("**************TestStandardVar ReadByte");
}
public void readShort2FloatMissing() throws Exception {
Variable v = null;
assert (null != (v = ncfileRead.findVariable("t5")));
assert (v.getDataType() == DataType.SHORT);
// standard convert with missing data
assert (null != (v = dsRead.findVariable("t5")));
assert v instanceof VariableEnhanced;
assert v instanceof VariableDS;
VariableDS vs = (VariableDS) v;
assert (vs.getDataType() == DataType.FLOAT);
assert (vs.hasMissing());
assert (vs.hasMissingValue());
double mv = 2 * (-9999) + 77;
assert (vs.isMissing((double) mv));
assert (vs.isMissingValue((double) mv));
Array A = vs.read();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
assert (vs.isMissing(A.getFloat(ima.set(0, 0))));
for (i = 0; i < shape[0]; i++) {
for (j = 1; j < shape[1]; j++) {
float val = A.getFloat(ima.set(i, j));
float want = 2 * (i * 10 + j) + 77;
if (val != want) System.out.println(i + " " + j + " " + val + " " + want);
assert (val == want);
}
}
// useNaNs
vs.setUseNaNs(true);
assert (vs.getDataType() == DataType.FLOAT);
assert (vs.hasMissing());
assert (vs.hasMissingValue());
double mv2 = 2 * (-9999) + 77;
assert (vs.isMissing((double) mv2));
assert (vs.isMissingValue((double) mv2));
Array A2 = vs.read();
Index ima2 = A2.getIndex();
int[] shape2 = A2.getShape();
double mval = A2.getFloat(ima2.set(0, 0));
assert vs.isMissing(mval);
assert Double.isNaN(mval);
for (i = 0; i < shape2[0]; i++) {
for (j = 1; j < shape2[1]; j++) {
float val = A2.getFloat(ima2.set(i, j));
float want = 2 * (i * 10 + j) + 77;
if (val != want) System.out.println(i + " " + j + " " + val + " " + want);
assert (val == want) : val + " != " + want;
}
}
assert (null == vs.findAttribute(CDM.SCALE_FACTOR));
assert (null == vs.findAttribute("add_offset"));
assert (null == vs.findAttribute(CDM.MISSING_VALUE));
System.out.println("**************TestStandardVar Read readShort2FloatMissing");
}
/**
* Implement NcML Forecast Model Run Collection Aggregation with files that are complete runs (have
* all forecast times in the same file)
*
* @author caron
*/
public class AggregationFmrc extends AggregationOuterDimension {
protected static Set<NetcdfDataset.Enhance> fmrcEnhanceMode =
NetcdfDataset.getDefaultEnhanceMode();
private Fmrc fmrc;
private String runMatcher; // , forecastMatcher, offsetMatcher; // scanFmrc
public AggregationFmrc(NetcdfDataset ncd, String dimName, String recheckS) {
super(ncd, dimName, Type.forecastModelRunCollection, recheckS);
}
public void addDirectoryScanFmrc(
String dirName,
String suffix,
String regexpPatternString,
String subdirs,
String olderThan,
String runMatcher,
String forecastMatcher,
String offsetMatcher)
throws IOException {
// only one
this.runMatcher = runMatcher;
// this.forecastMatcher = forecastMatcher;
// this.offsetMatcher = offsetMatcher;
// this.enhance = NetcdfDataset.getDefaultEnhanceMode();
isDate = true;
// DatasetScanner d = new DatasetScanner(null, dirName, suffix, regexpPatternString, subdirs,
// olderThan);
// datasetManager.addDirectoryScan(d);
datasetManager.addDirectoryScan(dirName, suffix, regexpPatternString, subdirs, olderThan, null);
if (runMatcher != null) {
DateExtractor dateExtractor = new DateExtractorFromName(runMatcher, false);
datasetManager.setDateExtractor(dateExtractor);
}
}
@Override
protected void makeDatasets(CancelTask cancelTask) throws IOException {
super.makeDatasets(cancelTask);
for (Dataset ds : datasets) ds.enhance = fmrcEnhanceMode;
}
@Override
public void getDetailInfo(Formatter f) {
super.getDetailInfo(f);
if (runMatcher != null) f.format(" runMatcher=%s%n", runMatcher);
/* if (forecastMatcher != null)
f.format(" forecastMatcher=%s%n", forecastMatcher);
if (offsetMatcher != null)
f.format(" offsetMatcher=%s%n", offsetMatcher); */
}
@Override
protected void buildNetcdfDataset(CancelTask cancelTask) throws IOException {
DateExtractor dateExtractor = null;
if (runMatcher != null)
dateExtractor = new DateExtractorFromName(runMatcher, false); // uses path
if (dateExtractor == null && dateFormatMark != null)
dateExtractor = new DateExtractorFromName(dateFormatMark, true);
fmrc = new Fmrc(datasetManager, new FeatureCollectionConfig());
// fill in the ncDataset
fmrc.getDataset2D(ncDataset);
}
// we assume the variables are complete, but the time dimensions and values have to be recomputed
@Override
protected void rebuildDataset() throws IOException {
throw new UnsupportedOperationException();
// ncDataset.empty();
// fmrc.getDataset2D(false, true, ncDataset);
}
}
