public void readShortMissing() throws Exception {
Variable v = null;
assert (null != (v = ncfileRead.findVariable("t4")));
assert (v.getDataType() == DataType.SHORT);
// default use of missing_value
assert (null != (v = dsRead.findVariable("t4")));
assert v instanceof VariableEnhanced;
assert v instanceof VariableDS;
VariableDS vs = (VariableDS) v;
assert (vs.getDataType() == DataType.SHORT);
Attribute att = vs.findAttribute(CDM.MISSING_VALUE);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
System.out.println("missing_value = " + att.getNumericValue().shortValue());
assert (((short) -9999) == att.getNumericValue().shortValue());
assert (DataType.SHORT == att.getDataType());
assert (vs.hasMissing());
assert (vs.hasMissingValue());
assert (vs.isMissing((double) ((short) -9999)));
assert (vs.isMissingValue((double) ((short) -9999)));
Array A = vs.read();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
}
}
// turn off missing data
vs.setMissingDataIsMissing(false);
assert (vs.getDataType() == DataType.SHORT);
assert (!vs.hasMissing());
assert (vs.hasMissingValue());
assert (!vs.isMissing((double) ((short) -9999)));
assert (vs.isMissingValue((double) ((short) -9999)));
vs.setMissingDataIsMissing(true);
assert (vs.hasMissing());
assert (vs.isMissing((double) ((short) -9999)));
System.out.println("**************TestStandardVar Read readShortMissing");
}
public void readByte2Short() throws Exception {
Variable t2 = null;
assert (null != (t2 = ncfileRead.findVariable("t2")));
assert (t2.getDataType() == DataType.BYTE);
Attribute att = t2.findAttribute(CDM.SCALE_FACTOR);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
assert (2 == att.getNumericValue().doubleValue());
assert (DataType.SHORT == att.getDataType());
assert (null != (t2 = dsRead.findVariable("t2")));
assert t2 instanceof VariableEnhanced;
VariableDS vs = (VariableDS) t2;
assert (vs.getDataType() == DataType.SHORT) : vs.getDataType();
assert (!vs.hasMissing());
Array A = vs.read();
assert (A.getElementType() == short.class) : A.getElementType();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getShort(ima.set(i, j)) == (2 * (i * 10 + j) + 77));
}
}
System.out.println("**************TestStandardVar readByte2Short");
}
private double[] getValueAsDouble(Attribute att) {
int n = att.getLength();
double[] value = new double[n];
if (debugMissing) System.out.printf("missing_data: ");
for (int i = 0; i < n; i++) {
if (isUnsigned && att.getDataType() == DataType.BYTE)
value[i] = convertScaleOffsetMissing(att.getNumericValue(i).byteValue());
else if (isUnsigned && att.getDataType() == DataType.SHORT)
value[i] = convertScaleOffsetMissing(att.getNumericValue(i).shortValue());
else if (isUnsigned && att.getDataType() == DataType.INT)
value[i] = convertScaleOffsetMissing(att.getNumericValue(i).intValue());
else value[i] = scale * att.getNumericValue(i).doubleValue() + offset;
if (debugMissing) System.out.print(" " + value[i]);
}
if (debugMissing) System.out.println();
return value;
}
public void readByte() throws Exception {
Variable v = null;
assert (null != (v = ncfileRead.findVariable("t3")));
assert (v.getDataType() == DataType.BYTE);
assert (null != (v = dsRead.findVariable("t3")));
assert v instanceof VariableEnhanced;
assert v instanceof VariableDS;
VariableDS vs = (VariableDS) v;
assert (vs.getDataType() == DataType.BYTE);
Attribute att = vs.findAttribute("_FillValue");
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
System.out.println("_FillValue = " + att.getNumericValue().byteValue());
assert (((byte) 255) == att.getNumericValue().byteValue());
assert (DataType.BYTE == att.getDataType());
assert (vs.hasMissing());
assert (vs.hasFillValue());
assert (vs.isMissing((double) ((byte) 255)));
assert (vs.isFillValue((double) ((byte) 255)));
Array A = vs.read();
assert (A.getElementType() == byte.class) : A.getElementType();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
}
}
System.out.println("**************TestStandardVar ReadByte");
}
public void testCoordVar(NetcdfFile ncfile) {
Variable lat = ncfile.findVariable("lat");
assert null != lat;
assert lat.getShortName().equals("lat");
assert lat.getRank() == 1;
assert lat.getSize() == 3;
assert lat.getShape()[0] == 3;
assert lat.getDataType() == DataType.FLOAT;
assert !lat.isUnlimited();
assert lat.getDimension(0).equals(ncfile.findDimension("lat"));
Attribute att = lat.findAttribute("units");
assert null != att;
assert !att.isArray();
assert att.isString();
assert att.getDataType() == DataType.STRING;
assert att.getStringValue().equals("degrees_north");
assert att.getNumericValue() == null;
assert att.getNumericValue(3) == null;
try {
Array data = lat.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == float.class;
IndexIterator dataI = data.getIndexIterator();
assert TestUtils.close(dataI.getDoubleNext(), 41.0);
assert TestUtils.close(dataI.getDoubleNext(), 40.0);
assert TestUtils.close(dataI.getDoubleNext(), 39.0);
} catch (IOException io) {
}
}
public void readDouble() throws Exception {
Variable t1 = null;
assert (null != (t1 = ncfileRead.findVariable("t1")));
assert (t1.getDataType() == DataType.DOUBLE);
Attribute att = t1.findAttribute(CDM.SCALE_FACTOR);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
assert (2.0 == att.getNumericValue().doubleValue());
assert (DataType.DOUBLE == att.getDataType());
// read
Array A = t1.read();
int i, j;
Index ima = A.getIndex();
int[] shape = A.getShape();
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getDouble(ima.set(i, j)) == (double) (i * 10.0 + j));
}
}
assert (null != (t1 = dsRead.findVariable("t1")));
assert t1 instanceof VariableEnhanced;
VariableEnhanced dsVar = (VariableEnhanced) t1;
assert (dsVar.getDataType() == DataType.DOUBLE);
A = dsVar.read();
ima = A.getIndex();
shape = A.getShape();
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getDouble(ima.set(i, j)) == (2.0 * (i * 10.0 + j) + 77.0));
}
}
assert (null == t1.findAttribute(CDM.SCALE_FACTOR));
assert (null == t1.findAttribute("add_offset"));
System.out.println("**************TestStandardVar ReadDouble");
}
/** Instances which have same content are equal. */
@Override
public boolean equals(Object o) {
if (this == o) return true;
if ((o == null) || !(o instanceof Attribute)) return false;
final Attribute att = (Attribute) o;
if (!name.equals(att.name)) return false;
if (nelems != att.nelems) return false;
if (!dataType.equals(att.dataType)) return false;
if (svalue != null) return svalue.equals(att.getStringValue());
if (values != null) {
for (int i = 0; i < getLength(); i++) {
int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
int r2 =
att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
if (r1 != r2) return false;
}
}
return true;
}
/**
* Constructor. If scale/offset attributes are found, remove them from the decorated variable.
*
* @param forVar the Variable to decorate.
* @param useNaNs pre-fill isMissing() data with NaNs
* @param fillValueIsMissing use _FillValue for isMissing()
* @param invalidDataIsMissing use valid_range for isMissing()
* @param missingDataIsMissing use missing_value for isMissing()
*/
EnhanceScaleMissingImpl(
VariableDS forVar,
boolean useNaNs,
boolean fillValueIsMissing,
boolean invalidDataIsMissing,
boolean missingDataIsMissing) {
this.fillValueIsMissing = fillValueIsMissing;
this.invalidDataIsMissing = invalidDataIsMissing;
this.missingDataIsMissing = missingDataIsMissing;
// see if underlying variable has scale/offset already applied
Variable orgVar = forVar.getOriginalVariable();
if (orgVar instanceof VariableDS) {
VariableDS orgVarDS = (VariableDS) orgVar;
EnumSet<NetcdfDataset.Enhance> orgEnhanceMode = orgVarDS.getEnhanceMode();
if ((orgEnhanceMode != null) && orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing))
return;
}
// the other possibility is that you want to apply scale and offset to a signed value, then
// declare the result unsigned
// this.isUnsigned = (orgVar != null) ? orgVar.isUnsigned() : forVar.isUnsigned();
this.isUnsigned = forVar.isUnsigned();
this.convertedDataType = forVar.getDataType();
DataType scaleType = null, missType = null, validType = null, fillType = null;
if (debug) System.out.println("EnhancementsImpl for Variable = " + forVar.getFullName());
Attribute att;
// scale and offset
if (null != (att = forVar.findAttribute(CDM.SCALE_FACTOR))) {
if (!att.isString()) {
scale = att.getNumericValue().doubleValue();
hasScaleOffset = true;
scaleType = att.getDataType();
forVar.remove(att);
if (debug) System.out.println("scale = " + scale + " type " + scaleType);
}
}
if (null != (att = forVar.findAttribute(CDM.ADD_OFFSET))) {
if (!att.isString()) {
offset = att.getNumericValue().doubleValue();
hasScaleOffset = true;
DataType offType = att.getDataType();
if (rank(offType) > rank(scaleType)) scaleType = offType;
forVar.remove(att);
if (debug) System.out.println("offset = " + offset);
}
}
////// missing data : valid_range. assume here its in units of unpacked data. correct this below
Attribute validRangeAtt;
if (null != (validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE))) {
if (!validRangeAtt.isString() && validRangeAtt.getLength() > 1) {
valid_min = validRangeAtt.getNumericValue(0).doubleValue();
valid_max = validRangeAtt.getNumericValue(1).doubleValue();
hasValidRange = true;
validType = validRangeAtt.getDataType();
if (hasScaleOffset) forVar.remove(validRangeAtt);
if (debug) System.out.println("valid_range = " + valid_min + " " + valid_max);
}
}
Attribute validMinAtt = null, validMaxAtt = null;
if (!hasValidRange) {
if (null != (validMinAtt = forVar.findAttribute("valid_min"))) {
if (!validMinAtt.isString()) {
valid_min = validMinAtt.getNumericValue().doubleValue();
hasValidMin = true;
validType = validMinAtt.getDataType();
if (hasScaleOffset) forVar.remove(validMinAtt);
if (debug) System.out.println("valid_min = " + valid_min);
}
}
if (null != (validMaxAtt = forVar.findAttribute("valid_max"))) {
if (!validMaxAtt.isString()) {
valid_max = validMaxAtt.getNumericValue().doubleValue();
hasValidMax = true;
DataType t = validMaxAtt.getDataType();
if (rank(t) > rank(validType)) validType = t;
if (hasScaleOffset) forVar.remove(validMaxAtt);
if (debug) System.out.println("valid_min = " + valid_max);
}
}
}
boolean hasValidData = hasValidMin || hasValidMax || hasValidRange;
if (hasValidMin && hasValidMax) hasValidRange = true;
/// _FillValue
if ((null != (att = forVar.findAttribute(CDM.FILL_VALUE))) && !att.isString()) {
double[] values = getValueAsDouble(att); // LOOK double WTF ??
if (values.length > 0) {
fillValue = values[0];
hasFillValue = true;
fillType = att.getDataType();
if (hasScaleOffset) forVar.remove(att);
if (debug) System.out.println("missing_datum from _FillValue = " + fillValue);
}
}
/// missing_value
if (null != (att = forVar.findAttribute(CDM.MISSING_VALUE))) {
if (att.isString()) {
String svalue = att.getStringValue();
if (forVar.getDataType() == DataType.CHAR) {
missingValue = new double[1];
if (svalue.length() == 0) missingValue[0] = 0;
else missingValue[0] = svalue.charAt(0);
missType = DataType.CHAR;
hasMissingValue = true;
} else { // not a CHAR - try to fix problem where they use a numeric value as a String
// attribute
try {
missingValue = new double[1];
missingValue[0] = Double.parseDouble(svalue);
missType = att.getDataType();
hasMissingValue = true;
} catch (NumberFormatException ex) {
if (debug)
System.out.println(
"String missing_value not parsable as double= " + att.getStringValue());
}
}
} else { // not a string
missingValue = getValueAsDouble(att);
missType = att.getDataType();
for (double mv : missingValue)
if (!Double.isNaN(mv))
hasMissingValue = true; // dont need to do anything if its already a NaN
}
if (hasScaleOffset) forVar.remove(att);
}
// missing
boolean hasMissing =
(invalidDataIsMissing && hasValidData)
|| (fillValueIsMissing && hasFillValue)
|| (missingDataIsMissing && hasMissingValue);
/// assign convertedDataType if needed
if (hasScaleOffset) {
convertedDataType = forVar.getDataType();
if (hasMissing) {
// has missing data : must be float or double
if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
if (missingDataIsMissing && rank(missType) > rank(convertedDataType))
convertedDataType = missType;
if (fillValueIsMissing && rank(fillType) > rank(convertedDataType))
convertedDataType = fillType;
if (invalidDataIsMissing && rank(validType) > rank(convertedDataType))
convertedDataType = validType;
if (rank(convertedDataType) < rank(DataType.DOUBLE)) convertedDataType = DataType.FLOAT;
} else {
// no missing data; can use wider of data and scale
if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
}
if (debug) System.out.println("assign dataType = " + convertedDataType);
// validData may be external or internal
if (hasValidData) {
DataType orgType = forVar.getDataType();
// If valid_range is the same type as scale_factor (actually the wider of
// scale_factor and add_offset) and this is wider than the external data, then it
// will be interpreted as being in the units of the internal (unpacked) data.
// Otherwise it is in the units of the external (unpacked) data.
// we assumed unpacked data above, redo if its really packed data
if (!((rank(validType) == rank(scaleType)) && (rank(scaleType) >= rank(orgType)))) {
if (validRangeAtt != null) {
double[] values = getValueAsDouble(validRangeAtt);
valid_min = values[0];
valid_max = values[1];
} else {
if (validMinAtt != null) {
double[] values = getValueAsDouble(validMinAtt);
valid_min = values[0];
}
if (validMaxAtt != null) {
double[] values = getValueAsDouble(validMaxAtt);
valid_max = values[0];
}
}
}
}
}
if (hasMissing
&& ((convertedDataType == DataType.DOUBLE) || (convertedDataType == DataType.FLOAT)))
this.useNaNs = useNaNs;
if (debug) System.out.println("this.useNaNs = " + this.useNaNs);
}
Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
throws IOException, InvalidRangeException {
NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
if (debug) {
System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
}
// global attributes
List<Attribute> glist = bufr.getGlobalAttributes();
for (Attribute att : glist) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
Attribute useAtt;
if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
if (debug) System.out.println("add gatt= " + useAtt);
}
// global dimensions
Dimension recordDim = null;
Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
for (Dimension oldD : bufr.getDimensions()) {
String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
boolean isRecord = useName.equals("record");
Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
dimHash.put(newD.getName(), newD);
if (isRecord) recordDim = newD;
if (debug) System.out.println("add dim= " + newD);
}
// Variables
Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
for (Variable oldVar : recordStruct.getVariables()) {
if (oldVar.getDataType() == DataType.STRUCTURE) continue;
String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
DataType newType = oldVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
for (Dimension dim : oldVar.getDimensions()) {
newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = oldVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
// int max_seq = countSeq(recordStruct);
// Dimension seqD = ncfile.addDimension("level", max_seq);
for (Variable v : recordStruct.getVariables()) {
if (v.getDataType() != DataType.STRUCTURE) continue;
String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
int shape[] = v.getShape();
Dimension structDim = ncfile.addDimension(structName, shape[0]);
Structure struct = (Structure) v;
for (Variable seqVar : struct.getVariables()) {
String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
DataType newType = seqVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
newDims.add(structDim);
for (Dimension dim : seqVar.getDimensions()) {
newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = seqVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
}
// create the file
ncfile.create();
if (debug) System.out.println("File Out= " + ncfile.toString());
// boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
double total = copyVarData(ncfile, recordStruct);
ncfile.flush();
System.out.println("FileWriter done total bytes = " + total);
ncfile.close();
}