/* Structure {
int LAT[0];
...
int LAT[149];
} IMAGE_LAT_ARRAY(3600);
type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
*/
@Test
public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 3600);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 3600);
Structure s = (Structure) dset;
// read last one - chunked
StructureData sd = s.readStructure(3599);
assert sd.getScalarInt("LAT[0]") == 70862722;
assert sd.getScalarInt("LAT[149]") == 85302263;
// read one at a time
for (int i = 3590; i < d.getLength(); i++) {
s.readStructure(i);
System.out.println(" read structure " + i);
}
}
System.out.println("*** testReadIASI ok");
}
/**
* Wrap the given Variable, making it into a VariableDS. Delegate data reading to the original
* variable. Take all metadata from original variable. Does not share cache, iosp.
*
* @param g logical container, if null use orgVar's group
* @param orgVar the original Variable to wrap. The original Variable is not modified. Must not be
* a Structure, use StructureDS instead.
* @param enhance if true, use NetcdfDataset.defaultEnhanceMode to define what enhancements are
* made. Note that this can change DataType and data values. You can also call enhance()
* later. If orgVar is VariableDS, then enhance is inherited from there, and this parameter is
* ignored.
*/
public VariableDS(Group g, Variable orgVar, boolean enhance) {
super(orgVar);
if (g != null)
this.group =
g; // otherwise super() sets group; this affects the long name and the dimensions.
setDimensions(getDimensionsString()); // reset the dimensions
if (orgVar instanceof Structure)
throw new IllegalArgumentException(
"VariableDS must not wrap a Structure; name=" + orgVar.getName());
// dont share cache, iosp : all IO is delegated
this.ncfile = null;
this.spiObject = null;
createNewCache();
this.orgVar = orgVar;
this.orgDataType = orgVar.getDataType();
if (orgVar instanceof VariableDS) {
VariableDS ncVarDS = (VariableDS) orgVar;
this.enhanceProxy = ncVarDS.enhanceProxy;
this.scaleMissingProxy = ncVarDS.scaleMissingProxy;
this.enhanceMode = ncVarDS.enhanceMode;
} else {
this.enhanceProxy = new EnhancementsImpl(this);
if (enhance) {
enhance(NetcdfDataset.getDefaultEnhanceMode());
} else {
this.scaleMissingProxy = new EnhanceScaleMissingImpl();
}
}
}
private void copySome(NetcdfFileWriteable ncfile, Variable oldVar, int nelems)
throws IOException {
String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
int[] shape = oldVar.getShape();
int[] origin = new int[oldVar.getRank()];
int size = shape[0];
for (int i = 0; i < size; i += nelems) {
origin[0] = i;
int left = size - i;
shape[0] = Math.min(nelems, left);
Array data;
try {
data = oldVar.read(origin, shape);
if (oldVar.getDataType() == DataType.STRING) {
data = convertToChar(ncfile.findVariable(newName), data);
}
if (data.getSize() > 0) { // zero when record dimension = 0
ncfile.write(newName, origin, data);
if (debug) System.out.println("write " + data.getSize() + " bytes");
}
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
}
}
/*
Structure {
int a_name;
byte b_name(3);
byte c_name(3);
short d_name(3);
int e_name(3);
long f_name(3);
int g_name(3);
short h_name(3);
int i_name(3);
long j_name(3);
float k_name(3);
double l_name(3);
} CompoundNative(15);
type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
*/
@Test
public void testReadH5StructureArrayMembers() throws java.io.IOException {
try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {
Variable dset = ncfile.findVariable("CompoundNative");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 15);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 15);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testReadH5StructureArrayMembers ok");
}
public void readByte2Short() throws Exception {
Variable t2 = null;
assert (null != (t2 = ncfileRead.findVariable("t2")));
assert (t2.getDataType() == DataType.BYTE);
Attribute att = t2.findAttribute(CDM.SCALE_FACTOR);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
assert (2 == att.getNumericValue().doubleValue());
assert (DataType.SHORT == att.getDataType());
assert (null != (t2 = dsRead.findVariable("t2")));
assert t2 instanceof VariableEnhanced;
VariableDS vs = (VariableDS) t2;
assert (vs.getDataType() == DataType.SHORT) : vs.getDataType();
assert (!vs.hasMissing());
Array A = vs.read();
assert (A.getElementType() == short.class) : A.getElementType();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getShort(ima.set(i, j)) == (2 * (i * 10 + j) + 77));
}
}
System.out.println("**************TestStandardVar readByte2Short");
}
/**
* Make the missing variable
*
* @return the missing variable
*/
protected Variable makeMissingVariable() {
Variable var = new Variable(ncfile, null, null, MISSING_VAR);
var.setDataType(DataType.BYTE);
var.setDimensions((List<Dimension>) null);
var.addAttribute(new Attribute("description", "missing flag - 1 means all params are missing"));
var.addAttribute(new Attribute("missing_value", new Byte((byte) 1)));
return var;
}
public boolean compareVariables(
Variable org, Variable copy, boolean compareData, boolean justOne) {
boolean ok = true;
if (showCompare)
f.format("compare Variable %s to %s %n", org.getFullName(), copy.getFullName());
if (!org.getFullName().equals(copy.getFullName())) {
f.format(" ** names are different %s != %s %n", org.getFullName(), copy.getFullName());
ok = false;
}
// dimensions
ok &= checkAll(org.getDimensions(), copy.getDimensions(), null);
// attributes
ok &= checkAll(org.getAttributes(), copy.getAttributes(), null);
// coord sys
if ((org instanceof VariableEnhanced) && (copy instanceof VariableEnhanced)) {
VariableEnhanced orge = (VariableEnhanced) org;
VariableEnhanced copye = (VariableEnhanced) copy;
ok &= checkAll(orge.getCoordinateSystems(), copye.getCoordinateSystems(), null);
}
// data !!
if (compareData) {
try {
compareVariableData(org, copy, showCompare, justOne);
} catch (IOException e) {
ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
e.printStackTrace(new PrintStream(bos));
f.format("%s", bos.toString());
}
}
// nested variables
if (org instanceof Structure) {
if (!(copy instanceof Structure)) {
f.format(" ** %s not Structure%n", org);
ok = false;
} else {
Structure orgS = (Structure) org;
Structure ncmlS = (Structure) copy;
List vars = new ArrayList();
ok &= checkAll(orgS.getVariables(), ncmlS.getVariables(), vars);
for (int i = 0; i < vars.size(); i += 2) {
Variable orgV = (Variable) vars.get(i);
Variable ncmlV = (Variable) vars.get(i + 1);
ok &= compareVariables(orgV, ncmlV, false, true);
}
}
}
return ok;
}
void CheckS(Variable v) throws IOException {
// string
// assert(null != (v = dodsfile.findVariable("types.strings.s")));
// assert v.getName().equals("types.strings.s");
assert v.getRank() == 0;
assert v.getDataType() == DataType.STRING : v.getDataType();
CheckSValue(v.read());
}
void CheckInt16(Variable v) throws IOException {
// int16
// assert(null != (v = dodsfile.findVariable("types.integers.i16")));
// assert v.getName().equals("types.integers.i16");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.SHORT;
CheckInt16Value(v.read());
}
void CheckLong32(Variable v) throws IOException {
// uint32
// assert(null != (v = dodsfile.findVariable("types.integers.ui32")));
// assert v.getName().equals("types.integers.ui32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.LONG : v.getDataType();
CheckLongValue(v.read());
}
void CheckF(Variable v) throws IOException {
// float
// assert(null != (v = dodsfile.findVariable("types.floats.f32")));
// assert v.getName().equals("types.floats.f32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.FLOAT : v.getDataType();
CheckFValue(v.read());
}
void CheckD(Variable v) throws IOException {
// double
// assert(null != (v = dodsfile.findVariable("types.floats.f64")));
// assert v.getName().equals("types.floats.f64");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.DOUBLE : v.getDataType();
CheckDValue(v.read());
}
void CheckUrl(Variable v) throws IOException {
// url
// assert(null != (v = dodsfile.findVariable("types.strings.u")));
// assert v.getName().equals("types.strings.u");
assert v.getRank() == 0;
assert v.getDataType() == DataType.STRING : v.getDataType();
String str = v.readScalarString();
assert str.equals("http://www.opendap.org") || str.equals("http://www.dods.org") : str;
}
public void testStride() throws IOException, InvalidRangeException {
System.out.println("ncfile opened = " + location + "\n" + ncfile);
Variable time = ncfile.findVariable("time");
ArrayInt all = (ArrayInt) time.read();
for (int i = 0; i < all.getSize(); i++) assert (all.getInt(i) == i + 1);
testStride("0:13:3");
for (int i = 1; i < 12; i++) testStride("0:13:" + i);
}
private void compareVariableData(
Variable var1, Variable var2, boolean showCompare, boolean justOne) throws IOException {
Array data1 = var1.read();
Array data2 = var2.read();
if (showCompare)
f.format(
" compareArrays %s unlimited=%s size=%d%n",
var1.getNameAndDimensions(), var1.isUnlimited(), data1.getSize());
compareData(var1.getFullName(), data1, data2, justOne);
if (showCompare) f.format(" ok%n");
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt.D1) : data.getClass().getName();
ArrayInt.D1 dataI = (ArrayInt.D1) data;
assert dataI.get(0) == 0;
assert dataI.get(1) == 10;
assert dataI.get(2) == 99;
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testAggCoordVar2(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt);
IndexIterator dataI = data.getIndexIterator();
assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
assert dataI.getIntNext() == 2 : dataI.getIntCurrent();
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void readShortMissing() throws Exception {
Variable v = null;
assert (null != (v = ncfileRead.findVariable("t4")));
assert (v.getDataType() == DataType.SHORT);
// default use of missing_value
assert (null != (v = dsRead.findVariable("t4")));
assert v instanceof VariableEnhanced;
assert v instanceof VariableDS;
VariableDS vs = (VariableDS) v;
assert (vs.getDataType() == DataType.SHORT);
Attribute att = vs.findAttribute(CDM.MISSING_VALUE);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
System.out.println("missing_value = " + att.getNumericValue().shortValue());
assert (((short) -9999) == att.getNumericValue().shortValue());
assert (DataType.SHORT == att.getDataType());
assert (vs.hasMissing());
assert (vs.hasMissingValue());
assert (vs.isMissing((double) ((short) -9999)));
assert (vs.isMissingValue((double) ((short) -9999)));
Array A = vs.read();
Index ima = A.getIndex();
int[] shape = A.getShape();
int i, j;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
}
}
// turn off missing data
vs.setMissingDataIsMissing(false);
assert (vs.getDataType() == DataType.SHORT);
assert (!vs.hasMissing());
assert (vs.hasMissingValue());
assert (!vs.isMissing((double) ((short) -9999)));
assert (vs.isMissingValue((double) ((short) -9999)));
vs.setMissingDataIsMissing(true);
assert (vs.hasMissing());
assert (vs.isMissing((double) ((short) -9999)));
System.out.println("**************TestStandardVar Read readShortMissing");
}
private void doWrite2(NetcdfFileWriteable ncfile, String varName) throws Exception {
Variable v = ncfile.findVariable(varName);
int[] w = getWeights(v);
int[] shape = v.getShape();
Array aa = Array.factory(v.getDataType().getPrimitiveClassType(), shape);
Index ima = aa.getIndex();
for (int i = 0; i < shape[0]; i++) {
for (int j = 0; j < shape[1]; j++) {
aa.setDouble(ima.set(i, j), (double) (i * w[0] + j * w[1]));
}
}
ncfile.write(varName, aa);
}
private int[] getWeights(Variable v) {
int rank = v.getRank();
int[] w = new int[rank];
for (int n = 0; n < rank; n++) {
Dimension dim = v.getDimension(n);
String dimName = dim.getName();
if (dimName.equals("time")) w[n] = 1000;
if (dimName.equals("z")) w[n] = 100;
if (dimName.equals("y")) w[n] = 10;
if (dimName.equals("x")) w[n] = 1;
}
return w;
}
private void copyAll(NetcdfFileWriteable ncfile, Variable oldVar) throws IOException {
String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
Array data = oldVar.read();
try {
if (oldVar.getDataType() == DataType.STRING) {
data = convertToChar(ncfile.findVariable(newName), data);
}
if (data.getSize() > 0) // zero when record dimension = 0
ncfile.write(newName, data);
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IOException(e.getMessage() + " for Variable " + oldVar.getFullName());
}
}
/** Read the value (parameters are ignored). */
public boolean read(String datasetName, Object specialO) throws IOException {
ArrayDouble.D0 a = (ArrayDouble.D0) ncVar.read();
setValue(a.get());
setRead(true);
return (false);
}
@Test
public void testH5StructureDS() throws java.io.IOException {
int a_name = 0;
String[] b_name =
new String[] {
"A fight is a contract that takes two people to honor.",
"A combative stance means that you've accepted the contract.",
"In which case, you deserve what you get.",
" -- Professor Cheng Man-ch'ing"
};
String c_name = "Hello!";
// H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
try (NetcdfDataset ncfile =
NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {
Variable dset = ncfile.findVariable("CompoundComplex");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 6);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 6);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
assert sd.getScalarInt("a_name") == a_name;
a_name++;
assert sd.getScalarString("c_name").equals(c_name);
String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
assert results.length == b_name.length;
int count = 0;
for (String r : results) assert r.equals(b_name[count++]);
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testH5StructureDS ok");
}
public static Map<String, Metadata> readFile(String file) throws Exception {
NetcdfFile n = NetcdfFile.open(file);
System.out.println("Opened: " + file);
/* Determine the size of our grid */
int xLen = n.findDimension("x").getLength();
int yLen = n.findDimension("y").getLength();
System.out.println("Grid size: " + xLen + "x" + yLen);
/* What time is this set of readings for? */
Variable timeVar = n.findVariable("time");
String timeStr = timeVar.getUnitsString().toUpperCase();
timeStr = timeStr.replace("HOURS SINCE ", "");
timeStr = timeStr.replace("HOUR SINCE ", "");
/* Find the base date (the day) the reading was taken */
Date baseDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX").parse(timeStr);
/* Get the number of hours since the base date this reading was taken */
int offset = timeVar.read().getInt(0);
/* Generate the actual date for this reading */
Calendar calendar = Calendar.getInstance();
calendar.setTime(baseDate);
calendar.set(Calendar.HOUR, calendar.get(Calendar.HOUR) + offset);
System.out.println("Time of collection: " + calendar.getTime());
/* We'll keep a mapping of geolocations -> Galileo Metadata */
Map<String, Metadata> metaMap = new HashMap<>();
/* Determine the lat, lon coordinates for the grid points, and get each
* reading at each grid point. */
NetcdfDataset dataset = new NetcdfDataset(n);
@SuppressWarnings("resource")
GridDataset gridData = new GridDataset(dataset);
for (GridDatatype g : gridData.getGrids()) {
/* Let's look at 3D variables: these have WxH dimensions, plus a
* single plane. A 4D variable would contain elevation
* and multiple planes as a result */
if (g.getShape().length == 3) {
convert3DVariable(g, calendar.getTime(), metaMap);
}
}
return metaMap;
}
public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
Vgroup vgroup = (Vgroup) v2.getSPobject();
Range scanRange = section.getRange(0);
Range radialRange = section.getRange(1);
Range gateRange = section.getRange(2);
Array data = Array.factory(v2.getDataType().getPrimitiveClassType(), section.getShape());
IndexIterator ii = data.getIndexIterator();
for (int i = scanRange.first(); i <= scanRange.last(); i += scanRange.stride()) {
Cinrad2Record[] mapScan = vgroup.map[i];
readOneScan(mapScan, radialRange, gateRange, vgroup.datatype, ii);
}
return data;
}
public Array getData(Range range, String parameterName)
throws IOException, InvalidRangeException {
Variable variable = ncfile.getRootGroup().findVariable(parameterName);
int varRank = variable.getRank();
int[] varShape = variable.getShape();
List section = new ArrayList(varRank);
section.add(range);
for (int i = 1; i < varRank; i++) {
section.add(new Range(0, varShape[i] - 1));
}
Array array = variable.read(section);
if (array.getShape()[0] == 1) {
return (array.reduce(0));
} else {
return (array);
}
// return( array.getShape()[0] == 1 ? array.reduce( 0 ) : array);
}
private String showMissing(Variable v) {
if (!(v instanceof VariableDS)) return "";
VariableDS ve = (VariableDS) v;
Formatter buff = new Formatter();
buff.format("%s:", v.getFullName());
EnumSet<NetcdfDataset.Enhance> enhanceMode = ve.getEnhanceMode();
buff.format("enhanceMode= %s%n", enhanceMode);
ve.showScaleMissingProxy(buff);
return buff.toString();
}
/**
* Make a new VariableDS, delegate data reading to the original variable, but otherwise dont take
* any info from it. This is used by NcML explicit mode.
*
* @param group the containing group; may not be null
* @param parent parent Structure, may be null
* @param shortName variable shortName, must be unique within the Group
* @param orgVar the original Variable to wrap. The original Variable is not modified. Must not be
* a Structure, use StructureDS instead.
*/
public VariableDS(Group group, Structure parent, String shortName, Variable orgVar) {
super(null, group, parent, shortName);
setDimensions(getDimensionsString()); // reset the dimensions
if (orgVar instanceof Structure)
throw new IllegalArgumentException(
"VariableDS must not wrap a Structure; name=" + orgVar.getName());
// dont share cache, iosp : all IO is delegated
this.ncfile = null;
this.spiObject = null;
createNewCache();
this.orgVar = orgVar;
this.orgDataType = orgVar.getDataType();
this.enhanceProxy = new EnhancementsImpl(this);
this.scaleMissingProxy = new EnhanceScaleMissingImpl();
}
public void testStride(String stride) throws IOException, InvalidRangeException {
Variable time = ncfile.findVariable("time");
ArrayInt all = (ArrayInt) time.read();
ArrayInt correct = (ArrayInt) all.section(new Section(stride).getRanges());
System.out.printf("correct(%s) %s", stride, NCdumpW.toString(correct));
ArrayInt data = (ArrayInt) time.read(stride);
System.out.printf("data(%s) %s%n", stride, NCdumpW.toString(data));
Index ci = correct.getIndex();
Index di = data.getIndex();
for (int i = 0; i < data.getSize(); i++)
assert (data.getInt(di.set(i)) == correct.getInt(ci.set(i)))
: stride
+ " index "
+ i
+ " = "
+ data.getInt(di.set(i))
+ " != "
+ correct.getInt(ci.set(i));
}
/**
* Get the 1D vertical coordinate array for this time step and the specified X,Y index for Lat-Lon
* point.
*
* @param timeIndex the time index. Ignored if !isTimeDependent().
* @param xIndex the x index
* @param yIndex the y index
* @return vertical coordinate array
* @throws java.io.IOException problem reading data
* @throws ucar.ma2.InvalidRangeException _more_
*/
public ArrayDouble.D1 getCoordinateArray1D(int timeIndex, int xIndex, int yIndex)
throws IOException, InvalidRangeException {
Array etaArray = readArray(etaVar, timeIndex);
Array sArray = readArray(sVar, timeIndex);
Array depthArray = readArray(depthVar, timeIndex);
Array cArray = readArray(cVar, timeIndex);
depth_c = depthCVar.readScalarDouble();
return makeHeight1D(etaArray, sArray, depthArray, cArray, depth_c, xIndex, yIndex);
}