private void check(NetcdfFile ncfile, int n) throws IOException, InvalidRangeException {
Variable v = ncfile.findVariable("time");
assert v != null;
System.out.printf(" time= %s%n", v.getNameAndDimensions());
assert v.getSize() == n : v.getSize();
v = ncfile.findVariable("eta");
assert v != null;
assert v.getRank() == 3 : v.getRank();
}
/* Structure {
int LAT[0];
...
int LAT[149];
} IMAGE_LAT_ARRAY(3600);
type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
*/
@Test
public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 3600);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 3600);
Structure s = (Structure) dset;
// read last one - chunked
StructureData sd = s.readStructure(3599);
assert sd.getScalarInt("LAT[0]") == 70862722;
assert sd.getScalarInt("LAT[149]") == 85302263;
// read one at a time
for (int i = 3590; i < d.getLength(); i++) {
s.readStructure(i);
System.out.println(" read structure " + i);
}
}
System.out.println("*** testReadIASI ok");
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count]);
count++;
}
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
/*
Structure {
int a_name;
byte b_name(3);
byte c_name(3);
short d_name(3);
int e_name(3);
long f_name(3);
int g_name(3);
short h_name(3);
int i_name(3);
long j_name(3);
float k_name(3);
double l_name(3);
} CompoundNative(15);
type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
*/
@Test
public void testReadH5StructureArrayMembers() throws java.io.IOException {
try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {
Variable dset = ncfile.findVariable("CompoundNative");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 15);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 15);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testReadH5StructureArrayMembers ok");
}
private void testReadData(Variable v) throws IOException {
if (show) System.out.printf(" read %s%n", v.getNameAndDimensions());
assert (null != v);
assert (null != v.getDimension(0));
Array a = v.read();
assert (null != a);
assert (v.getSize() == a.getSize());
}
void CheckInt16(Variable v) throws IOException {
// int16
// assert(null != (v = dodsfile.findVariable("types.integers.i16")));
// assert v.getName().equals("types.integers.i16");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.SHORT;
CheckInt16Value(v.read());
}
void CheckLong32(Variable v) throws IOException {
// uint32
// assert(null != (v = dodsfile.findVariable("types.integers.ui32")));
// assert v.getName().equals("types.integers.ui32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.LONG : v.getDataType();
CheckLongValue(v.read());
}
void CheckD(Variable v) throws IOException {
// double
// assert(null != (v = dodsfile.findVariable("types.floats.f64")));
// assert v.getName().equals("types.floats.f64");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.DOUBLE : v.getDataType();
CheckDValue(v.read());
}
void CheckF(Variable v) throws IOException {
// float
// assert(null != (v = dodsfile.findVariable("types.floats.f32")));
// assert v.getName().equals("types.floats.f32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.FLOAT : v.getDataType();
CheckFValue(v.read());
}
/* Structure {
char EntryName(64);
char Definition(1024);
char Unit(1024);
char Scale Factor(1024);
} TIME_DESCR(60);
type = Layout(8); type= 2 (chunked) storageSize = (1,3136) dataSize=0 dataAddress=684294
*/
@Test
public void testReadManyAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/TIME_DESCR");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 60);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 60);
ArrayStructure data = (ArrayStructure) dset.read();
StructureMembers.Member m = data.getStructureMembers().findMember("EntryName");
assert m != null;
for (int i = 0; i < dset.getSize(); i++) {
String r = data.getScalarString(i, m);
if (i % 2 == 0) assert r.equals("TIME[" + i / 2 + "]-days") : r + " at " + i;
else assert r.equals("TIME[" + i / 2 + "]-milliseconds") : r + " at " + i;
}
}
System.out.println("*** testReadManyAtATime ok");
}
public void testReadData(NetcdfFile ncfile, String name) throws IOException {
Variable v = ncfile.findVariable(name);
assert null != v;
assert v.getShortName().equals(name);
assert v.getRank() == 3;
assert v.getSize() == 36 : v.getSize();
assert v.getShape()[0] == 3;
assert v.getShape()[1] == 3;
assert v.getShape()[2] == 4;
assert v.getDataType() == DataType.DOUBLE;
assert !v.isCoordinateVariable();
assert v.getDimension(0) == ncfile.findDimension("time");
assert v.getDimension(1) == ncfile.findDimension("lat");
assert v.getDimension(2) == ncfile.findDimension("lon");
Array data = v.read();
assert data.getRank() == 3;
assert data.getSize() == 36;
assert data.getShape()[0] == 3;
assert data.getShape()[1] == 3;
assert data.getShape()[2] == 4;
assert data.getElementType() == double.class;
int[] shape = data.getShape();
Index tIndex = data.getIndex();
for (int i = 0; i < shape[0]; i++)
for (int j = 0; j < shape[1]; j++)
for (int k = 0; k < shape[2]; k++) {
double val = data.getDouble(tIndex.set(i, j, k));
// System.out.println(" "+val);
assert TestUtils.close(val, 100 * i + 10 * j + k) : val;
}
}
private void doOne(String urlString) throws IOException {
System.out.println("Open and read " + urlString);
NetcdfFile ncd = NetcdfDataset.openFile(urlString, null);
assert ncd != null;
// pick a random variable to read
List vlist = ncd.getVariables();
int n = vlist.size();
assert n > 0;
Variable v = (Variable) vlist.get(n / 2);
Array data = v.read();
assert data.getSize() == v.getSize();
ncd.close();
}
@Test
public void testH5StructureDS() throws java.io.IOException {
int a_name = 0;
String[] b_name =
new String[] {
"A fight is a contract that takes two people to honor.",
"A combative stance means that you've accepted the contract.",
"In which case, you deserve what you get.",
" -- Professor Cheng Man-ch'ing"
};
String c_name = "Hello!";
// H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
try (NetcdfDataset ncfile =
NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {
Variable dset = ncfile.findVariable("CompoundComplex");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 6);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 6);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
assert sd.getScalarInt("a_name") == a_name;
a_name++;
assert sd.getScalarString("c_name").equals(c_name);
String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
assert results.length == b_name.length;
int count = 0;
for (String r : results) assert r.equals(b_name[count++]);
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testH5StructureDS ok");
}
public static void doVar(NetcdfDataset netcdfdataset, String varS, String limits) {
System.out.println("\r---------\r\r");
ucar.nc2.Variable var = netcdfdataset.findVariable(varS);
System.out.println(var.getName() + " " + var.getSize());
Array arr;
try {
arr = var.read(limits);
NCdump.printArray(arr, "array", System.out, null);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InvalidRangeException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void testAggCoordVarSubsetDefeatLocalCache(NetcdfFile ncfile)
throws InvalidRangeException, IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
time.setCachedData(null, false);
Array data = time.read("1:2");
assert data.getRank() == 1;
assert data.getSize() == 2;
assert data.getShape()[0] == 2;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count + 1]);
count++;
}
time.setCachedData(null, false);
data = time.read("0:2:2");
assert data.getRank() == 1;
assert data.getSize() == 2;
assert data.getShape()[0] == 2;
assert data.getElementType() == double.class;
count = 0;
dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count * 2]);
count++;
}
}
public void testCoordVar(NetcdfFile ncfile) {
Variable lat = ncfile.findVariable("lat");
assert null != lat;
assert lat.getShortName().equals("lat");
assert lat.getRank() == 1;
assert lat.getSize() == 3;
assert lat.getShape()[0] == 3;
assert lat.getDataType() == DataType.FLOAT;
assert !lat.isUnlimited();
assert lat.getDimension(0).equals(ncfile.findDimension("lat"));
Attribute att = lat.findAttribute("units");
assert null != att;
assert !att.isArray();
assert att.isString();
assert att.getDataType() == DataType.STRING;
assert att.getStringValue().equals("degrees_north");
assert att.getNumericValue() == null;
assert att.getNumericValue(3) == null;
try {
Array data = lat.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == float.class;
IndexIterator dataI = data.getIndexIterator();
assert TestUtils.close(dataI.getDoubleNext(), 41.0);
assert TestUtils.close(dataI.getDoubleNext(), 40.0);
assert TestUtils.close(dataI.getDoubleNext(), 39.0);
} catch (IOException io) {
}
}
@Test
/**
* Find min/max/average precipitation for a randomly-positioned but fixed-size region from a nc
* file output: filename,origin,size key: value:min, max, average
*
* @throws Exception
*/
public void testProcessingNASANexDataInNetCDF() throws Exception {
final int SIZE = 100;
File file = new File(this.getClass().getResource("/ncar.nc").getPath());
byte[] netcdfinbyte = FileUtils.readFileToByteArray(file);
// use any dummy filename for file in memory
NetcdfFile netCDFfile = NetcdfFile.openInMemory("inmemory.nc", netcdfinbyte);
Variable time = netCDFfile.findVariable("time");
ArrayDouble.D1 days = (ArrayDouble.D1) time.read();
Variable lat = netCDFfile.findVariable("lat");
if (lat == null) {
logger.error("Cannot find Variable latitude(lat)");
return;
}
ArrayFloat.D1 absolutelat = (ArrayFloat.D1) lat.read();
Variable lon = netCDFfile.findVariable("lon");
if (lon == null) {
logger.error("Cannot find Variable longitude(lon)");
return;
}
ArrayFloat.D1 absolutelon = (ArrayFloat.D1) lon.read();
Variable pres = netCDFfile.findVariable("pr");
if (pres == null) {
logger.error("Cannot find Variable precipitation(pr)");
return;
}
Random rand = new Random();
int orig_lat = rand.nextInt((int) lat.getSize());
orig_lat = Math.min(orig_lat, (int) (lat.getSize() - SIZE));
int orig_lon = rand.nextInt((int) lon.getSize());
orig_lon = Math.min(orig_lon, (int) (lon.getSize() - SIZE));
int[] origin = new int[] {0, orig_lat, orig_lon};
int[] size = new int[] {1, SIZE, SIZE};
ArrayFloat.D3 data3D = (ArrayFloat.D3) pres.read(origin, size);
double max = Double.NEGATIVE_INFINITY;
double min = Double.POSITIVE_INFINITY;
double sum = 0;
for (int j = 0; j < SIZE; j++) {
for (int k = 0; k < SIZE; k++) {
double current = data3D.get(0, j, k);
max = (current > max ? current : max);
min = (current < min ? current : min);
sum += current;
}
}
logger.info(
days
+ ","
+ absolutelat.get(orig_lat)
+ ","
+ absolutelon.get(orig_lon)
+ ","
+ SIZE
+ ":"
+ min
+ ","
+ max
+ ","
+ sum / (SIZE * SIZE));
}
@Test
public void testArraySubset() throws IOException {
DODSNetcdfFile dodsfile = TestDODSRead.open("test.02?i32[1:10],f64[2:2:10]");
Variable v = null;
Array a = null;
// int32
assert (null != (v = dodsfile.findVariable("i32")));
assert v.getFullName().equals("i32");
assert v.getRank() == 1;
assert v.getSize() == 10;
assert v.getDataType() == DataType.INT;
a = v.read();
assert a.getRank() == 1;
assert a.getSize() == 25;
assert a.getElementType() == int.class;
assert a instanceof ArrayInt.D1;
ArrayInt.D1 ai = (ArrayInt.D1) a;
for (int i = 0; i < 10; i++) {
int val = ai.get(i);
assert (val == i * 2048) : val + " != " + (i * 2048);
}
// uint16
assert null == (v = dodsfile.findVariable("ui16"));
assert null == (v = dodsfile.findVariable("ui32"));
// double
assert (null != (v = dodsfile.findVariable("f64")));
assert v.getFullName().equals("f64");
assert v.getRank() == 1;
assert v.getSize() == 5;
assert v.getDataType() == DataType.DOUBLE : v.getDataType();
a = v.read();
assert a.getRank() == 1;
assert a.getSize() == 25;
assert a.getElementType() == double.class;
assert a instanceof ArrayDouble.D1;
ArrayDouble.D1 ad = (ArrayDouble.D1) a;
double[] tFloat64 =
new double[] {
1.0,
0.9999500004166653,
0.9998000066665778,
0.9995500337489875,
0.9992001066609779,
0.9987502603949663,
0.9982005399352042,
0.9975510002532796,
0.9968017063026194,
0.9959527330119943,
0.9950041652780257,
0.9939560979566968,
0.9928086358538663,
0.9915618937147881,
0.9902159962126371,
0.9887710779360422,
0.9872272833756269,
0.9855847669095608,
0.9838436927881214,
0.9820042351172703,
0.9800665778412416,
0.9780309147241483,
0.9758974493306055,
0.9736663950053749,
0.9713379748520297
};
for (int i = 0; i < 5; i++) {
double val = ad.get(i);
assert Misc.closeEnough(val, tFloat64[i], 1.0e-9);
}
dodsfile.close();
}
private FeatureType amendSwath(NetcdfFile ncfile, Element swathElem, Group parent) {
FeatureType featureType = FeatureType.SWATH;
List<Dimension> unknownDims = new ArrayList<>();
// Dimensions
Element d = swathElem.getChild("Dimension");
List<Element> dims = d.getChildren();
for (Element elem : dims) {
String name = elem.getChild("DimensionName").getText().trim();
name = NetcdfFile.makeValidCdmObjectName(name);
if (name.equalsIgnoreCase("scalar")) continue;
String sizeS = elem.getChild("Size").getText().trim();
int length = Integer.parseInt(sizeS);
if (length > 0) {
Dimension dim = parent.findDimensionLocal(name);
if (dim != null) { // already added - may be dimension scale ?
if (dim.getLength() != length) { // ok as long as it matches
log.error("Conflicting Dimensions = {} {}", dim, ncfile.getLocation());
throw new IllegalStateException("Conflicting Dimensions = " + name);
}
} else {
dim = new Dimension(name, length);
if (parent.addDimensionIfNotExists(dim) && showWork)
System.out.printf(" Add dimension %s %n", dim);
}
} else {
log.warn("Dimension " + name + " has size " + sizeS, ncfile.getLocation());
Dimension udim = new Dimension(name, 1);
udim.setGroup(parent);
unknownDims.add(udim);
if (showWork) System.out.printf(" Add dimension %s %n", udim);
}
}
// Dimension Maps
Element dmap = swathElem.getChild("DimensionMap");
List<Element> dimMaps = dmap.getChildren();
for (Element elem : dimMaps) {
String geoDimName = elem.getChild("GeoDimension").getText().trim();
geoDimName = NetcdfFile.makeValidCdmObjectName(geoDimName);
String dataDimName = elem.getChild("DataDimension").getText().trim();
dataDimName = NetcdfFile.makeValidCdmObjectName(dataDimName);
String offsetS = elem.getChild("Offset").getText().trim();
String incrS = elem.getChild("Increment").getText().trim();
int offset = Integer.parseInt(offsetS);
int incr = Integer.parseInt(incrS);
// make new variable for this dimension map
Variable v = new Variable(ncfile, parent, null, dataDimName);
v.setDimensions(geoDimName);
v.setDataType(DataType.INT);
int npts = (int) v.getSize();
Array data = Array.makeArray(v.getDataType(), npts, offset, incr);
v.setCachedData(data, true);
v.addAttribute(new Attribute("_DimensionMap", ""));
parent.addVariable(v);
if (showWork) System.out.printf(" Add dimensionMap %s %n", v);
}
// Geolocation Variables
Group geoFieldsG = parent.findGroup(GEOLOC_FIELDS);
if (geoFieldsG == null) geoFieldsG = parent.findGroup(GEOLOC_FIELDS2);
if (geoFieldsG != null) {
Variable latAxis = null, lonAxis = null;
Element floc = swathElem.getChild("GeoField");
List<Element> varsLoc = floc.getChildren();
for (Element elem : varsLoc) {
String varname = elem.getChild("GeoFieldName").getText().trim();
Variable v = geoFieldsG.findVariable(varname);
// if (v == null)
// v = geoFieldsG.findVariable( H4header.createValidObjectName(varname));
assert v != null : varname;
AxisType axis = addAxisType(ncfile, v);
if (axis == AxisType.Lat) latAxis = v;
if (axis == AxisType.Lon) lonAxis = v;
Element dimList = elem.getChild("DimList");
List<Element> values = dimList.getChildren("value");
setSharedDimensions(v, values, unknownDims, ncfile.getLocation());
if (showWork) System.out.printf(" set coordinate %s %n", v);
}
if ((latAxis != null) && (lonAxis != null)) {
List<Dimension> xyDomain = CoordinateSystem.makeDomain(new Variable[] {latAxis, lonAxis});
if (xyDomain.size() < 2) featureType = FeatureType.PROFILE; // ??
}
}
// Data Variables
Group dataG = parent.findGroup(DATA_FIELDS);
if (dataG == null) dataG = parent.findGroup(DATA_FIELDS2);
if (dataG != null) {
Element f = swathElem.getChild("DataField");
List<Element> vars = f.getChildren();
for (Element elem : vars) {
Element dataFieldNameElem = elem.getChild("DataFieldName");
if (dataFieldNameElem == null) continue;
String varname = NetcdfFile.makeValidCdmObjectName(dataFieldNameElem.getText().trim());
Variable v = dataG.findVariable(varname);
// if (v == null)
// v = dataG.findVariable( H4header.createValidObjectName(varname));
if (v == null) {
log.error("Cant find variable {} {}", varname, ncfile.getLocation());
continue;
}
Element dimList = elem.getChild("DimList");
List<Element> values = dimList.getChildren("value");
setSharedDimensions(v, values, unknownDims, ncfile.getLocation());
}
}
return featureType;
}
void CheckByte(Variable v) throws IOException {
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.BYTE;
CheckByteValue(v.read());
}