/* Structure {
int LAT[0];
...
int LAT[149];
} IMAGE_LAT_ARRAY(3600);
type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
*/
@Test
public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 3600);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 3600);
Structure s = (Structure) dset;
// read last one - chunked
StructureData sd = s.readStructure(3599);
assert sd.getScalarInt("LAT[0]") == 70862722;
assert sd.getScalarInt("LAT[149]") == 85302263;
// read one at a time
for (int i = 3590; i < d.getLength(); i++) {
s.readStructure(i);
System.out.println(" read structure " + i);
}
}
System.out.println("*** testReadIASI ok");
}
/*
Structure {
int a_name;
byte b_name(3);
byte c_name(3);
short d_name(3);
int e_name(3);
long f_name(3);
int g_name(3);
short h_name(3);
int i_name(3);
long j_name(3);
float k_name(3);
double l_name(3);
} CompoundNative(15);
type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
*/
@Test
public void testReadH5StructureArrayMembers() throws java.io.IOException {
try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {
Variable dset = ncfile.findVariable("CompoundNative");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 15);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 15);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testReadH5StructureArrayMembers ok");
}
@Test
public void testH5StructureDS() throws java.io.IOException {
int a_name = 0;
String[] b_name =
new String[] {
"A fight is a contract that takes two people to honor.",
"A combative stance means that you've accepted the contract.",
"In which case, you deserve what you get.",
" -- Professor Cheng Man-ch'ing"
};
String c_name = "Hello!";
// H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
try (NetcdfDataset ncfile =
NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {
Variable dset = ncfile.findVariable("CompoundComplex");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 6);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 6);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
assert sd.getScalarInt("a_name") == a_name;
a_name++;
assert sd.getScalarString("c_name").equals(c_name);
String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
assert results.length == b_name.length;
int count = 0;
for (String r : results) assert r.equals(b_name[count++]);
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testH5StructureDS ok");
}
private void createDataVariables(List<VariableSimpleIF> dataVars) throws IOException {
/* height variable
Variable heightVar = ncfile.addStringVariable(altName, recordDims, 20);
ncfile.addVariableAttribute(heightVar, new Attribute("long_name", "height of observation"));
ncfile.addVariableAttribute(heightVar, new Attribute("units", altUnits)); */
Variable v = ncfile.addVariable(parentProfileIndex, DataType.INT, recordDimName);
ncfile.addVariableAttribute(v, new Attribute("long_name", "index of parent profile"));
v = ncfile.addVariable(nextObsName, DataType.INT, recordDimName);
ncfile.addVariableAttribute(
v, new Attribute("long_name", "record number of next obs in linked list for this profile"));
// find all dimensions needed by the data variables
for (VariableSimpleIF var : dataVars) {
List<Dimension> dims = var.getDimensions();
dimSet.addAll(dims);
}
// add them
for (Dimension d : dimSet) {
if (!d.isUnlimited())
ncfile.addDimension(d.getName(), d.getLength(), d.isShared(), false, d.isVariableLength());
}
// add the data variables all using the record dimension
for (VariableSimpleIF oldVar : dataVars) {
List<Dimension> dims = oldVar.getDimensions();
StringBuffer dimNames = new StringBuffer(recordDimName);
for (Dimension d : dims) {
if (!d.isUnlimited()) dimNames.append(" ").append(d.getName());
}
Variable newVar =
ncfile.addVariable(oldVar.getName(), oldVar.getDataType(), dimNames.toString());
List<Attribute> atts = oldVar.getAttributes();
for (Attribute att : atts) {
ncfile.addVariableAttribute(newVar, att);
}
}
}
/* Structure {
char EntryName(64);
char Definition(1024);
char Unit(1024);
char Scale Factor(1024);
} TIME_DESCR(60);
type = Layout(8); type= 2 (chunked) storageSize = (1,3136) dataSize=0 dataAddress=684294
*/
@Test
public void testReadManyAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/TIME_DESCR");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 60);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 60);
ArrayStructure data = (ArrayStructure) dset.read();
StructureMembers.Member m = data.getStructureMembers().findMember("EntryName");
assert m != null;
for (int i = 0; i < dset.getSize(); i++) {
String r = data.getScalarString(i, m);
if (i % 2 == 0) assert r.equals("TIME[" + i / 2 + "]-days") : r + " at " + i;
else assert r.equals("TIME[" + i / 2 + "]-milliseconds") : r + " at " + i;
}
}
System.out.println("*** testReadManyAtATime ok");
}
public static void main(String args[]) throws Exception {
long start = System.currentTimeMillis();
Map<String, ucar.unidata.geoloc.Station> staHash =
new HashMap<String, ucar.unidata.geoloc.Station>();
String location = "R:/testdata/sounding/netcdf/Upperair_20070401_0000.nc";
NetcdfDataset ncfile = NetcdfDataset.openDataset(location);
ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
// look through record varibles, for those that have "manLevel" dimension
// make a StructureData object for those
StructureMembers sm = new StructureMembers("manLevel");
Dimension manDim = ncfile.findDimension("manLevel");
Structure record = (Structure) ncfile.findVariable("record");
List<Variable> allList = record.getVariables();
List<VariableSimpleIF> varList = new ArrayList<VariableSimpleIF>();
for (Variable v : allList) {
if ((v.getRank() == 1) && v.getDimension(0).equals(manDim)) {
// public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String
// shortName, DataType dataType,
// String dims, String units, String desc) {
varList.add(
new VariableDS(
ncfile,
null,
null,
v.getShortName(),
v.getDataType(),
"",
v.getUnitsString(),
v.getDescription()));
// (String name, String desc, String units, DataType dtype, int []shape)
sm.addMember(
v.getShortName(),
v.getDescription(),
v.getUnitsString(),
v.getDataType(),
new int[0]); // scalar
}
}
ArrayStructureMA manAS = new ArrayStructureMA(sm, new int[] {manDim.getLength()});
// need the date units
Variable time = ncfile.findVariable("synTime");
String timeUnits = ncfile.findAttValueIgnoreCase(time, "units", null);
timeUnits = StringUtil.remove(timeUnits, '('); // crappy fsl'ism
timeUnits = StringUtil.remove(timeUnits, ')');
DateUnit timeUnit = new DateUnit(timeUnits);
// extract stations
int nrecs = 0;
StructureDataIterator iter = record.getStructureIterator();
while (iter.hasNext()) {
StructureData sdata = iter.next();
String name = sdata.getScalarString("staName");
ucar.unidata.geoloc.Station s = staHash.get(name);
if (s == null) {
float lat = sdata.convertScalarFloat("staLat");
float lon = sdata.convertScalarFloat("staLon");
float elev = sdata.convertScalarFloat("staElev");
s = new StationImpl(name, "", lat, lon, elev);
staHash.put(name, s);
}
nrecs++;
}
List<ucar.unidata.geoloc.Station> stnList =
Arrays.asList(staHash.values().toArray(new ucar.unidata.geoloc.Station[staHash.size()]));
Collections.sort(stnList);
// create the writer
WriterProfileObsDataset writer =
new WriterProfileObsDataset(location + ".out", "rewrite " + location);
writer.writeHeader(stnList, varList, nrecs, "prMan");
// extract records
iter = record.getStructureIterator();
while (iter.hasNext()) {
StructureData sdata = iter.next();
String name = sdata.getScalarString("staName");
double timeValue = sdata.convertScalarDouble("synTime");
Date date = timeUnit.makeDate(timeValue);
// transfer to the ArrayStructure
List<String> names = sm.getMemberNames();
for (String mname : names) {
manAS.setMemberArray(mname, sdata.getArray(mname));
}
// each level is weritten as a seperate structure
int numMand = sdata.getScalarInt("numMand");
if (numMand >= manDim.getLength()) continue;
for (int i = 0; i < numMand; i++) {
StructureData useData = manAS.getStructureData(i);
writer.writeRecord(name, date, useData);
}
}
writer.finish();
long took = System.currentTimeMillis() - start;
System.out.println("That took = " + took);
}
public void testWritePermute() throws Exception {
NetcdfFileWriteable ncfile = new NetcdfFileWriteable();
ncfile.setName(TestLocal.cdmTestDataDir + "permuteTest.nc");
// define dimensions
Dimension xDim = ncfile.addDimension("x", 3);
Dimension yDim = ncfile.addDimension("y", 5);
Dimension zDim = ncfile.addDimension("z", 4);
Dimension tDim = ncfile.addDimension("time", 2);
// define Variables
ncfile.addVariable("time", double.class, new Dimension[] {tDim});
ncfile.addVariableAttribute("time", "units", "secs since 1-1-1 00:00");
ncfile.addVariable("z", double.class, new Dimension[] {zDim});
ncfile.addVariableAttribute("z", "units", "meters");
ncfile.addVariableAttribute("z", "positive", "up");
ncfile.addVariable("y", double.class, new Dimension[] {yDim});
ncfile.addVariableAttribute("y", "units", "degrees_north");
ncfile.addVariable("x", double.class, new Dimension[] {xDim});
ncfile.addVariableAttribute("x", "units", "degrees_east");
ncfile.addVariable("tzyx", double.class, new Dimension[] {tDim, zDim, yDim, xDim});
ncfile.addVariableAttribute("tzyx", "units", "K");
ncfile.addVariable("tzxy", double.class, new Dimension[] {tDim, zDim, xDim, yDim});
ncfile.addVariableAttribute("tzxy", "units", "K");
ncfile.addVariable("tyxz", double.class, new Dimension[] {tDim, yDim, xDim, zDim});
ncfile.addVariableAttribute("tyxz", "units", "K");
ncfile.addVariable("txyz", double.class, new Dimension[] {tDim, xDim, yDim, zDim});
ncfile.addVariableAttribute("txyz", "units", "K");
ncfile.addVariable("zyxt", double.class, new Dimension[] {zDim, yDim, xDim, tDim});
ncfile.addVariableAttribute("zyxt", "units", "K");
ncfile.addVariable("zxyt", double.class, new Dimension[] {zDim, xDim, yDim, tDim});
ncfile.addVariableAttribute("zxyt", "units", "K");
ncfile.addVariable("yxzt", double.class, new Dimension[] {yDim, xDim, zDim, tDim});
ncfile.addVariableAttribute("yxzt", "units", "K");
ncfile.addVariable("xyzt", double.class, new Dimension[] {xDim, yDim, zDim, tDim});
ncfile.addVariableAttribute("xyzt", "units", "K");
// missing one dimension
ncfile.addVariable("zyx", double.class, new Dimension[] {zDim, yDim, xDim});
ncfile.addVariable("txy", double.class, new Dimension[] {tDim, xDim, yDim});
ncfile.addVariable("yxz", double.class, new Dimension[] {yDim, xDim, zDim});
ncfile.addVariable("xzy", double.class, new Dimension[] {xDim, zDim, yDim});
ncfile.addVariable("yxt", double.class, new Dimension[] {yDim, xDim, tDim});
ncfile.addVariable("xyt", double.class, new Dimension[] {xDim, yDim, tDim});
ncfile.addVariable("xyz", double.class, new Dimension[] {xDim, yDim, zDim});
// missing two dimension
ncfile.addVariable("yx", double.class, new Dimension[] {yDim, xDim});
ncfile.addVariable("xy", double.class, new Dimension[] {xDim, yDim});
ncfile.addVariable("yz", double.class, new Dimension[] {yDim, zDim});
ncfile.addVariable("xz", double.class, new Dimension[] {xDim, zDim});
ncfile.addVariable("yt", double.class, new Dimension[] {yDim, tDim});
ncfile.addVariable("xt", double.class, new Dimension[] {xDim, tDim});
ncfile.addVariable("ty", double.class, new Dimension[] {tDim, yDim});
ncfile.addVariable("tx", double.class, new Dimension[] {tDim, xDim});
// add global attributes
ncfile.addGlobalAttribute("Convention", "COARDS");
// create the file
try {
ncfile.create();
} catch (IOException e) {
System.err.println("ERROR creating file");
assert (false);
}
// write time data
int len = tDim.getLength();
ArrayDouble A = new ArrayDouble.D1(len);
Index ima = A.getIndex();
for (int i = 0; i < len; i++) A.setDouble(ima.set(i), (double) (i * 3600));
int[] origin = new int[1];
try {
ncfile.write("time", origin, A);
} catch (IOException e) {
System.err.println("ERROR writing time");
assert (false);
}
// write z data
len = zDim.getLength();
A = new ArrayDouble.D1(len);
ima = A.getIndex();
for (int i = 0; i < len; i++) A.setDouble(ima.set(i), (double) (i * 10));
try {
ncfile.write("z", origin, A);
} catch (IOException e) {
System.err.println("ERROR writing z");
assert (false);
}
// write y data
len = yDim.getLength();
A = new ArrayDouble.D1(len);
ima = A.getIndex();
for (int i = 0; i < len; i++) A.setDouble(ima.set(i), (double) (i * 3));
try {
ncfile.write("y", origin, A);
} catch (IOException e) {
System.err.println("ERROR writing y");
assert (false);
}
// write x data
len = xDim.getLength();
A = new ArrayDouble.D1(len);
ima = A.getIndex();
for (int i = 0; i < len; i++) A.setDouble(ima.set(i), (double) (i * 5));
try {
ncfile.write("x", origin, A);
} catch (IOException e) {
System.err.println("ERROR writing x");
assert (false);
}
// write tzyx data
doWrite4(ncfile, "tzyx");
doWrite4(ncfile, "tzxy");
doWrite4(ncfile, "txyz");
doWrite4(ncfile, "tyxz");
doWrite4(ncfile, "zyxt");
doWrite4(ncfile, "zxyt");
doWrite4(ncfile, "xyzt");
doWrite4(ncfile, "yxzt");
doWrite3(ncfile, "zyx");
doWrite3(ncfile, "txy");
doWrite3(ncfile, "yxz");
doWrite3(ncfile, "xzy");
doWrite3(ncfile, "yxt");
doWrite3(ncfile, "xyt");
doWrite3(ncfile, "yxt");
doWrite3(ncfile, "xyz");
doWrite2(ncfile, "yx");
doWrite2(ncfile, "xy");
doWrite2(ncfile, "yz");
doWrite2(ncfile, "xz");
doWrite2(ncfile, "yt");
doWrite2(ncfile, "xt");
doWrite2(ncfile, "ty");
doWrite2(ncfile, "tx");
if (show) System.out.println("ncfile = " + ncfile);
// all done
try {
ncfile.close();
} catch (IOException e) {
System.err.println("ERROR writing file");
assert (false);
}
System.out.println("*****************Test Write done");
}
public int getRecordCount() {
Dimension unlimitedDim = ncfile.getUnlimitedDimension();
return unlimitedDim.getLength();
}