/* Structure {
int LAT[0];
...
int LAT[149];
} IMAGE_LAT_ARRAY(3600);
type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
*/
@Test
public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 3600);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 3600);
Structure s = (Structure) dset;
// read last one - chunked
StructureData sd = s.readStructure(3599);
assert sd.getScalarInt("LAT[0]") == 70862722;
assert sd.getScalarInt("LAT[149]") == 85302263;
// read one at a time
for (int i = 3590; i < d.getLength(); i++) {
s.readStructure(i);
System.out.println(" read structure " + i);
}
}
System.out.println("*** testReadIASI ok");
}
private int countSeq(Structure recordStruct) throws IOException {
int total = 0;
int count = 0;
int max = 0;
StructureDataIterator iter = recordStruct.getStructureIterator();
try {
while (iter.hasNext()) {
StructureData sdata = iter.next();
ArraySequence seq1 = sdata.getArraySequence("seq1");
int n = seq1.getStructureDataCount();
total += n;
count++;
max = Math.max(max, n);
}
} finally {
iter.finish();
}
double avg = total / count;
int wasted = count * max - total;
double wp = (double) wasted / (count * max);
System.out.println(" Max = " + max + " avg = " + avg + " wasted = " + wasted + " %= " + wp);
return max;
}
/*
Structure {
int a_name;
byte b_name(3);
byte c_name(3);
short d_name(3);
int e_name(3);
long f_name(3);
int g_name(3);
short h_name(3);
int i_name(3);
long j_name(3);
float k_name(3);
double l_name(3);
} CompoundNative(15);
type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
*/
@Test
public void testReadH5StructureArrayMembers() throws java.io.IOException {
try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {
Variable dset = ncfile.findVariable("CompoundNative");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 15);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 15);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testReadH5StructureArrayMembers ok");
}
public void writeRecord(String stnName, Date obsDate, StructureData sdata) throws IOException {
StationTracker stnTracker = stationMap.get(stnName);
ProfileTracker proTracker = stnTracker.profileMap.get(obsDate);
if (proTracker == null) {
proTracker = new ProfileTracker(profileIndex);
stnTracker.profileMap.put(obsDate, proTracker);
stnTracker.link.add(profileIndex);
stnTracker.lastChild = profileIndex;
stnTracker.numChildren++;
try {
originTime[0] = profileIndex; // 2d index
timeArray.set(0, dateFormatter.toDateTimeStringISO(obsDate));
parentArray.set(0, stnTracker.parent_index);
ncfile.writeStringData(timeName, originTime, timeArray);
ncfile.write(parentStationIndex, originTime, parentArray);
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IllegalStateException(e);
}
profileIndex++;
}
// needs to be wrapped as an ArrayStructure, even though we are only writing one at a time.
ArrayStructureW sArray = new ArrayStructureW(sdata.getStructureMembers(), new int[] {1});
sArray.setStructureData(sdata, 0);
// track the min and max date
if ((minDate == null) || minDate.after(obsDate)) minDate = obsDate;
if ((maxDate == null) || maxDate.before(obsDate)) maxDate = obsDate;
timeArray.set(0, dateFormatter.toDateTimeStringISO(obsDate));
parentArray.set(0, proTracker.parent_index);
proTracker.link.add(recno);
proTracker.lastChild = recno; // not currently used
proTracker.numChildren++;
// write the recno record
origin[0] = recno; // 1d index
originTime[0] = recno; // 2d index
try {
ncfile.write("record", origin, sArray);
ncfile.write(parentProfileIndex, originTime, parentArray);
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IllegalStateException(e);
}
recno++;
}
public boolean compareStructureData(
StructureData sdata1, StructureData sdata2, double tol, boolean justOne) {
boolean ok = true;
StructureMembers sm1 = sdata1.getStructureMembers();
StructureMembers sm2 = sdata2.getStructureMembers();
if (sm1.getMembers().size() != sm2.getMembers().size()) {
f.format(" size %d !== %d%n", sm1.getMembers().size(), sm2.getMembers().size());
ok = false;
}
for (StructureMembers.Member m1 : sm1.getMembers()) {
if (m1.getName().equals("time")) continue;
StructureMembers.Member m2 = sm2.findMember(m1.getName());
Array data1 = sdata1.getArray(m1);
Array data2 = sdata2.getArray(m2);
ok &= compareData(m1.getName(), data1, data2, tol, justOne);
}
return ok;
}
StationObs(Station s, StructureData data) {
this.station = s;
this.data = data;
int cyd = data.getScalarInt("DAY");
int year = cyd / 1000;
int doy = cyd % 1000;
int hms = data.getScalarInt("TIME");
int hour = hms / 10000;
hms %= 10000;
int min = hms / 100;
int sec = hms % 100;
calendar.clear();
calendar.set(GregorianCalendar.YEAR, year);
calendar.set(GregorianCalendar.DAY_OF_YEAR, doy);
calendar.set(GregorianCalendar.HOUR_OF_DAY, hour);
calendar.set(GregorianCalendar.MINUTE, min);
calendar.set(GregorianCalendar.SECOND, sec);
// Date check = calendar.getTime();
// System.out.println(" check="+DateUnit.getStandardDateString(check));
obsTime = calendar.getTimeInMillis() / 1000.0;
nomTime = obsTime; // LOOK get real nominal time
}
protected void makeFeatureVariables(StructureData featureData, boolean isExtended)
throws IOException {
// LOOK why not unlimited here ?
Dimension profileDim = writer.addDimension(null, profileDimName, nfeatures);
// Dimension profileDim = isExtendedModel ? writer.addUnlimitedDimension(profileDimName) :
// writer.addDimension(null, profileDimName, nprofiles);
// add the profile Variables using the profile dimension
List<VariableSimpleIF> profileVars = new ArrayList<>();
profileVars.add(
VariableSimpleImpl.makeScalar(latName, "profile latitude", CDM.LAT_UNITS, DataType.DOUBLE));
profileVars.add(
VariableSimpleImpl.makeScalar(
lonName, "profile longitude", CDM.LON_UNITS, DataType.DOUBLE));
profileVars.add(
VariableSimpleImpl.makeString(profileIdName, "profile identifier", null, id_strlen)
.add(new Attribute(CF.CF_ROLE, CF.PROFILE_ID))); // profileId:cf_role = "profile_id";
profileVars.add(
VariableSimpleImpl.makeScalar(
numberOfObsName, "number of obs for this profile", null, DataType.INT)
.add(
new Attribute(
CF.SAMPLE_DIMENSION, recordDimName))); // rowSize:sample_dimension = "obs"
profileVars.add(
VariableSimpleImpl.makeScalar(
profileTimeName,
"nominal time of profile",
timeUnit.getUnitsString(),
DataType.DOUBLE));
for (StructureMembers.Member m : featureData.getMembers()) {
VariableSimpleIF dv = getDataVar(m.getName());
if (dv != null) profileVars.add(dv);
}
if (isExtended) {
profileStruct =
(Structure)
writer.addVariable(null, profileStructName, DataType.STRUCTURE, profileDimName);
addCoordinatesExtended(profileStruct, profileVars);
} else {
addCoordinatesClassic(profileDim, profileVars, featureVarMap);
}
}
@Test
public void testH5StructureDS() throws java.io.IOException {
int a_name = 0;
String[] b_name =
new String[] {
"A fight is a contract that takes two people to honor.",
"A combative stance means that you've accepted the contract.",
"In which case, you deserve what you get.",
" -- Professor Cheng Man-ch'ing"
};
String c_name = "Hello!";
// H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
try (NetcdfDataset ncfile =
NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {
Variable dset = ncfile.findVariable("CompoundComplex");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 6);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 6);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
assert sd.getScalarInt("a_name") == a_name;
a_name++;
assert sd.getScalarString("c_name").equals(c_name);
String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
assert results.length == b_name.length;
int count = 0;
for (String r : results) assert r.equals(b_name[count++]);
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testH5StructureDS ok");
}
public static void main(String args[]) throws Exception {
long start = System.currentTimeMillis();
Map<String, ucar.unidata.geoloc.Station> staHash =
new HashMap<String, ucar.unidata.geoloc.Station>();
String location = "R:/testdata/sounding/netcdf/Upperair_20070401_0000.nc";
NetcdfDataset ncfile = NetcdfDataset.openDataset(location);
ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
// look through record varibles, for those that have "manLevel" dimension
// make a StructureData object for those
StructureMembers sm = new StructureMembers("manLevel");
Dimension manDim = ncfile.findDimension("manLevel");
Structure record = (Structure) ncfile.findVariable("record");
List<Variable> allList = record.getVariables();
List<VariableSimpleIF> varList = new ArrayList<VariableSimpleIF>();
for (Variable v : allList) {
if ((v.getRank() == 1) && v.getDimension(0).equals(manDim)) {
// public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String
// shortName, DataType dataType,
// String dims, String units, String desc) {
varList.add(
new VariableDS(
ncfile,
null,
null,
v.getShortName(),
v.getDataType(),
"",
v.getUnitsString(),
v.getDescription()));
// (String name, String desc, String units, DataType dtype, int []shape)
sm.addMember(
v.getShortName(),
v.getDescription(),
v.getUnitsString(),
v.getDataType(),
new int[0]); // scalar
}
}
ArrayStructureMA manAS = new ArrayStructureMA(sm, new int[] {manDim.getLength()});
// need the date units
Variable time = ncfile.findVariable("synTime");
String timeUnits = ncfile.findAttValueIgnoreCase(time, "units", null);
timeUnits = StringUtil.remove(timeUnits, '('); // crappy fsl'ism
timeUnits = StringUtil.remove(timeUnits, ')');
DateUnit timeUnit = new DateUnit(timeUnits);
// extract stations
int nrecs = 0;
StructureDataIterator iter = record.getStructureIterator();
while (iter.hasNext()) {
StructureData sdata = iter.next();
String name = sdata.getScalarString("staName");
ucar.unidata.geoloc.Station s = staHash.get(name);
if (s == null) {
float lat = sdata.convertScalarFloat("staLat");
float lon = sdata.convertScalarFloat("staLon");
float elev = sdata.convertScalarFloat("staElev");
s = new StationImpl(name, "", lat, lon, elev);
staHash.put(name, s);
}
nrecs++;
}
List<ucar.unidata.geoloc.Station> stnList =
Arrays.asList(staHash.values().toArray(new ucar.unidata.geoloc.Station[staHash.size()]));
Collections.sort(stnList);
// create the writer
WriterProfileObsDataset writer =
new WriterProfileObsDataset(location + ".out", "rewrite " + location);
writer.writeHeader(stnList, varList, nrecs, "prMan");
// extract records
iter = record.getStructureIterator();
while (iter.hasNext()) {
StructureData sdata = iter.next();
String name = sdata.getScalarString("staName");
double timeValue = sdata.convertScalarDouble("synTime");
Date date = timeUnit.makeDate(timeValue);
// transfer to the ArrayStructure
List<String> names = sm.getMemberNames();
for (String mname : names) {
manAS.setMemberArray(mname, sdata.getArray(mname));
}
// each level is weritten as a seperate structure
int numMand = sdata.getScalarInt("numMand");
if (numMand >= manDim.getLength()) continue;
for (int i = 0; i < numMand; i++) {
StructureData useData = manAS.getStructureData(i);
writer.writeRecord(name, date, useData);
}
}
writer.finish();
long took = System.currentTimeMillis() - start;
System.out.println("That took = " + took);
}
private double copyVarData(NetcdfFileWriteable ncfile, Structure recordStruct)
throws IOException, InvalidRangeException {
int nrecs = (int) recordStruct.getSize();
int sdataSize = recordStruct.getElementSize();
double total = 0;
double totalRecordBytes = 0;
for (int count = 0; count < nrecs; count++) {
StructureData recordData = recordStruct.readStructure(count);
for (StructureMembers.Member m : recordData.getMembers()) {
if (m.getDataType() == DataType.STRUCTURE) {
int countLevel = 0;
ArrayStructure seq1 = recordData.getArrayStructure(m);
StructureDataIterator iter = seq1.getStructureDataIterator();
try {
while (iter.hasNext()) {
StructureData seqData = iter.next();
for (StructureMembers.Member seqm : seqData.getMembers()) {
Array data = seqData.getArray(seqm);
int[] shape = data.getShape();
int[] newShape = new int[data.getRank() + 2];
newShape[0] = 1;
newShape[1] = 1;
for (int i = 0; i < data.getRank(); i++) newShape[i + 1] = shape[i];
int[] origin = new int[data.getRank() + 2];
origin[0] = count;
origin[1] = countLevel;
String mname = seqm.getName() + "-" + m.getName();
if (debug && (count == 0) && (countLevel == 0))
System.out.println("write to = " + mname);
ncfile.write(mname, origin, data.reshape(newShape));
}
countLevel++;
}
} finally {
iter.finish();
}
} else {
Array data = recordData.getArray(m);
int[] shape = data.getShape();
int[] newShape = new int[data.getRank() + 1];
newShape[0] = 1;
for (int i = 0; i < data.getRank(); i++) newShape[i + 1] = shape[i];
int[] origin = new int[data.getRank() + 1];
origin[0] = count;
if (debug && (count == 0)) System.out.println("write to = " + m.getName());
ncfile.write(m.getName(), origin, data.reshape(newShape));
}
}
totalRecordBytes += sdataSize;
}
total += totalRecordBytes;
totalRecordBytes /= 1000 * 1000;
if (debug)
System.out.println(
"write record var; total = " + totalRecordBytes + " Mbytes # recs=" + nrecs);
return total;
}
private MyEarthLocation(StructureData sdata) {
this.latitude = sdata.convertScalarDouble(SingleTrajectory.this.latVar.getShortName());
this.longitude = sdata.convertScalarDouble(SingleTrajectory.this.lonVar.getShortName());
this.elevation = sdata.convertScalarDouble(SingleTrajectory.this.elevVar.getShortName());
if (elevVarUnitsConversionFactor != 1.0) this.elevation *= elevVarUnitsConversionFactor;
}
private MyPointObsDatatype(int point, StructureData sdata) {
this.point = point;
this.sdata = sdata;
this.time = sdata.convertScalarDouble(SingleTrajectory.this.timeVar.getShortName());
this.earthLoc = new MyEarthLocation(sdata);
}
public void serialize(DataOutputStream sink, StructureData sdata, StructureMembers.Member m)
throws IOException {
setValue(sdata.getScalarShort(m));
externalize(sink);
}