private void compareVariableData(
Variable var1, Variable var2, boolean showCompare, boolean justOne) throws IOException {
Array data1 = var1.read();
Array data2 = var2.read();
if (showCompare)
f.format(
" compareArrays %s unlimited=%s size=%d%n",
var1.getNameAndDimensions(), var1.isUnlimited(), data1.getSize());
compareData(var1.getFullName(), data1, data2, justOne);
if (showCompare) f.format(" ok%n");
}
// @todo Make sure units are meters
public Array getElevation(Range range) throws IOException, InvalidRangeException {
List section = new ArrayList(1);
section.add(range);
Array a = elevVar.read(section);
if (elevVarUnitsConversionFactor == 1.0) return (a);
for (IndexIterator it = a.getIndexIterator(); it.hasNext(); ) {
if (elevVar.getDataType() == DataType.DOUBLE) {
double val = it.getDoubleNext();
it.setDoubleCurrent(val * elevVarUnitsConversionFactor);
} else if (elevVar.getDataType() == DataType.FLOAT) {
float val = it.getFloatNext();
it.setFloatCurrent((float) (val * elevVarUnitsConversionFactor));
} else if (elevVar.getDataType() == DataType.INT) {
int val = it.getIntNext();
it.setIntCurrent((int) (val * elevVarUnitsConversionFactor));
} else if (elevVar.getDataType() == DataType.LONG) {
long val = it.getLongNext();
it.setLongCurrent((long) (val * elevVarUnitsConversionFactor));
} else {
throw new IllegalStateException(
"Elevation variable type <"
+ elevVar.getDataType().toString()
+ "> not double, float, int, or long.");
}
}
return (a);
}
/** Read the value (parameters are ignored). */
public boolean read(String datasetName, Object specialO) throws IOException {
ArrayDouble.D0 a = (ArrayDouble.D0) ncVar.read();
setValue(a.get());
setRead(true);
return (false);
}
private void copySome(NetcdfFileWriteable ncfile, Variable oldVar, int nelems)
throws IOException {
String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
int[] shape = oldVar.getShape();
int[] origin = new int[oldVar.getRank()];
int size = shape[0];
for (int i = 0; i < size; i += nelems) {
origin[0] = i;
int left = size - i;
shape[0] = Math.min(nelems, left);
Array data;
try {
data = oldVar.read(origin, shape);
if (oldVar.getDataType() == DataType.STRING) {
data = convertToChar(ncfile.findVariable(newName), data);
}
if (data.getSize() > 0) { // zero when record dimension = 0
ncfile.write(newName, origin, data);
if (debug) System.out.println("write " + data.getSize() + " bytes");
}
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
}
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt.D1) : data.getClass().getName();
ArrayInt.D1 dataI = (ArrayInt.D1) data;
assert dataI.get(0) == 0;
assert dataI.get(1) == 10;
assert dataI.get(2) == 99;
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testAggCoordVar2(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt);
IndexIterator dataI = data.getIndexIterator();
assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
assert dataI.getIntNext() == 2 : dataI.getIntCurrent();
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
void CheckInt16(Variable v) throws IOException {
// int16
// assert(null != (v = dodsfile.findVariable("types.integers.i16")));
// assert v.getName().equals("types.integers.i16");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.SHORT;
CheckInt16Value(v.read());
}
void CheckS(Variable v) throws IOException {
// string
// assert(null != (v = dodsfile.findVariable("types.strings.s")));
// assert v.getName().equals("types.strings.s");
assert v.getRank() == 0;
assert v.getDataType() == DataType.STRING : v.getDataType();
CheckSValue(v.read());
}
void CheckLong32(Variable v) throws IOException {
// uint32
// assert(null != (v = dodsfile.findVariable("types.integers.ui32")));
// assert v.getName().equals("types.integers.ui32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.LONG : v.getDataType();
CheckLongValue(v.read());
}
void CheckF(Variable v) throws IOException {
// float
// assert(null != (v = dodsfile.findVariable("types.floats.f32")));
// assert v.getName().equals("types.floats.f32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.FLOAT : v.getDataType();
CheckFValue(v.read());
}
void CheckD(Variable v) throws IOException {
// double
// assert(null != (v = dodsfile.findVariable("types.floats.f64")));
// assert v.getName().equals("types.floats.f64");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.DOUBLE : v.getDataType();
CheckDValue(v.read());
}
public void testStride(String stride) throws IOException, InvalidRangeException {
Variable time = ncfile.findVariable("time");
ArrayInt all = (ArrayInt) time.read();
ArrayInt correct = (ArrayInt) all.section(new Section(stride).getRanges());
System.out.printf("correct(%s) %s", stride, NCdumpW.toString(correct));
ArrayInt data = (ArrayInt) time.read(stride);
System.out.printf("data(%s) %s%n", stride, NCdumpW.toString(data));
Index ci = correct.getIndex();
Index di = data.getIndex();
for (int i = 0; i < data.getSize(); i++)
assert (data.getInt(di.set(i)) == correct.getInt(ci.set(i)))
: stride
+ " index "
+ i
+ " = "
+ data.getInt(di.set(i))
+ " != "
+ correct.getInt(ci.set(i));
}
public void testStride() throws IOException, InvalidRangeException {
System.out.println("ncfile opened = " + location + "\n" + ncfile);
Variable time = ncfile.findVariable("time");
ArrayInt all = (ArrayInt) time.read();
for (int i = 0; i < all.getSize(); i++) assert (all.getInt(i) == i + 1);
testStride("0:13:3");
for (int i = 1; i < 12; i++) testStride("0:13:" + i);
}
// do not call directly
@Override
public Array reallyRead(Variable client, Section section, CancelTask cancelTask)
throws IOException, InvalidRangeException {
// see if its really a full read
if ((null == section) || section.computeSize() == getSize())
return reallyRead(client, cancelTask);
if (orgVar == null) return getMissingDataArray(section.getShape());
return orgVar.read(section);
}
private void copyAll(NetcdfFileWriteable ncfile, Variable oldVar) throws IOException {
String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
Array data = oldVar.read();
try {
if (oldVar.getDataType() == DataType.STRING) {
data = convertToChar(ncfile.findVariable(newName), data);
}
if (data.getSize() > 0) // zero when record dimension = 0
ncfile.write(newName, data);
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IOException(e.getMessage() + " for Variable " + oldVar.getFullName());
}
}
public static Map<String, Metadata> readFile(String file) throws Exception {
NetcdfFile n = NetcdfFile.open(file);
System.out.println("Opened: " + file);
/* Determine the size of our grid */
int xLen = n.findDimension("x").getLength();
int yLen = n.findDimension("y").getLength();
System.out.println("Grid size: " + xLen + "x" + yLen);
/* What time is this set of readings for? */
Variable timeVar = n.findVariable("time");
String timeStr = timeVar.getUnitsString().toUpperCase();
timeStr = timeStr.replace("HOURS SINCE ", "");
timeStr = timeStr.replace("HOUR SINCE ", "");
/* Find the base date (the day) the reading was taken */
Date baseDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX").parse(timeStr);
/* Get the number of hours since the base date this reading was taken */
int offset = timeVar.read().getInt(0);
/* Generate the actual date for this reading */
Calendar calendar = Calendar.getInstance();
calendar.setTime(baseDate);
calendar.set(Calendar.HOUR, calendar.get(Calendar.HOUR) + offset);
System.out.println("Time of collection: " + calendar.getTime());
/* We'll keep a mapping of geolocations -> Galileo Metadata */
Map<String, Metadata> metaMap = new HashMap<>();
/* Determine the lat, lon coordinates for the grid points, and get each
* reading at each grid point. */
NetcdfDataset dataset = new NetcdfDataset(n);
@SuppressWarnings("resource")
GridDataset gridData = new GridDataset(dataset);
for (GridDatatype g : gridData.getGrids()) {
/* Let's look at 3D variables: these have WxH dimensions, plus a
* single plane. A 4D variable would contain elevation
* and multiple planes as a result */
if (g.getShape().length == 3) {
convert3DVariable(g, calendar.getTime(), metaMap);
}
}
return metaMap;
}
public void readDouble() throws Exception {
Variable t1 = null;
assert (null != (t1 = ncfileRead.findVariable("t1")));
assert (t1.getDataType() == DataType.DOUBLE);
Attribute att = t1.findAttribute(CDM.SCALE_FACTOR);
assert (null != att);
assert (!att.isArray());
assert (1 == att.getLength());
assert (2.0 == att.getNumericValue().doubleValue());
assert (DataType.DOUBLE == att.getDataType());
// read
Array A = t1.read();
int i, j;
Index ima = A.getIndex();
int[] shape = A.getShape();
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getDouble(ima.set(i, j)) == (double) (i * 10.0 + j));
}
}
assert (null != (t1 = dsRead.findVariable("t1")));
assert t1 instanceof VariableEnhanced;
VariableEnhanced dsVar = (VariableEnhanced) t1;
assert (dsVar.getDataType() == DataType.DOUBLE);
A = dsVar.read();
ima = A.getIndex();
shape = A.getShape();
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
assert (A.getDouble(ima.set(i, j)) == (2.0 * (i * 10.0 + j) + 77.0));
}
}
assert (null == t1.findAttribute(CDM.SCALE_FACTOR));
assert (null == t1.findAttribute("add_offset"));
System.out.println("**************TestStandardVar ReadDouble");
}
public Array getData(Range range, String parameterName)
throws IOException, InvalidRangeException {
Variable variable = ncfile.getRootGroup().findVariable(parameterName);
int varRank = variable.getRank();
int[] varShape = variable.getShape();
List section = new ArrayList(varRank);
section.add(range);
for (int i = 1; i < varRank; i++) {
section.add(new Range(0, varShape[i] - 1));
}
Array array = variable.read(section);
if (array.getShape()[0] == 1) {
return (array.reduce(0));
} else {
return (array);
}
// return( array.getShape()[0] == 1 ? array.reduce( 0 ) : array);
}
public void testAggCoordVarScan(NetcdfFile ncfile) throws IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT : time.getDataType();
assert time.getDimension(0) == ncfile.findDimension("time");
int count = 0;
Array data = time.read();
assert (data instanceof ArrayInt);
while (data.hasNext()) {
int val = data.nextInt();
assert val == count * 10 : val + "!=" + count * 10;
count++;
}
}
public void testAggCoordVar3(NetcdfFile ncfile) throws IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE : time.getDataType();
assert time.getDimension(0) == ncfile.findDimension("time");
Array data = time.read();
assert (data instanceof ArrayDouble);
IndexIterator dataI = data.getIndexIterator();
double val = dataI.getDoubleNext();
assert Misc.closeEnough(val, 0.0) : val;
assert Misc.closeEnough(dataI.getDoubleNext(), 10.0) : dataI.getDoubleCurrent();
assert Misc.closeEnough(dataI.getDoubleNext(), 99.0) : dataI.getDoubleCurrent();
}
/** Converts a 3D variable to a {@link Metadata} representation */
private static void convert3DVariable(GridDatatype g, Date date, Map<String, Metadata> metaMap)
throws IOException {
Variable v = g.getVariable();
System.out.println("Reading: " + v.getFullName());
Array values = v.read();
int h = v.getShape(1);
int w = v.getShape(2);
for (int i = 0; i < h; ++i) {
for (int j = 0; j < w; ++j) {
LatLonPoint pt = g.getCoordinateSystem().getLatLon(j, i);
String hash =
GeoHash.encode((float) pt.getLatitude(), (float) pt.getLongitude(), 10).toLowerCase();
Metadata meta = metaMap.get(hash);
if (meta == null) {
/* We need to create Metadata for this location */
meta = new Metadata();
UUID metaUUID = UUID.nameUUIDFromBytes(hash.getBytes());
meta.setName(metaUUID.toString());
SpatialProperties location =
new SpatialProperties((float) pt.getLatitude(), (float) pt.getLongitude());
meta.setSpatialProperties(location);
TemporalProperties time = new TemporalProperties(date.getTime());
meta.setTemporalProperties(time);
metaMap.put(hash, meta);
}
String featureName = v.getFullName().toLowerCase();
float featureValue = values.getFloat(i * w + j);
Feature feature = new Feature(featureName, featureValue);
meta.putAttribute(feature);
}
}
}
public void testAggCoordVarNoCoordsDir(NetcdfFile ncfile) throws IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.STRING : time.getDataType();
assert time.getDimension(0) == ncfile.findDimension("time");
Array data = time.read();
assert (data instanceof ArrayObject);
IndexIterator dataI = data.getIndexIterator();
String coordName = (String) dataI.getObjectNext();
assert coordName.equals("time0Dir.nc") : coordName;
coordName = (String) dataI.getObjectNext();
assert coordName.equals("time1Dir.nc") : coordName;
coordName = (String) dataI.getObjectNext();
assert coordName.equals("time2Dir.nc") : coordName;
}
public void readSlice(NetcdfFile ncfile, int[] origin, int[] shape, String name)
throws IOException, InvalidRangeException {
Variable v = ncfile.findVariable(name);
Array data = v.read(origin, shape);
assert data.getRank() == 3;
assert data.getSize() == shape[0] * shape[1] * shape[2];
assert data.getShape()[0] == shape[0] : data.getShape()[0] + " " + shape[0];
assert data.getShape()[1] == shape[1];
assert data.getShape()[2] == shape[2];
assert data.getElementType() == double.class;
Index tIndex = data.getIndex();
for (int i = 0; i < shape[0]; i++)
for (int j = 0; j < shape[1]; j++)
for (int k = 0; k < shape[2]; k++) {
double val = data.getDouble(tIndex.set(i, j, k));
// System.out.println(" "+val);
assert TestUtils.close(val, 100 * (i + origin[0]) + 10 * j + k) : val;
}
}
/* Structure {
char EntryName(64);
char Definition(1024);
char Unit(1024);
char Scale Factor(1024);
} TIME_DESCR(60);
type = Layout(8); type= 2 (chunked) storageSize = (1,3136) dataSize=0 dataAddress=684294
*/
@Test
public void testReadManyAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/TIME_DESCR");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 60);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 60);
ArrayStructure data = (ArrayStructure) dset.read();
StructureMembers.Member m = data.getStructureMembers().findMember("EntryName");
assert m != null;
for (int i = 0; i < dset.getSize(); i++) {
String r = data.getScalarString(i, m);
if (i % 2 == 0) assert r.equals("TIME[" + i / 2 + "]-days") : r + " at " + i;
else assert r.equals("TIME[" + i / 2 + "]-milliseconds") : r + " at " + i;
}
}
System.out.println("*** testReadManyAtATime ok");
}
public void testCoordVar(NetcdfFile ncfile) {
Variable lat = ncfile.findVariable("lat");
assert null != lat;
assert lat.getShortName().equals("lat");
assert lat.getRank() == 1;
assert lat.getSize() == 3;
assert lat.getShape()[0] == 3;
assert lat.getDataType() == DataType.FLOAT;
assert !lat.isUnlimited();
assert lat.getDimension(0).equals(ncfile.findDimension("lat"));
Attribute att = lat.findAttribute("units");
assert null != att;
assert !att.isArray();
assert att.isString();
assert att.getDataType() == DataType.STRING;
assert att.getStringValue().equals("degrees_north");
assert att.getNumericValue() == null;
assert att.getNumericValue(3) == null;
try {
Array data = lat.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == float.class;
IndexIterator dataI = data.getIndexIterator();
assert TestUtils.close(dataI.getDoubleNext(), 41.0);
assert TestUtils.close(dataI.getDoubleNext(), 40.0);
assert TestUtils.close(dataI.getDoubleNext(), 39.0);
} catch (IOException io) {
}
}
public void testReadData(NetcdfFile ncfile, String name) throws IOException {
Variable v = ncfile.findVariable(name);
assert null != v;
assert v.getShortName().equals(name);
assert v.getRank() == 3;
assert v.getSize() == 36 : v.getSize();
assert v.getShape()[0] == 3;
assert v.getShape()[1] == 3;
assert v.getShape()[2] == 4;
assert v.getDataType() == DataType.DOUBLE;
assert !v.isCoordinateVariable();
assert v.getDimension(0) == ncfile.findDimension("time");
assert v.getDimension(1) == ncfile.findDimension("lat");
assert v.getDimension(2) == ncfile.findDimension("lon");
Array data = v.read();
assert data.getRank() == 3;
assert data.getSize() == 36;
assert data.getShape()[0] == 3;
assert data.getShape()[1] == 3;
assert data.getShape()[2] == 4;
assert data.getElementType() == double.class;
int[] shape = data.getShape();
Index tIndex = data.getIndex();
for (int i = 0; i < shape[0]; i++)
for (int j = 0; j < shape[1]; j++)
for (int k = 0; k < shape[2]; k++) {
double val = data.getDouble(tIndex.set(i, j, k));
// System.out.println(" "+val);
assert TestUtils.close(val, 100 * i + 10 * j + k) : val;
}
}
public void testNC3Read() throws IOException {
NetcdfFile ncfile = TestDir.openFileLocal("testWrite.nc");
assert (null != ncfile.findDimension("lat"));
assert (null != ncfile.findDimension("lon"));
Variable temp = null;
assert (null != (temp = ncfile.findVariable("temperature")));
// read entire array
Array A;
try {
A = temp.read();
} catch (IOException e) {
System.err.println("ERROR reading file");
assert (false);
return;
}
assert (A.getRank() == 2);
int i, j;
Index ima = A.getIndex();
int[] shape = A.getShape();
assert shape[0] == 64;
assert shape[1] == 128;
for (i = 0; i < shape[0]; i++) {
for (j = 0; j < shape[1]; j++) {
double dval = A.getDouble(ima.set(i, j));
assert (dval == (double) (i * 1000000 + j * 1000)) : dval;
}
}
// read part of array
int[] origin2 = new int[2];
int[] shape2 = new int[2];
shape2[0] = 1;
shape2[1] = temp.getShape()[1];
try {
A = temp.read(origin2, shape2);
} catch (InvalidRangeException e) {
System.err.println("ERROR reading file " + e);
assert (false);
return;
} catch (IOException e) {
System.err.println("ERROR reading file");
assert (false);
return;
}
assert (A.getRank() == 2);
for (j = 0; j < shape2[1]; j++) {
assert (A.getDouble(ima.set(0, j)) == (double) (j * 1000));
}
// rank reduction
Array Areduce = A.reduce();
Index ima2 = Areduce.getIndex();
assert (Areduce.getRank() == 1);
for (j = 0; j < shape2[1]; j++) {
assert (Areduce.getDouble(ima2.set(j)) == (double) (j * 1000));
}
// read char variable
Variable c = null;
assert (null != (c = ncfile.findVariable("svar")));
try {
A = c.read();
} catch (IOException e) {
assert (false);
}
assert (A instanceof ArrayChar);
ArrayChar ac = (ArrayChar) A;
String val = ac.getString(ac.getIndex());
assert val.equals("Testing 1-2-3") : val;
// System.out.println( "val = "+ val);
// read char variable 2
Variable c2 = null;
assert (null != (c2 = ncfile.findVariable("svar2")));
try {
A = c2.read();
} catch (IOException e) {
assert (false);
}
assert (A instanceof ArrayChar);
ArrayChar ac2 = (ArrayChar) A;
assert (ac2.getString().equals("Two pairs of ladies stockings!"));
// read String Array
Variable c3 = null;
assert (null != (c3 = ncfile.findVariable("names")));
try {
A = c3.read();
} catch (IOException e) {
assert (false);
}
assert (A instanceof ArrayChar);
ArrayChar ac3 = (ArrayChar) A;
ima = ac3.getIndex();
assert (ac3.getString(ima.set(0)).equals("No pairs of ladies stockings!"));
assert (ac3.getString(ima.set(1)).equals("One pair of ladies stockings!"));
assert (ac3.getString(ima.set(2)).equals("Two pairs of ladies stockings!"));
// read String Array - 2
Variable c4 = null;
assert (null != (c4 = ncfile.findVariable("names2")));
try {
A = c4.read();
} catch (IOException e) {
assert (false);
}
assert (A instanceof ArrayChar);
ArrayChar ac4 = (ArrayChar) A;
ima = ac4.getIndex();
assert (ac4.getString(0).equals("0 pairs of ladies stockings!"));
assert (ac4.getString(1).equals("1 pair of ladies stockings!"));
assert (ac4.getString(2).equals("2 pairs of ladies stockings!"));
// System.out.println( "ncfile = "+ ncfile);
ncfile.close();
System.out.println("**************TestRead done");
}
// do not call directly
@Override
public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException {
if (orgVar == null) return getMissingDataArray(shape);
return orgVar.read();
}
/** Read the value (parameters are ignored). */
public boolean read(String datasetName, Object specialO) throws IOException {
setData(ncVar.read());
return (false);
}
// @todo Make sure units are degrees_east
public Array getLongitude(Range range) throws IOException, InvalidRangeException {
List section = new ArrayList(1);
section.add(range);
return (lonVar.read(section));
}
