public boolean compareVariables(NetcdfFile org, NetcdfFile copy) {
f.format("Original = %s%n", org.getLocation());
f.format("CompareTo= %s%n", copy.getLocation());
boolean ok = true;
for (Variable orgV : org.getVariables()) {
if (orgV.isCoordinateVariable()) continue;
Variable copyVar = copy.findVariable(orgV.getShortName());
if (copyVar == null) {
f.format(" MISSING '%s' in 2nd file%n", orgV.getFullName());
ok = false;
} else {
List<Dimension> dims1 = orgV.getDimensions();
List<Dimension> dims2 = copyVar.getDimensions();
if (!compare(dims1, dims2)) {
f.format(" %s != %s%n", orgV.getNameAndDimensions(), copyVar.getNameAndDimensions());
} else {
// f.format(" ok %s%n", orgV.getName());
}
}
}
f.format("%n");
for (Variable orgV : copy.getVariables()) {
if (orgV.isCoordinateVariable()) continue;
Variable copyVar = org.findVariable(orgV.getShortName());
if (copyVar == null) {
f.format(" MISSING '%s' in 1st file%n", orgV.getFullName());
ok = false;
}
}
return ok;
}
public void testReadData(NetcdfFile ncfile, String name) throws IOException {
Variable v = ncfile.findVariable(name);
assert null != v;
assert v.getShortName().equals(name);
assert v.getRank() == 3;
assert v.getSize() == 36 : v.getSize();
assert v.getShape()[0] == 3;
assert v.getShape()[1] == 3;
assert v.getShape()[2] == 4;
assert v.getDataType() == DataType.DOUBLE;
assert !v.isCoordinateVariable();
assert v.getDimension(0) == ncfile.findDimension("time");
assert v.getDimension(1) == ncfile.findDimension("lat");
assert v.getDimension(2) == ncfile.findDimension("lon");
Array data = v.read();
assert data.getRank() == 3;
assert data.getSize() == 36;
assert data.getShape()[0] == 3;
assert data.getShape()[1] == 3;
assert data.getShape()[2] == 4;
assert data.getElementType() == double.class;
int[] shape = data.getShape();
Index tIndex = data.getIndex();
for (int i = 0; i < shape[0]; i++)
for (int j = 0; j < shape[1]; j++)
for (int k = 0; k < shape[2]; k++) {
double val = data.getDouble(tIndex.set(i, j, k));
// System.out.println(" "+val);
assert TestUtils.close(val, 100 * i + 10 * j + k) : val;
}
}
private FeatureType amendGrid(
Element gridElem, NetcdfFile ncfile, Group parent, String location) {
List<Dimension> unknownDims = new ArrayList<>();
// always has x and y dimension
String xdimSizeS = gridElem.getChild("XDim").getText().trim();
String ydimSizeS = gridElem.getChild("YDim").getText().trim();
int xdimSize = Integer.parseInt(xdimSizeS);
int ydimSize = Integer.parseInt(ydimSizeS);
parent.addDimensionIfNotExists(new Dimension("XDim", xdimSize));
parent.addDimensionIfNotExists(new Dimension("YDim", ydimSize));
/* see HdfEosModisConvention
UpperLeftPointMtrs=(-20015109.354000,1111950.519667)
LowerRightMtrs=(-18903158.834333,-0.000000)
Projection=GCTP_SNSOID
ProjParams=(6371007.181000,0,0,0,0,0,0,0,0,0,0,0,0)
SphereCode=-1
*/
Element proj = gridElem.getChild("Projection");
if (proj != null) {
Variable crs = new Variable(ncfile, parent, null, HDFEOS_CRS);
crs.setDataType(DataType.SHORT);
crs.setDimensions(""); // scalar
crs.setCachedData(Array.makeArray(DataType.SHORT, 1, 0, 0)); // fake data
parent.addVariable(crs);
addAttributeIfExists(gridElem, HDFEOS_CRS_Projection, crs, false);
addAttributeIfExists(gridElem, HDFEOS_CRS_UpperLeft, crs, true);
addAttributeIfExists(gridElem, HDFEOS_CRS_LowerRight, crs, true);
addAttributeIfExists(gridElem, HDFEOS_CRS_ProjParams, crs, true);
addAttributeIfExists(gridElem, HDFEOS_CRS_SphereCode, crs, false);
}
// global Dimensions
Element d = gridElem.getChild("Dimension");
List<Element> dims = d.getChildren();
for (Element elem : dims) {
String name = elem.getChild("DimensionName").getText().trim();
name = NetcdfFile.makeValidCdmObjectName(name);
if (name.equalsIgnoreCase("scalar")) continue;
String sizeS = elem.getChild("Size").getText().trim();
int length = Integer.parseInt(sizeS);
Dimension old = parent.findDimension(name);
if ((old == null) || (old.getLength() != length)) {
if (length > 0) {
Dimension dim = new Dimension(name, length);
if (parent.addDimensionIfNotExists(dim) && showWork)
System.out.printf(" Add dimension %s %n", dim);
} else {
log.warn("Dimension {} has size {} {} ", sizeS, name, location);
Dimension udim = new Dimension(name, 1);
udim.setGroup(parent);
unknownDims.add(udim);
if (showWork) System.out.printf(" Add dimension %s %n", udim);
}
}
}
// Geolocation Variables
Group geoFieldsG = parent.findGroup(GEOLOC_FIELDS);
if (geoFieldsG == null) geoFieldsG = parent.findGroup(GEOLOC_FIELDS2);
if (geoFieldsG != null) {
Element floc = gridElem.getChild("GeoField");
List<Element> varsLoc = floc.getChildren();
for (Element elem : varsLoc) {
String varname = elem.getChild("GeoFieldName").getText().trim();
Variable v = geoFieldsG.findVariable(varname);
// if (v == null)
// v = geoFieldsG.findVariable( H4header.createValidObjectName(varname));
assert v != null : varname;
Element dimList = elem.getChild("DimList");
List<Element> values = dimList.getChildren("value");
setSharedDimensions(v, values, unknownDims, location);
}
}
// Data Variables
Group dataG = parent.findGroup(DATA_FIELDS);
if (dataG == null)
dataG =
parent.findGroup(
DATA_FIELDS2); // eg C:\data\formats\hdf4\eos\mopitt\MOP03M-200501-L3V81.0.1.hdf
if (dataG != null) {
Element f = gridElem.getChild("DataField");
List<Element> vars = f.getChildren();
for (Element elem : vars) {
String varname = elem.getChild("DataFieldName").getText().trim();
varname = NetcdfFile.makeValidCdmObjectName(varname);
Variable v = dataG.findVariable(varname);
// if (v == null)
// v = dataG.findVariable( H4header.createValidObjectName(varname));
assert v != null : varname;
Element dimList = elem.getChild("DimList");
List<Element> values = dimList.getChildren("value");
setSharedDimensions(v, values, unknownDims, location);
}
// get projection
String projS = null;
Element projElem = gridElem.getChild("Projection");
if (projElem != null) projS = projElem.getText().trim();
boolean isLatLon = "GCTP_GEO".equals(projS);
// look for XDim, YDim coordinate variables
if (isLatLon) {
for (Variable v : dataG.getVariables()) {
if (v.isCoordinateVariable()) {
if (v.getShortName().equals("YDim")) {
v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lat.toString()));
v.addAttribute(new Attribute(CDM.UNITS, CDM.LAT_UNITS));
}
if (v.getShortName().equals("XDim"))
v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Lon.toString()));
}
}
}
}
return FeatureType.GRID;
}
/**
* Opens the NetCDF dataset at the given location, using the dataset cache if {@code location}
* represents an NcML aggregation. We cannot use the cache for OPeNDAP or single NetCDF files
* because the underlying data may have changed and the NetcdfDataset cache may cache a dataset
* forever. In the case of NcML we rely on the fact that server administrators ought to have set a
* "recheckEvery" parameter for NcML aggregations that may change with time. It is desirable to
* use the dataset cache for NcML aggregations because they can be time-consuming to assemble and
* we don't want to do this every time a map is drawn.
*
* @param location The location of the data: a local NetCDF file, an NcML aggregation file or an
* OPeNDAP location, {@literal i.e.} anything that can be passed to
* NetcdfDataset.openDataset(location).
* @return a {@link NetcdfDataset} object for accessing the data at the given location.
* @throws IOException if there was an error reading from the data source.
*/
private NetcdfDataset openAndAggregateDataset(String location) throws IOException, EdalException {
NetcdfDataset nc;
if (location.startsWith("dods://") || location.startsWith("http://")) {
/*
* We have a remote dataset
*/
nc = CdmUtils.openDataset(location);
} else {
/*
* We have a local dataset
*/
List<File> files = null;
try {
files = CdmUtils.expandGlobExpression(location);
} catch (NullPointerException e) {
System.out.println("NPE processing location: " + location);
throw e;
}
if (files.size() == 0) {
throw new EdalException(
"The location " + location + " doesn't refer to any existing files.");
}
if (files.size() == 1) {
location = files.get(0).getAbsolutePath();
nc = CdmUtils.openDataset(location);
} else {
/*
* We have multiple files in a glob expression. We write some
* NcML and use the NetCDF aggregation libs to parse this into
* an aggregated dataset.
*
* If we have already generated the ncML on a previous call,
* just use that.
*/
if (ncmlString == null) {
/*
* Find the name of the time dimension
*/
NetcdfDataset first = openAndAggregateDataset(files.get(0).getAbsolutePath());
String timeDimName = null;
for (Variable var : first.getVariables()) {
if (var.isCoordinateVariable()) {
for (Attribute attr : var.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("units")
&& attr.getStringValue().contains(" since ")) {
/*
* This is the time dimension. Since this is
* a co-ordinate variable, there is only 1
* dimension
*/
Dimension timeDimension = var.getDimension(0);
timeDimName = timeDimension.getFullName();
}
}
}
}
first.close();
if (timeDimName == null) {
throw new EdalException("Cannot join multiple files without time dimensions");
}
/*
* We can't assume that the glob expression will have
* returned the files in time order.
*
* We could assume that alphabetical == time ordered (and
* for properly named files it will - but let's not rely on
* our users having sensible naming conventions...
*
* Sort the list using a comparator which opens the file and
* gets the first value of the time dimension
*/
final String aggDimName = timeDimName;
Collections.sort(
files,
new Comparator<File>() {
@Override
public int compare(File ncFile1, File ncFile2) {
NetcdfFile nc1 = null;
NetcdfFile nc2 = null;
try {
nc1 = NetcdfFile.open(ncFile1.getAbsolutePath());
nc2 = NetcdfFile.open(ncFile2.getAbsolutePath());
Variable timeVar1 = nc1.findVariable(aggDimName);
Variable timeVar2 = nc2.findVariable(aggDimName);
long time1 = timeVar1.read().getLong(0);
long time2 = timeVar2.read().getLong(0);
return Long.compare(time1, time2);
} catch (Exception e) {
/*
* There was a problem reading the data. Sort
* alphanumerically by filename and hope for the
* best...
*
* This catches all exceptions because however
* it fails this is still our best option.
*
* If the error is a genuine problem, it'll show
* up as soon as we try and aggregate.
*/
return ncFile1.getAbsolutePath().compareTo(ncFile2.getAbsolutePath());
} finally {
if (nc1 != null) {
try {
nc1.close();
} catch (IOException e) {
log.error("Problem closing netcdf file", e);
}
}
if (nc2 != null) {
try {
nc2.close();
} catch (IOException e) {
log.error("Problem closing netcdf file", e);
}
}
}
}
});
/*
* Now create the NcML string and use it to create an
* aggregated dataset
*/
StringBuffer ncmlStringBuffer = new StringBuffer();
ncmlStringBuffer.append(
"<netcdf xmlns=\"http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2\">");
ncmlStringBuffer.append(
"<aggregation dimName=\"" + timeDimName + "\" type=\"joinExisting\">");
for (File file : files) {
ncmlStringBuffer.append("<netcdf location=\"" + file.getAbsolutePath() + "\"/>");
}
ncmlStringBuffer.append("</aggregation>");
ncmlStringBuffer.append("</netcdf>");
ncmlString = ncmlStringBuffer.toString();
}
nc = NcMLReader.readNcML(new StringReader(ncmlString), null);
}
}
return nc;
}
