public void testSingleDataset() throws IOException {
InvCatalogImpl cat = TestTDSAll.open(null);
InvDataset ds = cat.findDatasetByID("testSingleDataset");
assert (ds != null) : "cant find dataset 'testSingleDataset'";
assert ds.getDataType() == FeatureType.GRID;
ThreddsDataFactory fac = new ThreddsDataFactory();
ThreddsDataFactory.Result dataResult = fac.openFeatureDataset(ds, null);
assert dataResult != null;
assert !dataResult.fatalError;
assert dataResult.featureDataset != null;
GridDataset gds = (GridDataset) dataResult.featureDataset;
GridDatatype grid = gds.findGridDatatype("Z_sfc");
assert grid != null;
GridCoordSystem gcs = grid.getCoordinateSystem();
assert gcs != null;
assert null == gcs.getVerticalAxis();
CoordinateAxis1D time = gcs.getTimeAxis1D();
assert time != null;
assert time.getSize() == 1;
assert 102840.0 == time.readScalarDouble();
dataResult.featureDataset.close();
}
@Test
public void testFloatingPointCompare() throws Exception {
String spec =
TestDir.cdmUnitTestDir + "ft/fmrc/fp_precision/sediment_thickness_#yyMMddHHmm#.*\\.nc$";
System.out.printf("%n====================FMRC dataset %s%n", spec);
Formatter errlog = new Formatter();
Fmrc fmrc = Fmrc.open(spec, errlog);
assert (fmrc != null) : errlog;
try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDatasetBest()) {
GridDatatype v = gridDs.findGridByShortName("thickness_of_sediment");
assert v != null;
GridCoordSystem gcs = v.getCoordinateSystem();
CoordinateAxis1DTime time = gcs.getTimeAxis1D();
Assert.assertEquals("hours since 2015-03-08 12:51:00.000 UTC", time.getUnitsString());
Assert.assertEquals(74, time.getSize());
Array data = time.read();
System.out.printf("%s%n", NCdumpW.toString(data));
for (CalendarDate cd : time.getCalendarDates()) {
assert cd.getFieldValue(CalendarPeriod.Field.Minute) == 0 : System.out.printf("%s%n", cd);
}
}
}
@Test
public void testConventionsAttribute() throws Exception {
String path = TestDir.cdmUnitTestDir + "ncml/AggForecastModel.ncml";
Formatter errlog = new Formatter();
Fmrc fmrc = Fmrc.open(path, errlog);
assert (fmrc != null) : errlog;
try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDataset2D(null)) {
NetcdfDataset ncd = (NetcdfDataset) gridDs.getNetcdfFile();
Attribute att = ncd.findGlobalAttribute(CDM.CONVENTIONS);
assert att != null;
System.out.printf("%s%n", att);
}
}
public static ThreddsMetadata.Variables extractVariables(
InvDatasetImpl threddsDataset, GridDataset gridDataset) {
thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType();
if ((fileFormat != null)
&& (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) {
boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1);
ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString());
for (GridDatatype grid : gridDataset.getGrids()) {
ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
v.setName(grid.getName());
v.setDescription(grid.getDescription());
v.setUnits(grid.getUnitsString());
// ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number");
// String paramNumber = (att != null) ? att.getNumericValue().toString() : null;
if (isGrib1) {
v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR"));
v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
} else {
String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", "");
String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", "");
String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", "");
v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName);
v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
}
vars.addVariable(v);
}
vars.sort();
return vars;
} else { // GRID but not GRIB
ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0");
for (GridDatatype grid : gridDataset.getGrids()) {
ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
vars.addVariable(v);
v.setName(grid.getName());
v.setDescription(grid.getDescription());
v.setUnits(grid.getUnitsString());
ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name");
v.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
}
vars.sort();
return vars;
}
}
public static DateRange extractDateRange(GridDataset gridDataset) {
DateRange maxDateRange = null;
for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
GridCoordSystem gsys = gridset.getGeoCoordSystem();
DateRange dateRange;
CoordinateAxis1DTime time1D = gsys.getTimeAxis1D();
if (time1D != null) {
dateRange = time1D.getDateRange();
} else {
CoordinateAxis time = gsys.getTimeAxis();
if (time == null) continue;
try {
DateUnit du = new DateUnit(time.getUnitsString());
Date minDate = du.makeDate(time.getMinValue());
Date maxDate = du.makeDate(time.getMaxValue());
dateRange = new DateRange(minDate, maxDate);
} catch (Exception e) {
logger.warn("Illegal Date Unit " + time.getUnitsString());
continue;
}
}
if (maxDateRange == null) maxDateRange = dateRange;
else maxDateRange.extend(dateRange);
}
return maxDateRange;
}
public static ThreddsMetadata.GeospatialCoverage extractGeospatial(GridDataset gridDataset) {
ThreddsMetadata.GeospatialCoverage gc = new ThreddsMetadata.GeospatialCoverage();
LatLonRect llbb = null;
CoordinateAxis1D vaxis = null;
for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
GridCoordSystem gsys = gridset.getGeoCoordSystem();
if (llbb == null) llbb = gsys.getLatLonBoundingBox();
CoordinateAxis1D vaxis2 = gsys.getVerticalAxis();
if (vaxis == null) vaxis = vaxis2;
else if ((vaxis2 != null) && (vaxis2.getSize() > vaxis.getSize())) vaxis = vaxis2;
}
if (llbb != null) gc.setBoundingBox(llbb);
if (vaxis != null) gc.setVertical(vaxis);
return gc;
}
@Override
public AbstractGridDataset createDataset(String id, String location)
throws IOException, EdalException {
NetcdfDataset nc = null;
try {
/*
* Open the dataset, using the cache for NcML aggregations
*/
nc = openAndAggregateDataset(location);
/*-
* We may in future be able to use forecast model run collection aggregations for
* dealing with the case of overlapping time axes. To do this the code will look
* something like this:
*
* StringBuilder sb = new StringBuilder();
* Formatter formatter = new Formatter(sb, Locale.UK);
* Fmrc f = Fmrc.open(location, formatter);
*
* in openAndAggregateDataset. It will need to build up an NcML document which
* does this. It should look something like:
*
* <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="true">
* <aggregation dimName="run" type="forecastModelRunCollection" timeUnitsChange="true">
* <!-- scanFmrc actually works, but what we want is something like the following bit -->
* <scanFmrc location="/home/guy/Data/POLCOMS_IRISH/" regExp=".*\.nc"/>
* <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090320.nc" coordValue="2009-03-20T00:00:00Z" enhance="true" />
* <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090321.nc" coordValue="2009-03-21T00:00:00Z" enhance="true" />
* <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090322.nc" coordValue="2009-03-22T00:00:00Z" enhance="true" />
* </aggregation>
* </netcdf>
*
* For more documentation see:
* http://mailman.unidata.ucar.edu/software/thredds/current/netcdf-java/ncml/FmrcAggregation.html
*
* We then can do stuff like:
*
* ucar.nc2.dt.GridDataset gridDataset = f.getDatasetBest();
*
* To get the single best aggregation of the overlapping time axis
*
* Then we need to work with GridDatasets in place of NetcdfDatasets. Stuff like:
*
* for(Variable variable : gridDataset.getNetcdfFile().getVariables()) {
* // blah blah
* }
*
* will be necessary. We need to check that that works with remote datasets too
*/
/*
* We look for NetCDF-U variables to group mean/standard-deviation.
*
* We need to do this here because we want to subsequently ignore
* parent variables
*/
Map<String, String[]> varId2AncillaryVars = new HashMap<String, String[]>();
for (Variable variable : nc.getVariables()) {
/*
* Just look for parent variables, since these may not have a
* grid directly associated with them
*/
for (Attribute attr : variable.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("ancillary_variables")) {
varId2AncillaryVars.put(variable.getFullName(), attr.getStringValue().split(" "));
continue;
}
}
}
ucar.nc2.dt.GridDataset gridDataset = CdmUtils.getGridDataset(nc);
List<GridVariableMetadata> vars = new ArrayList<GridVariableMetadata>();
/*
* Store a map of component names. Key is the compound name, value
* is a 2-element String array with x, y component IDs
*
* Also store a map of whether these components are really
* eastward/northward, or whether they are locally u/v
*/
Map<String, String[]> xyComponentPairs = new HashMap<String, String[]>();
Map<String, Boolean> xyNameToTrueEN = new HashMap<String, Boolean>();
/*
* Store a map of variable IDs to UncertML URLs. This will be used
* to determine which components are mean/std/etc.
*
* TODO implement more than just Mean/SD
*/
Map<String, String> varId2UncertMLRefs = new HashMap<String, String>();
/*
* Here we store the parent variable IDs and their corresponding
* title.
*/
Map<String, String> parentVarId2Title = new HashMap<String, String>();
for (Gridset gridset : gridDataset.getGridsets()) {
GridCoordSystem coordSys = gridset.getGeoCoordSystem();
HorizontalGrid hDomain = CdmUtils.createHorizontalGrid(coordSys);
VerticalAxis zDomain = CdmUtils.createVerticalAxis(coordSys);
TimeAxis tDomain = CdmUtils.createTimeAxis(coordSys);
/*
* Create a VariableMetadata object for each GridDatatype
*/
for (GridDatatype grid : gridset.getGrids()) {
VariableDS variable = grid.getVariable();
String varId = variable.getFullName();
String name = getVariableName(variable);
/*
* If this is a parent variable for a stats collection, we
* don't want it to be a normal variable as well.
*/
if (varId2AncillaryVars.containsKey(varId)) {
parentVarId2Title.put(varId, name);
continue;
}
/*
* If it is a child variable is (potentially) referenced by
* UncertML, store its ID and the (possible) UncertML URI
*/
for (Attribute attr : variable.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("ref")) {
varId2UncertMLRefs.put(varId, attr.getStringValue());
}
}
Parameter parameter =
new Parameter(
varId,
variable.getShortName(),
variable.getDescription(),
variable.getUnitsString(),
name);
GridVariableMetadata metadata =
new GridVariableMetadata(
variable.getFullName(), parameter, hDomain, zDomain, tDomain, true);
vars.add(metadata);
if (name != null) {
/*
* Check for vector components
*/
if (name.contains("eastward_")) {
String compoundName = name.replaceFirst("eastward_", "");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, true);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[0] = varId;
} else if (name.contains("northward_")) {
String compoundName = name.replaceFirst("northward_", "");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, true);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[1] = varId;
} else if (name.matches("u-.*component")) {
String compoundName = name.replaceFirst("u-(.*)component", "$1");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, false);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[0] = varId;
} else if (name.matches("v-.*component")) {
String compoundName = name.replaceFirst("v-(.*)component", "$1");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, false);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[1] = varId;
}
/*
* We could potentially add a check for zonal/meridional
* here if required.
*/
}
}
}
CdmGridDataset cdmGridDataset =
new CdmGridDataset(id, location, vars, CdmUtils.getOptimumDataReadingStrategy(nc));
for (Entry<String, String[]> componentData : xyComponentPairs.entrySet()) {
String commonName = componentData.getKey();
String[] comps = componentData.getValue();
if (comps[0] != null && comps[1] != null) {
cdmGridDataset.addVariablePlugin(
new VectorPlugin(comps[0], comps[1], commonName, xyNameToTrueEN.get(commonName)));
}
}
for (String statsCollectionId : varId2AncillaryVars.keySet()) {
String[] ids = varId2AncillaryVars.get(statsCollectionId);
String meanId = null;
String stddevId = null;
for (String statsVarIds : ids) {
String uncertRef = varId2UncertMLRefs.get(statsVarIds);
if (uncertRef != null
&& uncertRef.equalsIgnoreCase("http://www.uncertml.org/statistics/mean")) {
meanId = statsVarIds;
}
if (uncertRef != null
&& uncertRef.equalsIgnoreCase(
"http://www.uncertml.org/statistics/standard-deviation")) {
stddevId = statsVarIds;
}
}
if (meanId != null && stddevId != null) {
MeanSDPlugin meanSDPlugin =
new MeanSDPlugin(meanId, stddevId, parentVarId2Title.get(statsCollectionId));
cdmGridDataset.addVariablePlugin(meanSDPlugin);
}
}
return cdmGridDataset;
} finally {
CdmUtils.closeDataset(nc);
}
}