public void testSingleDataset() throws IOException { InvCatalogImpl cat = TestTDSAll.open(null); InvDataset ds = cat.findDatasetByID("testSingleDataset"); assert (ds != null) : "cant find dataset 'testSingleDataset'"; assert ds.getDataType() == FeatureType.GRID; ThreddsDataFactory fac = new ThreddsDataFactory(); ThreddsDataFactory.Result dataResult = fac.openFeatureDataset(ds, null); assert dataResult != null; assert !dataResult.fatalError; assert dataResult.featureDataset != null; GridDataset gds = (GridDataset) dataResult.featureDataset; GridDatatype grid = gds.findGridDatatype("Z_sfc"); assert grid != null; GridCoordSystem gcs = grid.getCoordinateSystem(); assert gcs != null; assert null == gcs.getVerticalAxis(); CoordinateAxis1D time = gcs.getTimeAxis1D(); assert time != null; assert time.getSize() == 1; assert 102840.0 == time.readScalarDouble(); dataResult.featureDataset.close(); }
@Test public void testFloatingPointCompare() throws Exception { String spec = TestDir.cdmUnitTestDir + "ft/fmrc/fp_precision/sediment_thickness_#yyMMddHHmm#.*\\.nc$"; System.out.printf("%n====================FMRC dataset %s%n", spec); Formatter errlog = new Formatter(); Fmrc fmrc = Fmrc.open(spec, errlog); assert (fmrc != null) : errlog; try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDatasetBest()) { GridDatatype v = gridDs.findGridByShortName("thickness_of_sediment"); assert v != null; GridCoordSystem gcs = v.getCoordinateSystem(); CoordinateAxis1DTime time = gcs.getTimeAxis1D(); Assert.assertEquals("hours since 2015-03-08 12:51:00.000 UTC", time.getUnitsString()); Assert.assertEquals(74, time.getSize()); Array data = time.read(); System.out.printf("%s%n", NCdumpW.toString(data)); for (CalendarDate cd : time.getCalendarDates()) { assert cd.getFieldValue(CalendarPeriod.Field.Minute) == 0 : System.out.printf("%s%n", cd); } } }
@Test public void testConventionsAttribute() throws Exception { String path = TestDir.cdmUnitTestDir + "ncml/AggForecastModel.ncml"; Formatter errlog = new Formatter(); Fmrc fmrc = Fmrc.open(path, errlog); assert (fmrc != null) : errlog; try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDataset2D(null)) { NetcdfDataset ncd = (NetcdfDataset) gridDs.getNetcdfFile(); Attribute att = ncd.findGlobalAttribute(CDM.CONVENTIONS); assert att != null; System.out.printf("%s%n", att); } }
public static ThreddsMetadata.Variables extractVariables( InvDatasetImpl threddsDataset, GridDataset gridDataset) { thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType(); if ((fileFormat != null) && (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) { boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1); ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString()); for (GridDatatype grid : gridDataset.getGrids()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); v.setName(grid.getName()); v.setDescription(grid.getDescription()); v.setUnits(grid.getUnitsString()); // ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number"); // String paramNumber = (att != null) ? att.getNumericValue().toString() : null; if (isGrib1) { v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR")); v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id")); } else { String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", ""); String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", ""); String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", ""); v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName); v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id")); } vars.addVariable(v); } vars.sort(); return vars; } else { // GRID but not GRIB ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0"); for (GridDatatype grid : gridDataset.getGrids()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); vars.addVariable(v); v.setName(grid.getName()); v.setDescription(grid.getDescription()); v.setUnits(grid.getUnitsString()); ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name"); v.setVocabularyName((att != null) ? att.getStringValue() : "N/A"); } vars.sort(); return vars; } }
public static DateRange extractDateRange(GridDataset gridDataset) { DateRange maxDateRange = null; for (GridDataset.Gridset gridset : gridDataset.getGridsets()) { GridCoordSystem gsys = gridset.getGeoCoordSystem(); DateRange dateRange; CoordinateAxis1DTime time1D = gsys.getTimeAxis1D(); if (time1D != null) { dateRange = time1D.getDateRange(); } else { CoordinateAxis time = gsys.getTimeAxis(); if (time == null) continue; try { DateUnit du = new DateUnit(time.getUnitsString()); Date minDate = du.makeDate(time.getMinValue()); Date maxDate = du.makeDate(time.getMaxValue()); dateRange = new DateRange(minDate, maxDate); } catch (Exception e) { logger.warn("Illegal Date Unit " + time.getUnitsString()); continue; } } if (maxDateRange == null) maxDateRange = dateRange; else maxDateRange.extend(dateRange); } return maxDateRange; }
public static ThreddsMetadata.GeospatialCoverage extractGeospatial(GridDataset gridDataset) { ThreddsMetadata.GeospatialCoverage gc = new ThreddsMetadata.GeospatialCoverage(); LatLonRect llbb = null; CoordinateAxis1D vaxis = null; for (GridDataset.Gridset gridset : gridDataset.getGridsets()) { GridCoordSystem gsys = gridset.getGeoCoordSystem(); if (llbb == null) llbb = gsys.getLatLonBoundingBox(); CoordinateAxis1D vaxis2 = gsys.getVerticalAxis(); if (vaxis == null) vaxis = vaxis2; else if ((vaxis2 != null) && (vaxis2.getSize() > vaxis.getSize())) vaxis = vaxis2; } if (llbb != null) gc.setBoundingBox(llbb); if (vaxis != null) gc.setVertical(vaxis); return gc; }
@Override public AbstractGridDataset createDataset(String id, String location) throws IOException, EdalException { NetcdfDataset nc = null; try { /* * Open the dataset, using the cache for NcML aggregations */ nc = openAndAggregateDataset(location); /*- * We may in future be able to use forecast model run collection aggregations for * dealing with the case of overlapping time axes. To do this the code will look * something like this: * * StringBuilder sb = new StringBuilder(); * Formatter formatter = new Formatter(sb, Locale.UK); * Fmrc f = Fmrc.open(location, formatter); * * in openAndAggregateDataset. It will need to build up an NcML document which * does this. It should look something like: * * <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="true"> * <aggregation dimName="run" type="forecastModelRunCollection" timeUnitsChange="true"> * <!-- scanFmrc actually works, but what we want is something like the following bit --> * <scanFmrc location="/home/guy/Data/POLCOMS_IRISH/" regExp=".*\.nc"/> * <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090320.nc" coordValue="2009-03-20T00:00:00Z" enhance="true" /> * <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090321.nc" coordValue="2009-03-21T00:00:00Z" enhance="true" /> * <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090322.nc" coordValue="2009-03-22T00:00:00Z" enhance="true" /> * </aggregation> * </netcdf> * * For more documentation see: * http://mailman.unidata.ucar.edu/software/thredds/current/netcdf-java/ncml/FmrcAggregation.html * * We then can do stuff like: * * ucar.nc2.dt.GridDataset gridDataset = f.getDatasetBest(); * * To get the single best aggregation of the overlapping time axis * * Then we need to work with GridDatasets in place of NetcdfDatasets. Stuff like: * * for(Variable variable : gridDataset.getNetcdfFile().getVariables()) { * // blah blah * } * * will be necessary. We need to check that that works with remote datasets too */ /* * We look for NetCDF-U variables to group mean/standard-deviation. * * We need to do this here because we want to subsequently ignore * parent variables */ Map<String, String[]> varId2AncillaryVars = new HashMap<String, String[]>(); for (Variable variable : nc.getVariables()) { /* * Just look for parent variables, since these may not have a * grid directly associated with them */ for (Attribute attr : variable.getAttributes()) { if (attr.getFullName().equalsIgnoreCase("ancillary_variables")) { varId2AncillaryVars.put(variable.getFullName(), attr.getStringValue().split(" ")); continue; } } } ucar.nc2.dt.GridDataset gridDataset = CdmUtils.getGridDataset(nc); List<GridVariableMetadata> vars = new ArrayList<GridVariableMetadata>(); /* * Store a map of component names. Key is the compound name, value * is a 2-element String array with x, y component IDs * * Also store a map of whether these components are really * eastward/northward, or whether they are locally u/v */ Map<String, String[]> xyComponentPairs = new HashMap<String, String[]>(); Map<String, Boolean> xyNameToTrueEN = new HashMap<String, Boolean>(); /* * Store a map of variable IDs to UncertML URLs. This will be used * to determine which components are mean/std/etc. * * TODO implement more than just Mean/SD */ Map<String, String> varId2UncertMLRefs = new HashMap<String, String>(); /* * Here we store the parent variable IDs and their corresponding * title. */ Map<String, String> parentVarId2Title = new HashMap<String, String>(); for (Gridset gridset : gridDataset.getGridsets()) { GridCoordSystem coordSys = gridset.getGeoCoordSystem(); HorizontalGrid hDomain = CdmUtils.createHorizontalGrid(coordSys); VerticalAxis zDomain = CdmUtils.createVerticalAxis(coordSys); TimeAxis tDomain = CdmUtils.createTimeAxis(coordSys); /* * Create a VariableMetadata object for each GridDatatype */ for (GridDatatype grid : gridset.getGrids()) { VariableDS variable = grid.getVariable(); String varId = variable.getFullName(); String name = getVariableName(variable); /* * If this is a parent variable for a stats collection, we * don't want it to be a normal variable as well. */ if (varId2AncillaryVars.containsKey(varId)) { parentVarId2Title.put(varId, name); continue; } /* * If it is a child variable is (potentially) referenced by * UncertML, store its ID and the (possible) UncertML URI */ for (Attribute attr : variable.getAttributes()) { if (attr.getFullName().equalsIgnoreCase("ref")) { varId2UncertMLRefs.put(varId, attr.getStringValue()); } } Parameter parameter = new Parameter( varId, variable.getShortName(), variable.getDescription(), variable.getUnitsString(), name); GridVariableMetadata metadata = new GridVariableMetadata( variable.getFullName(), parameter, hDomain, zDomain, tDomain, true); vars.add(metadata); if (name != null) { /* * Check for vector components */ if (name.contains("eastward_")) { String compoundName = name.replaceFirst("eastward_", ""); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, true); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[0] = varId; } else if (name.contains("northward_")) { String compoundName = name.replaceFirst("northward_", ""); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, true); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[1] = varId; } else if (name.matches("u-.*component")) { String compoundName = name.replaceFirst("u-(.*)component", "$1"); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, false); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[0] = varId; } else if (name.matches("v-.*component")) { String compoundName = name.replaceFirst("v-(.*)component", "$1"); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, false); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[1] = varId; } /* * We could potentially add a check for zonal/meridional * here if required. */ } } } CdmGridDataset cdmGridDataset = new CdmGridDataset(id, location, vars, CdmUtils.getOptimumDataReadingStrategy(nc)); for (Entry<String, String[]> componentData : xyComponentPairs.entrySet()) { String commonName = componentData.getKey(); String[] comps = componentData.getValue(); if (comps[0] != null && comps[1] != null) { cdmGridDataset.addVariablePlugin( new VectorPlugin(comps[0], comps[1], commonName, xyNameToTrueEN.get(commonName))); } } for (String statsCollectionId : varId2AncillaryVars.keySet()) { String[] ids = varId2AncillaryVars.get(statsCollectionId); String meanId = null; String stddevId = null; for (String statsVarIds : ids) { String uncertRef = varId2UncertMLRefs.get(statsVarIds); if (uncertRef != null && uncertRef.equalsIgnoreCase("http://www.uncertml.org/statistics/mean")) { meanId = statsVarIds; } if (uncertRef != null && uncertRef.equalsIgnoreCase( "http://www.uncertml.org/statistics/standard-deviation")) { stddevId = statsVarIds; } } if (meanId != null && stddevId != null) { MeanSDPlugin meanSDPlugin = new MeanSDPlugin(meanId, stddevId, parentVarId2Title.get(statsCollectionId)); cdmGridDataset.addVariablePlugin(meanSDPlugin); } } return cdmGridDataset; } finally { CdmUtils.closeDataset(nc); } }