public void testSingleDataset() throws IOException {
    InvCatalogImpl cat = TestTDSAll.open(null);

    InvDataset ds = cat.findDatasetByID("testSingleDataset");
    assert (ds != null) : "cant find dataset 'testSingleDataset'";
    assert ds.getDataType() == FeatureType.GRID;

    ThreddsDataFactory fac = new ThreddsDataFactory();

    ThreddsDataFactory.Result dataResult = fac.openFeatureDataset(ds, null);

    assert dataResult != null;
    assert !dataResult.fatalError;
    assert dataResult.featureDataset != null;

    GridDataset gds = (GridDataset) dataResult.featureDataset;
    GridDatatype grid = gds.findGridDatatype("Z_sfc");
    assert grid != null;
    GridCoordSystem gcs = grid.getCoordinateSystem();
    assert gcs != null;
    assert null == gcs.getVerticalAxis();

    CoordinateAxis1D time = gcs.getTimeAxis1D();
    assert time != null;
    assert time.getSize() == 1;
    assert 102840.0 == time.readScalarDouble();

    dataResult.featureDataset.close();
  }
Example #2
0
  @Test
  public void testFloatingPointCompare() throws Exception {
    String spec =
        TestDir.cdmUnitTestDir + "ft/fmrc/fp_precision/sediment_thickness_#yyMMddHHmm#.*\\.nc$";
    System.out.printf("%n====================FMRC dataset %s%n", spec);
    Formatter errlog = new Formatter();
    Fmrc fmrc = Fmrc.open(spec, errlog);
    assert (fmrc != null) : errlog;

    try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDatasetBest()) {
      GridDatatype v = gridDs.findGridByShortName("thickness_of_sediment");
      assert v != null;
      GridCoordSystem gcs = v.getCoordinateSystem();
      CoordinateAxis1DTime time = gcs.getTimeAxis1D();

      Assert.assertEquals("hours since 2015-03-08 12:51:00.000 UTC", time.getUnitsString());
      Assert.assertEquals(74, time.getSize());
      Array data = time.read();
      System.out.printf("%s%n", NCdumpW.toString(data));

      for (CalendarDate cd : time.getCalendarDates()) {
        assert cd.getFieldValue(CalendarPeriod.Field.Minute) == 0 : System.out.printf("%s%n", cd);
      }
    }
  }
Example #3
0
  @Test
  public void testConventionsAttribute() throws Exception {
    String path = TestDir.cdmUnitTestDir + "ncml/AggForecastModel.ncml";
    Formatter errlog = new Formatter();
    Fmrc fmrc = Fmrc.open(path, errlog);
    assert (fmrc != null) : errlog;

    try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDataset2D(null)) {
      NetcdfDataset ncd = (NetcdfDataset) gridDs.getNetcdfFile();
      Attribute att = ncd.findGlobalAttribute(CDM.CONVENTIONS);
      assert att != null;
      System.out.printf("%s%n", att);
    }
  }
  public static ThreddsMetadata.Variables extractVariables(
      InvDatasetImpl threddsDataset, GridDataset gridDataset) {

    thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType();
    if ((fileFormat != null)
        && (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) {
      boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1);
      ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString());
      for (GridDatatype grid : gridDataset.getGrids()) {
        ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
        v.setName(grid.getName());
        v.setDescription(grid.getDescription());
        v.setUnits(grid.getUnitsString());

        // ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number");
        // String paramNumber = (att != null) ? att.getNumericValue().toString() : null;
        if (isGrib1) {
          v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR"));
          v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
        } else {
          String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", "");
          String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", "");
          String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", "");
          v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName);
          v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
        }
        vars.addVariable(v);
      }
      vars.sort();
      return vars;

    } else { // GRID but not GRIB
      ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0");
      for (GridDatatype grid : gridDataset.getGrids()) {
        ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
        vars.addVariable(v);

        v.setName(grid.getName());
        v.setDescription(grid.getDescription());
        v.setUnits(grid.getUnitsString());

        ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name");
        v.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
      }
      vars.sort();
      return vars;
    }
  }
  public static DateRange extractDateRange(GridDataset gridDataset) {
    DateRange maxDateRange = null;

    for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
      GridCoordSystem gsys = gridset.getGeoCoordSystem();
      DateRange dateRange;

      CoordinateAxis1DTime time1D = gsys.getTimeAxis1D();
      if (time1D != null) {
        dateRange = time1D.getDateRange();
      } else {
        CoordinateAxis time = gsys.getTimeAxis();
        if (time == null) continue;

        try {
          DateUnit du = new DateUnit(time.getUnitsString());
          Date minDate = du.makeDate(time.getMinValue());
          Date maxDate = du.makeDate(time.getMaxValue());
          dateRange = new DateRange(minDate, maxDate);
        } catch (Exception e) {
          logger.warn("Illegal Date Unit " + time.getUnitsString());
          continue;
        }
      }

      if (maxDateRange == null) maxDateRange = dateRange;
      else maxDateRange.extend(dateRange);
    }

    return maxDateRange;
  }
  public static ThreddsMetadata.GeospatialCoverage extractGeospatial(GridDataset gridDataset) {
    ThreddsMetadata.GeospatialCoverage gc = new ThreddsMetadata.GeospatialCoverage();
    LatLonRect llbb = null;
    CoordinateAxis1D vaxis = null;

    for (GridDataset.Gridset gridset : gridDataset.getGridsets()) {
      GridCoordSystem gsys = gridset.getGeoCoordSystem();
      if (llbb == null) llbb = gsys.getLatLonBoundingBox();

      CoordinateAxis1D vaxis2 = gsys.getVerticalAxis();
      if (vaxis == null) vaxis = vaxis2;
      else if ((vaxis2 != null) && (vaxis2.getSize() > vaxis.getSize())) vaxis = vaxis2;
    }

    if (llbb != null) gc.setBoundingBox(llbb);
    if (vaxis != null) gc.setVertical(vaxis);
    return gc;
  }
  @Override
  public AbstractGridDataset createDataset(String id, String location)
      throws IOException, EdalException {
    NetcdfDataset nc = null;
    try {
      /*
       * Open the dataset, using the cache for NcML aggregations
       */
      nc = openAndAggregateDataset(location);

      /*-
       * We may in future be able to use forecast model run collection aggregations for
       * dealing with the case of overlapping time axes.  To do this the code will look
       * something like this:
       *
       * StringBuilder sb = new StringBuilder();
       * Formatter formatter = new Formatter(sb, Locale.UK);
       * Fmrc f = Fmrc.open(location, formatter);
       *
       * in openAndAggregateDataset.  It will need to build up an NcML document which
       * does this.  It should look something like:
       *
       *  <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="true">
       *      <aggregation dimName="run" type="forecastModelRunCollection" timeUnitsChange="true">
       *           <!-- scanFmrc actually works, but what we want is something like the following bit -->
       *           <scanFmrc location="/home/guy/Data/POLCOMS_IRISH/" regExp=".*\.nc"/>
       *           <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090320.nc" coordValue="2009-03-20T00:00:00Z" enhance="true" />
       *           <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090321.nc" coordValue="2009-03-21T00:00:00Z" enhance="true" />
       *           <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090322.nc" coordValue="2009-03-22T00:00:00Z" enhance="true" />
       *      </aggregation>
       *  </netcdf>
       *
       * For more documentation see:
       * http://mailman.unidata.ucar.edu/software/thredds/current/netcdf-java/ncml/FmrcAggregation.html
       *
       * We then can do stuff like:
       *
       * ucar.nc2.dt.GridDataset gridDataset = f.getDatasetBest();
       *
       * To get the single best aggregation of the overlapping time axis
       *
       * Then we need to work with GridDatasets in place of NetcdfDatasets.  Stuff like:
       *
       * for(Variable variable : gridDataset.getNetcdfFile().getVariables()) {
       *    // blah blah
       * }
       *
       * will be necessary.  We need to check that that works with remote datasets too
       */

      /*
       * We look for NetCDF-U variables to group mean/standard-deviation.
       *
       * We need to do this here because we want to subsequently ignore
       * parent variables
       */
      Map<String, String[]> varId2AncillaryVars = new HashMap<String, String[]>();
      for (Variable variable : nc.getVariables()) {
        /*
         * Just look for parent variables, since these may not have a
         * grid directly associated with them
         */
        for (Attribute attr : variable.getAttributes()) {
          if (attr.getFullName().equalsIgnoreCase("ancillary_variables")) {
            varId2AncillaryVars.put(variable.getFullName(), attr.getStringValue().split(" "));
            continue;
          }
        }
      }

      ucar.nc2.dt.GridDataset gridDataset = CdmUtils.getGridDataset(nc);
      List<GridVariableMetadata> vars = new ArrayList<GridVariableMetadata>();
      /*
       * Store a map of component names. Key is the compound name, value
       * is a 2-element String array with x, y component IDs
       *
       * Also store a map of whether these components are really
       * eastward/northward, or whether they are locally u/v
       */
      Map<String, String[]> xyComponentPairs = new HashMap<String, String[]>();
      Map<String, Boolean> xyNameToTrueEN = new HashMap<String, Boolean>();
      /*
       * Store a map of variable IDs to UncertML URLs. This will be used
       * to determine which components are mean/std/etc.
       *
       * TODO implement more than just Mean/SD
       */
      Map<String, String> varId2UncertMLRefs = new HashMap<String, String>();
      /*
       * Here we store the parent variable IDs and their corresponding
       * title.
       */
      Map<String, String> parentVarId2Title = new HashMap<String, String>();
      for (Gridset gridset : gridDataset.getGridsets()) {
        GridCoordSystem coordSys = gridset.getGeoCoordSystem();
        HorizontalGrid hDomain = CdmUtils.createHorizontalGrid(coordSys);
        VerticalAxis zDomain = CdmUtils.createVerticalAxis(coordSys);
        TimeAxis tDomain = CdmUtils.createTimeAxis(coordSys);

        /*
         * Create a VariableMetadata object for each GridDatatype
         */
        for (GridDatatype grid : gridset.getGrids()) {
          VariableDS variable = grid.getVariable();
          String varId = variable.getFullName();
          String name = getVariableName(variable);

          /*
           * If this is a parent variable for a stats collection, we
           * don't want it to be a normal variable as well.
           */
          if (varId2AncillaryVars.containsKey(varId)) {
            parentVarId2Title.put(varId, name);
            continue;
          }

          /*
           * If it is a child variable is (potentially) referenced by
           * UncertML, store its ID and the (possible) UncertML URI
           */
          for (Attribute attr : variable.getAttributes()) {
            if (attr.getFullName().equalsIgnoreCase("ref")) {
              varId2UncertMLRefs.put(varId, attr.getStringValue());
            }
          }

          Parameter parameter =
              new Parameter(
                  varId,
                  variable.getShortName(),
                  variable.getDescription(),
                  variable.getUnitsString(),
                  name);
          GridVariableMetadata metadata =
              new GridVariableMetadata(
                  variable.getFullName(), parameter, hDomain, zDomain, tDomain, true);
          vars.add(metadata);

          if (name != null) {
            /*
             * Check for vector components
             */
            if (name.contains("eastward_")) {
              String compoundName = name.replaceFirst("eastward_", "");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, true);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[0] = varId;
            } else if (name.contains("northward_")) {
              String compoundName = name.replaceFirst("northward_", "");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, true);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[1] = varId;
            } else if (name.matches("u-.*component")) {
              String compoundName = name.replaceFirst("u-(.*)component", "$1");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, false);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[0] = varId;
            } else if (name.matches("v-.*component")) {
              String compoundName = name.replaceFirst("v-(.*)component", "$1");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, false);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[1] = varId;
            }
            /*
             * We could potentially add a check for zonal/meridional
             * here if required.
             */
          }
        }
      }

      CdmGridDataset cdmGridDataset =
          new CdmGridDataset(id, location, vars, CdmUtils.getOptimumDataReadingStrategy(nc));
      for (Entry<String, String[]> componentData : xyComponentPairs.entrySet()) {
        String commonName = componentData.getKey();
        String[] comps = componentData.getValue();
        if (comps[0] != null && comps[1] != null) {
          cdmGridDataset.addVariablePlugin(
              new VectorPlugin(comps[0], comps[1], commonName, xyNameToTrueEN.get(commonName)));
        }
      }

      for (String statsCollectionId : varId2AncillaryVars.keySet()) {
        String[] ids = varId2AncillaryVars.get(statsCollectionId);
        String meanId = null;
        String stddevId = null;
        for (String statsVarIds : ids) {
          String uncertRef = varId2UncertMLRefs.get(statsVarIds);
          if (uncertRef != null
              && uncertRef.equalsIgnoreCase("http://www.uncertml.org/statistics/mean")) {
            meanId = statsVarIds;
          }
          if (uncertRef != null
              && uncertRef.equalsIgnoreCase(
                  "http://www.uncertml.org/statistics/standard-deviation")) {
            stddevId = statsVarIds;
          }
        }
        if (meanId != null && stddevId != null) {
          MeanSDPlugin meanSDPlugin =
              new MeanSDPlugin(meanId, stddevId, parentVarId2Title.get(statsCollectionId));
          cdmGridDataset.addVariablePlugin(meanSDPlugin);
        }
      }

      return cdmGridDataset;
    } finally {
      CdmUtils.closeDataset(nc);
    }
  }