Esempio n. 1
0
  private void handleMetadataGroup(Group group, MetadataElement metadataElement)
      throws ProductIOException {
    List<Variable> variables = group.getVariables();
    for (Variable variable : variables) {
      final String name = variable.getShortName();
      final int dataType = getProductDataType(variable);
      Array array;
      try {
        array = variable.read();
      } catch (IOException e) {
        throw new ProductIOException(e.getMessage());
      }
      final ProductData data = ProductData.createInstance(dataType, array.getStorage());
      final MetadataAttribute attribute = new MetadataAttribute("data", data, true);

      final MetadataElement sdsElement = new MetadataElement(name);
      sdsElement.addAttribute(attribute);
      metadataElement.addElement(sdsElement);

      final List<Attribute> list = variable.getAttributes();
      for (Attribute hdfAttribute : list) {
        final String attribName = hdfAttribute.getShortName();
        if ("units".equals(attribName)) {
          attribute.setUnit(hdfAttribute.getStringValue());
        } else if ("long_name".equals(attribName)) {
          attribute.setDescription(hdfAttribute.getStringValue());
        } else {
          addAttributeToElement(sdsElement, hdfAttribute);
        }
      }
    }
  }
Esempio n. 2
0
  private int addAttributes(opendap.dap.AttributeTable table, Variable v, Iterator iter) {
    int count = 0;

    // add attribute table for this variable
    while (iter.hasNext()) {
      Attribute att = (Attribute) iter.next();
      int dods_type = DODSNetcdfFile.convertToDODSType(att.getDataType(), false);

      try {
        String attName = NcDDS.escapeName(att.getName());
        if (att.isString()) {
          /* FIX String value = escapeAttributeStringValues(att.getStringValue());
          table.appendAttribute(attName, dods_type, "\""+value+"\"");
          */
          table.appendAttribute(attName, dods_type, att.getStringValue());
        } else {
          // cant send signed bytes
          if (att.getDataType() == DataType.BYTE) {
            boolean signed = false;
            for (int i = 0; i < att.getLength(); i++) {
              if (att.getNumericValue(i).byteValue() < 0) signed = true;
            }
            if (signed) // promote to signed short
            dods_type = opendap.dap.Attribute.INT16;
          }

          for (int i = 0; i < att.getLength(); i++)
            table.appendAttribute(attName, dods_type, att.getNumericValue(i).toString());
        }
        count++;

      } catch (Exception e) {
        log.error(
            "Error appending attribute " + att.getName() + " = " + att.getStringValue() + "\n" + e);
      }
    } // loop over variable attributes

    // kludgy thing to map char arrays to DODS Strings
    if ((v != null) && (v.getDataType().getPrimitiveClassType() == char.class)) {
      int rank = v.getRank();
      int strlen = (rank == 0) ? 0 : v.getShape(rank - 1);
      Dimension dim = (rank == 0) ? null : v.getDimension(rank - 1);
      try {
        opendap.dap.AttributeTable dodsTable = table.appendContainer("DODS");
        dodsTable.appendAttribute("strlen", opendap.dap.Attribute.INT32, Integer.toString(strlen));
        if ((dim != null) && dim.isShared())
          dodsTable.appendAttribute("dimName", opendap.dap.Attribute.STRING, dim.getName());
        count++;
      } catch (Exception e) {
        log.error("Error appending attribute strlen\n" + e);
      }
    }

    return count;
  }
Esempio n. 3
0
 /**
  * Returns the phenomenon that the given variable represents.
  *
  * <p>This name will be, in order of preference:
  *
  * <p>The standard name
  *
  * <p>The long name
  *
  * <p>The variable name
  */
 private static String getVariableName(Variable var) {
   Attribute stdNameAtt = var.findAttributeIgnoreCase("standard_name");
   if (stdNameAtt == null || stdNameAtt.getStringValue().trim().equals("")) {
     Attribute longNameAtt = var.findAttributeIgnoreCase("long_name");
     if (longNameAtt == null || longNameAtt.getStringValue().trim().equals("")) {
       return var.getFullName();
     } else {
       return longNameAtt.getStringValue();
     }
   } else {
     return stdNameAtt.getStringValue();
   }
 }
Esempio n. 4
0
  private ProductData.UTC getUTCAttribute(String key, List<Attribute> globalAttributes) {
    Attribute attribute = findAttribute(key, globalAttributes);
    Boolean isModis = false;
    try {
      isModis = findAttribute("MODIS_Resolution", globalAttributes).isString();
    } catch (Exception ignored) {
    }

    if (attribute != null) {
      String timeString = attribute.getStringValue().trim();
      final DateFormat dateFormat = ProductData.UTC.createDateFormat("yyyyDDDHHmmssSSS");
      final DateFormat dateFormatModis =
          ProductData.UTC.createDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
      final DateFormat dateFormatOcts =
          ProductData.UTC.createDateFormat("yyyyMMdd HH:mm:ss.SSSSSS");
      try {
        if (isModis) {
          final Date date = dateFormatModis.parse(timeString);
          String milliSeconds = timeString.substring(timeString.length() - 3);
          return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000);
        } else if (productReader.getProductType() == SeadasProductReader.ProductType.Level1A_OCTS) {
          final Date date = dateFormatOcts.parse(timeString);
          String milliSeconds = timeString.substring(timeString.length() - 3);
          return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000);
        } else {
          final Date date = dateFormat.parse(timeString);
          String milliSeconds = timeString.substring(timeString.length() - 3);
          return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000);
        }
      } catch (ParseException ignored) {
      }
    }
    return null;
  }
Esempio n. 5
0
 public static String getRasterName(Variable variable) {
   Attribute attribute = variable.findAttribute(Constants.ORIG_NAME_ATT_NAME);
   if (attribute != null) {
     return attribute.getStringValue();
   } else {
     return variable.getName();
   }
 }
Esempio n. 6
0
 public String getStringAttribute(String key, List<Attribute> attributeList)
     throws ProductIOException {
   Attribute attribute = findAttribute(key, attributeList);
   if (attribute == null || attribute.getLength() != 1) {
     throw new ProductIOException("Global attribute '" + key + "' is missing.");
   } else {
     return attribute.getStringValue().trim();
   }
 }
Esempio n. 7
0
  protected Band addNewBand(Product product, Variable variable) {
    final int sceneRasterWidth = product.getSceneRasterWidth();
    final int sceneRasterHeight = product.getSceneRasterHeight();
    Band band = null;

    int variableRank = variable.getRank();
    if (variableRank == 2) {
      final int[] dimensions = variable.getShape();
      final int height = dimensions[0] - leadLineSkip - tailLineSkip;
      final int width = dimensions[1];
      if (height == sceneRasterHeight && width == sceneRasterWidth) {
        final String name = variable.getShortName();
        final int dataType = getProductDataType(variable);
        band = new Band(name, dataType, width, height);
        final String validExpression = bandInfoMap.get(name);
        if (validExpression != null && !validExpression.equals("")) {
          band.setValidPixelExpression(validExpression);
        }
        product.addBand(band);

        try {
          band.setNoDataValue(
              (double) variable.findAttribute("bad_value_scaled").getNumericValue().floatValue());
          band.setNoDataValueUsed(true);
        } catch (Exception ignored) {
        }

        final List<Attribute> list = variable.getAttributes();
        for (Attribute hdfAttribute : list) {
          final String attribName = hdfAttribute.getShortName();
          if ("units".equals(attribName)) {
            band.setUnit(hdfAttribute.getStringValue());
          } else if ("long_name".equals(attribName)) {
            band.setDescription(hdfAttribute.getStringValue());
          } else if ("slope".equals(attribName)) {
            band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
          } else if ("intercept".equals(attribName)) {
            band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
          }
        }
      }
    }
    return band;
  }
  private static IndexCoding readIndexCoding(Variable variable, String indexCodingName)
      throws ProductIOException {
    final IndexCoding indexCoding = CfIndexCodingPart.readIndexCoding(variable, indexCodingName);

    if (indexCoding != null) {
      final Attribute descriptionsAtt = variable.findAttributeIgnoreCase(INDEX_DESCRIPTIONS);
      if (descriptionsAtt != null) {
        final String[] descriptions = descriptionsAtt.getStringValue().split(DESCRIPTION_SEPARATOR);
        if (indexCoding.getNumAttributes() == descriptions.length) {
          for (int i = 0; i < descriptions.length; i++) {
            indexCoding.getAttributeAt(i).setDescription(descriptions[i]);
          }
        }
      }
      final Attribute nameAtt = variable.findAttributeIgnoreCase(INDEX_CODING_NAME);
      if (nameAtt != null) {
        indexCoding.setName(nameAtt.getStringValue());
      }
    }
    return indexCoding;
  }
  public static boolean isValidFile(NetcdfFile ds) {
    Attribute cdmDtAtt = ds.findGlobalAttributeIgnoreCase("cdm_data_type");
    if (cdmDtAtt == null) cdmDtAtt = ds.findGlobalAttributeIgnoreCase("cdm_datatype");
    if (cdmDtAtt == null) return false;
    if (!cdmDtAtt.isString()) return false;

    String cdmDtString = cdmDtAtt.getStringValue();
    if (cdmDtString == null) return false;
    if (!cdmDtString.equalsIgnoreCase(FeatureType.TRAJECTORY.toString())) return false;

    Attribute conventionsAtt = ds.findGlobalAttributeIgnoreCase("Conventions");
    if (conventionsAtt == null) return (false);
    if (!conventionsAtt.isString()) return (false);
    String convString = conventionsAtt.getStringValue();

    StringTokenizer stoke = new StringTokenizer(convString, ",");
    while (stoke.hasMoreTokens()) {
      String toke = stoke.nextToken().trim();
      if (toke.equalsIgnoreCase("Unidata Observation Dataset v1.0")) return true;
    }

    return false;
  }
Esempio n. 10
0
  /** Instances which have same content are equal. */
  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if ((o == null) || !(o instanceof Attribute)) return false;

    final Attribute att = (Attribute) o;

    if (!name.equals(att.name)) return false;
    if (nelems != att.nelems) return false;
    if (!dataType.equals(att.dataType)) return false;

    if (svalue != null) return svalue.equals(att.getStringValue());

    if (values != null) {
      for (int i = 0; i < getLength(); i++) {
        int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
        int r2 =
            att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
        if (r1 != r2) return false;
      }
    }

    return true;
  }
 /**
  * Is this the right format for this adapter
  *
  * @param ncd NetcdfDataset to check
  * @return true if the right format
  */
 public static boolean isValidFile(NetcdfDataset ncd) {
   // return (buildConfig(ncd) != null);
   // Check for "center" attribute w/ value of "UCAR/CDAAC".
   Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center");
   if (attrib == null) {
     return false;
   }
   if (!attrib.isString()) {
     return false;
   }
   if (!attrib.getStringValue().equals("UCAR/CDAAC")) {
     return false;
   }
   return true;
 }
Esempio n. 12
0
  public static ThreddsMetadata.Variables extractVariables(
      InvDatasetImpl threddsDataset, GridDataset gridDataset) {

    thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType();
    if ((fileFormat != null)
        && (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) {
      boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1);
      ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString());
      for (GridDatatype grid : gridDataset.getGrids()) {
        ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
        v.setName(grid.getName());
        v.setDescription(grid.getDescription());
        v.setUnits(grid.getUnitsString());

        // ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number");
        // String paramNumber = (att != null) ? att.getNumericValue().toString() : null;
        if (isGrib1) {
          v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR"));
          v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
        } else {
          String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", "");
          String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", "");
          String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", "");
          v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName);
          v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
        }
        vars.addVariable(v);
      }
      vars.sort();
      return vars;

    } else { // GRID but not GRIB
      ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0");
      for (GridDatatype grid : gridDataset.getGrids()) {
        ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
        vars.addVariable(v);

        v.setName(grid.getName());
        v.setDescription(grid.getDescription());
        v.setUnits(grid.getUnitsString());

        ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name");
        v.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
      }
      vars.sort();
      return vars;
    }
  }
Esempio n. 13
0
 MetadataAttribute attributeToMetadata(Attribute attribute) {
   final int productDataType = getProductDataType(attribute.getDataType(), false, false);
   if (productDataType != -1) {
     ProductData productData;
     if (attribute.isString()) {
       productData = ProductData.createInstance(attribute.getStringValue());
     } else if (attribute.isArray()) {
       productData = ProductData.createInstance(productDataType, attribute.getLength());
       productData.setElems(attribute.getValues().getStorage());
     } else {
       productData = ProductData.createInstance(productDataType, 1);
       productData.setElems(attribute.getValues().getStorage());
     }
     return new MetadataAttribute(attribute.getShortName(), productData, true);
   }
   return null;
 }
Esempio n. 14
0
  /**
   * Extract a list of data variables (and their canonical names if possible) from the dataset.
   *
   * @param threddsDataset open this dataset
   * @return ThreddsMetadata.Variables, or null if unable.
   * @throws IOException on read error
   */
  public static ThreddsMetadata.Variables extractVariables(InvDatasetImpl threddsDataset)
      throws IOException {
    ThreddsDataFactory.Result result = null;

    try {
      result = new ThreddsDataFactory().openFeatureDataset(threddsDataset, null);
      if (result.fatalError) {
        System.out.println(" openDatatype errs=" + result.errLog);
        return null;
      }

      if (result.featureType == FeatureType.GRID) {
        // System.out.println(" extractVariables GRID=" + result.location);
        GridDataset gridDataset = (GridDataset) result.featureDataset;
        return extractVariables(threddsDataset, gridDataset);

      } else if ((result.featureType == FeatureType.STATION)
          || (result.featureType == FeatureType.POINT)) {
        PointObsDataset pobsDataset = (PointObsDataset) result.featureDataset;
        ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0");
        for (VariableSimpleIF vs : pobsDataset.getDataVariables()) {
          ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
          vars.addVariable(v);

          v.setName(vs.getName());
          v.setDescription(vs.getDescription());
          v.setUnits(vs.getUnitsString());

          ucar.nc2.Attribute att = vs.findAttributeIgnoreCase("standard_name");
          v.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
        }
        vars.sort();
        return vars;
      }

    } finally {
      try {
        if ((result != null) && (result.featureDataset != null)) result.featureDataset.close();
      } catch (IOException ioe) {
        logger.error("Closing dataset " + result.featureDataset, ioe);
      }
    }

    return null;
  }
Esempio n. 15
0
  public static ThreddsMetadata.Variables extractVariables(FeatureDatasetPoint fd) {
    ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.5");
    List<VariableSimpleIF> dataVars = fd.getDataVariables();
    if (dataVars == null) return vars;

    for (VariableSimpleIF v : dataVars) {
      ThreddsMetadata.Variable tv = new ThreddsMetadata.Variable();
      vars.addVariable(tv);

      tv.setName(v.getName());
      tv.setDescription(v.getDescription());
      tv.setUnits(v.getUnitsString());

      ucar.nc2.Attribute att = v.findAttributeIgnoreCase("standard_name");
      tv.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
    }
    vars.sort();
    return vars;
  }
Esempio n. 16
0
  public void testCoordVar(NetcdfFile ncfile) {

    Variable lat = ncfile.findVariable("lat");
    assert null != lat;
    assert lat.getShortName().equals("lat");
    assert lat.getRank() == 1;
    assert lat.getSize() == 3;
    assert lat.getShape()[0] == 3;
    assert lat.getDataType() == DataType.FLOAT;

    assert !lat.isUnlimited();
    assert lat.getDimension(0).equals(ncfile.findDimension("lat"));

    Attribute att = lat.findAttribute("units");
    assert null != att;
    assert !att.isArray();
    assert att.isString();
    assert att.getDataType() == DataType.STRING;
    assert att.getStringValue().equals("degrees_north");
    assert att.getNumericValue() == null;
    assert att.getNumericValue(3) == null;

    try {
      Array data = lat.read();
      assert data.getRank() == 1;
      assert data.getSize() == 3;
      assert data.getShape()[0] == 3;
      assert data.getElementType() == float.class;

      IndexIterator dataI = data.getIndexIterator();
      assert TestUtils.close(dataI.getDoubleNext(), 41.0);
      assert TestUtils.close(dataI.getDoubleNext(), 40.0);
      assert TestUtils.close(dataI.getDoubleNext(), 39.0);
    } catch (IOException io) {
    }
  }
Esempio n. 17
0
  Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
      throws IOException, InvalidRangeException {

    NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
    if (debug) {
      System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
    }

    // global attributes
    List<Attribute> glist = bufr.getGlobalAttributes();
    for (Attribute att : glist) {
      String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
      Attribute useAtt;
      if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
      else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
      else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
      if (debug) System.out.println("add gatt= " + useAtt);
    }

    // global dimensions
    Dimension recordDim = null;
    Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
    for (Dimension oldD : bufr.getDimensions()) {
      String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
      boolean isRecord = useName.equals("record");
      Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
      dimHash.put(newD.getName(), newD);
      if (isRecord) recordDim = newD;
      if (debug) System.out.println("add dim= " + newD);
    }

    // Variables
    Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
    for (Variable oldVar : recordStruct.getVariables()) {
      if (oldVar.getDataType() == DataType.STRUCTURE) continue;

      String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
      DataType newType = oldVar.getDataType();

      List<Dimension> newDims = new ArrayList<Dimension>();
      newDims.add(recordDim);
      for (Dimension dim : oldVar.getDimensions()) {
        newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
      }

      Variable newVar = ncfile.addVariable(varName, newType, newDims);
      if (debug) System.out.println("add var= " + newVar);

      // attributes
      List<Attribute> attList = oldVar.getAttributes();
      for (Attribute att : attList) {
        String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
        if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
        else if (att.isString())
          ncfile.addVariableAttribute(varName, useName, att.getStringValue());
        else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
      }
    }

    // int max_seq = countSeq(recordStruct);
    // Dimension seqD = ncfile.addDimension("level", max_seq);

    for (Variable v : recordStruct.getVariables()) {
      if (v.getDataType() != DataType.STRUCTURE) continue;
      String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
      int shape[] = v.getShape();

      Dimension structDim = ncfile.addDimension(structName, shape[0]);

      Structure struct = (Structure) v;
      for (Variable seqVar : struct.getVariables()) {
        String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
        DataType newType = seqVar.getDataType();

        List<Dimension> newDims = new ArrayList<Dimension>();
        newDims.add(recordDim);
        newDims.add(structDim);
        for (Dimension dim : seqVar.getDimensions()) {
          newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
        }

        Variable newVar = ncfile.addVariable(varName, newType, newDims);
        if (debug) System.out.println("add var= " + newVar);

        // attributes
        List<Attribute> attList = seqVar.getAttributes();
        for (Attribute att : attList) {
          String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
          if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
          else if (att.isString())
            ncfile.addVariableAttribute(varName, useName, att.getStringValue());
          else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
        }
      }
    }

    // create the file
    ncfile.create();
    if (debug) System.out.println("File Out= " + ncfile.toString());

    // boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

    double total = copyVarData(ncfile, recordStruct);
    ncfile.flush();
    System.out.println("FileWriter done total bytes = " + total);
    ncfile.close();
  }
  /**
   * Build the configuration from the dataset
   *
   * @param ncd NetcdfDataset
   * @return the trajectory configuration
   */
  private static Config buildConfig(NetcdfDataset ncd) {

    // already did this in isValid, but we'll keep here for later refactor
    Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center");
    if (attrib == null) {
      return null;
    }
    if (!attrib.isString()) {
      return null;
    }
    if (!attrib.getStringValue().equals("UCAR/CDAAC")) {
      return null;
    }

    // Check for start_time, stop_time
    attrib = ncd.findGlobalAttributeIgnoreCase("start_time");
    if (attrib == null) {
      return null;
    }
    if (attrib.isString()) {
      return null;
    }
    double startTime = attrib.getNumericValue().doubleValue();
    attrib = ncd.findGlobalAttributeIgnoreCase("stop_time");
    if (attrib == null) {
      return null;
    }
    if (attrib.isString()) {
      return null;
    }
    double endTime = attrib.getNumericValue().doubleValue();

    // Check that only one dimension and that it is the alt dimension.
    List list = ncd.getRootGroup().getDimensions();
    if (list.size() != 1) {
      return null;
    }
    Dimension d = (Dimension) list.get(0);
    if (!d.getName().equals(timeDimName)) {
      return null;
    }

    Config trajConfig = new Config();
    trajConfig.setTimeDim(d);

    // Check for latitude variable with time dimension and units convertable to "degrees_north".
    Variable var = ncd.getRootGroup().findVariable(latVarName);
    if (var == null) {
      return null;
    }
    list = var.getDimensions();
    if (list.size() != 1) {
      return null;
    }
    d = (Dimension) list.get(0);
    if (!d.getName().equals(timeDimName)) {
      return null;
    }
    String units = var.findAttribute("units").getStringValue();
    if (!SimpleUnit.isCompatible(units, "degrees_north")) {
      return null;
    }

    trajConfig.setLatVar(var);

    // Make the time Variable
    int numTimes = d.getLength();
    double[] times = new double[numTimes];
    // Variable timeVar = new Variable(var);
    // timeVar.setName(timeVarName);
    VariableDS timeVar =
        new VariableDS(
            ncd,
            ncd.getRootGroup(),
            null,
            timeVarName,
            DataType.DOUBLE,
            timeDimName,
            "seconds since 1980-01-06 00:00:00",
            "Time coordinate");
    // Variable timeVar = new Variable(ncd, ncd.getRootGroup(), null,
    //                              timeVarName);
    // timeVar.setDataType(DataType.DOUBLE);
    // timeVar.setDimensions(list);
    // Attribute newUnits =
    //    new Attribute("units", "seconds since 1980-01-06 00:00:00");
    // timeVar.addAttribute(newUnits);
    timeVar.setCachedData(
        Array.makeArray(DataType.DOUBLE, numTimes, endTime, ((startTime - endTime) / numTimes)),
        true);
    ncd.addVariable(ncd.getRootGroup(), timeVar);
    trajConfig.setTimeVar(timeVar);

    // Check for longitude variable with time dimension and units convertable to "degrees_east".
    var = ncd.getRootGroup().findVariable(lonVarName);
    if (var == null) {
      return null;
    }
    list = var.getDimensions();
    if (list.size() != 1) {
      return null;
    }
    d = (Dimension) list.get(0);
    if (!d.getName().equals(timeDimName)) {
      return null;
    }
    units = var.findAttribute("units").getStringValue();
    if (!SimpleUnit.isCompatible(units, "degrees_east")) {
      return null;
    }

    trajConfig.setLonVar(var);

    // Check for altitude variable with time dimension and units convertable to "m".
    var = ncd.getRootGroup().findVariable(elevVarName);
    if (var == null) {
      return null;
    }
    list = var.getDimensions();
    if (list.size() != 1) {
      return null;
    }
    d = (Dimension) list.get(0);
    if (!d.getName().equals(timeDimName)) {
      return null;
    }
    units = var.findAttribute("units").getStringValue();
    if (!SimpleUnit.isCompatible(units, "meters")) {
      return null;
    }

    trajConfig.setElevVar(var);

    trajConfig.setTrajectoryId(trajId);

    return trajConfig;
  }
    // create from a dataset
    public ObsBean(Structure obs, StructureData sdata) {
      // first choice
      for (Variable v : obs.getVariables()) {
        Attribute att = v.findAttribute("BUFR:TableB_descriptor");
        if (att == null) continue;
        String val = att.getStringValue();
        if (val.equals("0-5-1") && Double.isNaN(lat)) {
          lat = sdata.convertScalarDouble(v.getShortName());
        } else if (val.equals("0-6-1") && Double.isNaN(lon)) {
          lon = sdata.convertScalarDouble(v.getShortName());
        } else if (val.equals("0-7-30") && Double.isNaN(alt)) {

          alt = sdata.convertScalarDouble(v.getShortName());
        } else if (val.equals("0-4-1") && (year < 0)) {
          year = sdata.convertScalarInt(v.getShortName());
        } else if (val.equals("0-4-2") && (month < 0)) {
          month = sdata.convertScalarInt(v.getShortName());
        } else if (val.equals("0-4-3") && (day < 0)) {
          day = sdata.convertScalarInt(v.getShortName());
        } else if (val.equals("0-4-4") && (hour < 0)) {
          hour = sdata.convertScalarInt(v.getShortName());
        } else if (val.equals("0-4-5") && (minute < 0)) {
          minute = sdata.convertScalarInt(v.getShortName());
        } else if (val.equals("0-4-6") && (sec < 0)) {
          sec = sdata.convertScalarInt(v.getShortName());

        } else if (val.equals("0-1-1") && (wmo_block < 0)) {
          wmo_block = sdata.convertScalarInt(v.getShortName());
        } else if (val.equals("0-1-2") && (wmo_id < 0)) {
          wmo_id = sdata.convertScalarInt(v.getShortName());

        } else if ((stn == null)
            && (val.equals("0-1-7")
                || val.equals("0-1-194")
                || val.equals("0-1-11")
                || val.equals("0-1-18"))) {
          if (v.getDataType().isString()) stn = sdata.getScalarString(v.getShortName());
          else stn = Integer.toString(sdata.convertScalarInt(v.getShortName()));
        }
      }

      // second choice
      for (Variable v : obs.getVariables()) {
        Attribute att = v.findAttribute("BUFR:TableB_descriptor");
        if (att == null) continue;
        String val = att.getStringValue();
        if (val.equals("0-5-2") && Double.isNaN(lat)) {
          lat = sdata.convertScalarDouble(v.getShortName());
        } else if (val.equals("0-6-2") && Double.isNaN(lon)) {
          lon = sdata.convertScalarDouble(v.getShortName());
        } else if (val.equals("0-7-1") && Double.isNaN(alt)) {
          alt = sdata.convertScalarDouble(v.getShortName());
        } else if ((val.equals("0-4-7")) && (sec < 0)) {
          sec = sdata.convertScalarInt(v.getShortName());
        }
      }

      // third choice
      for (Variable v : obs.getVariables()) {
        Attribute att = v.findAttribute("BUFR:TableB_descriptor");
        if (att == null) continue;
        String val = att.getStringValue();
        if (val.equals("0-7-10") && Double.isNaN(alt)) {
          alt = sdata.convertScalarDouble(v.getShortName());
        } else if (val.equals("0-7-2") && Double.isNaN(alt)) {
          alt = sdata.convertScalarDouble(v.getShortName());
        }
      }
    }
Esempio n. 20
0
 public String getAbbrev() {
   Attribute att = ve.findAttributeIgnoreCase(CDM.ABBREV);
   return (att == null) ? null : att.getStringValue();
 }
Esempio n. 21
0
 private String getCalendarAttribute(VariableEnhanced vds) {
   Attribute cal = vds.findAttribute("calendar");
   return (cal == null) ? null : cal.getStringValue();
 }
Esempio n. 22
0
  protected Map<Band, Variable> addSmiBands(Product product, List<Variable> variables) {
    final int sceneRasterWidth = product.getSceneRasterWidth();
    final int sceneRasterHeight = product.getSceneRasterHeight();
    Map<Band, Variable> bandToVariableMap = new HashMap<Band, Variable>();
    for (Variable variable : variables) {
      int variableRank = variable.getRank();
      if (variableRank == 2) {
        final int[] dimensions = variable.getShape();
        final int height = dimensions[0];
        final int width = dimensions[1];
        if (height == sceneRasterHeight && width == sceneRasterWidth) {
          String name = variable.getShortName();
          if (name.equals("l3m_data")) {
            try {
              name = getStringAttribute("Parameter") + " " + getStringAttribute("Measure");
            } catch (Exception e) {
              e.printStackTrace();
            }
          }

          final int dataType = getProductDataType(variable);
          final Band band = new Band(name, dataType, width, height);
          //                    band = new Band(name, dataType, width, height);

          product.addBand(band);

          try {
            Attribute fillvalue = variable.findAttribute("_FillValue");
            if (fillvalue == null) {
              fillvalue = variable.findAttribute("Fill");
            }
            if (fillvalue != null) {
              band.setNoDataValue((double) fillvalue.getNumericValue().floatValue());
              band.setNoDataValueUsed(true);
            }
          } catch (Exception ignored) {

          }
          bandToVariableMap.put(band, variable);
          // Set units, if defined
          try {
            band.setUnit(getStringAttribute("Units"));
          } catch (Exception ignored) {

          }

          final List<Attribute> list = variable.getAttributes();
          double[] validMinMax = {0.0, 0.0};
          for (Attribute hdfAttribute : list) {
            final String attribName = hdfAttribute.getShortName();
            if ("units".equals(attribName)) {
              band.setUnit(hdfAttribute.getStringValue());
            } else if ("long_name".equalsIgnoreCase(attribName)) {
              band.setDescription(hdfAttribute.getStringValue());
            } else if ("slope".equalsIgnoreCase(attribName)) {
              band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
            } else if ("intercept".equalsIgnoreCase(attribName)) {
              band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
            } else if ("scale_factor".equals(attribName)) {
              band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
            } else if ("add_offset".equals(attribName)) {
              band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
            } else if (attribName.startsWith("valid_")) {
              if ("valid_min".equals(attribName)) {
                validMinMax[0] = hdfAttribute.getNumericValue(0).doubleValue();
              } else if ("valid_max".equals(attribName)) {
                validMinMax[1] = hdfAttribute.getNumericValue(0).doubleValue();
              } else if ("valid_range".equals(attribName)) {
                validMinMax[0] = hdfAttribute.getNumericValue(0).doubleValue();
                validMinMax[1] = hdfAttribute.getNumericValue(1).doubleValue();
              }
            }
          }
          if (validMinMax[0] != validMinMax[1]) {
            double[] minmax = {0.0, 0.0};
            minmax[0] = validMinMax[0];
            minmax[1] = validMinMax[1];

            if (band.getScalingFactor() != 1.0) {
              minmax[0] *= band.getScalingFactor();
              minmax[1] *= band.getScalingFactor();
            }
            if (band.getScalingOffset() != 0.0) {
              minmax[0] += band.getScalingOffset();
              minmax[1] += band.getScalingOffset();
            }

            String validExp = format("%s >= %.2f && %s <= %.2f", name, minmax[0], name, minmax[1]);
            band.setValidPixelExpression(
                validExp); // .format(name, validMinMax[0], name, validMinMax[1]));
          }
        }
      } else if (variableRank == 4) {
        final int[] dimensions = variable.getShape();
        final int height = dimensions[2];
        final int width = dimensions[3];
        if (height == sceneRasterHeight && width == sceneRasterWidth) {
          String name = variable.getShortName();

          final int dataType = getProductDataType(variable);
          final Band band = new Band(name, dataType, width, height);
          //                    band = new Band(name, dataType, width, height);

          Variable sliced = null;
          try {
            sliced = variable.slice(0, 0).slice(0, 0);
          } catch (InvalidRangeException e) {
            e.printStackTrace(); // Todo change body of catch statement.
          }

          bandToVariableMap.put(band, sliced);
          product.addBand(band);

          try {
            Attribute fillvalue = variable.findAttribute("_FillValue");
            if (fillvalue != null) {
              band.setNoDataValue((double) fillvalue.getNumericValue().floatValue());
              band.setNoDataValueUsed(true);
            }
          } catch (Exception ignored) {

          }
          // Set units, if defined
          try {
            band.setUnit(getStringAttribute("units"));
          } catch (Exception ignored) {

          }

          final List<Attribute> list = variable.getAttributes();
          for (Attribute hdfAttribute : list) {
            final String attribName = hdfAttribute.getShortName();
            if ("scale_factor".equals(attribName)) {
              band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
            } else if ("add_offset".equals(attribName)) {
              band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
            }
          }
        }
      }
    }
    return bandToVariableMap;
  }
Esempio n. 23
0
  @Override
  public AbstractGridDataset createDataset(String id, String location)
      throws IOException, EdalException {
    NetcdfDataset nc = null;
    try {
      /*
       * Open the dataset, using the cache for NcML aggregations
       */
      nc = openAndAggregateDataset(location);

      /*-
       * We may in future be able to use forecast model run collection aggregations for
       * dealing with the case of overlapping time axes.  To do this the code will look
       * something like this:
       *
       * StringBuilder sb = new StringBuilder();
       * Formatter formatter = new Formatter(sb, Locale.UK);
       * Fmrc f = Fmrc.open(location, formatter);
       *
       * in openAndAggregateDataset.  It will need to build up an NcML document which
       * does this.  It should look something like:
       *
       *  <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="true">
       *      <aggregation dimName="run" type="forecastModelRunCollection" timeUnitsChange="true">
       *           <!-- scanFmrc actually works, but what we want is something like the following bit -->
       *           <scanFmrc location="/home/guy/Data/POLCOMS_IRISH/" regExp=".*\.nc"/>
       *           <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090320.nc" coordValue="2009-03-20T00:00:00Z" enhance="true" />
       *           <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090321.nc" coordValue="2009-03-21T00:00:00Z" enhance="true" />
       *           <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090322.nc" coordValue="2009-03-22T00:00:00Z" enhance="true" />
       *      </aggregation>
       *  </netcdf>
       *
       * For more documentation see:
       * http://mailman.unidata.ucar.edu/software/thredds/current/netcdf-java/ncml/FmrcAggregation.html
       *
       * We then can do stuff like:
       *
       * ucar.nc2.dt.GridDataset gridDataset = f.getDatasetBest();
       *
       * To get the single best aggregation of the overlapping time axis
       *
       * Then we need to work with GridDatasets in place of NetcdfDatasets.  Stuff like:
       *
       * for(Variable variable : gridDataset.getNetcdfFile().getVariables()) {
       *    // blah blah
       * }
       *
       * will be necessary.  We need to check that that works with remote datasets too
       */

      /*
       * We look for NetCDF-U variables to group mean/standard-deviation.
       *
       * We need to do this here because we want to subsequently ignore
       * parent variables
       */
      Map<String, String[]> varId2AncillaryVars = new HashMap<String, String[]>();
      for (Variable variable : nc.getVariables()) {
        /*
         * Just look for parent variables, since these may not have a
         * grid directly associated with them
         */
        for (Attribute attr : variable.getAttributes()) {
          if (attr.getFullName().equalsIgnoreCase("ancillary_variables")) {
            varId2AncillaryVars.put(variable.getFullName(), attr.getStringValue().split(" "));
            continue;
          }
        }
      }

      ucar.nc2.dt.GridDataset gridDataset = CdmUtils.getGridDataset(nc);
      List<GridVariableMetadata> vars = new ArrayList<GridVariableMetadata>();
      /*
       * Store a map of component names. Key is the compound name, value
       * is a 2-element String array with x, y component IDs
       *
       * Also store a map of whether these components are really
       * eastward/northward, or whether they are locally u/v
       */
      Map<String, String[]> xyComponentPairs = new HashMap<String, String[]>();
      Map<String, Boolean> xyNameToTrueEN = new HashMap<String, Boolean>();
      /*
       * Store a map of variable IDs to UncertML URLs. This will be used
       * to determine which components are mean/std/etc.
       *
       * TODO implement more than just Mean/SD
       */
      Map<String, String> varId2UncertMLRefs = new HashMap<String, String>();
      /*
       * Here we store the parent variable IDs and their corresponding
       * title.
       */
      Map<String, String> parentVarId2Title = new HashMap<String, String>();
      for (Gridset gridset : gridDataset.getGridsets()) {
        GridCoordSystem coordSys = gridset.getGeoCoordSystem();
        HorizontalGrid hDomain = CdmUtils.createHorizontalGrid(coordSys);
        VerticalAxis zDomain = CdmUtils.createVerticalAxis(coordSys);
        TimeAxis tDomain = CdmUtils.createTimeAxis(coordSys);

        /*
         * Create a VariableMetadata object for each GridDatatype
         */
        for (GridDatatype grid : gridset.getGrids()) {
          VariableDS variable = grid.getVariable();
          String varId = variable.getFullName();
          String name = getVariableName(variable);

          /*
           * If this is a parent variable for a stats collection, we
           * don't want it to be a normal variable as well.
           */
          if (varId2AncillaryVars.containsKey(varId)) {
            parentVarId2Title.put(varId, name);
            continue;
          }

          /*
           * If it is a child variable is (potentially) referenced by
           * UncertML, store its ID and the (possible) UncertML URI
           */
          for (Attribute attr : variable.getAttributes()) {
            if (attr.getFullName().equalsIgnoreCase("ref")) {
              varId2UncertMLRefs.put(varId, attr.getStringValue());
            }
          }

          Parameter parameter =
              new Parameter(
                  varId,
                  variable.getShortName(),
                  variable.getDescription(),
                  variable.getUnitsString(),
                  name);
          GridVariableMetadata metadata =
              new GridVariableMetadata(
                  variable.getFullName(), parameter, hDomain, zDomain, tDomain, true);
          vars.add(metadata);

          if (name != null) {
            /*
             * Check for vector components
             */
            if (name.contains("eastward_")) {
              String compoundName = name.replaceFirst("eastward_", "");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, true);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[0] = varId;
            } else if (name.contains("northward_")) {
              String compoundName = name.replaceFirst("northward_", "");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, true);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[1] = varId;
            } else if (name.matches("u-.*component")) {
              String compoundName = name.replaceFirst("u-(.*)component", "$1");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, false);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[0] = varId;
            } else if (name.matches("v-.*component")) {
              String compoundName = name.replaceFirst("v-(.*)component", "$1");
              String[] cData;
              if (!xyComponentPairs.containsKey(compoundName)) {
                cData = new String[2];
                xyComponentPairs.put(compoundName, cData);
                xyNameToTrueEN.put(compoundName, false);
              }
              cData = xyComponentPairs.get(compoundName);
              /*
               * By doing this, we will end up with the merged
               * coverage
               */
              cData[1] = varId;
            }
            /*
             * We could potentially add a check for zonal/meridional
             * here if required.
             */
          }
        }
      }

      CdmGridDataset cdmGridDataset =
          new CdmGridDataset(id, location, vars, CdmUtils.getOptimumDataReadingStrategy(nc));
      for (Entry<String, String[]> componentData : xyComponentPairs.entrySet()) {
        String commonName = componentData.getKey();
        String[] comps = componentData.getValue();
        if (comps[0] != null && comps[1] != null) {
          cdmGridDataset.addVariablePlugin(
              new VectorPlugin(comps[0], comps[1], commonName, xyNameToTrueEN.get(commonName)));
        }
      }

      for (String statsCollectionId : varId2AncillaryVars.keySet()) {
        String[] ids = varId2AncillaryVars.get(statsCollectionId);
        String meanId = null;
        String stddevId = null;
        for (String statsVarIds : ids) {
          String uncertRef = varId2UncertMLRefs.get(statsVarIds);
          if (uncertRef != null
              && uncertRef.equalsIgnoreCase("http://www.uncertml.org/statistics/mean")) {
            meanId = statsVarIds;
          }
          if (uncertRef != null
              && uncertRef.equalsIgnoreCase(
                  "http://www.uncertml.org/statistics/standard-deviation")) {
            stddevId = statsVarIds;
          }
        }
        if (meanId != null && stddevId != null) {
          MeanSDPlugin meanSDPlugin =
              new MeanSDPlugin(meanId, stddevId, parentVarId2Title.get(statsCollectionId));
          cdmGridDataset.addVariablePlugin(meanSDPlugin);
        }
      }

      return cdmGridDataset;
    } finally {
      CdmUtils.closeDataset(nc);
    }
  }
Esempio n. 24
0
  /**
   * Opens the NetCDF dataset at the given location, using the dataset cache if {@code location}
   * represents an NcML aggregation. We cannot use the cache for OPeNDAP or single NetCDF files
   * because the underlying data may have changed and the NetcdfDataset cache may cache a dataset
   * forever. In the case of NcML we rely on the fact that server administrators ought to have set a
   * "recheckEvery" parameter for NcML aggregations that may change with time. It is desirable to
   * use the dataset cache for NcML aggregations because they can be time-consuming to assemble and
   * we don't want to do this every time a map is drawn.
   *
   * @param location The location of the data: a local NetCDF file, an NcML aggregation file or an
   *     OPeNDAP location, {@literal i.e.} anything that can be passed to
   *     NetcdfDataset.openDataset(location).
   * @return a {@link NetcdfDataset} object for accessing the data at the given location.
   * @throws IOException if there was an error reading from the data source.
   */
  private NetcdfDataset openAndAggregateDataset(String location) throws IOException, EdalException {
    NetcdfDataset nc;
    if (location.startsWith("dods://") || location.startsWith("http://")) {
      /*
       * We have a remote dataset
       */
      nc = CdmUtils.openDataset(location);
    } else {
      /*
       * We have a local dataset
       */
      List<File> files = null;
      try {
        files = CdmUtils.expandGlobExpression(location);
      } catch (NullPointerException e) {
        System.out.println("NPE processing location: " + location);
        throw e;
      }
      if (files.size() == 0) {
        throw new EdalException(
            "The location " + location + " doesn't refer to any existing files.");
      }
      if (files.size() == 1) {
        location = files.get(0).getAbsolutePath();
        nc = CdmUtils.openDataset(location);
      } else {
        /*
         * We have multiple files in a glob expression. We write some
         * NcML and use the NetCDF aggregation libs to parse this into
         * an aggregated dataset.
         *
         * If we have already generated the ncML on a previous call,
         * just use that.
         */
        if (ncmlString == null) {
          /*
           * Find the name of the time dimension
           */
          NetcdfDataset first = openAndAggregateDataset(files.get(0).getAbsolutePath());
          String timeDimName = null;
          for (Variable var : first.getVariables()) {
            if (var.isCoordinateVariable()) {
              for (Attribute attr : var.getAttributes()) {
                if (attr.getFullName().equalsIgnoreCase("units")
                    && attr.getStringValue().contains(" since ")) {
                  /*
                   * This is the time dimension. Since this is
                   * a co-ordinate variable, there is only 1
                   * dimension
                   */
                  Dimension timeDimension = var.getDimension(0);
                  timeDimName = timeDimension.getFullName();
                }
              }
            }
          }
          first.close();
          if (timeDimName == null) {
            throw new EdalException("Cannot join multiple files without time dimensions");
          }
          /*
           * We can't assume that the glob expression will have
           * returned the files in time order.
           *
           * We could assume that alphabetical == time ordered (and
           * for properly named files it will - but let's not rely on
           * our users having sensible naming conventions...
           *
           * Sort the list using a comparator which opens the file and
           * gets the first value of the time dimension
           */
          final String aggDimName = timeDimName;
          Collections.sort(
              files,
              new Comparator<File>() {
                @Override
                public int compare(File ncFile1, File ncFile2) {
                  NetcdfFile nc1 = null;
                  NetcdfFile nc2 = null;
                  try {
                    nc1 = NetcdfFile.open(ncFile1.getAbsolutePath());
                    nc2 = NetcdfFile.open(ncFile2.getAbsolutePath());
                    Variable timeVar1 = nc1.findVariable(aggDimName);
                    Variable timeVar2 = nc2.findVariable(aggDimName);
                    long time1 = timeVar1.read().getLong(0);
                    long time2 = timeVar2.read().getLong(0);
                    return Long.compare(time1, time2);
                  } catch (Exception e) {
                    /*
                     * There was a problem reading the data. Sort
                     * alphanumerically by filename and hope for the
                     * best...
                     *
                     * This catches all exceptions because however
                     * it fails this is still our best option.
                     *
                     * If the error is a genuine problem, it'll show
                     * up as soon as we try and aggregate.
                     */
                    return ncFile1.getAbsolutePath().compareTo(ncFile2.getAbsolutePath());
                  } finally {
                    if (nc1 != null) {
                      try {
                        nc1.close();
                      } catch (IOException e) {
                        log.error("Problem closing netcdf file", e);
                      }
                    }
                    if (nc2 != null) {
                      try {
                        nc2.close();
                      } catch (IOException e) {
                        log.error("Problem closing netcdf file", e);
                      }
                    }
                  }
                }
              });

          /*
           * Now create the NcML string and use it to create an
           * aggregated dataset
           */
          StringBuffer ncmlStringBuffer = new StringBuffer();
          ncmlStringBuffer.append(
              "<netcdf xmlns=\"http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2\">");
          ncmlStringBuffer.append(
              "<aggregation dimName=\"" + timeDimName + "\" type=\"joinExisting\">");
          for (File file : files) {
            ncmlStringBuffer.append("<netcdf location=\"" + file.getAbsolutePath() + "\"/>");
          }
          ncmlStringBuffer.append("</aggregation>");
          ncmlStringBuffer.append("</netcdf>");

          ncmlString = ncmlStringBuffer.toString();
        }
        nc = NcMLReader.readNcML(new StringReader(ncmlString), null);
      }
    }

    return nc;
  }
Esempio n. 25
0
  @Test
  public void testDoradeGround() throws IOException {

    System.out.println("**** Open " + groundDoradeFile);
    try (NetcdfFile ncfile = NetcdfFile.open(groundDoradeFile)) {

      for (Variable v : ncfile.getVariables()) {
        System.out.println(v.getFullName());
      }

      /* test both gate and radial dimension */
      Dimension gateDim = ncfile.getRootGroup().findDimension("gate_1");

      assert (gateDim.getLength() == 1008);

      Dimension radialDim = ncfile.getRootGroup().findDimension("radial");

      assert (radialDim.getLength() == 439);

      /* test some att  */
      Attribute testAtt = ncfile.getRootGroup().findAttribute("Conventions");
      assert (testAtt.getStringValue().equals(_Coordinate.Convention));

      testAtt = ncfile.getRootGroup().findAttribute("format");
      assert (testAtt.getStringValue().equals("Unidata/netCDF/Dorade"));

      testAtt = ncfile.getRootGroup().findAttribute("Project_name");
      assert (testAtt.getStringValue().equalsIgnoreCase("IHOP_2002"));

      testAtt = ncfile.getRootGroup().findAttribute("Radar_Name");
      assert (testAtt.getStringValue().equalsIgnoreCase("SPOL"));

      testAtt = ncfile.getRootGroup().findAttribute("VolumeCoveragePatternName");
      assert (testAtt.getStringValue().equalsIgnoreCase("SUR"));

      testAtt = ncfile.getRootGroup().findAttribute("Volume_Number");
      assert (testAtt.getStringValue().equalsIgnoreCase("1"));

      testAtt = ncfile.getRootGroup().findAttribute("Sweep_Number");
      assert (testAtt.getStringValue().equalsIgnoreCase("2"));

      testAtt = ncfile.getRootGroup().findAttribute("Sweep_Date");
      assert (testAtt.getStringValue().equalsIgnoreCase("2002-05-11 01:58:15Z"));

      Variable var;
      var = ncfile.findVariable("elevation");
      testReadData(var);
      var = ncfile.findVariable("azimuth");
      testReadData(var);
      var = ncfile.findVariable("distance_1");
      testReadData(var);
      var = ncfile.findVariable("latitudes_1");
      testReadData(var);
      var = ncfile.findVariable("longitudes_1");
      testReadData(var);
      var = ncfile.findVariable("altitudes_1");
      testReadData(var);
      var = ncfile.findVariable("rays_time");
      testReadData(var);
      var = ncfile.findVariable("Range_to_First_Cell");
      float t = testReadScalar(var);
      assert (t == (float) 150.0);

      var = ncfile.findVariable("Cell_Spacing");
      t = testReadScalar(var);
      assert (t == (float) 149.89624);
      var = ncfile.findVariable("Fixed_Angle");
      t = testReadScalar(var);
      assert (t == (float) 1.1975098);
      var = ncfile.findVariable("Nyquist_Velocity");
      t = testReadScalar(var);
      assert (t == (float) 25.618269);
      var = ncfile.findVariable("Unambiguous_Range");
      t = testReadScalar(var);
      assert (t == (float) 156.11694);
      var = ncfile.findVariable("Radar_Constant");
      t = testReadScalar(var);
      assert (t == (float) 70.325195);
      var = ncfile.findVariable("rcvr_gain");
      t = testReadScalar(var);
      assert (t == (float) 46.95);
      var = ncfile.findVariable("ant_gain");
      t = testReadScalar(var);
      assert (t == (float) 45.58);
      var = ncfile.findVariable("sys_gain");
      t = testReadScalar(var);
      assert (t == (float) 46.95);
      var = ncfile.findVariable("bm_width");
      t = testReadScalar(var);
      assert (t == (float) 0.92);

      var = ncfile.findVariable("VE");
      testReadData(var);
      var = ncfile.findVariable("DM");
      testReadData(var);
      var = ncfile.findVariable("NCP");
      testReadData(var);
      var = ncfile.findVariable("SW");
      testReadData(var);
      var = ncfile.findVariable("DZ");
      testReadData(var);
      var = ncfile.findVariable("DCZ");
      testReadData(var);
      var = ncfile.findVariable("LVDR");
      testReadData(var);
      var = ncfile.findVariable("NIQ");
      testReadData(var);
      var = ncfile.findVariable("AIQ");
      testReadData(var);
      var = ncfile.findVariable("CH");
      testReadData(var);
      var = ncfile.findVariable("AH");
      testReadData(var);
      var = ncfile.findVariable("CV");
      testReadData(var);
      var = ncfile.findVariable("AV");
      testReadData(var);
      var = ncfile.findVariable("RHOHV");
      testReadData(var);
      var = ncfile.findVariable("LDR");
      testReadData(var);
      var = ncfile.findVariable("DL");
      testReadData(var);
      var = ncfile.findVariable("DX");
      testReadData(var);
      var = ncfile.findVariable("ZDR");
      testReadData(var);
      var = ncfile.findVariable("PHI");
      testReadData(var);
      var = ncfile.findVariable("KDP");
      testReadData(var);

      assert (0 == var.findDimensionIndex("radial"));
      assert (1 == var.findDimensionIndex("gate_1"));
    }
  }
  /**
   * Constructor. If scale/offset attributes are found, remove them from the decorated variable.
   *
   * @param forVar the Variable to decorate.
   * @param useNaNs pre-fill isMissing() data with NaNs
   * @param fillValueIsMissing use _FillValue for isMissing()
   * @param invalidDataIsMissing use valid_range for isMissing()
   * @param missingDataIsMissing use missing_value for isMissing()
   */
  EnhanceScaleMissingImpl(
      VariableDS forVar,
      boolean useNaNs,
      boolean fillValueIsMissing,
      boolean invalidDataIsMissing,
      boolean missingDataIsMissing) {

    this.fillValueIsMissing = fillValueIsMissing;
    this.invalidDataIsMissing = invalidDataIsMissing;
    this.missingDataIsMissing = missingDataIsMissing;

    // see if underlying variable has scale/offset already applied
    Variable orgVar = forVar.getOriginalVariable();
    if (orgVar instanceof VariableDS) {
      VariableDS orgVarDS = (VariableDS) orgVar;
      EnumSet<NetcdfDataset.Enhance> orgEnhanceMode = orgVarDS.getEnhanceMode();
      if ((orgEnhanceMode != null) && orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing))
        return;
    }

    // the other possibility is that you want to apply scale and offset to a signed value, then
    // declare the result unsigned
    // this.isUnsigned = (orgVar != null) ? orgVar.isUnsigned() : forVar.isUnsigned();
    this.isUnsigned = forVar.isUnsigned();
    this.convertedDataType = forVar.getDataType();

    DataType scaleType = null, missType = null, validType = null, fillType = null;
    if (debug) System.out.println("EnhancementsImpl for Variable = " + forVar.getFullName());
    Attribute att;

    // scale and offset
    if (null != (att = forVar.findAttribute(CDM.SCALE_FACTOR))) {
      if (!att.isString()) {
        scale = att.getNumericValue().doubleValue();
        hasScaleOffset = true;
        scaleType = att.getDataType();
        forVar.remove(att);
        if (debug) System.out.println("scale = " + scale + " type " + scaleType);
      }
    }
    if (null != (att = forVar.findAttribute(CDM.ADD_OFFSET))) {
      if (!att.isString()) {
        offset = att.getNumericValue().doubleValue();
        hasScaleOffset = true;
        DataType offType = att.getDataType();
        if (rank(offType) > rank(scaleType)) scaleType = offType;
        forVar.remove(att);
        if (debug) System.out.println("offset = " + offset);
      }
    }

    ////// missing data : valid_range. assume here its in units of unpacked data. correct this below
    Attribute validRangeAtt;
    if (null != (validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE))) {
      if (!validRangeAtt.isString() && validRangeAtt.getLength() > 1) {
        valid_min = validRangeAtt.getNumericValue(0).doubleValue();
        valid_max = validRangeAtt.getNumericValue(1).doubleValue();
        hasValidRange = true;
        validType = validRangeAtt.getDataType();
        if (hasScaleOffset) forVar.remove(validRangeAtt);
        if (debug) System.out.println("valid_range = " + valid_min + " " + valid_max);
      }
    }

    Attribute validMinAtt = null, validMaxAtt = null;
    if (!hasValidRange) {
      if (null != (validMinAtt = forVar.findAttribute("valid_min"))) {
        if (!validMinAtt.isString()) {
          valid_min = validMinAtt.getNumericValue().doubleValue();
          hasValidMin = true;
          validType = validMinAtt.getDataType();
          if (hasScaleOffset) forVar.remove(validMinAtt);
          if (debug) System.out.println("valid_min = " + valid_min);
        }
      }

      if (null != (validMaxAtt = forVar.findAttribute("valid_max"))) {
        if (!validMaxAtt.isString()) {
          valid_max = validMaxAtt.getNumericValue().doubleValue();
          hasValidMax = true;
          DataType t = validMaxAtt.getDataType();
          if (rank(t) > rank(validType)) validType = t;
          if (hasScaleOffset) forVar.remove(validMaxAtt);
          if (debug) System.out.println("valid_min = " + valid_max);
        }
      }
    }
    boolean hasValidData = hasValidMin || hasValidMax || hasValidRange;
    if (hasValidMin && hasValidMax) hasValidRange = true;

    /// _FillValue
    if ((null != (att = forVar.findAttribute(CDM.FILL_VALUE))) && !att.isString()) {
      double[] values = getValueAsDouble(att); // LOOK double WTF ??
      if (values.length > 0) {
        fillValue = values[0];
        hasFillValue = true;
        fillType = att.getDataType();
        if (hasScaleOffset) forVar.remove(att);
        if (debug) System.out.println("missing_datum from _FillValue = " + fillValue);
      }
    }

    /// missing_value
    if (null != (att = forVar.findAttribute(CDM.MISSING_VALUE))) {
      if (att.isString()) {
        String svalue = att.getStringValue();
        if (forVar.getDataType() == DataType.CHAR) {
          missingValue = new double[1];
          if (svalue.length() == 0) missingValue[0] = 0;
          else missingValue[0] = svalue.charAt(0);

          missType = DataType.CHAR;
          hasMissingValue = true;

        } else { // not a CHAR - try to fix problem where they use a numeric value as a String
                 // attribute

          try {
            missingValue = new double[1];
            missingValue[0] = Double.parseDouble(svalue);
            missType = att.getDataType();
            hasMissingValue = true;
          } catch (NumberFormatException ex) {
            if (debug)
              System.out.println(
                  "String missing_value not parsable as double= " + att.getStringValue());
          }
        }

      } else { // not a string
        missingValue = getValueAsDouble(att);
        missType = att.getDataType();
        for (double mv : missingValue)
          if (!Double.isNaN(mv))
            hasMissingValue = true; // dont need to do anything if its already a NaN
      }
      if (hasScaleOffset) forVar.remove(att);
    }

    // missing
    boolean hasMissing =
        (invalidDataIsMissing && hasValidData)
            || (fillValueIsMissing && hasFillValue)
            || (missingDataIsMissing && hasMissingValue);

    /// assign convertedDataType if needed
    if (hasScaleOffset) {

      convertedDataType = forVar.getDataType();
      if (hasMissing) {
        // has missing data : must be float or double
        if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
        if (missingDataIsMissing && rank(missType) > rank(convertedDataType))
          convertedDataType = missType;
        if (fillValueIsMissing && rank(fillType) > rank(convertedDataType))
          convertedDataType = fillType;
        if (invalidDataIsMissing && rank(validType) > rank(convertedDataType))
          convertedDataType = validType;
        if (rank(convertedDataType) < rank(DataType.DOUBLE)) convertedDataType = DataType.FLOAT;

      } else {
        // no missing data; can use wider of data and scale
        if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
      }
      if (debug) System.out.println("assign dataType = " + convertedDataType);

      // validData may be external or internal
      if (hasValidData) {
        DataType orgType = forVar.getDataType();

        // If valid_range is the same type as scale_factor (actually the wider of
        // scale_factor and add_offset) and this is wider than the external data, then it
        // will be interpreted as being in the units of the internal (unpacked) data.
        // Otherwise it is in the units of the external (unpacked) data.
        // we assumed unpacked data above, redo if its really packed data
        if (!((rank(validType) == rank(scaleType)) && (rank(scaleType) >= rank(orgType)))) {
          if (validRangeAtt != null) {
            double[] values = getValueAsDouble(validRangeAtt);
            valid_min = values[0];
            valid_max = values[1];
          } else {
            if (validMinAtt != null) {
              double[] values = getValueAsDouble(validMinAtt);
              valid_min = values[0];
            }
            if (validMaxAtt != null) {
              double[] values = getValueAsDouble(validMaxAtt);
              valid_max = values[0];
            }
          }
        }
      }
    }

    if (hasMissing
        && ((convertedDataType == DataType.DOUBLE) || (convertedDataType == DataType.FLOAT)))
      this.useNaNs = useNaNs;
    if (debug) System.out.println("this.useNaNs = " + this.useNaNs);
  }