コード例 #1
0
ファイル: TestStandardVar.java プロジェクト: kenkyu/thredds
  public void readShortMissing() throws Exception {
    Variable v = null;
    assert (null != (v = ncfileRead.findVariable("t4")));
    assert (v.getDataType() == DataType.SHORT);

    // default use of missing_value
    assert (null != (v = dsRead.findVariable("t4")));
    assert v instanceof VariableEnhanced;
    assert v instanceof VariableDS;
    VariableDS vs = (VariableDS) v;
    assert (vs.getDataType() == DataType.SHORT);

    Attribute att = vs.findAttribute(CDM.MISSING_VALUE);
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    System.out.println("missing_value = " + att.getNumericValue().shortValue());
    assert (((short) -9999) == att.getNumericValue().shortValue());
    assert (DataType.SHORT == att.getDataType());

    assert (vs.hasMissing());
    assert (vs.hasMissingValue());
    assert (vs.isMissing((double) ((short) -9999)));
    assert (vs.isMissingValue((double) ((short) -9999)));

    Array A = vs.read();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;
    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
      }
    }

    // turn off missing data
    vs.setMissingDataIsMissing(false);
    assert (vs.getDataType() == DataType.SHORT);

    assert (!vs.hasMissing());
    assert (vs.hasMissingValue());
    assert (!vs.isMissing((double) ((short) -9999)));
    assert (vs.isMissingValue((double) ((short) -9999)));

    vs.setMissingDataIsMissing(true);
    assert (vs.hasMissing());
    assert (vs.isMissing((double) ((short) -9999)));

    System.out.println("**************TestStandardVar Read readShortMissing");
  }
コード例 #2
0
ファイル: TestStandardVar.java プロジェクト: kenkyu/thredds
  public void readByte2Short() throws Exception {
    Variable t2 = null;
    assert (null != (t2 = ncfileRead.findVariable("t2")));
    assert (t2.getDataType() == DataType.BYTE);

    Attribute att = t2.findAttribute(CDM.SCALE_FACTOR);
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    assert (2 == att.getNumericValue().doubleValue());
    assert (DataType.SHORT == att.getDataType());

    assert (null != (t2 = dsRead.findVariable("t2")));
    assert t2 instanceof VariableEnhanced;
    VariableDS vs = (VariableDS) t2;
    assert (vs.getDataType() == DataType.SHORT) : vs.getDataType();
    assert (!vs.hasMissing());

    Array A = vs.read();
    assert (A.getElementType() == short.class) : A.getElementType();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;
    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getShort(ima.set(i, j)) == (2 * (i * 10 + j) + 77));
      }
    }
    System.out.println("**************TestStandardVar readByte2Short");
  }
コード例 #3
0
  private double[] getValueAsDouble(Attribute att) {
    int n = att.getLength();
    double[] value = new double[n];

    if (debugMissing) System.out.printf("missing_data: ");
    for (int i = 0; i < n; i++) {
      if (isUnsigned && att.getDataType() == DataType.BYTE)
        value[i] = convertScaleOffsetMissing(att.getNumericValue(i).byteValue());
      else if (isUnsigned && att.getDataType() == DataType.SHORT)
        value[i] = convertScaleOffsetMissing(att.getNumericValue(i).shortValue());
      else if (isUnsigned && att.getDataType() == DataType.INT)
        value[i] = convertScaleOffsetMissing(att.getNumericValue(i).intValue());
      else value[i] = scale * att.getNumericValue(i).doubleValue() + offset;
      if (debugMissing) System.out.print(" " + value[i]);
    }
    if (debugMissing) System.out.println();
    return value;
  }
コード例 #4
0
ファイル: TestStandardVar.java プロジェクト: kenkyu/thredds
  public void readByte() throws Exception {
    Variable v = null;
    assert (null != (v = ncfileRead.findVariable("t3")));
    assert (v.getDataType() == DataType.BYTE);

    assert (null != (v = dsRead.findVariable("t3")));
    assert v instanceof VariableEnhanced;
    assert v instanceof VariableDS;
    VariableDS vs = (VariableDS) v;
    assert (vs.getDataType() == DataType.BYTE);

    Attribute att = vs.findAttribute("_FillValue");
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    System.out.println("_FillValue = " + att.getNumericValue().byteValue());
    assert (((byte) 255) == att.getNumericValue().byteValue());
    assert (DataType.BYTE == att.getDataType());

    assert (vs.hasMissing());
    assert (vs.hasFillValue());
    assert (vs.isMissing((double) ((byte) 255)));
    assert (vs.isFillValue((double) ((byte) 255)));

    Array A = vs.read();
    assert (A.getElementType() == byte.class) : A.getElementType();
    Index ima = A.getIndex();
    int[] shape = A.getShape();
    int i, j;
    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getFloat(ima.set(i, j)) == (i * 10 + j));
      }
    }
    System.out.println("**************TestStandardVar ReadByte");
  }
コード例 #5
0
  public void testCoordVar(NetcdfFile ncfile) {

    Variable lat = ncfile.findVariable("lat");
    assert null != lat;
    assert lat.getShortName().equals("lat");
    assert lat.getRank() == 1;
    assert lat.getSize() == 3;
    assert lat.getShape()[0] == 3;
    assert lat.getDataType() == DataType.FLOAT;

    assert !lat.isUnlimited();
    assert lat.getDimension(0).equals(ncfile.findDimension("lat"));

    Attribute att = lat.findAttribute("units");
    assert null != att;
    assert !att.isArray();
    assert att.isString();
    assert att.getDataType() == DataType.STRING;
    assert att.getStringValue().equals("degrees_north");
    assert att.getNumericValue() == null;
    assert att.getNumericValue(3) == null;

    try {
      Array data = lat.read();
      assert data.getRank() == 1;
      assert data.getSize() == 3;
      assert data.getShape()[0] == 3;
      assert data.getElementType() == float.class;

      IndexIterator dataI = data.getIndexIterator();
      assert TestUtils.close(dataI.getDoubleNext(), 41.0);
      assert TestUtils.close(dataI.getDoubleNext(), 40.0);
      assert TestUtils.close(dataI.getDoubleNext(), 39.0);
    } catch (IOException io) {
    }
  }
コード例 #6
0
ファイル: TestStandardVar.java プロジェクト: kenkyu/thredds
  public void readDouble() throws Exception {

    Variable t1 = null;
    assert (null != (t1 = ncfileRead.findVariable("t1")));
    assert (t1.getDataType() == DataType.DOUBLE);

    Attribute att = t1.findAttribute(CDM.SCALE_FACTOR);
    assert (null != att);
    assert (!att.isArray());
    assert (1 == att.getLength());
    assert (2.0 == att.getNumericValue().doubleValue());
    assert (DataType.DOUBLE == att.getDataType());

    // read
    Array A = t1.read();
    int i, j;
    Index ima = A.getIndex();
    int[] shape = A.getShape();

    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getDouble(ima.set(i, j)) == (double) (i * 10.0 + j));
      }
    }

    assert (null != (t1 = dsRead.findVariable("t1")));
    assert t1 instanceof VariableEnhanced;
    VariableEnhanced dsVar = (VariableEnhanced) t1;
    assert (dsVar.getDataType() == DataType.DOUBLE);

    A = dsVar.read();
    ima = A.getIndex();
    shape = A.getShape();

    for (i = 0; i < shape[0]; i++) {
      for (j = 0; j < shape[1]; j++) {
        assert (A.getDouble(ima.set(i, j)) == (2.0 * (i * 10.0 + j) + 77.0));
      }
    }

    assert (null == t1.findAttribute(CDM.SCALE_FACTOR));
    assert (null == t1.findAttribute("add_offset"));

    System.out.println("**************TestStandardVar ReadDouble");
  }
コード例 #7
0
ファイル: Attribute.java プロジェクト: kenkyu/thredds
  /** Instances which have same content are equal. */
  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if ((o == null) || !(o instanceof Attribute)) return false;

    final Attribute att = (Attribute) o;

    if (!name.equals(att.name)) return false;
    if (nelems != att.nelems) return false;
    if (!dataType.equals(att.dataType)) return false;

    if (svalue != null) return svalue.equals(att.getStringValue());

    if (values != null) {
      for (int i = 0; i < getLength(); i++) {
        int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
        int r2 =
            att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
        if (r1 != r2) return false;
      }
    }

    return true;
  }
コード例 #8
0
  /**
   * Constructor. If scale/offset attributes are found, remove them from the decorated variable.
   *
   * @param forVar the Variable to decorate.
   * @param useNaNs pre-fill isMissing() data with NaNs
   * @param fillValueIsMissing use _FillValue for isMissing()
   * @param invalidDataIsMissing use valid_range for isMissing()
   * @param missingDataIsMissing use missing_value for isMissing()
   */
  EnhanceScaleMissingImpl(
      VariableDS forVar,
      boolean useNaNs,
      boolean fillValueIsMissing,
      boolean invalidDataIsMissing,
      boolean missingDataIsMissing) {

    this.fillValueIsMissing = fillValueIsMissing;
    this.invalidDataIsMissing = invalidDataIsMissing;
    this.missingDataIsMissing = missingDataIsMissing;

    // see if underlying variable has scale/offset already applied
    Variable orgVar = forVar.getOriginalVariable();
    if (orgVar instanceof VariableDS) {
      VariableDS orgVarDS = (VariableDS) orgVar;
      EnumSet<NetcdfDataset.Enhance> orgEnhanceMode = orgVarDS.getEnhanceMode();
      if ((orgEnhanceMode != null) && orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing))
        return;
    }

    // the other possibility is that you want to apply scale and offset to a signed value, then
    // declare the result unsigned
    // this.isUnsigned = (orgVar != null) ? orgVar.isUnsigned() : forVar.isUnsigned();
    this.isUnsigned = forVar.isUnsigned();
    this.convertedDataType = forVar.getDataType();

    DataType scaleType = null, missType = null, validType = null, fillType = null;
    if (debug) System.out.println("EnhancementsImpl for Variable = " + forVar.getFullName());
    Attribute att;

    // scale and offset
    if (null != (att = forVar.findAttribute(CDM.SCALE_FACTOR))) {
      if (!att.isString()) {
        scale = att.getNumericValue().doubleValue();
        hasScaleOffset = true;
        scaleType = att.getDataType();
        forVar.remove(att);
        if (debug) System.out.println("scale = " + scale + " type " + scaleType);
      }
    }
    if (null != (att = forVar.findAttribute(CDM.ADD_OFFSET))) {
      if (!att.isString()) {
        offset = att.getNumericValue().doubleValue();
        hasScaleOffset = true;
        DataType offType = att.getDataType();
        if (rank(offType) > rank(scaleType)) scaleType = offType;
        forVar.remove(att);
        if (debug) System.out.println("offset = " + offset);
      }
    }

    ////// missing data : valid_range. assume here its in units of unpacked data. correct this below
    Attribute validRangeAtt;
    if (null != (validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE))) {
      if (!validRangeAtt.isString() && validRangeAtt.getLength() > 1) {
        valid_min = validRangeAtt.getNumericValue(0).doubleValue();
        valid_max = validRangeAtt.getNumericValue(1).doubleValue();
        hasValidRange = true;
        validType = validRangeAtt.getDataType();
        if (hasScaleOffset) forVar.remove(validRangeAtt);
        if (debug) System.out.println("valid_range = " + valid_min + " " + valid_max);
      }
    }

    Attribute validMinAtt = null, validMaxAtt = null;
    if (!hasValidRange) {
      if (null != (validMinAtt = forVar.findAttribute("valid_min"))) {
        if (!validMinAtt.isString()) {
          valid_min = validMinAtt.getNumericValue().doubleValue();
          hasValidMin = true;
          validType = validMinAtt.getDataType();
          if (hasScaleOffset) forVar.remove(validMinAtt);
          if (debug) System.out.println("valid_min = " + valid_min);
        }
      }

      if (null != (validMaxAtt = forVar.findAttribute("valid_max"))) {
        if (!validMaxAtt.isString()) {
          valid_max = validMaxAtt.getNumericValue().doubleValue();
          hasValidMax = true;
          DataType t = validMaxAtt.getDataType();
          if (rank(t) > rank(validType)) validType = t;
          if (hasScaleOffset) forVar.remove(validMaxAtt);
          if (debug) System.out.println("valid_min = " + valid_max);
        }
      }
    }
    boolean hasValidData = hasValidMin || hasValidMax || hasValidRange;
    if (hasValidMin && hasValidMax) hasValidRange = true;

    /// _FillValue
    if ((null != (att = forVar.findAttribute(CDM.FILL_VALUE))) && !att.isString()) {
      double[] values = getValueAsDouble(att); // LOOK double WTF ??
      if (values.length > 0) {
        fillValue = values[0];
        hasFillValue = true;
        fillType = att.getDataType();
        if (hasScaleOffset) forVar.remove(att);
        if (debug) System.out.println("missing_datum from _FillValue = " + fillValue);
      }
    }

    /// missing_value
    if (null != (att = forVar.findAttribute(CDM.MISSING_VALUE))) {
      if (att.isString()) {
        String svalue = att.getStringValue();
        if (forVar.getDataType() == DataType.CHAR) {
          missingValue = new double[1];
          if (svalue.length() == 0) missingValue[0] = 0;
          else missingValue[0] = svalue.charAt(0);

          missType = DataType.CHAR;
          hasMissingValue = true;

        } else { // not a CHAR - try to fix problem where they use a numeric value as a String
                 // attribute

          try {
            missingValue = new double[1];
            missingValue[0] = Double.parseDouble(svalue);
            missType = att.getDataType();
            hasMissingValue = true;
          } catch (NumberFormatException ex) {
            if (debug)
              System.out.println(
                  "String missing_value not parsable as double= " + att.getStringValue());
          }
        }

      } else { // not a string
        missingValue = getValueAsDouble(att);
        missType = att.getDataType();
        for (double mv : missingValue)
          if (!Double.isNaN(mv))
            hasMissingValue = true; // dont need to do anything if its already a NaN
      }
      if (hasScaleOffset) forVar.remove(att);
    }

    // missing
    boolean hasMissing =
        (invalidDataIsMissing && hasValidData)
            || (fillValueIsMissing && hasFillValue)
            || (missingDataIsMissing && hasMissingValue);

    /// assign convertedDataType if needed
    if (hasScaleOffset) {

      convertedDataType = forVar.getDataType();
      if (hasMissing) {
        // has missing data : must be float or double
        if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
        if (missingDataIsMissing && rank(missType) > rank(convertedDataType))
          convertedDataType = missType;
        if (fillValueIsMissing && rank(fillType) > rank(convertedDataType))
          convertedDataType = fillType;
        if (invalidDataIsMissing && rank(validType) > rank(convertedDataType))
          convertedDataType = validType;
        if (rank(convertedDataType) < rank(DataType.DOUBLE)) convertedDataType = DataType.FLOAT;

      } else {
        // no missing data; can use wider of data and scale
        if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
      }
      if (debug) System.out.println("assign dataType = " + convertedDataType);

      // validData may be external or internal
      if (hasValidData) {
        DataType orgType = forVar.getDataType();

        // If valid_range is the same type as scale_factor (actually the wider of
        // scale_factor and add_offset) and this is wider than the external data, then it
        // will be interpreted as being in the units of the internal (unpacked) data.
        // Otherwise it is in the units of the external (unpacked) data.
        // we assumed unpacked data above, redo if its really packed data
        if (!((rank(validType) == rank(scaleType)) && (rank(scaleType) >= rank(orgType)))) {
          if (validRangeAtt != null) {
            double[] values = getValueAsDouble(validRangeAtt);
            valid_min = values[0];
            valid_max = values[1];
          } else {
            if (validMinAtt != null) {
              double[] values = getValueAsDouble(validMinAtt);
              valid_min = values[0];
            }
            if (validMaxAtt != null) {
              double[] values = getValueAsDouble(validMaxAtt);
              valid_max = values[0];
            }
          }
        }
      }
    }

    if (hasMissing
        && ((convertedDataType == DataType.DOUBLE) || (convertedDataType == DataType.FLOAT)))
      this.useNaNs = useNaNs;
    if (debug) System.out.println("this.useNaNs = " + this.useNaNs);
  }
コード例 #9
0
ファイル: Write2ncRect.java プロジェクト: nbald/thredds
  Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
      throws IOException, InvalidRangeException {

    NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
    if (debug) {
      System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
    }

    // global attributes
    List<Attribute> glist = bufr.getGlobalAttributes();
    for (Attribute att : glist) {
      String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
      Attribute useAtt;
      if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
      else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
      else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
      if (debug) System.out.println("add gatt= " + useAtt);
    }

    // global dimensions
    Dimension recordDim = null;
    Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
    for (Dimension oldD : bufr.getDimensions()) {
      String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
      boolean isRecord = useName.equals("record");
      Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
      dimHash.put(newD.getName(), newD);
      if (isRecord) recordDim = newD;
      if (debug) System.out.println("add dim= " + newD);
    }

    // Variables
    Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
    for (Variable oldVar : recordStruct.getVariables()) {
      if (oldVar.getDataType() == DataType.STRUCTURE) continue;

      String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
      DataType newType = oldVar.getDataType();

      List<Dimension> newDims = new ArrayList<Dimension>();
      newDims.add(recordDim);
      for (Dimension dim : oldVar.getDimensions()) {
        newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
      }

      Variable newVar = ncfile.addVariable(varName, newType, newDims);
      if (debug) System.out.println("add var= " + newVar);

      // attributes
      List<Attribute> attList = oldVar.getAttributes();
      for (Attribute att : attList) {
        String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
        if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
        else if (att.isString())
          ncfile.addVariableAttribute(varName, useName, att.getStringValue());
        else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
      }
    }

    // int max_seq = countSeq(recordStruct);
    // Dimension seqD = ncfile.addDimension("level", max_seq);

    for (Variable v : recordStruct.getVariables()) {
      if (v.getDataType() != DataType.STRUCTURE) continue;
      String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
      int shape[] = v.getShape();

      Dimension structDim = ncfile.addDimension(structName, shape[0]);

      Structure struct = (Structure) v;
      for (Variable seqVar : struct.getVariables()) {
        String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
        DataType newType = seqVar.getDataType();

        List<Dimension> newDims = new ArrayList<Dimension>();
        newDims.add(recordDim);
        newDims.add(structDim);
        for (Dimension dim : seqVar.getDimensions()) {
          newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
        }

        Variable newVar = ncfile.addVariable(varName, newType, newDims);
        if (debug) System.out.println("add var= " + newVar);

        // attributes
        List<Attribute> attList = seqVar.getAttributes();
        for (Attribute att : attList) {
          String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
          if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
          else if (att.isString())
            ncfile.addVariableAttribute(varName, useName, att.getStringValue());
          else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
        }
      }
    }

    // create the file
    ncfile.create();
    if (debug) System.out.println("File Out= " + ncfile.toString());

    // boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

    double total = copyVarData(ncfile, recordStruct);
    ncfile.flush();
    System.out.println("FileWriter done total bytes = " + total);
    ncfile.close();
  }