Beispiel #1
0
  /**
   * Read data from encoded values and run len into regular data array
   *
   * @param ddata is encoded data values
   * @return the data array of row data
   */
  public byte[] readOneRowData(byte[] ddata, int rLen, int xt)
      throws IOException, InvalidRangeException {
    int run;
    byte[] bdata = new byte[xt];
    int nbin = 0;
    int total = 0;

    for (run = 0; run < rLen; run++) {
      int drun = DataType.unsignedByteToShort(ddata[run]) >> 4;
      byte dcode1 = (byte) (DataType.unsignedByteToShort(ddata[run]) & 0Xf);

      for (int i = 0; i < drun; i++) {
        bdata[nbin++] = dcode1;
        total++;
      }
    }

    if (total < xt) {
      for (run = total; run < xt; run++) {
        bdata[run] = 0;
      }
    }

    return bdata;
  }
Beispiel #2
0
 long computeFieldSize(BaseType bt, boolean isAscii) throws Exception {
   long fieldsize = 0;
   // Figure out what this field is (e.g. primitive or not)
   // Somewhat convoluted.
   if (bt instanceof DConstructor) {
     // simple struct, seq, or grid => recurse
     fieldsize = computeSize((DConstructor) bt, isAscii);
   } else if (bt instanceof DArray) {
     SDArray da = (SDArray) bt;
     // Separate structure arrays from primitive arrays
     if (da.getContainerVar() instanceof DPrimitive) {
       fieldsize = computeArraySize(da);
     } else if (da.getContainerVar() instanceof DStructure) {
       fieldsize = computeSize((DStructure) da.getContainerVar(), isAscii); // recurse
     } else { // Some kind of problem
       throw new NoSuchTypeException("Computesize: unexpected type for " + bt.getLongName());
     }
   } else if (bt instanceof DPrimitive) {
     DPrimitive dp = (DPrimitive) bt;
     if (dp instanceof DString) {
       String v = ((DString) dp).getValue();
       fieldsize = (v == null ? 0 : v.length());
     } else {
       DataType dtype = DODSNetcdfFile.convertToNCType(bt);
       fieldsize = dtype.getSize();
     }
   } else { // Some kind of problem
     throw new NoSuchTypeException("Computesize: unknown type for " + bt.getLongName());
   }
   return fieldsize;
 }
Beispiel #3
0
 /**
  * Create a scalar numeric-valued Attribute.
  *
  * @param name name of Attribute
  * @param val value of Attribute
  */
 public Attribute(String name, Number val) {
   this.name = name;
   int[] shape = new int[1];
   shape[0] = 1;
   DataType dt = DataType.getType(val.getClass());
   Array vala = Array.factory(dt.getPrimitiveClassType(), shape);
   Index ima = vala.getIndex();
   vala.setDouble(ima.set0(0), val.doubleValue());
   setValues(vala);
 }
 /**
  * Get a 1DArray for the type and length
  *
  * @param type DataType
  * @param len length
  * @return the array
  */
 private Array get1DArray(DataType type, int len) {
   Array varArray = null;
   if (type.equals(DataType.FLOAT)) {
     varArray = new ArrayFloat.D1(len);
   } else if (type.equals(DataType.DOUBLE)) {
     varArray = new ArrayDouble.D1(len);
   } else if (type.equals(DataType.INT)) {
     varArray = new ArrayInt.D1(len);
   }
   return varArray;
 }
  protected Array convertEnums(Array values) {
    DataType dt = DataType.getType(values.getElementType());
    if (!dt.isNumeric()) System.out.println("HEY");

    Array result = Array.factory(DataType.STRING, values.getShape());
    IndexIterator ii = result.getIndexIterator();
    values.resetLocalIterator();
    while (values.hasNext()) {
      String sval = lookupEnumString(values.nextInt());
      ii.setObjectNext(sval);
    }
    return result;
  }
Beispiel #6
0
 long computeArraySize(SDArray da) throws Exception {
   assert (da.getContainerVar() instanceof DPrimitive);
   BaseType base = da.getPrimitiveVector().getTemplate();
   DataType dtype = DODSNetcdfFile.convertToNCType(base);
   int elemSize = dtype.getSize();
   int n = da.numDimensions();
   List<Range> ranges = new ArrayList<Range>(n);
   long size = 0;
   for (int i = 0; i < n; i++) {
     ranges.add(new Range(da.getStart(i), da.getStop(i), da.getStride(i)));
     Section s = new Section(ranges);
     size += s.computeSize() * elemSize;
   }
   return size;
 }
Beispiel #7
0
  int getInt(byte[] b, int offset, int num) {
    int base = 1;
    int i;
    int word = 0;
    int bv[] = new int[num];

    for (i = 0; i < num; i++) {
      bv[i] = DataType.unsignedByteToShort(b[offset + i]);
    }

    if (bv[0] > 127) {
      bv[0] -= 128;
      base = -1;
    }

    /*
     * Calculate the integer value of the byte sequence
     */

    for (i = num - 1; i >= 0; i--) {
      word += base * bv[i];
      base *= 256;
    }

    return word;
  }
Beispiel #8
0
  /**
   * set the values from an Array
   *
   * @param arr value of Attribute
   */
  protected void setValues(Array arr) {
    if (arr == null) {
      dataType = DataType.STRING;
      return;
    }

    if (DataType.getType(arr.getElementType()) == null)
      throw new IllegalArgumentException("Cant set Attribute with type " + arr.getElementType());

    if (arr.getElementType() == char.class) { // turn CHAR into STRING
      ArrayChar carr = (ArrayChar) arr;
      if (carr.getRank() == 1) { // common case
        svalue = carr.getString();
        this.nelems = 1;
        this.dataType = DataType.STRING;
        return;
      }
      // otherwise its an array of Strings
      arr = carr.make1DStringArray();
    }

    // this should be a utility somewhere
    if (arr.getElementType() == ByteBuffer.class) { // turn OPAQUE into BYTE
      int totalLen = 0;
      arr.resetLocalIterator();
      while (arr.hasNext()) {
        ByteBuffer bb = (ByteBuffer) arr.next();
        totalLen += bb.limit();
      }
      byte[] ba = new byte[totalLen];
      int pos = 0;
      arr.resetLocalIterator();
      while (arr.hasNext()) {
        ByteBuffer bb = (ByteBuffer) arr.next();
        System.arraycopy(bb.array(), 0, ba, pos, bb.limit());
        pos += bb.limit();
      }
      arr = Array.factory(DataType.BYTE, new int[] {totalLen}, ba);
    }

    if (arr.getRank() > 1) arr = arr.reshape(new int[] {(int) arr.getSize()}); // make sure 1D

    this.values = arr;
    this.nelems = (int) arr.getSize();
    this.dataType = DataType.getType(arr.getElementType());
    hashCode = 0;
  }
 private void convertScaleOffsetUnsignedInt(IndexIterator iterIn, IndexIterator iterOut) {
   boolean checkMissing = useNaNs && hasMissing();
   while (iterIn.hasNext()) {
     int valb = iterIn.getIntNext();
     double val = scale * DataType.unsignedIntToLong(valb) + offset;
     iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
   }
 }
  public double convertScaleOffsetMissing(int vali) {
    if (!hasScaleOffset) return useNaNs && isMissing((double) vali) ? Double.NaN : (double) vali;

    double convertedValue;
    if (isUnsigned) convertedValue = scale * DataType.unsignedIntToLong(vali) + offset;
    else convertedValue = scale * vali + offset;

    return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
  }
  public double convertScaleOffsetMissing(byte valb) {
    if (!hasScaleOffset) return useNaNs && isMissing((double) valb) ? Double.NaN : (double) valb;

    double convertedValue;
    if (isUnsigned) convertedValue = scale * DataType.unsignedByteToShort(valb) + offset;
    else convertedValue = scale * valb + offset;

    return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
  }
  public double convertScaleOffsetMissing(short vals) {
    if (!hasScaleOffset) return useNaNs && isMissing((double) vals) ? Double.NaN : (double) vals;

    double convertedValue;
    if (isUnsigned) convertedValue = scale * DataType.unsignedShortToInt(vals) + offset;
    else convertedValue = scale * vals + offset;

    return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
  }
Beispiel #13
0
  /** Instances which have same content are equal. */
  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if ((o == null) || !(o instanceof Attribute)) return false;

    final Attribute att = (Attribute) o;

    if (!name.equals(att.name)) return false;
    if (nelems != att.nelems) return false;
    if (!dataType.equals(att.dataType)) return false;

    if (svalue != null) return svalue.equals(att.getStringValue());

    if (values != null) {
      for (int i = 0; i < getLength(); i++) {
        int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
        int r2 =
            att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
        if (r1 != r2) return false;
      }
    }

    return true;
  }
  /**
   * Convert Data with scale and offset. Also translate missing data to NaNs if useNaNs = true.
   *
   * @param in data to convert
   * @return converted data.
   */
  private Array convertScaleOffset(Array in) {
    if (!hasScaleOffset) return in;
    if (debugRead) System.out.println("convertScaleOffset ");

    Array out = Array.factory(convertedDataType.getPrimitiveClassType(), in.getShape());
    IndexIterator iterIn = in.getIndexIterator();
    IndexIterator iterOut = out.getIndexIterator();

    if (isUnsigned && in.getElementType() == byte.class)
      convertScaleOffsetUnsignedByte(iterIn, iterOut);
    else if (isUnsigned && in.getElementType() == short.class)
      convertScaleOffsetUnsignedShort(iterIn, iterOut);
    else if (isUnsigned && in.getElementType() == int.class)
      convertScaleOffsetUnsignedInt(iterIn, iterOut);
    else {
      boolean checkMissing = useNaNs && hasMissing();
      while (iterIn.hasNext()) {
        double val = scale * iterIn.getDoubleNext() + offset;
        iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
      }
    }

    return out;
  }
Beispiel #15
0
  private boolean compareData(String name, Array data1, Array data2, double tol, boolean justOne) {
    boolean ok = true;
    if (data1.getSize() != data2.getSize()) {
      f.format(" DIFF %s: size %d !== %d%n", name, data1.getSize(), data2.getSize());
      ok = false;
    }

    if (data1.getElementType() != data2.getElementType()) {
      f.format(
          " DIFF %s: element type %s !== %s%n",
          name, data1.getElementType(), data2.getElementType());
      ok = false;
    }

    if (!ok) return false;

    DataType dt = DataType.getType(data1.getElementType());

    IndexIterator iter1 = data1.getIndexIterator();
    IndexIterator iter2 = data2.getIndexIterator();

    if (dt == DataType.DOUBLE) {
      while (iter1.hasNext() && iter2.hasNext()) {
        double v1 = iter1.getDoubleNext();
        double v2 = iter2.getDoubleNext();
        if (!Double.isNaN(v1) || !Double.isNaN(v2))
          if (!closeEnough(v1, v2, tol)) {
            f.format(
                " DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n",
                name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
            ok = false;
            if (justOne) break;
          }
      }
    } else if (dt == DataType.FLOAT) {
      while (iter1.hasNext() && iter2.hasNext()) {
        float v1 = iter1.getFloatNext();
        float v2 = iter2.getFloatNext();
        if (!Float.isNaN(v1) || !Float.isNaN(v2))
          if (!closeEnough(v1, v2, (float) tol)) {
            f.format(
                " DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n",
                name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
            ok = false;
            if (justOne) break;
          }
      }
    } else if (dt == DataType.INT) {
      while (iter1.hasNext() && iter2.hasNext()) {
        int v1 = iter1.getIntNext();
        int v2 = iter2.getIntNext();
        if (v1 != v2) {
          f.format(
              " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n",
              name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt == DataType.SHORT) {
      while (iter1.hasNext() && iter2.hasNext()) {
        short v1 = iter1.getShortNext();
        short v2 = iter2.getShortNext();
        if (v1 != v2) {
          f.format(
              " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n",
              name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt == DataType.BYTE) {
      while (iter1.hasNext() && iter2.hasNext()) {
        byte v1 = iter1.getByteNext();
        byte v2 = iter2.getByteNext();
        if (v1 != v2) {
          f.format(
              " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n",
              name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
          ok = false;
          if (justOne) break;
        }
      }
    } else if (dt == DataType.STRUCTURE) {
      while (iter1.hasNext() && iter2.hasNext()) {
        compareStructureData(
            (StructureData) iter1.next(), (StructureData) iter2.next(), tol, justOne);
      }
    }

    return ok;
  }
Beispiel #16
0
  // all the work is here, so can be called recursively
  public Object readOneScanData(ByteBuffer bos, NOWRadheader.Vinfo vinfo, String vName)
      throws IOException, InvalidRangeException {
    int doff = (int) vinfo.hoff;
    int npixel = vinfo.yt * vinfo.xt;
    byte[] rdata = null;
    byte[] ldata = new byte[vinfo.xt];
    byte[] pdata = new byte[npixel];
    byte[] b2 = new byte[2];

    bos.position(doff);

    // begining of image data
    if ((DataType.unsignedByteToShort(bos.get()) != 0xF0) || (bos.get() != 0x0C)) {
      return null;
    }

    int ecode;
    int color;
    int datapos;
    int offset = 0;
    int roffset = 0;
    boolean newline = true;
    int linenum = 0;

    while (true) {

      // line number
      if (newline) {
        bos.get(b2);
        linenum = (DataType.unsignedByteToShort(b2[1]) << 8) + DataType.unsignedByteToShort(b2[0]);

        // System.out.println("Line Number = " + linenum);
      }

      // int linenum = bytesToInt(b2[0], b2[1], true);
      // System.out.println("Line Number = " + linenum);
      // if(linenum == 1225)
      //   System.out.println(" HHHHH");
      short b = DataType.unsignedByteToShort(bos.get());

      color = b & 0xF;
      ecode = b >> 4;
      datapos = bos.position();

      int datarun;

      if (ecode == 0xF) {
        byte bb1 = bos.get(datapos - 2);
        byte bb2 = bos.get(datapos);

        if ((color == 0x0) && (bb1 == 0x00) && (bb2 == 0x00)) {
          datapos += 1;
        }

        bos.position(datapos);
        datarun = 0;
      } else if (ecode == 0xE) {
        byte b0 = bos.get(datapos);

        datarun = DataType.unsignedByteToShort(b0) + 1;
        datapos += 1;
        bos.position(datapos);
      } else if (ecode == 0xD) {
        b2[0] = bos.get(datapos);
        b2[1] = bos.get(datapos + 1);
        datarun =
            (DataType.unsignedByteToShort(b2[1]) << 8) + DataType.unsignedByteToShort(b2[0]) + 1;
        datapos += 2;
        bos.position(datapos);
      } else {
        datarun = ecode + 1;
      }

      // move the unpacked data in the data line
      rdata = new byte[datarun];

      for (int i = 0; i < datarun; i++) {
        rdata[i] = (byte) color;
      }

      System.arraycopy(rdata, 0, ldata, roffset, datarun);
      roffset = roffset + datarun;

      // System.out.println("run ecode = " + ecode + " and data run " + datarun + " and totalrun " +
      // roffset);
      // check to see if the beginning of the next line or at the end of the file
      short c0 = DataType.unsignedByteToShort(bos.get());

      if (c0 == 0x00) {
        short c1 = DataType.unsignedByteToShort(bos.get());
        short c2 = DataType.unsignedByteToShort(bos.get());

        // System.out.println("c1 and c2 " + c1 + " " + c2);

        if ((c0 == 0x00) && (c1 == 0xF0) && (c2 == 0x0C)) {
          // beginning of next line
          //  System.out.println("linenum   " + linenum + "   and this line total " + roffset);
          //  if (roffset != 3661) {
          //      System.out.println("ERROR missing data, this line total only " + roffset);
          //  }
          System.arraycopy(ldata, 0, pdata, offset, roffset);
          offset = offset + vinfo.xt;
          roffset = 0;
          newline = true;
          ldata = new byte[vinfo.xt];
        } else if ((c1 == 0xF0) && (c2 == 0x02)) {
          // end of the file
          break;
        } else {
          datapos = bos.position() - 3;
          bos.position(datapos);
          newline = false;
        }
      } else {
        newline = false;
        datapos = bos.position();
        bos.position(datapos - 1);
      }
    }

    return pdata;
  }
  /**
   * Make a station variable
   *
   * @param varname variable name
   * @param firstDim station dimension
   * @return corresponding variable
   */
  protected Variable makeStationVariable(String varname, Dimension firstDim) {
    String longName = varname;
    String unit = null;
    DataType type = DataType.CHAR;
    List<Dimension> dims = new ArrayList<Dimension>();
    List<Attribute> attrs = new ArrayList<Attribute>();
    if (firstDim != null) {
      dims.add(firstDim);
    }

    if (varname.equals(GempakStation.STID)) {
      longName = "Station identifier";
      dims.add(DIM_LEN8);
    } else if (varname.equals(GempakStation.STNM)) {
      longName = "WMO station id";
      type = DataType.INT;
    } else if (varname.equals(GempakStation.SLAT)) {
      longName = "latitude";
      unit = "degrees_north";
      type = DataType.FLOAT;
      attrs.add(new Attribute("standard_name", "latitude"));
    } else if (varname.equals(GempakStation.SLON)) {
      longName = "longitude";
      unit = "degrees_east";
      type = DataType.FLOAT;
      attrs.add(new Attribute("standard_name", "longitude"));
    } else if (varname.equals(GempakStation.SELV)) {
      longName = "altitude";
      unit = "meter";
      type = DataType.FLOAT;
      attrs.add(new Attribute(CF.POSITIVE, CF.POSITIVE_UP));
      attrs.add(new Attribute("standard_name", "station_altitude"));
    } else if (varname.equals(GempakStation.STAT)) {
      longName = "state or province";
      dims.add(DIM_LEN2);
    } else if (varname.equals(GempakStation.COUN)) {
      longName = "country code";
      dims.add(DIM_LEN2);
    } else if (varname.equals(GempakStation.STD2)) {
      longName = "Extended station id";
      dims.add(DIM_LEN4);
    } else if (varname.equals(GempakStation.SPRI)) {
      longName = "Station priority";
      type = DataType.INT;
    } else if (varname.equals(GempakStation.SWFO)) {
      longName = "WFO code";
      dims.add(DIM_LEN4);
    } else if (varname.equals(GempakStation.WFO2)) {
      longName = "Second WFO code";
      dims.add(DIM_LEN4);
    }
    Variable v = new Variable(ncfile, null, null, varname);
    v.setDataType(type);
    v.addAttribute(new Attribute("long_name", longName));
    if (unit != null) {
      v.addAttribute(new Attribute(CDM.UNITS, unit));
    }
    if (type.equals(DataType.FLOAT)) {
      v.addAttribute(new Attribute("missing_value", RMISS));
    } else if (type.equals(DataType.INT)) {
      v.addAttribute(new Attribute("missing_value", IMISS));
    }
    if (!attrs.isEmpty()) {
      for (Attribute attr : attrs) {
        v.addAttribute(attr);
      }
    }
    if (!dims.isEmpty()) {
      v.setDimensions(dims);
    } else {
      v.setDimensions((String) null);
    }
    return v;
  }
  private int populateFile2(
      NetcdfFileWriteable ncFile,
      String varName,
      DataType dataType,
      int[] dimensions,
      int valueCounter,
      Random generator)
      throws IOException, InvalidRangeException {

    long singleWriteSize = maxMemory / dataType.getSize();
    int highestNonWriteDim = determineHighestNonWriteDimension(dimensions, singleWriteSize);
    int[] singleStep = createWriteStep(dimensions, singleWriteSize, highestNonWriteDim);
    int numStepWrites = determineNumberOfStepWrites(singleStep, singleWriteSize);

    System.out.println(
        "SingleWriteSize: "
            + singleWriteSize
            + " datatype size: "
            + dataType.getSize()
            + " singleStep: "
            + arrayToString(singleStep)
            + " numberStepsPerWrite: "
            + numStepWrites);

    int[] allOnes = new int[dimensions.length];
    int[] allZeros = new int[dimensions.length];

    for (int i = 0; i < allOnes.length; i++) {
      allOnes[i] = 1;
      allZeros[i] = 0;
    }

    Index origin = new Index(dimensions, allOnes);

    long totalSize = origin.getSize();
    long writtenSoFar = 0;

    boolean done = false;

    while (!done) {
      for (int i = 0; i < numStepWrites; i++) {
        // this is a crack at an optimization

        if ((highestNonWriteDim >= 0)
            && ((origin.getCurrentCounter()[highestNonWriteDim] + numStepWrites)
                <= dimensions[highestNonWriteDim])) {
          singleStep[highestNonWriteDim] = numStepWrites;
          System.out.println("JUST OPTIMIZED. New write step: " + arrayToString(singleStep));
          // keep 'i' right
          i += numStepWrites - 1;
        } else {
          singleStep[highestNonWriteDim] = 1;
        }

        Array array = new ArrayInt(singleStep);
        IndexIterator iter = array.getIndexIterator();

        while (iter.hasNext()) {
          iter.getIntNext();
          // uncomment the following line for a random distribution
          // iter.setIntCurrent((int)  (Math.abs(generator.nextGaussian()) * 40) );
          // uncomment this line for an incrementing value
          iter.setIntCurrent(valueCounter);
          valueCounter++;

          // book keeping
          writtenSoFar++;
          // origin.incr();
        }

        System.out.println(
            "Writing to file: "
                + ncFile.getLocation()
                + ". var_name: "
                + varName
                + " origin: "
                + arrayToString(origin.getCurrentCounter())
                + " writeSize: "
                + array.getSize()
                + " write shape: "
                + arrayToString(singleStep));

        // write to the actual file
        ncFile.write(varName, origin.getCurrentCounter(), array);

        // update origin accordingly

        for (int j = 0; j < Index.computeSize(singleStep); j++) {
          // writtenSoFar++;
          origin.incr();
        }

        System.out.println(
            "\tcurrentIndex: "
                + arrayToString(origin.getCurrentCounter())
                + " currentElement: "
                + origin.currentElement()
                + " totalsize: "
                + (totalSize)
                + " writtenSoFar: "
                + writtenSoFar);

        if (writtenSoFar >= totalSize) {
          done = true;
          break;
        }
      }
    }

    return valueCounter;
  }