/** * Read data from encoded values and run len into regular data array * * @param ddata is encoded data values * @return the data array of row data */ public byte[] readOneRowData(byte[] ddata, int rLen, int xt) throws IOException, InvalidRangeException { int run; byte[] bdata = new byte[xt]; int nbin = 0; int total = 0; for (run = 0; run < rLen; run++) { int drun = DataType.unsignedByteToShort(ddata[run]) >> 4; byte dcode1 = (byte) (DataType.unsignedByteToShort(ddata[run]) & 0Xf); for (int i = 0; i < drun; i++) { bdata[nbin++] = dcode1; total++; } } if (total < xt) { for (run = total; run < xt; run++) { bdata[run] = 0; } } return bdata; }
long computeFieldSize(BaseType bt, boolean isAscii) throws Exception { long fieldsize = 0; // Figure out what this field is (e.g. primitive or not) // Somewhat convoluted. if (bt instanceof DConstructor) { // simple struct, seq, or grid => recurse fieldsize = computeSize((DConstructor) bt, isAscii); } else if (bt instanceof DArray) { SDArray da = (SDArray) bt; // Separate structure arrays from primitive arrays if (da.getContainerVar() instanceof DPrimitive) { fieldsize = computeArraySize(da); } else if (da.getContainerVar() instanceof DStructure) { fieldsize = computeSize((DStructure) da.getContainerVar(), isAscii); // recurse } else { // Some kind of problem throw new NoSuchTypeException("Computesize: unexpected type for " + bt.getLongName()); } } else if (bt instanceof DPrimitive) { DPrimitive dp = (DPrimitive) bt; if (dp instanceof DString) { String v = ((DString) dp).getValue(); fieldsize = (v == null ? 0 : v.length()); } else { DataType dtype = DODSNetcdfFile.convertToNCType(bt); fieldsize = dtype.getSize(); } } else { // Some kind of problem throw new NoSuchTypeException("Computesize: unknown type for " + bt.getLongName()); } return fieldsize; }
/** * Create a scalar numeric-valued Attribute. * * @param name name of Attribute * @param val value of Attribute */ public Attribute(String name, Number val) { this.name = name; int[] shape = new int[1]; shape[0] = 1; DataType dt = DataType.getType(val.getClass()); Array vala = Array.factory(dt.getPrimitiveClassType(), shape); Index ima = vala.getIndex(); vala.setDouble(ima.set0(0), val.doubleValue()); setValues(vala); }
/** * Get a 1DArray for the type and length * * @param type DataType * @param len length * @return the array */ private Array get1DArray(DataType type, int len) { Array varArray = null; if (type.equals(DataType.FLOAT)) { varArray = new ArrayFloat.D1(len); } else if (type.equals(DataType.DOUBLE)) { varArray = new ArrayDouble.D1(len); } else if (type.equals(DataType.INT)) { varArray = new ArrayInt.D1(len); } return varArray; }
protected Array convertEnums(Array values) { DataType dt = DataType.getType(values.getElementType()); if (!dt.isNumeric()) System.out.println("HEY"); Array result = Array.factory(DataType.STRING, values.getShape()); IndexIterator ii = result.getIndexIterator(); values.resetLocalIterator(); while (values.hasNext()) { String sval = lookupEnumString(values.nextInt()); ii.setObjectNext(sval); } return result; }
long computeArraySize(SDArray da) throws Exception { assert (da.getContainerVar() instanceof DPrimitive); BaseType base = da.getPrimitiveVector().getTemplate(); DataType dtype = DODSNetcdfFile.convertToNCType(base); int elemSize = dtype.getSize(); int n = da.numDimensions(); List<Range> ranges = new ArrayList<Range>(n); long size = 0; for (int i = 0; i < n; i++) { ranges.add(new Range(da.getStart(i), da.getStop(i), da.getStride(i))); Section s = new Section(ranges); size += s.computeSize() * elemSize; } return size; }
int getInt(byte[] b, int offset, int num) { int base = 1; int i; int word = 0; int bv[] = new int[num]; for (i = 0; i < num; i++) { bv[i] = DataType.unsignedByteToShort(b[offset + i]); } if (bv[0] > 127) { bv[0] -= 128; base = -1; } /* * Calculate the integer value of the byte sequence */ for (i = num - 1; i >= 0; i--) { word += base * bv[i]; base *= 256; } return word; }
/** * set the values from an Array * * @param arr value of Attribute */ protected void setValues(Array arr) { if (arr == null) { dataType = DataType.STRING; return; } if (DataType.getType(arr.getElementType()) == null) throw new IllegalArgumentException("Cant set Attribute with type " + arr.getElementType()); if (arr.getElementType() == char.class) { // turn CHAR into STRING ArrayChar carr = (ArrayChar) arr; if (carr.getRank() == 1) { // common case svalue = carr.getString(); this.nelems = 1; this.dataType = DataType.STRING; return; } // otherwise its an array of Strings arr = carr.make1DStringArray(); } // this should be a utility somewhere if (arr.getElementType() == ByteBuffer.class) { // turn OPAQUE into BYTE int totalLen = 0; arr.resetLocalIterator(); while (arr.hasNext()) { ByteBuffer bb = (ByteBuffer) arr.next(); totalLen += bb.limit(); } byte[] ba = new byte[totalLen]; int pos = 0; arr.resetLocalIterator(); while (arr.hasNext()) { ByteBuffer bb = (ByteBuffer) arr.next(); System.arraycopy(bb.array(), 0, ba, pos, bb.limit()); pos += bb.limit(); } arr = Array.factory(DataType.BYTE, new int[] {totalLen}, ba); } if (arr.getRank() > 1) arr = arr.reshape(new int[] {(int) arr.getSize()}); // make sure 1D this.values = arr; this.nelems = (int) arr.getSize(); this.dataType = DataType.getType(arr.getElementType()); hashCode = 0; }
private void convertScaleOffsetUnsignedInt(IndexIterator iterIn, IndexIterator iterOut) { boolean checkMissing = useNaNs && hasMissing(); while (iterIn.hasNext()) { int valb = iterIn.getIntNext(); double val = scale * DataType.unsignedIntToLong(valb) + offset; iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val); } }
public double convertScaleOffsetMissing(int vali) { if (!hasScaleOffset) return useNaNs && isMissing((double) vali) ? Double.NaN : (double) vali; double convertedValue; if (isUnsigned) convertedValue = scale * DataType.unsignedIntToLong(vali) + offset; else convertedValue = scale * vali + offset; return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue; }
public double convertScaleOffsetMissing(byte valb) { if (!hasScaleOffset) return useNaNs && isMissing((double) valb) ? Double.NaN : (double) valb; double convertedValue; if (isUnsigned) convertedValue = scale * DataType.unsignedByteToShort(valb) + offset; else convertedValue = scale * valb + offset; return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue; }
public double convertScaleOffsetMissing(short vals) { if (!hasScaleOffset) return useNaNs && isMissing((double) vals) ? Double.NaN : (double) vals; double convertedValue; if (isUnsigned) convertedValue = scale * DataType.unsignedShortToInt(vals) + offset; else convertedValue = scale * vals + offset; return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue; }
/** Instances which have same content are equal. */ @Override public boolean equals(Object o) { if (this == o) return true; if ((o == null) || !(o instanceof Attribute)) return false; final Attribute att = (Attribute) o; if (!name.equals(att.name)) return false; if (nelems != att.nelems) return false; if (!dataType.equals(att.dataType)) return false; if (svalue != null) return svalue.equals(att.getStringValue()); if (values != null) { for (int i = 0; i < getLength(); i++) { int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode(); int r2 = att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode(); if (r1 != r2) return false; } } return true; }
/** * Convert Data with scale and offset. Also translate missing data to NaNs if useNaNs = true. * * @param in data to convert * @return converted data. */ private Array convertScaleOffset(Array in) { if (!hasScaleOffset) return in; if (debugRead) System.out.println("convertScaleOffset "); Array out = Array.factory(convertedDataType.getPrimitiveClassType(), in.getShape()); IndexIterator iterIn = in.getIndexIterator(); IndexIterator iterOut = out.getIndexIterator(); if (isUnsigned && in.getElementType() == byte.class) convertScaleOffsetUnsignedByte(iterIn, iterOut); else if (isUnsigned && in.getElementType() == short.class) convertScaleOffsetUnsignedShort(iterIn, iterOut); else if (isUnsigned && in.getElementType() == int.class) convertScaleOffsetUnsignedInt(iterIn, iterOut); else { boolean checkMissing = useNaNs && hasMissing(); while (iterIn.hasNext()) { double val = scale * iterIn.getDoubleNext() + offset; iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val); } } return out; }
private boolean compareData(String name, Array data1, Array data2, double tol, boolean justOne) { boolean ok = true; if (data1.getSize() != data2.getSize()) { f.format(" DIFF %s: size %d !== %d%n", name, data1.getSize(), data2.getSize()); ok = false; } if (data1.getElementType() != data2.getElementType()) { f.format( " DIFF %s: element type %s !== %s%n", name, data1.getElementType(), data2.getElementType()); ok = false; } if (!ok) return false; DataType dt = DataType.getType(data1.getElementType()); IndexIterator iter1 = data1.getIndexIterator(); IndexIterator iter2 = data2.getIndexIterator(); if (dt == DataType.DOUBLE) { while (iter1.hasNext() && iter2.hasNext()) { double v1 = iter1.getDoubleNext(); double v2 = iter2.getDoubleNext(); if (!Double.isNaN(v1) || !Double.isNaN(v2)) if (!closeEnough(v1, v2, tol)) { f.format( " DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.FLOAT) { while (iter1.hasNext() && iter2.hasNext()) { float v1 = iter1.getFloatNext(); float v2 = iter2.getFloatNext(); if (!Float.isNaN(v1) || !Float.isNaN(v2)) if (!closeEnough(v1, v2, (float) tol)) { f.format( " DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.INT) { while (iter1.hasNext() && iter2.hasNext()) { int v1 = iter1.getIntNext(); int v2 = iter2.getIntNext(); if (v1 != v2) { f.format( " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.SHORT) { while (iter1.hasNext() && iter2.hasNext()) { short v1 = iter1.getShortNext(); short v2 = iter2.getShortNext(); if (v1 != v2) { f.format( " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.BYTE) { while (iter1.hasNext() && iter2.hasNext()) { byte v1 = iter1.getByteNext(); byte v2 = iter2.getByteNext(); if (v1 != v2) { f.format( " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.STRUCTURE) { while (iter1.hasNext() && iter2.hasNext()) { compareStructureData( (StructureData) iter1.next(), (StructureData) iter2.next(), tol, justOne); } } return ok; }
// all the work is here, so can be called recursively public Object readOneScanData(ByteBuffer bos, NOWRadheader.Vinfo vinfo, String vName) throws IOException, InvalidRangeException { int doff = (int) vinfo.hoff; int npixel = vinfo.yt * vinfo.xt; byte[] rdata = null; byte[] ldata = new byte[vinfo.xt]; byte[] pdata = new byte[npixel]; byte[] b2 = new byte[2]; bos.position(doff); // begining of image data if ((DataType.unsignedByteToShort(bos.get()) != 0xF0) || (bos.get() != 0x0C)) { return null; } int ecode; int color; int datapos; int offset = 0; int roffset = 0; boolean newline = true; int linenum = 0; while (true) { // line number if (newline) { bos.get(b2); linenum = (DataType.unsignedByteToShort(b2[1]) << 8) + DataType.unsignedByteToShort(b2[0]); // System.out.println("Line Number = " + linenum); } // int linenum = bytesToInt(b2[0], b2[1], true); // System.out.println("Line Number = " + linenum); // if(linenum == 1225) // System.out.println(" HHHHH"); short b = DataType.unsignedByteToShort(bos.get()); color = b & 0xF; ecode = b >> 4; datapos = bos.position(); int datarun; if (ecode == 0xF) { byte bb1 = bos.get(datapos - 2); byte bb2 = bos.get(datapos); if ((color == 0x0) && (bb1 == 0x00) && (bb2 == 0x00)) { datapos += 1; } bos.position(datapos); datarun = 0; } else if (ecode == 0xE) { byte b0 = bos.get(datapos); datarun = DataType.unsignedByteToShort(b0) + 1; datapos += 1; bos.position(datapos); } else if (ecode == 0xD) { b2[0] = bos.get(datapos); b2[1] = bos.get(datapos + 1); datarun = (DataType.unsignedByteToShort(b2[1]) << 8) + DataType.unsignedByteToShort(b2[0]) + 1; datapos += 2; bos.position(datapos); } else { datarun = ecode + 1; } // move the unpacked data in the data line rdata = new byte[datarun]; for (int i = 0; i < datarun; i++) { rdata[i] = (byte) color; } System.arraycopy(rdata, 0, ldata, roffset, datarun); roffset = roffset + datarun; // System.out.println("run ecode = " + ecode + " and data run " + datarun + " and totalrun " + // roffset); // check to see if the beginning of the next line or at the end of the file short c0 = DataType.unsignedByteToShort(bos.get()); if (c0 == 0x00) { short c1 = DataType.unsignedByteToShort(bos.get()); short c2 = DataType.unsignedByteToShort(bos.get()); // System.out.println("c1 and c2 " + c1 + " " + c2); if ((c0 == 0x00) && (c1 == 0xF0) && (c2 == 0x0C)) { // beginning of next line // System.out.println("linenum " + linenum + " and this line total " + roffset); // if (roffset != 3661) { // System.out.println("ERROR missing data, this line total only " + roffset); // } System.arraycopy(ldata, 0, pdata, offset, roffset); offset = offset + vinfo.xt; roffset = 0; newline = true; ldata = new byte[vinfo.xt]; } else if ((c1 == 0xF0) && (c2 == 0x02)) { // end of the file break; } else { datapos = bos.position() - 3; bos.position(datapos); newline = false; } } else { newline = false; datapos = bos.position(); bos.position(datapos - 1); } } return pdata; }
/** * Make a station variable * * @param varname variable name * @param firstDim station dimension * @return corresponding variable */ protected Variable makeStationVariable(String varname, Dimension firstDim) { String longName = varname; String unit = null; DataType type = DataType.CHAR; List<Dimension> dims = new ArrayList<Dimension>(); List<Attribute> attrs = new ArrayList<Attribute>(); if (firstDim != null) { dims.add(firstDim); } if (varname.equals(GempakStation.STID)) { longName = "Station identifier"; dims.add(DIM_LEN8); } else if (varname.equals(GempakStation.STNM)) { longName = "WMO station id"; type = DataType.INT; } else if (varname.equals(GempakStation.SLAT)) { longName = "latitude"; unit = "degrees_north"; type = DataType.FLOAT; attrs.add(new Attribute("standard_name", "latitude")); } else if (varname.equals(GempakStation.SLON)) { longName = "longitude"; unit = "degrees_east"; type = DataType.FLOAT; attrs.add(new Attribute("standard_name", "longitude")); } else if (varname.equals(GempakStation.SELV)) { longName = "altitude"; unit = "meter"; type = DataType.FLOAT; attrs.add(new Attribute(CF.POSITIVE, CF.POSITIVE_UP)); attrs.add(new Attribute("standard_name", "station_altitude")); } else if (varname.equals(GempakStation.STAT)) { longName = "state or province"; dims.add(DIM_LEN2); } else if (varname.equals(GempakStation.COUN)) { longName = "country code"; dims.add(DIM_LEN2); } else if (varname.equals(GempakStation.STD2)) { longName = "Extended station id"; dims.add(DIM_LEN4); } else if (varname.equals(GempakStation.SPRI)) { longName = "Station priority"; type = DataType.INT; } else if (varname.equals(GempakStation.SWFO)) { longName = "WFO code"; dims.add(DIM_LEN4); } else if (varname.equals(GempakStation.WFO2)) { longName = "Second WFO code"; dims.add(DIM_LEN4); } Variable v = new Variable(ncfile, null, null, varname); v.setDataType(type); v.addAttribute(new Attribute("long_name", longName)); if (unit != null) { v.addAttribute(new Attribute(CDM.UNITS, unit)); } if (type.equals(DataType.FLOAT)) { v.addAttribute(new Attribute("missing_value", RMISS)); } else if (type.equals(DataType.INT)) { v.addAttribute(new Attribute("missing_value", IMISS)); } if (!attrs.isEmpty()) { for (Attribute attr : attrs) { v.addAttribute(attr); } } if (!dims.isEmpty()) { v.setDimensions(dims); } else { v.setDimensions((String) null); } return v; }
private int populateFile2( NetcdfFileWriteable ncFile, String varName, DataType dataType, int[] dimensions, int valueCounter, Random generator) throws IOException, InvalidRangeException { long singleWriteSize = maxMemory / dataType.getSize(); int highestNonWriteDim = determineHighestNonWriteDimension(dimensions, singleWriteSize); int[] singleStep = createWriteStep(dimensions, singleWriteSize, highestNonWriteDim); int numStepWrites = determineNumberOfStepWrites(singleStep, singleWriteSize); System.out.println( "SingleWriteSize: " + singleWriteSize + " datatype size: " + dataType.getSize() + " singleStep: " + arrayToString(singleStep) + " numberStepsPerWrite: " + numStepWrites); int[] allOnes = new int[dimensions.length]; int[] allZeros = new int[dimensions.length]; for (int i = 0; i < allOnes.length; i++) { allOnes[i] = 1; allZeros[i] = 0; } Index origin = new Index(dimensions, allOnes); long totalSize = origin.getSize(); long writtenSoFar = 0; boolean done = false; while (!done) { for (int i = 0; i < numStepWrites; i++) { // this is a crack at an optimization if ((highestNonWriteDim >= 0) && ((origin.getCurrentCounter()[highestNonWriteDim] + numStepWrites) <= dimensions[highestNonWriteDim])) { singleStep[highestNonWriteDim] = numStepWrites; System.out.println("JUST OPTIMIZED. New write step: " + arrayToString(singleStep)); // keep 'i' right i += numStepWrites - 1; } else { singleStep[highestNonWriteDim] = 1; } Array array = new ArrayInt(singleStep); IndexIterator iter = array.getIndexIterator(); while (iter.hasNext()) { iter.getIntNext(); // uncomment the following line for a random distribution // iter.setIntCurrent((int) (Math.abs(generator.nextGaussian()) * 40) ); // uncomment this line for an incrementing value iter.setIntCurrent(valueCounter); valueCounter++; // book keeping writtenSoFar++; // origin.incr(); } System.out.println( "Writing to file: " + ncFile.getLocation() + ". var_name: " + varName + " origin: " + arrayToString(origin.getCurrentCounter()) + " writeSize: " + array.getSize() + " write shape: " + arrayToString(singleStep)); // write to the actual file ncFile.write(varName, origin.getCurrentCounter(), array); // update origin accordingly for (int j = 0; j < Index.computeSize(singleStep); j++) { // writtenSoFar++; origin.incr(); } System.out.println( "\tcurrentIndex: " + arrayToString(origin.getCurrentCounter()) + " currentElement: " + origin.currentElement() + " totalsize: " + (totalSize) + " writtenSoFar: " + writtenSoFar); if (writtenSoFar >= totalSize) { done = true; break; } } } return valueCounter; }