private void showValues2D(CoordinateAxis2D axis2D) { Formatter f = new Formatter(); if (axis2D.isInterval()) { ArrayDouble.D2 coords = axis2D.getCoordValuesArray(); ArrayDouble.D3 bounds = axis2D.getCoordBoundsArray(); if (bounds == null) { infoTA.appendLine("No bounds for interval " + axis2D.getFullName()); return; } IndexIterator coordIter = coords.getIndexIterator(); IndexIterator boundsIter = bounds.getIndexIterator(); while (coordIter.hasNext()) { double coordValue = coordIter.getDoubleNext(); if (!boundsIter.hasNext()) break; double bounds1 = boundsIter.getDoubleNext(); if (!boundsIter.hasNext()) break; double bounds2 = boundsIter.getDoubleNext(); f.format("%f (%f,%f) = %f%n", coordValue, bounds1, bounds2, bounds2 - bounds1); } } else { ArrayDouble.D2 coords = axis2D.getCoordValuesArray(); IndexIterator coordIter = coords.getIndexIterator(); while (coordIter.hasNext()) { double coordValue = coordIter.getDoubleNext(); f.format("%f%n", coordValue); } } infoTA.appendLine(f.toString()); }
private void writeCoordinateVariable(VariableEntry variable, NetcdfFileWriteable ncFile) throws IOException, InvalidRangeException { int[] dimensions = {variable.getSize()}; Array array; // so far, everything is a float or an int // way too much code duplication but I'm done fightin java for now if (variable.getType() == DataType.INT) { array = new ArrayInt(dimensions); int tempInt = 0; IndexIterator iter = array.getIndexIterator(); while (iter.hasNext()) { iter.getIntNext(); iter.setIntCurrent(tempInt); tempInt++; } ncFile.write(variable.getVariableName(), array); } else if (variable.getType() == DataType.FLOAT) { array = new ArrayFloat(dimensions); float tempFloat = 0; IndexIterator iter = array.getIndexIterator(); while (iter.hasNext()) { iter.getFloatNext(); iter.setFloatCurrent(tempFloat); tempFloat++; } ncFile.write(variable.getVariableName(), array); } else if (variable.getType() == DataType.LONG) { array = new ArrayLong(dimensions); long tempLong = 0; IndexIterator iter = array.getIndexIterator(); while (iter.hasNext()) { iter.getLongNext(); iter.setLongCurrent(tempLong); tempLong++; } ncFile.write(variable.getVariableName(), array); } // ncFile.write(variable.getVariableName(),array); }
// @todo Make sure units are meters public Array getElevation(Range range) throws IOException, InvalidRangeException { List section = new ArrayList(1); section.add(range); Array a = elevVar.read(section); if (elevVarUnitsConversionFactor == 1.0) return (a); for (IndexIterator it = a.getIndexIterator(); it.hasNext(); ) { if (elevVar.getDataType() == DataType.DOUBLE) { double val = it.getDoubleNext(); it.setDoubleCurrent(val * elevVarUnitsConversionFactor); } else if (elevVar.getDataType() == DataType.FLOAT) { float val = it.getFloatNext(); it.setFloatCurrent((float) (val * elevVarUnitsConversionFactor)); } else if (elevVar.getDataType() == DataType.INT) { int val = it.getIntNext(); it.setIntCurrent((int) (val * elevVarUnitsConversionFactor)); } else if (elevVar.getDataType() == DataType.LONG) { long val = it.getLongNext(); it.setLongCurrent((long) (val * elevVarUnitsConversionFactor)); } else { throw new IllegalStateException( "Elevation variable type <" + elevVar.getDataType().toString() + "> not double, float, int, or long."); } } return (a); }
public void testEnhanceDefer() throws IOException { NetcdfDataset ncd = NetcdfDataset.openDataset( filename, EnumSet.of(NetcdfDataset.Enhance.ScaleMissing), -1, null, null); VariableDS enhancedVar = (VariableDS) ncd.findVariable("t1"); NetcdfDataset ncdefer = NetcdfDataset.openDataset( filename, EnumSet.of(NetcdfDataset.Enhance.ScaleMissingDefer), -1, null, null); VariableDS deferVar = (VariableDS) ncdefer.findVariable("t1"); Array data = enhancedVar.read(); Array dataDefer = deferVar.read(); System.out.printf("Enhanced="); NCdumpW.printArray(data); System.out.printf("%nDeferred="); NCdumpW.printArray(dataDefer); System.out.printf("%nProcessed="); CompareNetcdf2 nc = new CompareNetcdf2(new Formatter(System.out), false, false, true); assert !nc.compareData(enhancedVar.getShortName(), data, dataDefer, false); IndexIterator ii = dataDefer.getIndexIterator(); while (ii.hasNext()) { double val = deferVar.convertScaleOffsetMissing(ii.getDoubleNext()); ii.setDoubleCurrent(val); } NCdumpW.printArray(dataDefer); assert nc.compareData(enhancedVar.getShortName(), data, dataDefer, false); ncd.close(); ncdefer.close(); }
public void testAggCoordVar(NetcdfFile ncfile) { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1; assert time.getSize() == 3; assert time.getShape()[0] == 3; assert time.getDataType() == DataType.DOUBLE; assert time.getDimension(0) == ncfile.findDimension("time"); try { Array data = time.read(); assert data.getRank() == 1; assert data.getSize() == 3; assert data.getShape()[0] == 3; assert data.getElementType() == double.class; int count = 0; IndexIterator dataI = data.getIndexIterator(); while (dataI.hasNext()) { assert Misc.closeEnough(dataI.getDoubleNext(), result[count]); count++; } } catch (IOException io) { io.printStackTrace(); assert false; } }
/** * Read data from a top level Variable and send data to a WritableByteChannel. * * @param v2 Variable * @param section wanted section of data of Variable. The section list is a list of ucar.ma2.Range * which define the requested data subset. * @param channel WritableByteChannel object - channel that can write bytes. * @return the number of bytes written, possibly zero. */ public long readToByteChannel11( ucar.nc2.Variable v2, Section section, WritableByteChannel channel) throws java.io.IOException, ucar.ma2.InvalidRangeException { Array data = readData(v2, section); float[] ftdata = new float[(int) data.getSize()]; byte[] bytedata = new byte[(int) data.getSize() * 4]; IndexIterator iter = data.getIndexIterator(); int i = 0; ByteBuffer buffer = ByteBuffer.allocateDirect(bytedata.length); while (iter.hasNext()) { ftdata[i] = iter.getFloatNext(); bytedata[i] = new Float(ftdata[i]).byteValue(); buffer.put(bytedata[i]); i++; } buffer = ByteBuffer.wrap(bytedata); // write the bytes to the channel int count = channel.write(buffer); System.out.println("COUNT=" + count); // check if all bytes where written if (buffer.hasRemaining()) { // if not all bytes were written, move the unwritten bytes to the beginning and // set position just after the last unwritten byte buffer.compact(); } else { buffer.clear(); } return (long) count; }
private void convertScaleOffsetUnsignedInt(IndexIterator iterIn, IndexIterator iterOut) { boolean checkMissing = useNaNs && hasMissing(); while (iterIn.hasNext()) { int valb = iterIn.getIntNext(); double val = scale * DataType.unsignedIntToLong(valb) + offset; iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val); } }
/** * Translate missing data to NaNs. Data must be DOUBLE or FLOAT * * @param in convert this array * @return same array, with missing values replaced by NaNs */ private Array convertMissing(Array in) { if (debugRead) System.out.println("convertMissing "); IndexIterator iterIn = in.getIndexIterator(); if (in.getElementType() == double.class) { while (iterIn.hasNext()) { double val = iterIn.getDoubleNext(); if (isMissing_(val)) iterIn.setDoubleCurrent(Double.NaN); } } else if (in.getElementType() == float.class) { while (iterIn.hasNext()) { float val = iterIn.getFloatNext(); if (isMissing_(val)) iterIn.setFloatCurrent(Float.NaN); } } return in; }
private void showValuesAsDates(CoordinateAxis axis) { String units = axis.getUnitsString(); String cal = getCalendarAttribute(axis); CalendarDateUnit cdu = CalendarDateUnit.of(cal, units); try { infoTA.appendLine(units); infoTA.appendLine(NCdumpW.printVariableData(axis, null)); if (axis.getDataType().isNumeric()) { if (axis instanceof CoordinateAxis2D) { showDates2D((CoordinateAxis2D) axis, cdu); } else if (axis instanceof CoordinateAxis1D) { // 1D showDates1D((CoordinateAxis1D) axis, cdu); } else { // > 2D Array data = axis.read(); IndexIterator ii = data.getIndexIterator(); while (ii.hasNext()) { double val = ii.getDoubleNext(); infoTA.appendLine(makeCalendarDateStringOrMissing(cdu, val)); } } } else { // must be iso dates Array data = axis.read(); Formatter f = new Formatter(); if (data instanceof ArrayChar) { ArrayChar dataS = (ArrayChar) data; ArrayChar.StringIterator iter = dataS.getStringIterator(); while (iter.hasNext()) f.format(" %s%n", iter.next()); infoTA.appendLine(f.toString()); } else if (data instanceof ArrayObject) { IndexIterator iter = data.getIndexIterator(); while (iter.hasNext()) f.format(" %s%n", iter.next()); infoTA.appendLine(f.toString()); } } } catch (Exception ex) { ex.printStackTrace(); infoTA.appendLine(ex.getMessage()); } }
// get a list of station obs for this station public List getData(Station s, CancelTask cancel) throws IOException { ArrayStructure stationData = readData("ID " + s.getName(), cancel); ArrayList stationObs = new ArrayList(); IndexIterator ii = stationData.getIndexIterator(); while (ii.hasNext()) { stationObs.add(new StationObs((Station) s, (StructureData) ii.getObjectNext())); } return stationObs; }
public void testAggCoordVarSubsetDefeatLocalCache(NetcdfFile ncfile) throws InvalidRangeException, IOException { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1; assert time.getSize() == 3; assert time.getShape()[0] == 3; assert time.getDataType() == DataType.DOUBLE; assert time.getDimension(0) == ncfile.findDimension("time"); time.setCachedData(null, false); Array data = time.read("1:2"); assert data.getRank() == 1; assert data.getSize() == 2; assert data.getShape()[0] == 2; assert data.getElementType() == double.class; int count = 0; IndexIterator dataI = data.getIndexIterator(); while (dataI.hasNext()) { assert Misc.closeEnough(dataI.getDoubleNext(), result[count + 1]); count++; } time.setCachedData(null, false); data = time.read("0:2:2"); assert data.getRank() == 1; assert data.getSize() == 2; assert data.getShape()[0] == 2; assert data.getElementType() == double.class; count = 0; dataI = data.getIndexIterator(); while (dataI.hasNext()) { assert Misc.closeEnough(dataI.getDoubleNext(), result[count * 2]); count++; } }
private Array convertToChar(Variable newVar, Array oldData) { ArrayChar newData = (ArrayChar) Array.factory(DataType.CHAR, newVar.getShape()); Index ima = newData.getIndex(); IndexIterator ii = oldData.getIndexIterator(); while (ii.hasNext()) { String s = (String) ii.getObjectNext(); int[] c = ii.getCurrentCounter(); for (int i = 0; i < c.length; i++) ima.setDim(i, c[i]); newData.setString(ima, s); } return newData; }
// for jon blower private Array getEnhancedArray(VariableDS vds) throws IOException { Array data = vds.read(); EnumSet<NetcdfDataset.Enhance> mode = vds.getEnhanceMode(); if (mode.contains(NetcdfDataset.Enhance.ScaleMissing)) return data; if (!mode.contains(NetcdfDataset.Enhance.ScaleMissingDefer)) throw new IllegalStateException("Must include " + NetcdfDataset.Enhance.ScaleMissingDefer); IndexIterator ii = data.getIndexIterator(); while (ii.hasNext()) { double val = vds.convertScaleOffsetMissing(ii.getDoubleNext()); ii.setDoubleCurrent(val); } return data; }
/** * Get the minimum and the maximum data value of the previously read Array, skipping missing * values as defined by isMissingData(double val). * * @param a Array to get min/max values * @return both min and max value. */ public MAMath.MinMax getMinMaxSkipMissingData(Array a) { if (!hasMissingData()) return MAMath.getMinMax(a); IndexIterator iter = a.getIndexIterator(); double max = -Double.MAX_VALUE; double min = Double.MAX_VALUE; while (iter.hasNext()) { double val = iter.getDoubleNext(); if (isMissingData(val)) continue; if (val > max) max = val; if (val < min) min = val; } return new MAMath.MinMax(min, max); }
private void showDates2D(CoordinateAxis2D axis2D, CalendarDateUnit cdu) { Formatter f = new Formatter(); if (axis2D.isInterval()) { ArrayDouble.D2 coords = axis2D.getCoordValuesArray(); ArrayDouble.D3 bounds = axis2D.getCoordBoundsArray(); if (bounds == null) { infoTA.appendLine("No bounds for interval " + axis2D.getFullName()); return; } IndexIterator coordIter = coords.getIndexIterator(); IndexIterator boundsIter = bounds.getIndexIterator(); while (coordIter.hasNext()) { double coordValue = coordIter.getDoubleNext(); if (!boundsIter.hasNext()) break; double bounds1 = boundsIter.getDoubleNext(); if (!boundsIter.hasNext()) break; double bounds2 = boundsIter.getDoubleNext(); f.format( "%s (%s,%s)%n", makeCalendarDateStringOrMissing(cdu, coordValue), makeCalendarDateStringOrMissing(cdu, bounds1), makeCalendarDateStringOrMissing(cdu, bounds2)); } } else { ArrayDouble.D2 coords = axis2D.getCoordValuesArray(); IndexIterator coordIter = coords.getIndexIterator(); while (coordIter.hasNext()) { double coordValue = coordIter.getDoubleNext(); f.format("%s%n", makeCalendarDateStringOrMissing(cdu, coordValue)); } } infoTA.appendLine(f.toString()); }
// test offset only gets applied once public void testWrapOnce() throws IOException { String filename = TestAll.cdmUnitTestDir + "ncml/coords/testCoordScaling.ncml"; System.out.printf("%s%n", filename); NetcdfDataset ncd = ucar.nc2.dataset.NetcdfDataset.openDataset(filename); Variable v = ncd.findCoordinateAxis("Longitude"); assert v != null; assert v instanceof CoordinateAxis1D; // if offset is applied twice, the result is not in +-180 range Array data = v.read(); NCdumpW.printArray(data); IndexIterator ii = data.getIndexIterator(); while (ii.hasNext()) { assert Math.abs(ii.getDoubleNext()) < 180 : ii.getDoubleCurrent(); } }
/** * Convert Data with scale and offset. Also translate missing data to NaNs if useNaNs = true. * * @param in data to convert * @return converted data. */ private Array convertScaleOffset(Array in) { if (!hasScaleOffset) return in; if (debugRead) System.out.println("convertScaleOffset "); Array out = Array.factory(convertedDataType.getPrimitiveClassType(), in.getShape()); IndexIterator iterIn = in.getIndexIterator(); IndexIterator iterOut = out.getIndexIterator(); if (isUnsigned && in.getElementType() == byte.class) convertScaleOffsetUnsignedByte(iterIn, iterOut); else if (isUnsigned && in.getElementType() == short.class) convertScaleOffsetUnsignedShort(iterIn, iterOut); else if (isUnsigned && in.getElementType() == int.class) convertScaleOffsetUnsignedInt(iterIn, iterOut); else { boolean checkMissing = useNaNs && hasMissing(); while (iterIn.hasNext()) { double val = scale * iterIn.getDoubleNext() + offset; iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val); } } return out; }
private boolean compareData(String name, Array data1, Array data2, double tol, boolean justOne) { boolean ok = true; if (data1.getSize() != data2.getSize()) { f.format(" DIFF %s: size %d !== %d%n", name, data1.getSize(), data2.getSize()); ok = false; } if (data1.getElementType() != data2.getElementType()) { f.format( " DIFF %s: element type %s !== %s%n", name, data1.getElementType(), data2.getElementType()); ok = false; } if (!ok) return false; DataType dt = DataType.getType(data1.getElementType()); IndexIterator iter1 = data1.getIndexIterator(); IndexIterator iter2 = data2.getIndexIterator(); if (dt == DataType.DOUBLE) { while (iter1.hasNext() && iter2.hasNext()) { double v1 = iter1.getDoubleNext(); double v2 = iter2.getDoubleNext(); if (!Double.isNaN(v1) || !Double.isNaN(v2)) if (!closeEnough(v1, v2, tol)) { f.format( " DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.FLOAT) { while (iter1.hasNext() && iter2.hasNext()) { float v1 = iter1.getFloatNext(); float v2 = iter2.getFloatNext(); if (!Float.isNaN(v1) || !Float.isNaN(v2)) if (!closeEnough(v1, v2, (float) tol)) { f.format( " DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.INT) { while (iter1.hasNext() && iter2.hasNext()) { int v1 = iter1.getIntNext(); int v2 = iter2.getIntNext(); if (v1 != v2) { f.format( " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.SHORT) { while (iter1.hasNext() && iter2.hasNext()) { short v1 = iter1.getShortNext(); short v2 = iter2.getShortNext(); if (v1 != v2) { f.format( " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.BYTE) { while (iter1.hasNext() && iter2.hasNext()) { byte v1 = iter1.getByteNext(); byte v2 = iter2.getByteNext(); if (v1 != v2) { f.format( " DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n", name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2)); ok = false; if (justOne) break; } } } else if (dt == DataType.STRUCTURE) { while (iter1.hasNext() && iter2.hasNext()) { compareStructureData( (StructureData) iter1.next(), (StructureData) iter2.next(), tol, justOne); } } return ok; }
private void makeCoordinateDataWithMissing( int datatype, Variable time, Variable elev, Variable azi, Variable nradialsVar, Variable ngatesVar, List groups) { Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape()); Index timeIndex = timeData.getIndex(); Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape()); Index elevIndex = elevData.getIndex(); Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape()); Index aziIndex = aziData.getIndex(); Array nradialsData = Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape()); IndexIterator nradialsIter = nradialsData.getIndexIterator(); Array ngatesData = Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape()); IndexIterator ngatesIter = ngatesData.getIndexIterator(); // first fill with missing data IndexIterator ii = timeData.getIndexIterator(); while (ii.hasNext()) ii.setIntNext(MISSING_INT); ii = elevData.getIndexIterator(); while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT); ii = aziData.getIndexIterator(); while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT); // now set the coordinate variables from the Cinrad2Record radial int last_msecs = Integer.MIN_VALUE; int nscans = groups.size(); try { for (int scan = 0; scan < nscans; scan++) { List scanGroup = (List) groups.get(scan); int nradials = scanGroup.size(); Cinrad2Record first = null; for (int j = 0; j < nradials; j++) { Cinrad2Record r = (Cinrad2Record) scanGroup.get(j); if (first == null) first = r; int radial = r.radial_num - 1; timeData.setInt(timeIndex.set(scan, radial), r.data_msecs); elevData.setFloat(elevIndex.set(scan, radial), r.getElevation()); aziData.setFloat(aziIndex.set(scan, radial), r.getAzimuth()); if (r.data_msecs < last_msecs) logger.warn("makeCoordinateData time out of order " + r.data_msecs); last_msecs = r.data_msecs; } nradialsIter.setIntNext(nradials); ngatesIter.setIntNext(first.getGateCount(datatype)); } } catch (java.lang.ArrayIndexOutOfBoundsException ae) { logger.debug("Cinrad2IOSP.uncompress ", ae); } time.setCachedData(timeData, false); elev.setCachedData(elevData, false); azi.setCachedData(aziData, false); nradialsVar.setCachedData(nradialsData, false); ngatesVar.setCachedData(ngatesData, false); }
private int populateFile2( NetcdfFileWriteable ncFile, String varName, DataType dataType, int[] dimensions, int valueCounter, Random generator) throws IOException, InvalidRangeException { long singleWriteSize = maxMemory / dataType.getSize(); int highestNonWriteDim = determineHighestNonWriteDimension(dimensions, singleWriteSize); int[] singleStep = createWriteStep(dimensions, singleWriteSize, highestNonWriteDim); int numStepWrites = determineNumberOfStepWrites(singleStep, singleWriteSize); System.out.println( "SingleWriteSize: " + singleWriteSize + " datatype size: " + dataType.getSize() + " singleStep: " + arrayToString(singleStep) + " numberStepsPerWrite: " + numStepWrites); int[] allOnes = new int[dimensions.length]; int[] allZeros = new int[dimensions.length]; for (int i = 0; i < allOnes.length; i++) { allOnes[i] = 1; allZeros[i] = 0; } Index origin = new Index(dimensions, allOnes); long totalSize = origin.getSize(); long writtenSoFar = 0; boolean done = false; while (!done) { for (int i = 0; i < numStepWrites; i++) { // this is a crack at an optimization if ((highestNonWriteDim >= 0) && ((origin.getCurrentCounter()[highestNonWriteDim] + numStepWrites) <= dimensions[highestNonWriteDim])) { singleStep[highestNonWriteDim] = numStepWrites; System.out.println("JUST OPTIMIZED. New write step: " + arrayToString(singleStep)); // keep 'i' right i += numStepWrites - 1; } else { singleStep[highestNonWriteDim] = 1; } Array array = new ArrayInt(singleStep); IndexIterator iter = array.getIndexIterator(); while (iter.hasNext()) { iter.getIntNext(); // uncomment the following line for a random distribution // iter.setIntCurrent((int) (Math.abs(generator.nextGaussian()) * 40) ); // uncomment this line for an incrementing value iter.setIntCurrent(valueCounter); valueCounter++; // book keeping writtenSoFar++; // origin.incr(); } System.out.println( "Writing to file: " + ncFile.getLocation() + ". var_name: " + varName + " origin: " + arrayToString(origin.getCurrentCounter()) + " writeSize: " + array.getSize() + " write shape: " + arrayToString(singleStep)); // write to the actual file ncFile.write(varName, origin.getCurrentCounter(), array); // update origin accordingly for (int j = 0; j < Index.computeSize(singleStep); j++) { // writtenSoFar++; origin.incr(); } System.out.println( "\tcurrentIndex: " + arrayToString(origin.getCurrentCounter()) + " currentElement: " + origin.currentElement() + " totalsize: " + (totalSize) + " writtenSoFar: " + writtenSoFar); if (writtenSoFar >= totalSize) { done = true; break; } } } return valueCounter; }