public void testAggCoordVar2(NetcdfFile ncfile) { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.INT; assert time.getDimension(0) == ncfile.findDimension("time"); try { Array data = time.read(); assert (data instanceof ArrayInt); IndexIterator dataI = data.getIndexIterator(); assert dataI.getIntNext() == 0 : dataI.getIntCurrent(); assert dataI.getIntNext() == 1 : dataI.getIntCurrent(); assert dataI.getIntNext() == 2 : dataI.getIntCurrent(); } catch (IOException io) { io.printStackTrace(); assert false; } }
public void testAggCoordVar(NetcdfFile ncfile) { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.INT; assert time.getDimension(0) == ncfile.findDimension("time"); try { Array data = time.read(); assert (data instanceof ArrayInt.D1) : data.getClass().getName(); ArrayInt.D1 dataI = (ArrayInt.D1) data; assert dataI.get(0) == 0; assert dataI.get(1) == 10; assert dataI.get(2) == 99; } catch (IOException io) { io.printStackTrace(); assert false; } }
private void copySome(NetcdfFileWriteable ncfile, Variable oldVar, int nelems) throws IOException { String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName()); int[] shape = oldVar.getShape(); int[] origin = new int[oldVar.getRank()]; int size = shape[0]; for (int i = 0; i < size; i += nelems) { origin[0] = i; int left = size - i; shape[0] = Math.min(nelems, left); Array data; try { data = oldVar.read(origin, shape); if (oldVar.getDataType() == DataType.STRING) { data = convertToChar(ncfile.findVariable(newName), data); } if (data.getSize() > 0) { // zero when record dimension = 0 ncfile.write(newName, origin, data); if (debug) System.out.println("write " + data.getSize() + " bytes"); } } catch (InvalidRangeException e) { e.printStackTrace(); throw new IOException(e.getMessage()); } } }
private Array convertToChar(Variable newVar, Array oldData) { ArrayChar newData = (ArrayChar) Array.factory(DataType.CHAR, newVar.getShape()); Index ima = newData.getIndex(); IndexIterator ii = oldData.getIndexIterator(); while (ii.hasNext()) { String s = (String) ii.getObjectNext(); int[] c = ii.getCurrentCounter(); for (int i = 0; i < c.length; i++) ima.setDim(i, c[i]); newData.setString(ima, s); } return newData; }
public void testReadData(NetcdfFile ncfile, String name) throws IOException { Variable v = ncfile.findVariable(name); assert null != v; assert v.getShortName().equals(name); assert v.getRank() == 3; assert v.getSize() == 36 : v.getSize(); assert v.getShape()[0] == 3; assert v.getShape()[1] == 3; assert v.getShape()[2] == 4; assert v.getDataType() == DataType.DOUBLE; assert !v.isCoordinateVariable(); assert v.getDimension(0) == ncfile.findDimension("time"); assert v.getDimension(1) == ncfile.findDimension("lat"); assert v.getDimension(2) == ncfile.findDimension("lon"); Array data = v.read(); assert data.getRank() == 3; assert data.getSize() == 36; assert data.getShape()[0] == 3; assert data.getShape()[1] == 3; assert data.getShape()[2] == 4; assert data.getElementType() == double.class; int[] shape = data.getShape(); Index tIndex = data.getIndex(); for (int i = 0; i < shape[0]; i++) for (int j = 0; j < shape[1]; j++) for (int k = 0; k < shape[2]; k++) { double val = data.getDouble(tIndex.set(i, j, k)); // System.out.println(" "+val); assert TestUtils.close(val, 100 * i + 10 * j + k) : val; } }
private void doWrite2(NetcdfFileWriteable ncfile, String varName) throws Exception { Variable v = ncfile.findVariable(varName); int[] w = getWeights(v); int[] shape = v.getShape(); Array aa = Array.factory(v.getDataType().getPrimitiveClassType(), shape); Index ima = aa.getIndex(); for (int i = 0; i < shape[0]; i++) { for (int j = 0; j < shape[1]; j++) { aa.setDouble(ima.set(i, j), (double) (i * w[0] + j * w[1])); } } ncfile.write(varName, aa); }
public Array getData(Range range, String parameterName) throws IOException, InvalidRangeException { Variable variable = ncfile.getRootGroup().findVariable(parameterName); int varRank = variable.getRank(); int[] varShape = variable.getShape(); List section = new ArrayList(varRank); section.add(range); for (int i = 1; i < varRank; i++) { section.add(new Range(0, varShape[i] - 1)); } Array array = variable.read(section); if (array.getShape()[0] == 1) { return (array.reduce(0)); } else { return (array); } // return( array.getShape()[0] == 1 ? array.reduce( 0 ) : array); }
public void testAggCoordVar3(NetcdfFile ncfile) throws IOException { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.DOUBLE : time.getDataType(); assert time.getDimension(0) == ncfile.findDimension("time"); Array data = time.read(); assert (data instanceof ArrayDouble); IndexIterator dataI = data.getIndexIterator(); double val = dataI.getDoubleNext(); assert Misc.closeEnough(val, 0.0) : val; assert Misc.closeEnough(dataI.getDoubleNext(), 10.0) : dataI.getDoubleCurrent(); assert Misc.closeEnough(dataI.getDoubleNext(), 99.0) : dataI.getDoubleCurrent(); }
public void testAggCoordVarScan(NetcdfFile ncfile) throws IOException { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.INT : time.getDataType(); assert time.getDimension(0) == ncfile.findDimension("time"); int count = 0; Array data = time.read(); assert (data instanceof ArrayInt); while (data.hasNext()) { int val = data.nextInt(); assert val == count * 10 : val + "!=" + count * 10; count++; } }
/** Converts a 3D variable to a {@link Metadata} representation */ private static void convert3DVariable(GridDatatype g, Date date, Map<String, Metadata> metaMap) throws IOException { Variable v = g.getVariable(); System.out.println("Reading: " + v.getFullName()); Array values = v.read(); int h = v.getShape(1); int w = v.getShape(2); for (int i = 0; i < h; ++i) { for (int j = 0; j < w; ++j) { LatLonPoint pt = g.getCoordinateSystem().getLatLon(j, i); String hash = GeoHash.encode((float) pt.getLatitude(), (float) pt.getLongitude(), 10).toLowerCase(); Metadata meta = metaMap.get(hash); if (meta == null) { /* We need to create Metadata for this location */ meta = new Metadata(); UUID metaUUID = UUID.nameUUIDFromBytes(hash.getBytes()); meta.setName(metaUUID.toString()); SpatialProperties location = new SpatialProperties((float) pt.getLatitude(), (float) pt.getLongitude()); meta.setSpatialProperties(location); TemporalProperties time = new TemporalProperties(date.getTime()); meta.setTemporalProperties(time); metaMap.put(hash, meta); } String featureName = v.getFullName().toLowerCase(); float featureValue = values.getFloat(i * w + j); Feature feature = new Feature(featureName, featureValue); meta.putAttribute(feature); } } }
public void testAggCoordVarNoCoordsDir(NetcdfFile ncfile) throws IOException { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.STRING : time.getDataType(); assert time.getDimension(0) == ncfile.findDimension("time"); Array data = time.read(); assert (data instanceof ArrayObject); IndexIterator dataI = data.getIndexIterator(); String coordName = (String) dataI.getObjectNext(); assert coordName.equals("time0Dir.nc") : coordName; coordName = (String) dataI.getObjectNext(); assert coordName.equals("time1Dir.nc") : coordName; coordName = (String) dataI.getObjectNext(); assert coordName.equals("time2Dir.nc") : coordName; }
public void testCoordVar(NetcdfFile ncfile) { Variable lat = ncfile.findVariable("lat"); assert null != lat; assert lat.getShortName().equals("lat"); assert lat.getRank() == 1; assert lat.getSize() == 3; assert lat.getShape()[0] == 3; assert lat.getDataType() == DataType.FLOAT; assert !lat.isUnlimited(); assert lat.getDimension(0).equals(ncfile.findDimension("lat")); Attribute att = lat.findAttribute("units"); assert null != att; assert !att.isArray(); assert att.isString(); assert att.getDataType() == DataType.STRING; assert att.getStringValue().equals("degrees_north"); assert att.getNumericValue() == null; assert att.getNumericValue(3) == null; try { Array data = lat.read(); assert data.getRank() == 1; assert data.getSize() == 3; assert data.getShape()[0] == 3; assert data.getElementType() == float.class; IndexIterator dataI = data.getIndexIterator(); assert TestUtils.close(dataI.getDoubleNext(), 41.0); assert TestUtils.close(dataI.getDoubleNext(), 40.0); assert TestUtils.close(dataI.getDoubleNext(), 39.0); } catch (IOException io) { } }
private void makeCoordinateData( int datatype, Variable time, Variable elev, Variable azi, Variable nradialsVar, Variable ngatesVar, List groups) { Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape()); IndexIterator timeDataIter = timeData.getIndexIterator(); Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape()); IndexIterator elevDataIter = elevData.getIndexIterator(); Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape()); IndexIterator aziDataIter = aziData.getIndexIterator(); Array nradialsData = Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape()); IndexIterator nradialsIter = nradialsData.getIndexIterator(); Array ngatesData = Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape()); IndexIterator ngatesIter = ngatesData.getIndexIterator(); int last_msecs = Integer.MIN_VALUE; int nscans = groups.size(); int maxRadials = volScan.getMaxRadials(); for (int i = 0; i < nscans; i++) { List scanGroup = (List) groups.get(i); int nradials = scanGroup.size(); Cinrad2Record first = null; for (int j = 0; j < nradials; j++) { Cinrad2Record r = (Cinrad2Record) scanGroup.get(j); if (first == null) first = r; timeDataIter.setIntNext(r.data_msecs); elevDataIter.setFloatNext(r.getElevation()); aziDataIter.setFloatNext(r.getAzimuth()); if (r.data_msecs < last_msecs) logger.warn("makeCoordinateData time out of order " + r.data_msecs); last_msecs = r.data_msecs; } for (int j = nradials; j < maxRadials; j++) { timeDataIter.setIntNext(MISSING_INT); elevDataIter.setFloatNext(MISSING_FLOAT); aziDataIter.setFloatNext(MISSING_FLOAT); } nradialsIter.setIntNext(nradials); ngatesIter.setIntNext(first.getGateCount(datatype)); } time.setCachedData(timeData, false); elev.setCachedData(elevData, false); azi.setCachedData(aziData, false); nradialsVar.setCachedData(nradialsData, false); ngatesVar.setCachedData(ngatesData, false); }
public void testNC3Read() throws IOException { NetcdfFile ncfile = TestDir.openFileLocal("testWrite.nc"); assert (null != ncfile.findDimension("lat")); assert (null != ncfile.findDimension("lon")); Variable temp = null; assert (null != (temp = ncfile.findVariable("temperature"))); // read entire array Array A; try { A = temp.read(); } catch (IOException e) { System.err.println("ERROR reading file"); assert (false); return; } assert (A.getRank() == 2); int i, j; Index ima = A.getIndex(); int[] shape = A.getShape(); assert shape[0] == 64; assert shape[1] == 128; for (i = 0; i < shape[0]; i++) { for (j = 0; j < shape[1]; j++) { double dval = A.getDouble(ima.set(i, j)); assert (dval == (double) (i * 1000000 + j * 1000)) : dval; } } // read part of array int[] origin2 = new int[2]; int[] shape2 = new int[2]; shape2[0] = 1; shape2[1] = temp.getShape()[1]; try { A = temp.read(origin2, shape2); } catch (InvalidRangeException e) { System.err.println("ERROR reading file " + e); assert (false); return; } catch (IOException e) { System.err.println("ERROR reading file"); assert (false); return; } assert (A.getRank() == 2); for (j = 0; j < shape2[1]; j++) { assert (A.getDouble(ima.set(0, j)) == (double) (j * 1000)); } // rank reduction Array Areduce = A.reduce(); Index ima2 = Areduce.getIndex(); assert (Areduce.getRank() == 1); for (j = 0; j < shape2[1]; j++) { assert (Areduce.getDouble(ima2.set(j)) == (double) (j * 1000)); } // read char variable Variable c = null; assert (null != (c = ncfile.findVariable("svar"))); try { A = c.read(); } catch (IOException e) { assert (false); } assert (A instanceof ArrayChar); ArrayChar ac = (ArrayChar) A; String val = ac.getString(ac.getIndex()); assert val.equals("Testing 1-2-3") : val; // System.out.println( "val = "+ val); // read char variable 2 Variable c2 = null; assert (null != (c2 = ncfile.findVariable("svar2"))); try { A = c2.read(); } catch (IOException e) { assert (false); } assert (A instanceof ArrayChar); ArrayChar ac2 = (ArrayChar) A; assert (ac2.getString().equals("Two pairs of ladies stockings!")); // read String Array Variable c3 = null; assert (null != (c3 = ncfile.findVariable("names"))); try { A = c3.read(); } catch (IOException e) { assert (false); } assert (A instanceof ArrayChar); ArrayChar ac3 = (ArrayChar) A; ima = ac3.getIndex(); assert (ac3.getString(ima.set(0)).equals("No pairs of ladies stockings!")); assert (ac3.getString(ima.set(1)).equals("One pair of ladies stockings!")); assert (ac3.getString(ima.set(2)).equals("Two pairs of ladies stockings!")); // read String Array - 2 Variable c4 = null; assert (null != (c4 = ncfile.findVariable("names2"))); try { A = c4.read(); } catch (IOException e) { assert (false); } assert (A instanceof ArrayChar); ArrayChar ac4 = (ArrayChar) A; ima = ac4.getIndex(); assert (ac4.getString(0).equals("0 pairs of ladies stockings!")); assert (ac4.getString(1).equals("1 pair of ladies stockings!")); assert (ac4.getString(2).equals("2 pairs of ladies stockings!")); // System.out.println( "ncfile = "+ ncfile); ncfile.close(); System.out.println("**************TestRead done"); }
/** * Make the station variables from a representative station * * @param stations list of stations * @param dim station dimension * @return the list of variables */ protected List<Variable> makeStationVars(List<GempakStation> stations, Dimension dim) { int numStations = stations.size(); boolean useSTID = true; for (GempakStation station : stations) { if (station.getSTID().equals("")) { useSTID = false; break; } } List<Variable> vars = new ArrayList<Variable>(); List<String> stnKeyNames = gemreader.getStationKeyNames(); for (String varName : stnKeyNames) { Variable v = makeStationVariable(varName, dim); // use STNM or STID as the name or description Attribute stIDAttr = new Attribute("standard_name", "station_id"); if (varName.equals(GempakStation.STID) && useSTID) { v.addAttribute(stIDAttr); } if (varName.equals(GempakStation.STNM) && !useSTID) { v.addAttribute(stIDAttr); } vars.add(v); } // see if we fill these in completely now if ((dim != null) && (numStations > 0)) { for (Variable v : vars) { Array varArray; if (v.getDataType().equals(DataType.CHAR)) { int[] shape = v.getShape(); varArray = new ArrayChar.D2(shape[0], shape[1]); } else { varArray = get1DArray(v.getDataType(), numStations); } int index = 0; String varname = v.getFullName(); for (GempakStation stn : stations) { String test = ""; if (varname.equals(GempakStation.STID)) { test = stn.getName(); } else if (varname.equals(GempakStation.STNM)) { ((ArrayInt.D1) varArray) .set( index, // (int) (stn.getSTNM() / 10)); (int) (stn.getSTNM())); } else if (varname.equals(GempakStation.SLAT)) { ((ArrayFloat.D1) varArray).set(index, (float) stn.getLatitude()); } else if (varname.equals(GempakStation.SLON)) { ((ArrayFloat.D1) varArray).set(index, (float) stn.getLongitude()); } else if (varname.equals(GempakStation.SELV)) { ((ArrayFloat.D1) varArray).set(index, (float) stn.getAltitude()); } else if (varname.equals(GempakStation.STAT)) { test = stn.getSTAT(); } else if (varname.equals(GempakStation.COUN)) { test = stn.getCOUN(); } else if (varname.equals(GempakStation.STD2)) { test = stn.getSTD2(); } else if (varname.equals(GempakStation.SPRI)) { ((ArrayInt.D1) varArray).set(index, stn.getSPRI()); } else if (varname.equals(GempakStation.SWFO)) { test = stn.getSWFO(); } else if (varname.equals(GempakStation.WFO2)) { test = stn.getWFO2(); } if (!test.equals("")) { ((ArrayChar.D2) varArray).setString(index, test); } index++; } v.setCachedData(varArray, false); } } return vars; }
Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill) throws IOException, InvalidRangeException { NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill); if (debug) { System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName); } // global attributes List<Attribute> glist = bufr.getGlobalAttributes(); for (Attribute att : glist) { String useName = N3iosp.makeValidNetcdfObjectName(att.getName()); Attribute useAtt; if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues()); else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue()); else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue()); if (debug) System.out.println("add gatt= " + useAtt); } // global dimensions Dimension recordDim = null; Map<String, Dimension> dimHash = new HashMap<String, Dimension>(); for (Dimension oldD : bufr.getDimensions()) { String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName()); boolean isRecord = useName.equals("record"); Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false); dimHash.put(newD.getName(), newD); if (isRecord) recordDim = newD; if (debug) System.out.println("add dim= " + newD); } // Variables Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord); for (Variable oldVar : recordStruct.getVariables()) { if (oldVar.getDataType() == DataType.STRUCTURE) continue; String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName()); DataType newType = oldVar.getDataType(); List<Dimension> newDims = new ArrayList<Dimension>(); newDims.add(recordDim); for (Dimension dim : oldVar.getDimensions()) { newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength())); } Variable newVar = ncfile.addVariable(varName, newType, newDims); if (debug) System.out.println("add var= " + newVar); // attributes List<Attribute> attList = oldVar.getAttributes(); for (Attribute att : attList) { String useName = N3iosp.makeValidNetcdfObjectName(att.getName()); if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues()); else if (att.isString()) ncfile.addVariableAttribute(varName, useName, att.getStringValue()); else ncfile.addVariableAttribute(varName, useName, att.getNumericValue()); } } // int max_seq = countSeq(recordStruct); // Dimension seqD = ncfile.addDimension("level", max_seq); for (Variable v : recordStruct.getVariables()) { if (v.getDataType() != DataType.STRUCTURE) continue; String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName()); int shape[] = v.getShape(); Dimension structDim = ncfile.addDimension(structName, shape[0]); Structure struct = (Structure) v; for (Variable seqVar : struct.getVariables()) { String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName); DataType newType = seqVar.getDataType(); List<Dimension> newDims = new ArrayList<Dimension>(); newDims.add(recordDim); newDims.add(structDim); for (Dimension dim : seqVar.getDimensions()) { newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength())); } Variable newVar = ncfile.addVariable(varName, newType, newDims); if (debug) System.out.println("add var= " + newVar); // attributes List<Attribute> attList = seqVar.getAttributes(); for (Attribute att : attList) { String useName = N3iosp.makeValidNetcdfObjectName(att.getName()); if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues()); else if (att.isString()) ncfile.addVariableAttribute(varName, useName, att.getStringValue()); else ncfile.addVariableAttribute(varName, useName, att.getNumericValue()); } } } // create the file ncfile.create(); if (debug) System.out.println("File Out= " + ncfile.toString()); // boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE); double total = copyVarData(ncfile, recordStruct); ncfile.flush(); System.out.println("FileWriter done total bytes = " + total); ncfile.close(); }
private void makeCoordinateDataWithMissing( int datatype, Variable time, Variable elev, Variable azi, Variable nradialsVar, Variable ngatesVar, List groups) { Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape()); Index timeIndex = timeData.getIndex(); Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape()); Index elevIndex = elevData.getIndex(); Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape()); Index aziIndex = aziData.getIndex(); Array nradialsData = Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape()); IndexIterator nradialsIter = nradialsData.getIndexIterator(); Array ngatesData = Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape()); IndexIterator ngatesIter = ngatesData.getIndexIterator(); // first fill with missing data IndexIterator ii = timeData.getIndexIterator(); while (ii.hasNext()) ii.setIntNext(MISSING_INT); ii = elevData.getIndexIterator(); while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT); ii = aziData.getIndexIterator(); while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT); // now set the coordinate variables from the Cinrad2Record radial int last_msecs = Integer.MIN_VALUE; int nscans = groups.size(); try { for (int scan = 0; scan < nscans; scan++) { List scanGroup = (List) groups.get(scan); int nradials = scanGroup.size(); Cinrad2Record first = null; for (int j = 0; j < nradials; j++) { Cinrad2Record r = (Cinrad2Record) scanGroup.get(j); if (first == null) first = r; int radial = r.radial_num - 1; timeData.setInt(timeIndex.set(scan, radial), r.data_msecs); elevData.setFloat(elevIndex.set(scan, radial), r.getElevation()); aziData.setFloat(aziIndex.set(scan, radial), r.getAzimuth()); if (r.data_msecs < last_msecs) logger.warn("makeCoordinateData time out of order " + r.data_msecs); last_msecs = r.data_msecs; } nradialsIter.setIntNext(nradials); ngatesIter.setIntNext(first.getGateCount(datatype)); } } catch (java.lang.ArrayIndexOutOfBoundsException ae) { logger.debug("Cinrad2IOSP.uncompress ", ae); } time.setCachedData(timeData, false); elev.setCachedData(elevData, false); azi.setCachedData(aziData, false); nradialsVar.setCachedData(nradialsData, false); ngatesVar.setCachedData(ngatesData, false); }