public void testAggCoordVar2(NetcdfFile ncfile) { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.INT; assert time.getDimension(0) == ncfile.findDimension("time"); try { Array data = time.read(); assert (data instanceof ArrayInt); IndexIterator dataI = data.getIndexIterator(); assert dataI.getIntNext() == 0 : dataI.getIntCurrent(); assert dataI.getIntNext() == 1 : dataI.getIntCurrent(); assert dataI.getIntNext() == 2 : dataI.getIntCurrent(); } catch (IOException io) { io.printStackTrace(); assert false; } }
public void testScan() throws IOException, InvalidRangeException { String xml = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<netcdf xmlns='http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2'>\n" + " <variable name='time' type='int' shape='time'>\n" + " <attribute name='long_name' type='string' value='time coordinate' />\n" + " <attribute name='units' type='string' value='days since 2001-8-31 00:00:00 UTC' />\n" + " <values start='0' increment='10' />\n" + " </variable>\n" + " <aggregation dimName='time' type='joinNew'>\n" + " <variableAgg name='T'/>\n" + " <scan location='src/test/data/ncml/nc/' suffix='Dir.nc' subdirs='false'/>\n" + " </aggregation>\n" + "</netcdf>"; String filename = "file:./" + TestNcML.topDir + "aggSynScan.xml"; NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(xml), null); testDimensions(ncfile); testCoordVar(ncfile); testAggCoordVarScan(ncfile); testReadData(ncfile, "T"); testReadSlice(ncfile, "T"); ncfile.close(); }
public void testAggCoordVar(NetcdfFile ncfile) { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1 : time.getRank(); assert time.getShape()[0] == 3; assert time.getDataType() == DataType.INT; assert time.getDimension(0) == ncfile.findDimension("time"); try { Array data = time.read(); assert (data instanceof ArrayInt.D1) : data.getClass().getName(); ArrayInt.D1 dataI = (ArrayInt.D1) data; assert dataI.get(0) == 0; assert dataI.get(1) == 10; assert dataI.get(2) == 99; } catch (IOException io) { io.printStackTrace(); assert false; } }
public void testNcmlCached() throws IOException, InvalidRangeException { System.out.println("\n TestNcmlAggExistingCached.acquire at " + new Date()); NetcdfFile ncfile = NetcdfDataset.acquireDataset(filename, null); testAggCoordVar(ncfile); ncfile.close(); System.out.println("\n TestNcmlAggExistingCached.acquire again at " + new Date()); ncfile = NetcdfDataset.acquireDataset(filename, null); testAggCoordVar(ncfile); ncfile.close(); try { Thread.sleep(5000); } catch (InterruptedException e) { e.printStackTrace(); } System.out.println("\n TestNcmlAggExistingCached.acquire after sleeping " + new Date()); ncfile = NetcdfDataset.acquireDataset(filename, null); testAggCoordVar(ncfile); ncfile.close(); NetcdfDataset.getNetcdfFileCache().clearCache(false); System.out.println("\n TestNcmlAggExistingCached.acquire after flushing cache " + new Date()); ncfile = NetcdfDataset.acquireDataset(filename, null); testAggCoordVar(ncfile); testAggCoordVarSubset(ncfile); ncfile.close(); }
public void testAggCoordVar(NetcdfFile ncfile) { Variable time = ncfile.findVariable("time"); assert null != time; assert time.getShortName().equals("time"); assert time.getRank() == 1; assert time.getSize() == 3; assert time.getShape()[0] == 3; assert time.getDataType() == DataType.DOUBLE; assert time.getDimension(0) == ncfile.findDimension("time"); try { Array data = time.read(); assert data.getRank() == 1; assert data.getSize() == 3; assert data.getShape()[0] == 3; assert data.getElementType() == double.class; int count = 0; IndexIterator dataI = data.getIndexIterator(); while (dataI.hasNext()) { assert Misc.closeEnough(dataI.getDoubleNext(), result[count]); count++; } } catch (IOException io) { io.printStackTrace(); assert false; } }
/** * Sets the v-velocity file to be used by this class * * @param _velocityFile - the v-velocity NetCDF object * @param vName - the name of the v variable * @throws IOException */ public void setVFile(String _velocityFile, String vName) throws IOException { this.vName = vName; vFile = NetcdfFile.open(_velocityFile); if (vFile != null) { vVar = vFile.findVariable(vName); } }
@Test public void testCacheIsUsed() throws IOException, InvalidRangeException { String filename = "file:TestAggExistingCache.xml"; System.out.printf("%s%n", filename); String cacheDirName = TestDir.temporaryLocalDataDir + "testAggExistingCache/"; System.out.printf("cacheDir=%s%n", cacheDirName); File cacheDir = new File(cacheDirName); FileUtils.deleteDirectory(cacheDir); // from commons-io assert !cacheDir.exists(); DiskCache2 cache = new DiskCache2(cacheDirName, false, 0, 0); cache.setAlwaysUseCache(true); Assert.assertEquals(cache.getRootDirectory(), cacheDirName); assert new File(cache.getRootDirectory()).exists(); Aggregation.setPersistenceCache(cache); AggregationExisting.countCacheUse = 0; try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), filename, null)) { System.out.println(" TestNcmlAggExisting.open " + filename); Array ATssta = ncfile.readSection("ATssta(:,0,0,0)"); Assert.assertEquals(4, ATssta.getSize()); } Assert.assertEquals(0, AggregationExisting.countCacheUse); AggregationExisting.countCacheUse = 0; try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), filename, null)) { System.out.println(" TestNcmlAggExisting.open " + filename); Array ATssta = ncfile.readSection("ATssta(:,0,0,0)"); Assert.assertEquals(4, ATssta.getSize()); } Assert.assertEquals(8, AggregationExisting.countCacheUse); }
/* Structure { int LAT[0]; ... int LAT[149]; } IMAGE_LAT_ARRAY(3600); type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046 */ @Test public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException { try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) { Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY"); assert (null != dset); assert (dset.getDataType() == DataType.STRUCTURE); assert (dset.getRank() == 1); assert (dset.getSize() == 3600); Dimension d = dset.getDimension(0); assert (d.getLength() == 3600); Structure s = (Structure) dset; // read last one - chunked StructureData sd = s.readStructure(3599); assert sd.getScalarInt("LAT[0]") == 70862722; assert sd.getScalarInt("LAT[149]") == 85302263; // read one at a time for (int i = 3590; i < d.getLength(); i++) { s.readStructure(i); System.out.println(" read structure " + i); } } System.out.println("*** testReadIASI ok"); }
/* Structure { int a_name; byte b_name(3); byte c_name(3); short d_name(3); int e_name(3); long f_name(3); int g_name(3); short h_name(3); int i_name(3); long j_name(3); float k_name(3); double l_name(3); } CompoundNative(15); type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048 */ @Test public void testReadH5StructureArrayMembers() throws java.io.IOException { try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) { Variable dset = ncfile.findVariable("CompoundNative"); assert (null != dset); assert (dset.getDataType() == DataType.STRUCTURE); assert (dset.getRank() == 1); assert (dset.getSize() == 15); Dimension d = dset.getDimension(0); assert (d.getLength() == 15); Structure s = (Structure) dset; // read all with the iterator StructureDataIterator iter = s.getStructureIterator(); while (iter.hasNext()) { StructureData sd = iter.next(); for (StructureMembers.Member m : sd.getMembers()) { Array data = sd.getArray(m); NCdumpW.printArray(data, m.getName(), out, null); } } } System.out.println("*** testReadH5StructureArrayMembers ok"); }
public void addBandMetadata(Product product) throws ProductIOException { Group group = ncFile.findGroup("Geophysical_Data"); if (productReader.getProductType() == SeadasProductReader.ProductType.Level2_Aquarius) { group = ncFile.findGroup("Aquarius_Data"); } if (productReader.getProductType() == SeadasProductReader.ProductType.Level1B_HICO) { group = ncFile.findGroup("products"); } if (group != null) { final MetadataElement bandAttributes = new MetadataElement("Band_Attributes"); List<Variable> variables = group.getVariables(); for (Variable variable : variables) { final String name = variable.getShortName(); final MetadataElement sdsElement = new MetadataElement(name + ".attributes"); final int dataType = getProductDataType(variable); final MetadataAttribute prodtypeattr = new MetadataAttribute("data_type", dataType); sdsElement.addAttribute(prodtypeattr); bandAttributes.addElement(sdsElement); final List<Attribute> list = variable.getAttributes(); for (Attribute varAttribute : list) { addAttributeToElement(sdsElement, varAttribute); } } final MetadataElement metadataRoot = product.getMetadataRoot(); metadataRoot.addElement(bandAttributes); } }
/** * Open a ucar.nc2.ft.PointFeatureCollection, write out in CF point format. * * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..) * @param fileOut write to this netcdf-3 file * @param inMemory if true, read file into memory for efficiency * @return true on success * @throws IOException on read/write error */ public static boolean rewritePointFeatureDataset(String fileIn, String fileOut, boolean inMemory) throws IOException { System.out.println( "Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory); long start = System.currentTimeMillis(); // do it in memory for speed NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn); NetcdfDataset ncd = new NetcdfDataset(ncfile); Formatter errlog = new Formatter(); FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.ANY_POINT, ncd, null, errlog); if (fd == null) return false; if (fd instanceof FeatureDatasetPoint) { writePointFeatureCollection((FeatureDatasetPoint) fd, fileOut); fd.close(); long took = System.currentTimeMillis() - start; System.out.println(" that took " + (took - start) + " msecs"); return true; } return false; }
private void initialize() throws IOException { List<Variable> list = ncf.getVariables(); List<Array> readArrays = ncf.readArrays(ncf.getVariables()); for (int i = 0; i < list.size(); i++) { elementList[i] = new CDFElement(list.get(i)); elementList[i].setArray(readArrays.get(i)); } }
/** * Sets the name of the lookup variable for the Time position * * @param name */ public void setTLookup(String name) { tVar = uFile.findVariable(tName); if (tVar == null) { System.out.println( "Incorrect variable match:\n\nVariables in local config file: " + tName + "."); System.out.println("Velocity file variables: " + uFile.getVariables().toString() + "\n"); } tloc = new IndexLookup_Nearest(tVar); }
/** * Build the netCDF file * * @throws IOException problem reading the file */ protected void buildNCFile() throws IOException { Trace.call1("GEMPAKSIOSP: buildNCFile"); ncfile.empty(); fillNCFile(); addGlobalAttributes(); ncfile.finish(); Trace.call2("GEMPAKSIOSP: buildNCFile"); // System.out.println(ncfile); }
/** Create a DAS for this netcdf file */ NcDAS(NetcdfFile ncfile) { // Variable attributes Iterator iter = ncfile.getVariables().iterator(); while (iter.hasNext()) { Variable v = (Variable) iter.next(); doVariable(v, null); } // Global attributes opendap.dap.AttributeTable gtable = new opendap.dap.AttributeTable("NC_GLOBAL"); int count = addAttributes(gtable, null, ncfile.getGlobalAttributes().iterator()); if (count > 0) try { addAttributeTable("NC_GLOBAL", gtable); } catch (AttributeExistsException e) { log.error("Cant add NC_GLOBAL", e); } // unlimited dimension iter = ncfile.getDimensions().iterator(); while (iter.hasNext()) { Dimension d = (Dimension) iter.next(); if (d.isUnlimited()) { opendap.dap.AttributeTable table = new opendap.dap.AttributeTable("DODS_EXTRA"); try { table.appendAttribute("Unlimited_Dimension", opendap.dap.Attribute.STRING, d.getName()); addAttributeTable("DODS_EXTRA", table); } catch (Exception e) { log.error("Error adding Unlimited_Dimension =" + e); } break; } } // unused dimensions opendap.dap.AttributeTable dimTable = null; iter = ncfile.getDimensions().iterator(); while (iter.hasNext()) { Dimension d = (Dimension) iter.next(); if (null == usedDims.get(d.getName())) { if (dimTable == null) dimTable = new opendap.dap.AttributeTable("EXTRA_DIMENSION"); try { dimTable.appendAttribute( d.getName(), opendap.dap.Attribute.INT32, Integer.toString(d.getLength())); } catch (Exception e) { log.error("Error adding Unlimited_Dimension =" + e); } } } if (dimTable != null) try { addAttributeTable("EXTRA_DIMENSION", dimTable); } catch (AttributeExistsException e) { log.error("Cant add EXTRA_DIMENSION", e); } }
public static boolean isMine(NetcdfFile ncfile) { String satName = ncfile.findAttValueIgnoreCase(null, "SATNAME", null); if ((satName == null) || !(satName.equalsIgnoreCase("Aqua"))) return false; String instName = ncfile.findAttValueIgnoreCase(null, "INTRUMENT_NAME", null); if ((instName == null) || !(instName.equalsIgnoreCase("modis"))) return false; return true; }
private void check(NetcdfFile ncfile, int n) throws IOException, InvalidRangeException { Variable v = ncfile.findVariable("time"); assert v != null; System.out.printf(" time= %s%n", v.getNameAndDimensions()); assert v.getSize() == n : v.getSize(); v = ncfile.findVariable("eta"); assert v != null; assert v.getRank() == 3 : v.getRank(); }
public void testUpdate() throws IOException, InvalidRangeException, InterruptedException { String dir = TestAll.cdmUnitTestDir + "agg/updating"; File dirFile = new File(dir); assert dirFile != null; assert dirFile.exists(); assert dirFile.isDirectory(); // make sure that the extra file is not in the agg for (File f : dirFile.listFiles()) { if (f.getName().equals("extra.nc")) { if (!f.renameTo(new File(dirFile, "extra.wait"))) { System.out.printf("Rename fails on %s.extra.nc %n", dirFile); } break; } } String ncml = "<?xml version='1.0' encoding='UTF-8'?>\n" + "<netcdf xmlns='http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2'>\n" + " <aggregation dimName='time' type='joinExisting' recheckEvery='1 nsec'>\n" + " <scan location='" + dir + "' suffix='*.nc' />\n" + " <variable name='depth'>\n" + " <attribute name='coordinates' value='lon lat'/>\n" + " </variable>\n" + " <variable name='wvh'>\n" + " <attribute name='coordinates' value='lon lat'/>\n" + " </variable>\n" + " </aggregation>\n" + " <attribute name='Conventions' type='String' value='CF-1.0'/>\n" + "</netcdf>"; String location = dir + "agg/updating.ncml"; System.out.println(" TestOffAggExistingTimeUnitsChange.testNarrGrib=\n" + ncml); NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), location, null); check(ncfile, 12); // make sure that the extra file is in the agg for (File f : dirFile.listFiles()) { if (f.getName().equals("extra.wait")) { if (!f.renameTo(new File(dirFile, "extra.nc"))) System.out.println(" rename fails on " + f.getPath()); break; } } ncfile.sync(); check(ncfile, 18); ncfile.close(); }
@Test public void testReadnetCDFinSequnceFileFormat() throws Exception { String path = this.getClass().getResource("/ncar.nc").getPath(); Util.writeToSequenceFile("file://" + path, hadoopMaster + "/tmp/ncar.seq", new DefaultCodec()); Map<Text, byte[]> netcdfsequnce = Util.readSequenceFile(hadoopMaster + "/tmp/ncar.seq"); for (Map.Entry<Text, byte[]> entry : netcdfsequnce.entrySet()) { NetcdfFile ncFile = NetcdfFile.openInMemory(entry.getKey().toString(), entry.getValue()); assertEquals(ncFile.getDimensions().size(), 5); } }
public static void main(String args[]) throws Exception { // String fileIn = // "C:/data/bufr/edition3/newIdd/IcingTropopause/IcingTropopause_20080529_0000.bufr"; String fileIn = "C:\\data\\bufr\\edition3\\meteosat\\METEOSAT7-MVIRI-MTPHRWW-NA-1-20080405123005.000000000Z-909326.bfr "; NetcdfFile ncf = NetcdfDataset.openFile(fileIn, null); System.out.println(ncf.toString()); new Write2ncRect(ncf, "C:/data/bufr2nc.meteosat.nc", true); }
void writeNetCDF(String filename) { try { FileWriter2 writer = new FileWriter2(ds, filename, NetcdfFileWriter.Version.netcdf3); NetcdfFile result = writer.write(); result.close(); JOptionPane.showMessageDialog(this, "File successfully written"); } catch (Exception ioe) { JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage()); ioe.printStackTrace(); } }
public static boolean isMine(NetcdfFile ncfile) { if (!ncfile.getFileTypeId().equals("HDF5")) return false; Group loc = ncfile.findGroup("VHRR/Geo-Location"); if (null == loc) return false; if (null == loc.findVariable("Latitude")) return false; if (null == loc.findVariable("Longitude")) return false; if (null == ncfile.findGroup("VHRR/Image Data")) return false; return true; }
public void testNcmlDirect() throws IOException, InvalidRangeException { NetcdfFile ncfile = NetcdfDataset.openDataset(filename, false, null); System.out.println("\n TestNcmlAggExistingCached.open " + filename); // System.out.println(" "+ ncfile); testAggCoordVar(ncfile); testAggCoordVarSubset(ncfile); testAggCoordVarSubsetDefeatLocalCache(ncfile); ncfile.close(); }
/** * Sets the name of the lookup variable for the North-South position * * @param name */ public void setYLookup(String name) { latVar = uFile.findVariable(latName); if (latVar == null) { System.out.println( "Incorrect variable match:\n\nVariables in local config file: " + latName + "."); System.out.println("Velocity file variables: " + uFile.getVariables().toString() + "\n"); } yloc = new IndexLookup_Nearest(latVar, 0); // dimension 0 because slices // have 2D lat/lon bounds[2][0] = yloc.getMinVal(); bounds[2][1] = yloc.getMaxVal(); }
public void testNoCoord() throws IOException, InvalidRangeException { String filename = "file:./" + TestNcML.topDir + "aggSynNoCoord.xml"; NetcdfFile ncfile = NcMLReader.readNcML(filename, null); testDimensions(ncfile); testCoordVar(ncfile); testAggCoordVarNoCoord(ncfile); testReadData(ncfile, "T"); testReadSlice(ncfile, "T"); ncfile.close(); }
@Test public void testForStructMetadata() throws IOException { System.out.printf("TestH4eosReadAll %s%n", filename); try (NetcdfFile ncfile = NetcdfFile.open(filename)) { Group root = ncfile.getRootGroup(); Group g = root.findGroup("HDFEOS INFORMATION"); if (g == null) g = ncfile.getRootGroup(); Variable dset = g.findVariable("StructMetadata.0"); assert (dset != null); } }
/* (non-Javadoc) * @see loci.formats.NetCDFService#setFile(java.lang.String) */ public void setFile(String file) throws IOException { this.currentFile = file; String currentId = Location.getMappedId(currentFile); netCDFFile = NetcdfFile.open(currentId); root = netCDFFile.getRootGroup(); attributeList = new Vector<String>(); variableList = new Vector<String>(); List<Group> groups = new ArrayList<Group>(); groups.add(root); parseAttributesAndVariables(groups); }
/** Add on global attributes for all types */ protected void addGlobalAttributes() { // global stuff ncfile.addAttribute(null, new Attribute("Conventions", getConventions())); String fileType = "GEMPAK " + gemreader.getFileType(); ncfile.addAttribute(null, new Attribute("file_format", fileType)); ncfile.addAttribute( null, new Attribute( "history", "Direct read of " + fileType + " into NetCDF-Java 4.2 API")); // at " + dateFormat.toDateTimeStringISO(new // Date()))); ncfile.addAttribute(null, new Attribute(CF.FEATURE_TYPE, getCFFeatureType())); }
public void testReadStandardVar() throws Exception { ncfileRead = NetcdfFile.open(filename); dsRead = NetcdfDataset.openDataset(filename); readDouble(); readByte2Short(); readByte(); readShortMissing(); readShort2FloatMissing(); readDoubleMissing(); ncfileRead.close(); dsRead.close(); }
/** * Clones the attributes of one NetCDF file into another. * * @param infile - The source NetCDF file * @param inVarName - The variable name to be copied * @param outfile - The destination NetCDF file * @param outVarName - The output variable name */ private void cloneAttributes( NetcdfFile infile, String inVarName, NetcdfFileWriteable outfile, String outVarName) { // Find the variable Variable vi = infile.findVariable(inVarName); // Grab all of its attributes - unchecked, but should be OK. List<Attribute> l = vi.getAttributes(); for (Attribute a : l) { if (a.getName().equalsIgnoreCase("units") && inVarName.equalsIgnoreCase("time")) { Attribute aa = new Attribute("units", "days since 1900-12-31 00:00:00"); outfile.addVariableAttribute(outVarName, aa); } else if (a.getName().equalsIgnoreCase("time_origin") && inVarName.equalsIgnoreCase("time")) { Attribute aa = new Attribute("time_origin", "1900-12-31 00:00:00"); outfile.addVariableAttribute(outVarName, aa); } else if (a.getName().equalsIgnoreCase("calendar")) { Attribute aa = new Attribute("calendar", "standard"); outfile.addVariableAttribute(outVarName, aa); } else { outfile.addVariableAttribute(outVarName, a); } } }