public void testAggCoordVar2(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt);
IndexIterator dataI = data.getIndexIterator();
assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
assert dataI.getIntNext() == 2 : dataI.getIntCurrent();
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testScan() throws IOException, InvalidRangeException {
String xml =
"<?xml version='1.0' encoding='UTF-8'?>\n"
+ "<netcdf xmlns='http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2'>\n"
+ " <variable name='time' type='int' shape='time'>\n"
+ " <attribute name='long_name' type='string' value='time coordinate' />\n"
+ " <attribute name='units' type='string' value='days since 2001-8-31 00:00:00 UTC' />\n"
+ " <values start='0' increment='10' />\n"
+ " </variable>\n"
+ " <aggregation dimName='time' type='joinNew'>\n"
+ " <variableAgg name='T'/>\n"
+ " <scan location='src/test/data/ncml/nc/' suffix='Dir.nc' subdirs='false'/>\n"
+ " </aggregation>\n"
+ "</netcdf>";
String filename = "file:./" + TestNcML.topDir + "aggSynScan.xml";
NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(xml), null);
testDimensions(ncfile);
testCoordVar(ncfile);
testAggCoordVarScan(ncfile);
testReadData(ncfile, "T");
testReadSlice(ncfile, "T");
ncfile.close();
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt.D1) : data.getClass().getName();
ArrayInt.D1 dataI = (ArrayInt.D1) data;
assert dataI.get(0) == 0;
assert dataI.get(1) == 10;
assert dataI.get(2) == 99;
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testNcmlCached() throws IOException, InvalidRangeException {
System.out.println("\n TestNcmlAggExistingCached.acquire at " + new Date());
NetcdfFile ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
ncfile.close();
System.out.println("\n TestNcmlAggExistingCached.acquire again at " + new Date());
ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
ncfile.close();
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("\n TestNcmlAggExistingCached.acquire after sleeping " + new Date());
ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
ncfile.close();
NetcdfDataset.getNetcdfFileCache().clearCache(false);
System.out.println("\n TestNcmlAggExistingCached.acquire after flushing cache " + new Date());
ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
testAggCoordVarSubset(ncfile);
ncfile.close();
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count]);
count++;
}
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
/**
* Sets the v-velocity file to be used by this class
*
* @param _velocityFile - the v-velocity NetCDF object
* @param vName - the name of the v variable
* @throws IOException
*/
public void setVFile(String _velocityFile, String vName) throws IOException {
this.vName = vName;
vFile = NetcdfFile.open(_velocityFile);
if (vFile != null) {
vVar = vFile.findVariable(vName);
}
}
@Test
public void testCacheIsUsed() throws IOException, InvalidRangeException {
String filename = "file:TestAggExistingCache.xml";
System.out.printf("%s%n", filename);
String cacheDirName = TestDir.temporaryLocalDataDir + "testAggExistingCache/";
System.out.printf("cacheDir=%s%n", cacheDirName);
File cacheDir = new File(cacheDirName);
FileUtils.deleteDirectory(cacheDir); // from commons-io
assert !cacheDir.exists();
DiskCache2 cache = new DiskCache2(cacheDirName, false, 0, 0);
cache.setAlwaysUseCache(true);
Assert.assertEquals(cache.getRootDirectory(), cacheDirName);
assert new File(cache.getRootDirectory()).exists();
Aggregation.setPersistenceCache(cache);
AggregationExisting.countCacheUse = 0;
try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), filename, null)) {
System.out.println(" TestNcmlAggExisting.open " + filename);
Array ATssta = ncfile.readSection("ATssta(:,0,0,0)");
Assert.assertEquals(4, ATssta.getSize());
}
Assert.assertEquals(0, AggregationExisting.countCacheUse);
AggregationExisting.countCacheUse = 0;
try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), filename, null)) {
System.out.println(" TestNcmlAggExisting.open " + filename);
Array ATssta = ncfile.readSection("ATssta(:,0,0,0)");
Assert.assertEquals(4, ATssta.getSize());
}
Assert.assertEquals(8, AggregationExisting.countCacheUse);
}
/* Structure {
int LAT[0];
...
int LAT[149];
} IMAGE_LAT_ARRAY(3600);
type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
*/
@Test
public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 3600);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 3600);
Structure s = (Structure) dset;
// read last one - chunked
StructureData sd = s.readStructure(3599);
assert sd.getScalarInt("LAT[0]") == 70862722;
assert sd.getScalarInt("LAT[149]") == 85302263;
// read one at a time
for (int i = 3590; i < d.getLength(); i++) {
s.readStructure(i);
System.out.println(" read structure " + i);
}
}
System.out.println("*** testReadIASI ok");
}
/*
Structure {
int a_name;
byte b_name(3);
byte c_name(3);
short d_name(3);
int e_name(3);
long f_name(3);
int g_name(3);
short h_name(3);
int i_name(3);
long j_name(3);
float k_name(3);
double l_name(3);
} CompoundNative(15);
type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
*/
@Test
public void testReadH5StructureArrayMembers() throws java.io.IOException {
try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {
Variable dset = ncfile.findVariable("CompoundNative");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 15);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 15);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testReadH5StructureArrayMembers ok");
}
public void addBandMetadata(Product product) throws ProductIOException {
Group group = ncFile.findGroup("Geophysical_Data");
if (productReader.getProductType() == SeadasProductReader.ProductType.Level2_Aquarius) {
group = ncFile.findGroup("Aquarius_Data");
}
if (productReader.getProductType() == SeadasProductReader.ProductType.Level1B_HICO) {
group = ncFile.findGroup("products");
}
if (group != null) {
final MetadataElement bandAttributes = new MetadataElement("Band_Attributes");
List<Variable> variables = group.getVariables();
for (Variable variable : variables) {
final String name = variable.getShortName();
final MetadataElement sdsElement = new MetadataElement(name + ".attributes");
final int dataType = getProductDataType(variable);
final MetadataAttribute prodtypeattr = new MetadataAttribute("data_type", dataType);
sdsElement.addAttribute(prodtypeattr);
bandAttributes.addElement(sdsElement);
final List<Attribute> list = variable.getAttributes();
for (Attribute varAttribute : list) {
addAttributeToElement(sdsElement, varAttribute);
}
}
final MetadataElement metadataRoot = product.getMetadataRoot();
metadataRoot.addElement(bandAttributes);
}
}
/**
* Open a ucar.nc2.ft.PointFeatureCollection, write out in CF point format.
*
* @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
* @param fileOut write to this netcdf-3 file
* @param inMemory if true, read file into memory for efficiency
* @return true on success
* @throws IOException on read/write error
*/
public static boolean rewritePointFeatureDataset(String fileIn, String fileOut, boolean inMemory)
throws IOException {
System.out.println(
"Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);
long start = System.currentTimeMillis();
// do it in memory for speed
NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
NetcdfDataset ncd = new NetcdfDataset(ncfile);
Formatter errlog = new Formatter();
FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.ANY_POINT, ncd, null, errlog);
if (fd == null) return false;
if (fd instanceof FeatureDatasetPoint) {
writePointFeatureCollection((FeatureDatasetPoint) fd, fileOut);
fd.close();
long took = System.currentTimeMillis() - start;
System.out.println(" that took " + (took - start) + " msecs");
return true;
}
return false;
}
private void initialize() throws IOException {
List<Variable> list = ncf.getVariables();
List<Array> readArrays = ncf.readArrays(ncf.getVariables());
for (int i = 0; i < list.size(); i++) {
elementList[i] = new CDFElement(list.get(i));
elementList[i].setArray(readArrays.get(i));
}
}
/**
* Sets the name of the lookup variable for the Time position
*
* @param name
*/
public void setTLookup(String name) {
tVar = uFile.findVariable(tName);
if (tVar == null) {
System.out.println(
"Incorrect variable match:\n\nVariables in local config file: " + tName + ".");
System.out.println("Velocity file variables: " + uFile.getVariables().toString() + "\n");
}
tloc = new IndexLookup_Nearest(tVar);
}
/**
* Build the netCDF file
*
* @throws IOException problem reading the file
*/
protected void buildNCFile() throws IOException {
Trace.call1("GEMPAKSIOSP: buildNCFile");
ncfile.empty();
fillNCFile();
addGlobalAttributes();
ncfile.finish();
Trace.call2("GEMPAKSIOSP: buildNCFile");
// System.out.println(ncfile);
}
/** Create a DAS for this netcdf file */
NcDAS(NetcdfFile ncfile) {
// Variable attributes
Iterator iter = ncfile.getVariables().iterator();
while (iter.hasNext()) {
Variable v = (Variable) iter.next();
doVariable(v, null);
}
// Global attributes
opendap.dap.AttributeTable gtable = new opendap.dap.AttributeTable("NC_GLOBAL");
int count = addAttributes(gtable, null, ncfile.getGlobalAttributes().iterator());
if (count > 0)
try {
addAttributeTable("NC_GLOBAL", gtable);
} catch (AttributeExistsException e) {
log.error("Cant add NC_GLOBAL", e);
}
// unlimited dimension
iter = ncfile.getDimensions().iterator();
while (iter.hasNext()) {
Dimension d = (Dimension) iter.next();
if (d.isUnlimited()) {
opendap.dap.AttributeTable table = new opendap.dap.AttributeTable("DODS_EXTRA");
try {
table.appendAttribute("Unlimited_Dimension", opendap.dap.Attribute.STRING, d.getName());
addAttributeTable("DODS_EXTRA", table);
} catch (Exception e) {
log.error("Error adding Unlimited_Dimension =" + e);
}
break;
}
}
// unused dimensions
opendap.dap.AttributeTable dimTable = null;
iter = ncfile.getDimensions().iterator();
while (iter.hasNext()) {
Dimension d = (Dimension) iter.next();
if (null == usedDims.get(d.getName())) {
if (dimTable == null) dimTable = new opendap.dap.AttributeTable("EXTRA_DIMENSION");
try {
dimTable.appendAttribute(
d.getName(), opendap.dap.Attribute.INT32, Integer.toString(d.getLength()));
} catch (Exception e) {
log.error("Error adding Unlimited_Dimension =" + e);
}
}
}
if (dimTable != null)
try {
addAttributeTable("EXTRA_DIMENSION", dimTable);
} catch (AttributeExistsException e) {
log.error("Cant add EXTRA_DIMENSION", e);
}
}
public static boolean isMine(NetcdfFile ncfile) {
String satName = ncfile.findAttValueIgnoreCase(null, "SATNAME", null);
if ((satName == null) || !(satName.equalsIgnoreCase("Aqua"))) return false;
String instName = ncfile.findAttValueIgnoreCase(null, "INTRUMENT_NAME", null);
if ((instName == null) || !(instName.equalsIgnoreCase("modis"))) return false;
return true;
}
private void check(NetcdfFile ncfile, int n) throws IOException, InvalidRangeException {
Variable v = ncfile.findVariable("time");
assert v != null;
System.out.printf(" time= %s%n", v.getNameAndDimensions());
assert v.getSize() == n : v.getSize();
v = ncfile.findVariable("eta");
assert v != null;
assert v.getRank() == 3 : v.getRank();
}
public void testUpdate() throws IOException, InvalidRangeException, InterruptedException {
String dir = TestAll.cdmUnitTestDir + "agg/updating";
File dirFile = new File(dir);
assert dirFile != null;
assert dirFile.exists();
assert dirFile.isDirectory();
// make sure that the extra file is not in the agg
for (File f : dirFile.listFiles()) {
if (f.getName().equals("extra.nc")) {
if (!f.renameTo(new File(dirFile, "extra.wait"))) {
System.out.printf("Rename fails on %s.extra.nc %n", dirFile);
}
break;
}
}
String ncml =
"<?xml version='1.0' encoding='UTF-8'?>\n"
+ "<netcdf xmlns='http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2'>\n"
+ " <aggregation dimName='time' type='joinExisting' recheckEvery='1 nsec'>\n"
+ " <scan location='"
+ dir
+ "' suffix='*.nc' />\n"
+ " <variable name='depth'>\n"
+ " <attribute name='coordinates' value='lon lat'/>\n"
+ " </variable>\n"
+ " <variable name='wvh'>\n"
+ " <attribute name='coordinates' value='lon lat'/>\n"
+ " </variable>\n"
+ " </aggregation>\n"
+ " <attribute name='Conventions' type='String' value='CF-1.0'/>\n"
+ "</netcdf>";
String location = dir + "agg/updating.ncml";
System.out.println(" TestOffAggExistingTimeUnitsChange.testNarrGrib=\n" + ncml);
NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), location, null);
check(ncfile, 12);
// make sure that the extra file is in the agg
for (File f : dirFile.listFiles()) {
if (f.getName().equals("extra.wait")) {
if (!f.renameTo(new File(dirFile, "extra.nc")))
System.out.println(" rename fails on " + f.getPath());
break;
}
}
ncfile.sync();
check(ncfile, 18);
ncfile.close();
}
@Test
public void testReadnetCDFinSequnceFileFormat() throws Exception {
String path = this.getClass().getResource("/ncar.nc").getPath();
Util.writeToSequenceFile("file://" + path, hadoopMaster + "/tmp/ncar.seq", new DefaultCodec());
Map<Text, byte[]> netcdfsequnce = Util.readSequenceFile(hadoopMaster + "/tmp/ncar.seq");
for (Map.Entry<Text, byte[]> entry : netcdfsequnce.entrySet()) {
NetcdfFile ncFile = NetcdfFile.openInMemory(entry.getKey().toString(), entry.getValue());
assertEquals(ncFile.getDimensions().size(), 5);
}
}
public static void main(String args[]) throws Exception {
// String fileIn =
// "C:/data/bufr/edition3/newIdd/IcingTropopause/IcingTropopause_20080529_0000.bufr";
String fileIn =
"C:\\data\\bufr\\edition3\\meteosat\\METEOSAT7-MVIRI-MTPHRWW-NA-1-20080405123005.000000000Z-909326.bfr ";
NetcdfFile ncf = NetcdfDataset.openFile(fileIn, null);
System.out.println(ncf.toString());
new Write2ncRect(ncf, "C:/data/bufr2nc.meteosat.nc", true);
}
void writeNetCDF(String filename) {
try {
FileWriter2 writer = new FileWriter2(ds, filename, NetcdfFileWriter.Version.netcdf3);
NetcdfFile result = writer.write();
result.close();
JOptionPane.showMessageDialog(this, "File successfully written");
} catch (Exception ioe) {
JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage());
ioe.printStackTrace();
}
}
public static boolean isMine(NetcdfFile ncfile) {
if (!ncfile.getFileTypeId().equals("HDF5")) return false;
Group loc = ncfile.findGroup("VHRR/Geo-Location");
if (null == loc) return false;
if (null == loc.findVariable("Latitude")) return false;
if (null == loc.findVariable("Longitude")) return false;
if (null == ncfile.findGroup("VHRR/Image Data")) return false;
return true;
}
public void testNcmlDirect() throws IOException, InvalidRangeException {
NetcdfFile ncfile = NetcdfDataset.openDataset(filename, false, null);
System.out.println("\n TestNcmlAggExistingCached.open " + filename);
// System.out.println(" "+ ncfile);
testAggCoordVar(ncfile);
testAggCoordVarSubset(ncfile);
testAggCoordVarSubsetDefeatLocalCache(ncfile);
ncfile.close();
}
/**
* Sets the name of the lookup variable for the North-South position
*
* @param name
*/
public void setYLookup(String name) {
latVar = uFile.findVariable(latName);
if (latVar == null) {
System.out.println(
"Incorrect variable match:\n\nVariables in local config file: " + latName + ".");
System.out.println("Velocity file variables: " + uFile.getVariables().toString() + "\n");
}
yloc = new IndexLookup_Nearest(latVar, 0); // dimension 0 because slices
// have 2D lat/lon
bounds[2][0] = yloc.getMinVal();
bounds[2][1] = yloc.getMaxVal();
}
public void testNoCoord() throws IOException, InvalidRangeException {
String filename = "file:./" + TestNcML.topDir + "aggSynNoCoord.xml";
NetcdfFile ncfile = NcMLReader.readNcML(filename, null);
testDimensions(ncfile);
testCoordVar(ncfile);
testAggCoordVarNoCoord(ncfile);
testReadData(ncfile, "T");
testReadSlice(ncfile, "T");
ncfile.close();
}
@Test
public void testForStructMetadata() throws IOException {
System.out.printf("TestH4eosReadAll %s%n", filename);
try (NetcdfFile ncfile = NetcdfFile.open(filename)) {
Group root = ncfile.getRootGroup();
Group g = root.findGroup("HDFEOS INFORMATION");
if (g == null) g = ncfile.getRootGroup();
Variable dset = g.findVariable("StructMetadata.0");
assert (dset != null);
}
}
/* (non-Javadoc)
* @see loci.formats.NetCDFService#setFile(java.lang.String)
*/
public void setFile(String file) throws IOException {
this.currentFile = file;
String currentId = Location.getMappedId(currentFile);
netCDFFile = NetcdfFile.open(currentId);
root = netCDFFile.getRootGroup();
attributeList = new Vector<String>();
variableList = new Vector<String>();
List<Group> groups = new ArrayList<Group>();
groups.add(root);
parseAttributesAndVariables(groups);
}
/** Add on global attributes for all types */
protected void addGlobalAttributes() {
// global stuff
ncfile.addAttribute(null, new Attribute("Conventions", getConventions()));
String fileType = "GEMPAK " + gemreader.getFileType();
ncfile.addAttribute(null, new Attribute("file_format", fileType));
ncfile.addAttribute(
null,
new Attribute(
"history",
"Direct read of "
+ fileType
+ " into NetCDF-Java 4.2 API")); // at " + dateFormat.toDateTimeStringISO(new
// Date())));
ncfile.addAttribute(null, new Attribute(CF.FEATURE_TYPE, getCFFeatureType()));
}
public void testReadStandardVar() throws Exception {
ncfileRead = NetcdfFile.open(filename);
dsRead = NetcdfDataset.openDataset(filename);
readDouble();
readByte2Short();
readByte();
readShortMissing();
readShort2FloatMissing();
readDoubleMissing();
ncfileRead.close();
dsRead.close();
}
/**
* Clones the attributes of one NetCDF file into another.
*
* @param infile - The source NetCDF file
* @param inVarName - The variable name to be copied
* @param outfile - The destination NetCDF file
* @param outVarName - The output variable name
*/
private void cloneAttributes(
NetcdfFile infile, String inVarName, NetcdfFileWriteable outfile, String outVarName) {
// Find the variable
Variable vi = infile.findVariable(inVarName);
// Grab all of its attributes - unchecked, but should be OK.
List<Attribute> l = vi.getAttributes();
for (Attribute a : l) {
if (a.getName().equalsIgnoreCase("units") && inVarName.equalsIgnoreCase("time")) {
Attribute aa = new Attribute("units", "days since 1900-12-31 00:00:00");
outfile.addVariableAttribute(outVarName, aa);
} else if (a.getName().equalsIgnoreCase("time_origin")
&& inVarName.equalsIgnoreCase("time")) {
Attribute aa = new Attribute("time_origin", "1900-12-31 00:00:00");
outfile.addVariableAttribute(outVarName, aa);
} else if (a.getName().equalsIgnoreCase("calendar")) {
Attribute aa = new Attribute("calendar", "standard");
outfile.addVariableAttribute(outVarName, aa);
} else {
outfile.addVariableAttribute(outVarName, a);
}
}
}
