public void testNcmlCached() throws IOException, InvalidRangeException {
System.out.println("\n TestNcmlAggExistingCached.acquire at " + new Date());
NetcdfFile ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
ncfile.close();
System.out.println("\n TestNcmlAggExistingCached.acquire again at " + new Date());
ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
ncfile.close();
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace();
}
System.out.println("\n TestNcmlAggExistingCached.acquire after sleeping " + new Date());
ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
ncfile.close();
NetcdfDataset.getNetcdfFileCache().clearCache(false);
System.out.println("\n TestNcmlAggExistingCached.acquire after flushing cache " + new Date());
ncfile = NetcdfDataset.acquireDataset(filename, null);
testAggCoordVar(ncfile);
testAggCoordVarSubset(ncfile);
ncfile.close();
}
public Float10TrajectoryObsDataset(NetcdfDataset ncd) throws IOException {
super(ncd);
// Get the names of the two coordinate variables and dimensions
// and grab the variables and dimensions themselves.
trajDimName = trajDimNameDefault;
trajVarName = trajVarNameDefault;
timeDimName = timeDimNameDefault;
timeVarName = timeVarNameDefault;
latVarName = latVarNameDefault;
lonVarName = lonVarNameDefault;
elevVarName = elevVarNameDefault;
Variable latVar = ncd.getRootGroup().findVariable(latVarName);
latVar.addAttribute(new Attribute("units", "degrees_north"));
Variable lonVar = ncd.getRootGroup().findVariable(lonVarName);
lonVar.addAttribute(new Attribute("units", "degrees_east"));
this.setTrajectoryInfo(
netcdfDataset.getRootGroup().findDimension(trajDimName),
netcdfDataset.getRootGroup().findVariable(trajVarName),
netcdfDataset.getRootGroup().findDimension(timeDimName),
netcdfDataset.getRootGroup().findVariable(timeVarName),
netcdfDataset.getRootGroup().findVariable(latVarName),
netcdfDataset.getRootGroup().findVariable(lonVarName),
netcdfDataset.getRootGroup().findVariable(elevVarName));
}
private void makeMultidimInner(
NetcdfDataset ds, TableConfig parentTable, TableConfig childTable) {
Dimension parentDim = ds.findDimension(parentTable.dimName);
Dimension childDim = ds.findDimension(childTable.innerName);
// divide up the variables between the parent and the child
List<String> obsVars;
List<Variable> vars = ds.getVariables();
List<String> parentVars = new ArrayList<>(vars.size());
obsVars = new ArrayList<>(vars.size());
for (Variable orgV : vars) {
if (orgV instanceof Structure) continue;
Dimension dim0 = orgV.getDimension(0);
if ((dim0 != null) && dim0.equals(parentDim)) {
if ((orgV.getRank() == 1)
|| ((orgV.getRank() == 2) && orgV.getDataType() == DataType.CHAR)) {
parentVars.add(orgV.getShortName());
} else {
Dimension dim1 = orgV.getDimension(1);
if ((dim1 != null) && dim1.equals(childDim)) obsVars.add(orgV.getShortName());
}
}
}
parentTable.vars = parentVars;
childTable.vars = obsVars;
}
public UnidataTrajectoryObsDataset(NetcdfDataset ncd) throws IOException {
super(ncd);
// coordinate variables
latVar = UnidataObsDatasetHelper.getCoordinate(ncd, AxisType.Lat);
lonVar = UnidataObsDatasetHelper.getCoordinate(ncd, AxisType.Lon);
timeVar = UnidataObsDatasetHelper.getCoordinate(ncd, AxisType.Time);
elevVar = UnidataObsDatasetHelper.getCoordinate(ncd, AxisType.Height);
if (latVar == null) throw new IllegalStateException("Missing latitude variable");
if (lonVar == null) throw new IllegalStateException("Missing longitude coordinate variable");
if (timeVar == null) throw new IllegalStateException("Missing time coordinate variable");
if (elevVar == null) throw new IllegalStateException("Missing height coordinate variable");
timeDimName = timeVar.getDimension(0).getName();
timeVarName = timeVar.getShortName();
latVarName = latVar.getShortName();
lonVarName = lonVar.getShortName();
elevVarName = elevVar.getShortName();
Config trajConfig =
new Config(
"1Hz data",
ncd.getRootGroup().findDimension(timeDimName),
ncd.getRootGroup().findVariable(timeVarName),
ncd.getRootGroup().findVariable(latVarName),
ncd.getRootGroup().findVariable(lonVarName),
ncd.getRootGroup().findVariable(elevVarName));
this.setTrajectoryInfo(trajConfig);
}
private boolean hasAxisType(NetcdfDataset ds, AxisType a) {
List<Variable> varList = ds.getVariables();
for (Variable v : varList) {
String axisType = ds.findAttValueIgnoreCase(v, "CoordinateAxisType", null);
if ((axisType != null) && axisType.equals(a.toString())) return true;
}
return false;
}
public static void main(String arg[]) throws IOException {
NetcdfFile ncfile1 =
NetcdfDataset.openFile(
"dods://thredds.cise-nsf.gov:8080/thredds/dodsC/satellite/SFC-T/SUPER-NATIONAL_1km/20090516/SUPER-NATIONAL_1km_SFC-T_20090516_2200.gini",
null);
NetcdfFile ncfile2 =
NetcdfDataset.openFile(
"dods://motherlode.ucar.edu:8080/thredds/dodsC/satellite/SFC-T/SUPER-NATIONAL_1km/20090516/SUPER-NATIONAL_1km_SFC-T_20090516_2200.gini",
null);
compareFiles(ncfile1, ncfile2, new Formatter(System.out), false, true, false);
}
// read text from textArea through NcMLReader
// then write it back out via resulting dataset
private void checkNcml(Formatter f) {
if (ncmlLocation == null) return;
try {
NetcdfDataset ncd = NetcdfDataset.openDataset(ncmlLocation);
ncd.check(f);
} catch (IOException ioe) {
JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage());
ioe.printStackTrace();
}
}
@Test
public void testNoValid2DVariable() throws Exception {
final File file = TestData.file(this, "noVars.nc");
NetcdfDataset dataset = NetcdfDataset.acquireDataset(file.getAbsolutePath(), null);
List<Variable> variables = dataset.getVariables();
boolean speedVariableIsPresent = false;
String speedVariableName = "";
for (Variable variable : variables) {
if (variable.getShortName().equals("spd")) {
speedVariableIsPresent = true;
speedVariableName = variable.getFullName();
break;
}
}
assertTrue(speedVariableIsPresent);
final NetCDFImageReaderSpi unidataImageReaderSpi = new NetCDFImageReaderSpi();
assertTrue(unidataImageReaderSpi.canDecodeInput(file));
NetCDFImageReader reader = null;
try {
// sample dataset containing a water_speed variable having
// only time, depth dimensions. No lon/lat dims are present
// resulting into variable not usable.
reader = (NetCDFImageReader) unidataImageReaderSpi.createReaderInstance();
reader.setInput(file);
final List<Name> names = reader.getCoveragesNames();
boolean isSpeedCoverageAvailable = false;
for (Name name : names) {
if (name.toString().equals(speedVariableName)) {
isSpeedCoverageAvailable = true;
break;
}
}
// Checking that only "mask" variable is found
assertFalse(isSpeedCoverageAvailable);
} finally {
if (dataset != null) {
dataset.close();
}
if (reader != null) {
try {
reader.dispose();
} catch (Throwable t) {
// Does nothing
}
}
}
}
/**
* Constructor.
*
* @param ncfile the netccdf file
* @param typedDataVariables list of data variables; all record variables will be added to this
* list, except . You can remove extra
* @param obsTimeVName observation time variable name (required)
* @param nomTimeVName nominal time variable name (may be null)
* @throws IllegalArgumentException if ncfile has no unlimited dimension and recDimName is null.
*/
public RecordDatasetHelper(
NetcdfDataset ncfile,
String obsTimeVName,
String nomTimeVName,
List<VariableSimpleIF> typedDataVariables,
String recDimName,
Formatter errBuffer) {
this.ncfile = ncfile;
this.obsTimeVName = obsTimeVName;
this.nomTimeVName = nomTimeVName;
this.errs = errBuffer;
// check if we already have a structure vs if we have to add it.
if (this.ncfile.hasUnlimitedDimension()) {
this.ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
this.recordVar = (StructureDS) this.ncfile.getRootGroup().findVariable("record");
this.obsDim = ncfile.getUnlimitedDimension();
} else {
if (recDimName == null)
throw new IllegalArgumentException(
"File <"
+ this.ncfile.getLocation()
+ "> has no unlimited dimension, specify psuedo record dimension with observationDimension global attribute.");
this.obsDim = this.ncfile.getRootGroup().findDimension(recDimName);
this.recordVar = new StructurePseudoDS(this.ncfile, null, "record", null, obsDim);
}
// create member variables
List<Variable> recordMembers = ncfile.getVariables();
for (Variable v : recordMembers) {
if (v == recordVar) continue;
if (v.isScalar()) continue;
if (v.getDimension(0) == this.obsDim) typedDataVariables.add(v);
}
// need the time units
Variable timeVar = ncfile.findVariable(obsTimeVName);
String timeUnitString =
ncfile.findAttValueIgnoreCase(timeVar, CDM.UNITS, "seconds since 1970-01-01");
try {
timeUnit = new DateUnit(timeUnitString);
} catch (Exception e) {
if (null != errs) errs.format("Error on string = %s == %s%n", timeUnitString, e.getMessage());
try {
timeUnit = new DateUnit("seconds since 1970-01-01");
} catch (Exception e1) {
// cant happen
}
}
}
@Test
public void testConventionsAttribute() throws Exception {
String path = TestDir.cdmUnitTestDir + "ncml/AggForecastModel.ncml";
Formatter errlog = new Formatter();
Fmrc fmrc = Fmrc.open(path, errlog);
assert (fmrc != null) : errlog;
try (ucar.nc2.dt.GridDataset gridDs = fmrc.getDataset2D(null)) {
NetcdfDataset ncd = (NetcdfDataset) gridDs.getNetcdfFile();
Attribute att = ncd.findGlobalAttribute(CDM.CONVENTIONS);
assert att != null;
System.out.printf("%s%n", att);
}
}
// read text from textArea through NcMLReader
// then write it back out via resulting dataset
void doTransform(String text) {
try {
StringReader reader = new StringReader(text);
NetcdfDataset ncd = NcMLReader.readNcML(reader, null);
StringWriter sw = new StringWriter(10000);
ncd.writeNcML(sw, null);
editor.setText(sw.toString());
editor.setCaretPosition(0);
JOptionPane.showMessageDialog(this, "File successfully transformed");
} catch (IOException ioe) {
JOptionPane.showMessageDialog(this, "ERROR: " + ioe.getMessage());
ioe.printStackTrace();
}
}
private boolean addAxisType(NetcdfDataset ds, String vname, AxisType a) {
if (vname == null) return false;
Variable v = ds.findVariable(vname);
if (v == null) return false;
addAxisType(v, a);
return true;
}
public static void main(String[] args) {
// when dods using no ending
// NetcdfFile doesnt work
// http://thredds1.pfeg.noaa.gov:8080/thredds/dodsC/satellite/BA/ssta/mday.dods?BAssta[0:1:20][0:1:0][0:10:0][0:10:0]
String url =
"file:/Users/bermudez/downloads/SADCec95-UNC_WANAFp50-UFL_20080921T1800_20080921T1900_20080926T1800_11dvel_Z.nc";
// BAssta[0:1:73][0:1:0][1000:1:1000][2400:1:2400]
try {
NetcdfDataset netcdfdataset = NetcdfDataset.openDataset(url);
NCdump.print(netcdfdataset, "", System.out, null);
// List<Variable> vars = netcdfdataset.getVariables();
// for (Iterator iterator = vars.iterator(); iterator.hasNext();) {
// ucar.nc2.Variable var = (ucar.nc2.Variable) iterator.next();
// doVar(netcdfdataset,var.getName(),"0:2:2");
//// NCdump.printVariableData(var,null);
// System.out.println(var.getName());
// }
// doVar(netcdfdataset, "BAssta", "1:73,0,1000:1010,2400");
// doVar(netcdfdataset, "lon", "0:100");
// doVar(netcdfdataset, "lat", "0:100");
// doVar(netcdfdataset, "time", "0:30");
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
// should be called when tomcat exits
public void destroy() throws Exception {
if (timer != null) timer.cancel();
NetcdfDataset.shutdown();
if (aggCache != null) aggCache.exit();
if (cacheManager != null) cacheManager.close();
thredds.inventory.bdb.MetadataManager.closeAll();
}
public void closeOpenFiles() {
try {
if (ds != null) ds.close();
} catch (IOException ioe) {
}
ds = null;
}
public void showAtts() {
if (ds == null) return;
if (attTable == null) {
// global attributes
attTable =
new BeanTable(AttributeBean.class, (PreferencesExt) prefs.node("AttributeBeans"), false);
PopupMenu varPopup = new ucar.nc2.ui.widget.PopupMenu(attTable.getJTable(), "Options");
varPopup.addAction(
"Show Attribute",
new AbstractAction() {
public void actionPerformed(ActionEvent e) {
AttributeBean bean = (AttributeBean) attTable.getSelectedBean();
if (bean != null) {
infoTA.setText(bean.att.toString());
infoTA.gotoTop();
infoWindow.show();
}
}
});
attWindow =
new IndependentWindow("Global Attributes", BAMutil.getImage("netcdfUI"), attTable);
attWindow.setBounds(
(Rectangle) prefs.getBean("AttWindowBounds", new Rectangle(300, 100, 500, 800)));
}
List<AttributeBean> attlist = new ArrayList<>();
for (Attribute att : ds.getGlobalAttributes()) {
attlist.add(new AttributeBean(att));
}
attTable.setBeans(attlist);
attWindow.show();
}
public List<CoordinateSystemBean> getCoordinateSystemBeans(NetcdfDataset ds) {
List<CoordinateSystemBean> vlist = new ArrayList<>();
for (CoordinateSystem elem : ds.getCoordinateSystems()) {
vlist.add(new CoordinateSystemBean(elem));
}
return vlist;
}
@Test
public void testH5StructureDS() throws java.io.IOException {
int a_name = 0;
String[] b_name =
new String[] {
"A fight is a contract that takes two people to honor.",
"A combative stance means that you've accepted the contract.",
"In which case, you deserve what you get.",
" -- Professor Cheng Man-ch'ing"
};
String c_name = "Hello!";
// H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
try (NetcdfDataset ncfile =
NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {
Variable dset = ncfile.findVariable("CompoundComplex");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 6);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 6);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
assert sd.getScalarInt("a_name") == a_name;
a_name++;
assert sd.getScalarString("c_name").equals(c_name);
String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
assert results.length == b_name.length;
int count = 0;
for (String r : results) assert r.equals(b_name[count++]);
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testH5StructureDS ok");
}
private GridProjected queryGrid(Dataset dataset, List<TimeSlice> tss) throws IOException {
// Create a bounding box that encompasses all time slices.
Box smbounds = timeSliceUtil.aggregateBounds(tss);
if (smbounds == null) throw new IOException("Could not determine bounds: no time slices.");
BoxReal bounds = new BoxReal(2);
bounds.getMin().setX(smbounds.getXMin());
bounds.getMin().setY(smbounds.getYMin());
bounds.getMax().setX(smbounds.getXMax());
bounds.getMax().setY(smbounds.getYMax());
// Resolution.
double cellSize = dataset.getResolution().toDouble();
VectorReal resolution = VectorReal.createEmpty(2);
resolution.setX(cellSize);
resolution.setY(cellSize);
// All files for a given dataset share a common coordinate system. So,
// we can just open one of the blank files and take a peek at it.
CoordinateSystem srs = null;
List<Band> bands = datasetDao.getBands(dataset.getId());
for (Band b : bands) {
Path blankTilePath = bandUtil.getBlankTilePath(dataset, b);
if (Files.notExists(blankTilePath)) continue;
NetcdfDataset ncd = NetcdfDataset.openDataset(blankTilePath.toString());
try {
ncd.enhance();
srs = ncd.getCoordinateSystems().get(0);
break;
} finally {
ncd.close();
}
}
if (srs == null) {
throw new IOException(
String.format(
"Could not determine "
+ "coordinate system for dataset %s. There may be no "
+ "tiles, or the bounds in the database may not match the "
+ "tiles stored on disk.",
dataset));
}
return new GridProjected(bounds, resolution, srs);
}
public void setLocationInfo(String latVName, String lonVName, String zcoordVName) {
this.latVName = latVName;
this.lonVName = lonVName;
this.zcoordVName = zcoordVName;
// check for meter conversion
if (zcoordVName != null) {
Variable v = ncfile.findVariable(zcoordVName);
zcoordUnits = ncfile.findAttValueIgnoreCase(v, CDM.UNITS, null);
if (zcoordUnits != null)
try {
altScaleFactor = getMetersConversionFactor(zcoordUnits);
} catch (Exception e) {
if (errs != null) errs.format("%s", e.getMessage());
}
}
}
@Override
public NetcdfDataset open(
String uri,
String referential,
BoxReal boundsHint,
DateTime timeMin,
DateTime timeMax,
List<String> bands)
throws IOException {
NetcdfDataset ds;
VectorReal min = boundsHint.getMin();
VectorReal max = boundsHint.getMax();
Box bounds = new Box(min.getX(), min.getY(), max.getX(), max.getY());
Dataset dataset = findDataset(uri, referential);
if (dataset == null) {
throw new IOException(String.format("Could not find dataset %s", uri));
}
if (ndgConfigManager.getConfig().isFilelockingOn()) {
ReadWriteLock lock = getLock(dataset);
// If filelockingOn, then ensure can get all required read locks
if (lock.readLock().tryLock()) {
try {
ds = open(uri, dataset, bounds, timeMin, timeMax, bands);
try {
ds = new RsaNetcdfDataset(ds, lock);
} catch (IOException | RuntimeException e) {
if (ds != null) ds.close();
throw e;
}
} catch (IOException | RuntimeException e) {
lock.readLock().unlock();
throw e;
}
} else {
throw new IOException(String.format("Could not lock dataset %s", uri));
}
} else {
ds = open(uri, dataset, bounds, timeMin, timeMax, bands);
}
return ds;
}
public String getReadOk() {
if (!getComplete().equals("true")) return "false";
if (!getBitsOk().equals("true")) return "false";
if (!doRead) return "N/A";
if (readOk == 0)
try {
NetcdfDataset ncd = getBufrMessageAsDataset(m);
SequenceDS v = (SequenceDS) ncd.findVariable(BufrIosp.obsRecord);
StructureDataIterator iter = v.getStructureIterator(-1);
while (iter.hasNext()) {
iter.next();
}
readOk = 1;
} catch (Exception e) {
readOk = 2;
}
return readOk == 1 ? "true" : "false";
}
/** Main-type execution */
public void go() {
System.out.println("Writing to " + outputWFile + "...");
try {
uFile = NetcdfDataset.openDataset(inputUFile);
vFile = NetcdfDataset.openDataset(inputVFile);
bathy = new NetCDF_FloatGrid_3D(inputBathyFile, inLatName, inLonName);
generate(uFile, vFile);
} catch (IOException e) {
e.printStackTrace();
} catch (InvalidRangeException e) {
e.printStackTrace();
}
System.out.println("Complete.");
}
private void setObs(Message m) {
java.util.List<ObsBean> beanList = new ArrayList<ObsBean>();
try {
NetcdfDataset ncd = getBufrMessageAsDataset(m);
Variable v = ncd.findVariable(BufrIosp.obsRecord);
if ((v != null) && (v instanceof Structure)) {
Structure obs = (Structure) v;
StructureDataIterator iter = obs.getStructureIterator();
while (iter.hasNext()) {
beanList.add(new ObsBean(obs, iter.next()));
}
}
} catch (Exception ex) {
JOptionPane.showMessageDialog(BufrMessageViewer.this, ex.getMessage());
ex.printStackTrace();
}
obsTable.setBeans(beanList);
}
public void testAlias() throws IOException {
String filename = TestAll.cdmUnitTestDir + "fmrc/ensemble/demeter/MM_cnrm_129_red.ncml";
NetcdfDataset ncd = ucar.nc2.dataset.NetcdfDataset.openDataset(filename);
Variable v = ncd.findCoordinateAxis("number");
assert v != null;
// assert v.isCoordinateVariable();
assert v instanceof CoordinateAxis1D;
assert null != ncd.findDimension("ensemble");
assert v.getDimension(0) == ncd.findDimension("ensemble");
}
public static void main(String args[]) throws Exception {
// String fileIn =
// "C:/data/bufr/edition3/newIdd/IcingTropopause/IcingTropopause_20080529_0000.bufr";
String fileIn =
"C:\\data\\bufr\\edition3\\meteosat\\METEOSAT7-MVIRI-MTPHRWW-NA-1-20080405123005.000000000Z-909326.bfr ";
NetcdfFile ncf = NetcdfDataset.openFile(fileIn, null);
System.out.println(ncf.toString());
new Write2ncRect(ncf, "C:/data/bufr2nc.meteosat.nc", true);
}
private JoinArray readJoinArray(NetcdfDataset ds, Element joinElement) {
JoinArray.Type type = JoinArray.Type.valueOf(joinElement.getAttributeValue("type"));
Element paramElem = joinElement.getChild("param");
String paramS = paramElem.getText();
Integer param = Integer.parseInt(paramS);
Element varElem = joinElement.getChild("variable");
String varName = varElem.getText();
VariableDS v = (VariableDS) ds.findVariable(varName);
return new JoinArray(v, type, param);
}
public void testNcmlDirect() throws IOException, InvalidRangeException {
NetcdfFile ncfile = NetcdfDataset.openDataset(filename, false, null);
System.out.println("\n TestNcmlAggExistingCached.open " + filename);
// System.out.println(" "+ ncfile);
testAggCoordVar(ncfile);
testAggCoordVarSubset(ncfile);
testAggCoordVarSubsetDefeatLocalCache(ncfile);
ncfile.close();
}
public void setDataset(NetcdfDataset ds) {
this.ds = ds;
parseInfo = new Formatter();
List<VariableBean> beanList = new ArrayList<>();
List<AxisBean> axisList = new ArrayList<>();
setVariables(ds.getVariables(), axisList, beanList);
varTable.setBeans(beanList);
axisTable.setBeans(axisList);
csTable.setBeans(getCoordinateSystemBeans(ds));
}
/**
* Set extra information used by station obs datasets. Use stnIdVName or stnIndexVName.
*
* @param stnIdVName the obs variable that is used to find the station in the stnHash; may be type
* int or a String (char).
* @param stnDescVName optional station var containing station description
*/
public void setStationInfo(
String stnIdVName, String stnDescVName, String stnIndexVName, StationHelper stationHelper) {
this.stnIdVName = stnIdVName;
this.stnDescVName = stnDescVName;
this.stnIndexVName = stnIndexVName;
this.stationHelper = stationHelper;
if (stnIdVName != null) {
Variable stationVar = ncfile.findVariable(stnIdVName);
stationIdType = stationVar.getDataType();
}
}
