private void makeMultidimInner(
NetcdfDataset ds, TableConfig parentTable, TableConfig childTable) {
Dimension parentDim = ds.findDimension(parentTable.dimName);
Dimension childDim = ds.findDimension(childTable.innerName);
// divide up the variables between the parent and the child
List<String> obsVars;
List<Variable> vars = ds.getVariables();
List<String> parentVars = new ArrayList<>(vars.size());
obsVars = new ArrayList<>(vars.size());
for (Variable orgV : vars) {
if (orgV instanceof Structure) continue;
Dimension dim0 = orgV.getDimension(0);
if ((dim0 != null) && dim0.equals(parentDim)) {
if ((orgV.getRank() == 1)
|| ((orgV.getRank() == 2) && orgV.getDataType() == DataType.CHAR)) {
parentVars.add(orgV.getShortName());
} else {
Dimension dim1 = orgV.getDimension(1);
if ((dim1 != null) && dim1.equals(childDim)) obsVars.add(orgV.getShortName());
}
}
}
parentTable.vars = parentVars;
childTable.vars = obsVars;
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count]);
count++;
}
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testAggCoordVar2(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt);
IndexIterator dataI = data.getIndexIterator();
assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
assert dataI.getIntNext() == 2 : dataI.getIntCurrent();
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt.D1) : data.getClass().getName();
ArrayInt.D1 dataI = (ArrayInt.D1) data;
assert dataI.get(0) == 0;
assert dataI.get(1) == 10;
assert dataI.get(2) == 99;
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
/* Structure {
int LAT[0];
...
int LAT[149];
} IMAGE_LAT_ARRAY(3600);
type = Layout(8); type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
*/
@Test
public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 3600);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 3600);
Structure s = (Structure) dset;
// read last one - chunked
StructureData sd = s.readStructure(3599);
assert sd.getScalarInt("LAT[0]") == 70862722;
assert sd.getScalarInt("LAT[149]") == 85302263;
// read one at a time
for (int i = 3590; i < d.getLength(); i++) {
s.readStructure(i);
System.out.println(" read structure " + i);
}
}
System.out.println("*** testReadIASI ok");
}
/*
Structure {
int a_name;
byte b_name(3);
byte c_name(3);
short d_name(3);
int e_name(3);
long f_name(3);
int g_name(3);
short h_name(3);
int i_name(3);
long j_name(3);
float k_name(3);
double l_name(3);
} CompoundNative(15);
type = Layout(8); type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
*/
@Test
public void testReadH5StructureArrayMembers() throws java.io.IOException {
try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {
Variable dset = ncfile.findVariable("CompoundNative");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 15);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 15);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testReadH5StructureArrayMembers ok");
}
private void testReadData(Variable v) throws IOException {
if (show) System.out.printf(" read %s%n", v.getNameAndDimensions());
assert (null != v);
assert (null != v.getDimension(0));
Array a = v.read();
assert (null != a);
assert (v.getSize() == a.getSize());
}
public void testAlias() throws IOException {
String filename = TestAll.cdmUnitTestDir + "fmrc/ensemble/demeter/MM_cnrm_129_red.ncml";
NetcdfDataset ncd = ucar.nc2.dataset.NetcdfDataset.openDataset(filename);
Variable v = ncd.findCoordinateAxis("number");
assert v != null;
// assert v.isCoordinateVariable();
assert v instanceof CoordinateAxis1D;
assert null != ncd.findDimension("ensemble");
assert v.getDimension(0) == ncd.findDimension("ensemble");
}
private int addAttributes(opendap.dap.AttributeTable table, Variable v, Iterator iter) {
int count = 0;
// add attribute table for this variable
while (iter.hasNext()) {
Attribute att = (Attribute) iter.next();
int dods_type = DODSNetcdfFile.convertToDODSType(att.getDataType(), false);
try {
String attName = NcDDS.escapeName(att.getName());
if (att.isString()) {
/* FIX String value = escapeAttributeStringValues(att.getStringValue());
table.appendAttribute(attName, dods_type, "\""+value+"\"");
*/
table.appendAttribute(attName, dods_type, att.getStringValue());
} else {
// cant send signed bytes
if (att.getDataType() == DataType.BYTE) {
boolean signed = false;
for (int i = 0; i < att.getLength(); i++) {
if (att.getNumericValue(i).byteValue() < 0) signed = true;
}
if (signed) // promote to signed short
dods_type = opendap.dap.Attribute.INT16;
}
for (int i = 0; i < att.getLength(); i++)
table.appendAttribute(attName, dods_type, att.getNumericValue(i).toString());
}
count++;
} catch (Exception e) {
log.error(
"Error appending attribute " + att.getName() + " = " + att.getStringValue() + "\n" + e);
}
} // loop over variable attributes
// kludgy thing to map char arrays to DODS Strings
if ((v != null) && (v.getDataType().getPrimitiveClassType() == char.class)) {
int rank = v.getRank();
int strlen = (rank == 0) ? 0 : v.getShape(rank - 1);
Dimension dim = (rank == 0) ? null : v.getDimension(rank - 1);
try {
opendap.dap.AttributeTable dodsTable = table.appendContainer("DODS");
dodsTable.appendAttribute("strlen", opendap.dap.Attribute.INT32, Integer.toString(strlen));
if ((dim != null) && dim.isShared())
dodsTable.appendAttribute("dimName", opendap.dap.Attribute.STRING, dim.getName());
count++;
} catch (Exception e) {
log.error("Error appending attribute strlen\n" + e);
}
}
return count;
}
/**
* Constructor.
*
* @param ncfile the netccdf file
* @param typedDataVariables list of data variables; all record variables will be added to this
* list, except . You can remove extra
* @param obsTimeVName observation time variable name (required)
* @param nomTimeVName nominal time variable name (may be null)
* @throws IllegalArgumentException if ncfile has no unlimited dimension and recDimName is null.
*/
public RecordDatasetHelper(
NetcdfDataset ncfile,
String obsTimeVName,
String nomTimeVName,
List<VariableSimpleIF> typedDataVariables,
String recDimName,
Formatter errBuffer) {
this.ncfile = ncfile;
this.obsTimeVName = obsTimeVName;
this.nomTimeVName = nomTimeVName;
this.errs = errBuffer;
// check if we already have a structure vs if we have to add it.
if (this.ncfile.hasUnlimitedDimension()) {
this.ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
this.recordVar = (StructureDS) this.ncfile.getRootGroup().findVariable("record");
this.obsDim = ncfile.getUnlimitedDimension();
} else {
if (recDimName == null)
throw new IllegalArgumentException(
"File <"
+ this.ncfile.getLocation()
+ "> has no unlimited dimension, specify psuedo record dimension with observationDimension global attribute.");
this.obsDim = this.ncfile.getRootGroup().findDimension(recDimName);
this.recordVar = new StructurePseudoDS(this.ncfile, null, "record", null, obsDim);
}
// create member variables
List<Variable> recordMembers = ncfile.getVariables();
for (Variable v : recordMembers) {
if (v == recordVar) continue;
if (v.isScalar()) continue;
if (v.getDimension(0) == this.obsDim) typedDataVariables.add(v);
}
// need the time units
Variable timeVar = ncfile.findVariable(obsTimeVName);
String timeUnitString =
ncfile.findAttValueIgnoreCase(timeVar, CDM.UNITS, "seconds since 1970-01-01");
try {
timeUnit = new DateUnit(timeUnitString);
} catch (Exception e) {
if (null != errs) errs.format("Error on string = %s == %s%n", timeUnitString, e.getMessage());
try {
timeUnit = new DateUnit("seconds since 1970-01-01");
} catch (Exception e1) {
// cant happen
}
}
}
public void testReadData(NetcdfFile ncfile, String name) throws IOException {
Variable v = ncfile.findVariable(name);
assert null != v;
assert v.getShortName().equals(name);
assert v.getRank() == 3;
assert v.getSize() == 36 : v.getSize();
assert v.getShape()[0] == 3;
assert v.getShape()[1] == 3;
assert v.getShape()[2] == 4;
assert v.getDataType() == DataType.DOUBLE;
assert !v.isCoordinateVariable();
assert v.getDimension(0) == ncfile.findDimension("time");
assert v.getDimension(1) == ncfile.findDimension("lat");
assert v.getDimension(2) == ncfile.findDimension("lon");
Array data = v.read();
assert data.getRank() == 3;
assert data.getSize() == 36;
assert data.getShape()[0] == 3;
assert data.getShape()[1] == 3;
assert data.getShape()[2] == 4;
assert data.getElementType() == double.class;
int[] shape = data.getShape();
Index tIndex = data.getIndex();
for (int i = 0; i < shape[0]; i++)
for (int j = 0; j < shape[1]; j++)
for (int k = 0; k < shape[2]; k++) {
double val = data.getDouble(tIndex.set(i, j, k));
// System.out.println(" "+val);
assert TestUtils.close(val, 100 * i + 10 * j + k) : val;
}
}
private int[] getWeights(Variable v) {
int rank = v.getRank();
int[] w = new int[rank];
for (int n = 0; n < rank; n++) {
Dimension dim = v.getDimension(n);
String dimName = dim.getName();
if (dimName.equals("time")) w[n] = 1000;
if (dimName.equals("z")) w[n] = 100;
if (dimName.equals("y")) w[n] = 10;
if (dimName.equals("x")) w[n] = 1;
}
return w;
}
@Test
public void testH5StructureDS() throws java.io.IOException {
int a_name = 0;
String[] b_name =
new String[] {
"A fight is a contract that takes two people to honor.",
"A combative stance means that you've accepted the contract.",
"In which case, you deserve what you get.",
" -- Professor Cheng Man-ch'ing"
};
String c_name = "Hello!";
// H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
try (NetcdfDataset ncfile =
NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {
Variable dset = ncfile.findVariable("CompoundComplex");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 6);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 6);
Structure s = (Structure) dset;
// read all with the iterator
StructureDataIterator iter = s.getStructureIterator();
while (iter.hasNext()) {
StructureData sd = iter.next();
assert sd.getScalarInt("a_name") == a_name;
a_name++;
assert sd.getScalarString("c_name").equals(c_name);
String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
assert results.length == b_name.length;
int count = 0;
for (String r : results) assert r.equals(b_name[count++]);
for (StructureMembers.Member m : sd.getMembers()) {
Array data = sd.getArray(m);
NCdumpW.printArray(data, m.getName(), out, null);
}
}
}
System.out.println("*** testH5StructureDS ok");
}
public void testAggCoordVar3(NetcdfFile ncfile) throws IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE : time.getDataType();
assert time.getDimension(0) == ncfile.findDimension("time");
Array data = time.read();
assert (data instanceof ArrayDouble);
IndexIterator dataI = data.getIndexIterator();
double val = dataI.getDoubleNext();
assert Misc.closeEnough(val, 0.0) : val;
assert Misc.closeEnough(dataI.getDoubleNext(), 10.0) : dataI.getDoubleCurrent();
assert Misc.closeEnough(dataI.getDoubleNext(), 99.0) : dataI.getDoubleCurrent();
}
public void testAggCoordVarScan(NetcdfFile ncfile) throws IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT : time.getDataType();
assert time.getDimension(0) == ncfile.findDimension("time");
int count = 0;
Array data = time.read();
assert (data instanceof ArrayInt);
while (data.hasNext()) {
int val = data.nextInt();
assert val == count * 10 : val + "!=" + count * 10;
count++;
}
}
public void testAggCoordVarSubsetDefeatLocalCache(NetcdfFile ncfile)
throws InvalidRangeException, IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
time.setCachedData(null, false);
Array data = time.read("1:2");
assert data.getRank() == 1;
assert data.getSize() == 2;
assert data.getShape()[0] == 2;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count + 1]);
count++;
}
time.setCachedData(null, false);
data = time.read("0:2:2");
assert data.getRank() == 1;
assert data.getSize() == 2;
assert data.getShape()[0] == 2;
assert data.getElementType() == double.class;
count = 0;
dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count * 2]);
count++;
}
}
public void testAggCoordVarNoCoordsDir(NetcdfFile ncfile) throws IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.STRING : time.getDataType();
assert time.getDimension(0) == ncfile.findDimension("time");
Array data = time.read();
assert (data instanceof ArrayObject);
IndexIterator dataI = data.getIndexIterator();
String coordName = (String) dataI.getObjectNext();
assert coordName.equals("time0Dir.nc") : coordName;
coordName = (String) dataI.getObjectNext();
assert coordName.equals("time1Dir.nc") : coordName;
coordName = (String) dataI.getObjectNext();
assert coordName.equals("time2Dir.nc") : coordName;
}
/* Structure {
char EntryName(64);
char Definition(1024);
char Unit(1024);
char Scale Factor(1024);
} TIME_DESCR(60);
type = Layout(8); type= 2 (chunked) storageSize = (1,3136) dataSize=0 dataAddress=684294
*/
@Test
public void testReadManyAtATime() throws java.io.IOException, InvalidRangeException {
try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {
Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/TIME_DESCR");
assert (null != dset);
assert (dset.getDataType() == DataType.STRUCTURE);
assert (dset.getRank() == 1);
assert (dset.getSize() == 60);
Dimension d = dset.getDimension(0);
assert (d.getLength() == 60);
ArrayStructure data = (ArrayStructure) dset.read();
StructureMembers.Member m = data.getStructureMembers().findMember("EntryName");
assert m != null;
for (int i = 0; i < dset.getSize(); i++) {
String r = data.getScalarString(i, m);
if (i % 2 == 0) assert r.equals("TIME[" + i / 2 + "]-days") : r + " at " + i;
else assert r.equals("TIME[" + i / 2 + "]-milliseconds") : r + " at " + i;
}
}
System.out.println("*** testReadManyAtATime ok");
}
public void testCoordVar(NetcdfFile ncfile) {
Variable lat = ncfile.findVariable("lat");
assert null != lat;
assert lat.getShortName().equals("lat");
assert lat.getRank() == 1;
assert lat.getSize() == 3;
assert lat.getShape()[0] == 3;
assert lat.getDataType() == DataType.FLOAT;
assert !lat.isUnlimited();
assert lat.getDimension(0).equals(ncfile.findDimension("lat"));
Attribute att = lat.findAttribute("units");
assert null != att;
assert !att.isArray();
assert att.isString();
assert att.getDataType() == DataType.STRING;
assert att.getStringValue().equals("degrees_north");
assert att.getNumericValue() == null;
assert att.getNumericValue(3) == null;
try {
Array data = lat.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == float.class;
IndexIterator dataI = data.getIndexIterator();
assert TestUtils.close(dataI.getDoubleNext(), 41.0);
assert TestUtils.close(dataI.getDoubleNext(), 40.0);
assert TestUtils.close(dataI.getDoubleNext(), 39.0);
} catch (IOException io) {
}
}
/**
* Opens the NetCDF dataset at the given location, using the dataset cache if {@code location}
* represents an NcML aggregation. We cannot use the cache for OPeNDAP or single NetCDF files
* because the underlying data may have changed and the NetcdfDataset cache may cache a dataset
* forever. In the case of NcML we rely on the fact that server administrators ought to have set a
* "recheckEvery" parameter for NcML aggregations that may change with time. It is desirable to
* use the dataset cache for NcML aggregations because they can be time-consuming to assemble and
* we don't want to do this every time a map is drawn.
*
* @param location The location of the data: a local NetCDF file, an NcML aggregation file or an
* OPeNDAP location, {@literal i.e.} anything that can be passed to
* NetcdfDataset.openDataset(location).
* @return a {@link NetcdfDataset} object for accessing the data at the given location.
* @throws IOException if there was an error reading from the data source.
*/
private NetcdfDataset openAndAggregateDataset(String location) throws IOException, EdalException {
NetcdfDataset nc;
if (location.startsWith("dods://") || location.startsWith("http://")) {
/*
* We have a remote dataset
*/
nc = CdmUtils.openDataset(location);
} else {
/*
* We have a local dataset
*/
List<File> files = null;
try {
files = CdmUtils.expandGlobExpression(location);
} catch (NullPointerException e) {
System.out.println("NPE processing location: " + location);
throw e;
}
if (files.size() == 0) {
throw new EdalException(
"The location " + location + " doesn't refer to any existing files.");
}
if (files.size() == 1) {
location = files.get(0).getAbsolutePath();
nc = CdmUtils.openDataset(location);
} else {
/*
* We have multiple files in a glob expression. We write some
* NcML and use the NetCDF aggregation libs to parse this into
* an aggregated dataset.
*
* If we have already generated the ncML on a previous call,
* just use that.
*/
if (ncmlString == null) {
/*
* Find the name of the time dimension
*/
NetcdfDataset first = openAndAggregateDataset(files.get(0).getAbsolutePath());
String timeDimName = null;
for (Variable var : first.getVariables()) {
if (var.isCoordinateVariable()) {
for (Attribute attr : var.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("units")
&& attr.getStringValue().contains(" since ")) {
/*
* This is the time dimension. Since this is
* a co-ordinate variable, there is only 1
* dimension
*/
Dimension timeDimension = var.getDimension(0);
timeDimName = timeDimension.getFullName();
}
}
}
}
first.close();
if (timeDimName == null) {
throw new EdalException("Cannot join multiple files without time dimensions");
}
/*
* We can't assume that the glob expression will have
* returned the files in time order.
*
* We could assume that alphabetical == time ordered (and
* for properly named files it will - but let's not rely on
* our users having sensible naming conventions...
*
* Sort the list using a comparator which opens the file and
* gets the first value of the time dimension
*/
final String aggDimName = timeDimName;
Collections.sort(
files,
new Comparator<File>() {
@Override
public int compare(File ncFile1, File ncFile2) {
NetcdfFile nc1 = null;
NetcdfFile nc2 = null;
try {
nc1 = NetcdfFile.open(ncFile1.getAbsolutePath());
nc2 = NetcdfFile.open(ncFile2.getAbsolutePath());
Variable timeVar1 = nc1.findVariable(aggDimName);
Variable timeVar2 = nc2.findVariable(aggDimName);
long time1 = timeVar1.read().getLong(0);
long time2 = timeVar2.read().getLong(0);
return Long.compare(time1, time2);
} catch (Exception e) {
/*
* There was a problem reading the data. Sort
* alphanumerically by filename and hope for the
* best...
*
* This catches all exceptions because however
* it fails this is still our best option.
*
* If the error is a genuine problem, it'll show
* up as soon as we try and aggregate.
*/
return ncFile1.getAbsolutePath().compareTo(ncFile2.getAbsolutePath());
} finally {
if (nc1 != null) {
try {
nc1.close();
} catch (IOException e) {
log.error("Problem closing netcdf file", e);
}
}
if (nc2 != null) {
try {
nc2.close();
} catch (IOException e) {
log.error("Problem closing netcdf file", e);
}
}
}
}
});
/*
* Now create the NcML string and use it to create an
* aggregated dataset
*/
StringBuffer ncmlStringBuffer = new StringBuffer();
ncmlStringBuffer.append(
"<netcdf xmlns=\"http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2\">");
ncmlStringBuffer.append(
"<aggregation dimName=\"" + timeDimName + "\" type=\"joinExisting\">");
for (File file : files) {
ncmlStringBuffer.append("<netcdf location=\"" + file.getAbsolutePath() + "\"/>");
}
ncmlStringBuffer.append("</aggregation>");
ncmlStringBuffer.append("</netcdf>");
ncmlString = ncmlStringBuffer.toString();
}
nc = NcMLReader.readNcML(new StringReader(ncmlString), null);
}
}
return nc;
}
public static void main(String args[]) throws Exception {
long start = System.currentTimeMillis();
Map<String, ucar.unidata.geoloc.Station> staHash =
new HashMap<String, ucar.unidata.geoloc.Station>();
String location = "R:/testdata/sounding/netcdf/Upperair_20070401_0000.nc";
NetcdfDataset ncfile = NetcdfDataset.openDataset(location);
ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
// look through record varibles, for those that have "manLevel" dimension
// make a StructureData object for those
StructureMembers sm = new StructureMembers("manLevel");
Dimension manDim = ncfile.findDimension("manLevel");
Structure record = (Structure) ncfile.findVariable("record");
List<Variable> allList = record.getVariables();
List<VariableSimpleIF> varList = new ArrayList<VariableSimpleIF>();
for (Variable v : allList) {
if ((v.getRank() == 1) && v.getDimension(0).equals(manDim)) {
// public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String
// shortName, DataType dataType,
// String dims, String units, String desc) {
varList.add(
new VariableDS(
ncfile,
null,
null,
v.getShortName(),
v.getDataType(),
"",
v.getUnitsString(),
v.getDescription()));
// (String name, String desc, String units, DataType dtype, int []shape)
sm.addMember(
v.getShortName(),
v.getDescription(),
v.getUnitsString(),
v.getDataType(),
new int[0]); // scalar
}
}
ArrayStructureMA manAS = new ArrayStructureMA(sm, new int[] {manDim.getLength()});
// need the date units
Variable time = ncfile.findVariable("synTime");
String timeUnits = ncfile.findAttValueIgnoreCase(time, "units", null);
timeUnits = StringUtil.remove(timeUnits, '('); // crappy fsl'ism
timeUnits = StringUtil.remove(timeUnits, ')');
DateUnit timeUnit = new DateUnit(timeUnits);
// extract stations
int nrecs = 0;
StructureDataIterator iter = record.getStructureIterator();
while (iter.hasNext()) {
StructureData sdata = iter.next();
String name = sdata.getScalarString("staName");
ucar.unidata.geoloc.Station s = staHash.get(name);
if (s == null) {
float lat = sdata.convertScalarFloat("staLat");
float lon = sdata.convertScalarFloat("staLon");
float elev = sdata.convertScalarFloat("staElev");
s = new StationImpl(name, "", lat, lon, elev);
staHash.put(name, s);
}
nrecs++;
}
List<ucar.unidata.geoloc.Station> stnList =
Arrays.asList(staHash.values().toArray(new ucar.unidata.geoloc.Station[staHash.size()]));
Collections.sort(stnList);
// create the writer
WriterProfileObsDataset writer =
new WriterProfileObsDataset(location + ".out", "rewrite " + location);
writer.writeHeader(stnList, varList, nrecs, "prMan");
// extract records
iter = record.getStructureIterator();
while (iter.hasNext()) {
StructureData sdata = iter.next();
String name = sdata.getScalarString("staName");
double timeValue = sdata.convertScalarDouble("synTime");
Date date = timeUnit.makeDate(timeValue);
// transfer to the ArrayStructure
List<String> names = sm.getMemberNames();
for (String mname : names) {
manAS.setMemberArray(mname, sdata.getArray(mname));
}
// each level is weritten as a seperate structure
int numMand = sdata.getScalarInt("numMand");
if (numMand >= manDim.getLength()) continue;
for (int i = 0; i < numMand; i++) {
StructureData useData = manAS.getStructureData(i);
writer.writeRecord(name, date, useData);
}
}
writer.finish();
long took = System.currentTimeMillis() - start;
System.out.println("That took = " + took);
}
/**
* Generates an empty template for a block
*
* @param files
* @param outputFile
* @return
*/
private NetcdfFileWriteable makeTemplate(TreeMap<Long, NetcdfFile> files, String outputFile) {
NetcdfFileWriteable ncf_out = null;
if (!(new File(outputDir).exists())) {
throw new IllegalArgumentException("Directory " + outputDir + " does not exist.");
}
try {
ncf_in = files.get((new ArrayList<Long>(files.keySet())).get(0));
Variable time = ncf_in.findVariable(inTimeName);
if (time == null) {
List<Variable> var = ncf_in.getVariables();
System.out.println(
"WARNING: Variable "
+ inTimeName
+ " was not found. File variables are:\n"
+ Arrays.toString(var.toArray()));
}
Variable depth = ncf_in.findVariable(inDepthName);
if (depth == null && ncf_in.getDimensions().size() > 3) {
List<Variable> var = ncf_in.getVariables();
System.out.println(
"WARNING: Depth variable "
+ inDepthName
+ " not found, and the number of dimensions is greater than 3."
+ " File variables are:\n"
+ Arrays.toString(var.toArray()));
}
Variable lat = ncf_in.findVariable(inLatName);
if (lat == null) {
List<Variable> var = ncf_in.getVariables();
System.out.println(
"WARNING: Variable "
+ inLatName
+ " was not found. File variables are:\n"
+ Arrays.toString(var.toArray()));
}
Variable lon = ncf_in.findVariable(inLonName);
if (lon == null) {
List<Variable> var = ncf_in.getVariables();
System.out.println(
"WARNING: Variable "
+ inLonName
+ " was not found. File variables are:\n"
+ Arrays.toString(var.toArray()));
}
int latidx = lat.findDimensionIndex(inVerticalDim);
int lonidx = lon.findDimensionIndex(inHorizontalDim);
int latlen = lat.getDimension(latidx).getLength();
int lonlen = lon.getDimension(lonidx).getLength();
Long[] la = files.keySet().toArray(new Long[files.size()]);
timeArr = Array.factory(DataType.DOUBLE, new int[] {files.size()});
for (int i = 0; i < la.length; i++) {
timeArr.setDouble(i, TimeConvert.millisToHYCOM(la[i]));
}
if (depth != null) {
depthArr = depth.read();
if (reverseDepth) {
for (int i = 0; i < depthArr.getShape()[0]; i++) {
if (i != 0) {
depthArr.setDouble(i, -depthArr.getDouble(i));
} else {
depthArr.setDouble(i, 0);
}
}
}
}
int[] latshape = ones(lat.getRank());
latshape[latidx] = latlen;
latArr = (lat.read(new int[lat.getRank()], latshape)).reduce();
int[] lonshape = ones(lon.getRank());
lonshape[lonidx] = lonlen;
lonArr = (lon.read(new int[lon.getRank()], lonshape)).reduce();
ncf_out = NetcdfFileWriteable.createNew(outputFile, false);
// Add Dimensions
Dimension timeDim = new Dimension(outTimeName, timeArr.getShape()[0]);
Dimension depthDim = null;
if (depthArr != null) {
depthDim = new Dimension(outDepthName, depthArr.getShape()[0]);
}
Dimension latDim = new Dimension(outLatName, latArr.getShape()[0]);
Dimension lonDim = new Dimension(outLonName, lonArr.getShape()[0]);
ncf_out.addDimension(null, timeDim);
if (depthDim != null) {
ncf_out.addDimension(null, depthDim);
}
ncf_out.addDimension(null, latDim);
ncf_out.addDimension(null, lonDim);
// Add Variables
ncf_out.addVariable(outTimeName, DataType.DOUBLE, new Dimension[] {timeDim});
if (depthDim != null) {
ncf_out.addVariable(outDepthName, DataType.DOUBLE, new Dimension[] {depthDim});
}
ncf_out.addVariable(outLatName, DataType.DOUBLE, new Dimension[] {latDim});
ncf_out.addVariable(outLonName, DataType.DOUBLE, new Dimension[] {lonDim});
Dimension[] dims = null;
if (depthDim == null) {
dims = new Dimension[] {timeDim, latDim, lonDim};
} else {
dims = new Dimension[] {timeDim, depthDim, latDim, lonDim};
}
ncf_out.addVariable(outVarName, DataType.FLOAT, dims);
// Add attribute information (cloned from source)
cloneAttributes(ncf_in, inTimeName, ncf_out, outTimeName);
cloneAttributes(ncf_in, inDepthName, ncf_out, outDepthName);
cloneAttributes(ncf_in, inLatName, ncf_out, outLatName);
cloneAttributes(ncf_in, inLonName, ncf_out, outLonName);
cloneAttributes(ncf_in, inVarName, ncf_out, outVarName);
ncf_out.create();
} catch (IOException e) {
e.printStackTrace();
} catch (InvalidRangeException e) {
e.printStackTrace();
}
return ncf_out;
}
@Test
public void testDoradeAir() throws IOException {
System.out.println("**** Open " + airDoradeFile);
try (NetcdfFile ncfile = NetcdfFile.open(airDoradeFile)) {
for (Variable v : ncfile.getVariables()) {
System.out.println(v.getFullName());
}
Dimension gateDim = ncfile.getRootGroup().findDimension("gate_1");
assert (gateDim.getLength() == 320);
Dimension radialDim = ncfile.getRootGroup().findDimension("radial");
assert (radialDim.getLength() == 274);
Variable var;
var = ncfile.findVariable("elevation");
testReadData(var);
var = ncfile.findVariable("azimuth");
testReadData(var);
var = ncfile.findVariable("distance_1");
testReadData(var);
var = ncfile.findVariable("latitudes_1");
testReadData(var);
var = ncfile.findVariable("longitudes_1");
testReadData(var);
var = ncfile.findVariable("altitudes_1");
testReadData(var);
var = ncfile.findVariable("rays_time");
testReadData(var);
var = ncfile.findVariable("Range_to_First_Cell");
float t = testReadScalar(var);
assert (t == (float) -2.0);
var = ncfile.findVariable("Cell_Spacing");
t = testReadScalar(var);
assert (t == (float) 150.0);
var = ncfile.findVariable("Fixed_Angle");
t = testReadScalar(var);
assert (t == (float) -16.53);
var = ncfile.findVariable("Nyquist_Velocity");
t = testReadScalar(var);
assert (t == (float) 78.03032);
var = ncfile.findVariable("Unambiguous_Range");
t = testReadScalar(var);
assert (t == (float) 60.0);
var = ncfile.findVariable("Radar_Constant");
t = testReadScalar(var);
assert (t == (float) -81.17389);
var = ncfile.findVariable("rcvr_gain");
t = testReadScalar(var);
assert (t == (float) 32.64);
var = ncfile.findVariable("ant_gain");
t = testReadScalar(var);
assert (t == (float) 39.35);
var = ncfile.findVariable("sys_gain");
t = testReadScalar(var);
assert (t == (float) 52.34);
var = ncfile.findVariable("bm_width");
t = testReadScalar(var);
assert (t == (float) 1.79);
var = ncfile.findVariable("VS");
testReadData(var);
var = ncfile.findVariable("VL");
testReadData(var);
var = ncfile.findVariable("SW");
testReadData(var);
var = ncfile.findVariable("VR");
testReadData(var);
var = ncfile.findVariable("NCP");
testReadData(var);
var = ncfile.findVariable("DBZ");
testReadData(var);
var = ncfile.findVariable("VG");
testReadData(var);
var = ncfile.findVariable("VT");
testReadData(var);
assert (null != var.getDimension(0));
assert (null != var.getDimension(1));
}
}
