// create from a dataset
public VariableBean(Variable vs) {
this.vs = vs;
// vs = (v instanceof VariableEnhanced) ? (VariableEnhanced) v : new VariableStandardized( v);
setName(vs.getShortName());
setDescription(vs.getDescription());
setUnits(vs.getUnitsString());
setDataType(vs.getDataType().toString());
// Attribute csAtt = vs.findAttribute("_coordSystems");
// if (csAtt != null)
// setCoordSys( csAtt.getStringValue());
// collect dimensions
StringBuilder lens = new StringBuilder();
StringBuilder names = new StringBuilder();
java.util.List dims = vs.getDimensions();
for (int j = 0; j < dims.size(); j++) {
ucar.nc2.Dimension dim = (ucar.nc2.Dimension) dims.get(j);
if (j > 0) {
lens.append(",");
names.append(",");
}
String name = dim.isShared() ? dim.getName() : "anon";
names.append(name);
lens.append(dim.getLength());
}
setDimensions(names.toString());
setShape(lens.toString());
}
public boolean compareVariables(NetcdfFile org, NetcdfFile copy) {
f.format("Original = %s%n", org.getLocation());
f.format("CompareTo= %s%n", copy.getLocation());
boolean ok = true;
for (Variable orgV : org.getVariables()) {
if (orgV.isCoordinateVariable()) continue;
Variable copyVar = copy.findVariable(orgV.getShortName());
if (copyVar == null) {
f.format(" MISSING '%s' in 2nd file%n", orgV.getFullName());
ok = false;
} else {
List<Dimension> dims1 = orgV.getDimensions();
List<Dimension> dims2 = copyVar.getDimensions();
if (!compare(dims1, dims2)) {
f.format(" %s != %s%n", orgV.getNameAndDimensions(), copyVar.getNameAndDimensions());
} else {
// f.format(" ok %s%n", orgV.getName());
}
}
}
f.format("%n");
for (Variable orgV : copy.getVariables()) {
if (orgV.isCoordinateVariable()) continue;
Variable copyVar = org.findVariable(orgV.getShortName());
if (copyVar == null) {
f.format(" MISSING '%s' in 1st file%n", orgV.getFullName());
ok = false;
}
}
return ok;
}
public boolean compareVariables(
Variable org, Variable copy, boolean compareData, boolean justOne) {
boolean ok = true;
if (showCompare)
f.format("compare Variable %s to %s %n", org.getFullName(), copy.getFullName());
if (!org.getFullName().equals(copy.getFullName())) {
f.format(" ** names are different %s != %s %n", org.getFullName(), copy.getFullName());
ok = false;
}
// dimensions
ok &= checkAll(org.getDimensions(), copy.getDimensions(), null);
// attributes
ok &= checkAll(org.getAttributes(), copy.getAttributes(), null);
// coord sys
if ((org instanceof VariableEnhanced) && (copy instanceof VariableEnhanced)) {
VariableEnhanced orge = (VariableEnhanced) org;
VariableEnhanced copye = (VariableEnhanced) copy;
ok &= checkAll(orge.getCoordinateSystems(), copye.getCoordinateSystems(), null);
}
// data !!
if (compareData) {
try {
compareVariableData(org, copy, showCompare, justOne);
} catch (IOException e) {
ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
e.printStackTrace(new PrintStream(bos));
f.format("%s", bos.toString());
}
}
// nested variables
if (org instanceof Structure) {
if (!(copy instanceof Structure)) {
f.format(" ** %s not Structure%n", org);
ok = false;
} else {
Structure orgS = (Structure) org;
Structure ncmlS = (Structure) copy;
List vars = new ArrayList();
ok &= checkAll(orgS.getVariables(), ncmlS.getVariables(), vars);
for (int i = 0; i < vars.size(); i += 2) {
Variable orgV = (Variable) vars.get(i);
Variable ncmlV = (Variable) vars.get(i + 1);
ok &= compareVariables(orgV, ncmlV, false, true);
}
}
}
return ok;
}
private void doVariable(Variable v, opendap.dap.AttributeTable parentTable) {
List dims = v.getDimensions();
for (int i = 0; i < dims.size(); i++) {
Dimension dim = (Dimension) dims.get(i);
if (dim.isShared()) usedDims.put(dim.getName(), dim);
}
// if (v.getAttributes().size() == 0) return; // LOOK DAP 2 say must have empty
String name = NcDDS.escapeName(v.getShortName());
opendap.dap.AttributeTable table;
if (parentTable == null) {
table = new opendap.dap.AttributeTable(name);
try {
addAttributeTable(name, table);
} catch (AttributeExistsException e) {
log.error("Cant add " + name, e);
}
} else {
table = parentTable.appendContainer(name);
}
addAttributes(table, v, v.getAttributes().iterator());
if (v instanceof Structure) {
Structure s = (Structure) v;
List nested = s.getVariables();
for (int i = 0; i < nested.size(); i++) {
Variable nv = (Variable) nested.get(i);
doVariable(nv, table);
}
}
}
private void makeVariableNoCoords(
NetcdfFile ncfile, int datatype, String shortName, String longName, Variable from) {
Variable v = new Variable(ncfile, null, null, shortName);
v.setDataType(DataType.BYTE);
v.setDimensions(from.getDimensions());
ncfile.addVariable(null, v);
v.addAttribute(new Attribute(CDM.UNITS, Cinrad2Record.getDatatypeUnits(datatype)));
v.addAttribute(new Attribute(CDM.LONG_NAME, longName));
byte[] b = new byte[2];
b[0] = Cinrad2Record.MISSING_DATA;
b[1] = Cinrad2Record.BELOW_THRESHOLD;
Array missingArray = Array.factory(DataType.BYTE.getPrimitiveClassType(), new int[] {2}, b);
v.addAttribute(new Attribute(CDM.MISSING_VALUE, missingArray));
v.addAttribute(
new Attribute("signal_below_threshold", new Byte(Cinrad2Record.BELOW_THRESHOLD)));
v.addAttribute(
new Attribute(CDM.SCALE_FACTOR, new Float(Cinrad2Record.getDatatypeScaleFactor(datatype))));
v.addAttribute(
new Attribute(CDM.ADD_OFFSET, new Float(Cinrad2Record.getDatatypeAddOffset(datatype))));
v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));
Attribute fromAtt = from.findAttribute(_Coordinate.Axes);
v.addAttribute(new Attribute(_Coordinate.Axes, fromAtt));
Vgroup vgFrom = (Vgroup) from.getSPobject();
Vgroup vg = new Vgroup(datatype, vgFrom.map);
v.setSPobject(vg);
}
// convert to shared dimensions
private void setSharedDimensions(
Variable v, List<Element> values, List<Dimension> unknownDims, String location) {
if (values.size() == 0) return;
// remove the "scalar" dumbension
Iterator<Element> iter = values.iterator();
while (iter.hasNext()) {
Element value = iter.next();
String dimName = value.getText().trim();
if (dimName.equalsIgnoreCase("scalar")) iter.remove();
}
// gotta have same number of dimensions
List<Dimension> oldDims = v.getDimensions();
if (oldDims.size() != values.size()) {
log.error("Different number of dimensions for {} {}", v, location);
return;
}
List<Dimension> newDims = new ArrayList<>();
Group group = v.getParentGroup();
for (int i = 0; i < values.size(); i++) {
Element value = values.get(i);
String dimName = value.getText().trim();
dimName = NetcdfFile.makeValidCdmObjectName(dimName);
Dimension dim = group.findDimension(dimName);
Dimension oldDim = oldDims.get(i);
if (dim == null) dim = checkUnknownDims(dimName, unknownDims, oldDim, location);
if (dim == null) {
log.error(
"Unknown Dimension= {} for variable = {} {} ", dimName, v.getFullName(), location);
return;
}
if (dim.getLength() != oldDim.getLength()) {
log.error(
"Shared dimension ("
+ dim.getShortName()
+ ") has different length than data dimension ("
+ oldDim.getShortName()
+ ") shared="
+ dim.getLength()
+ " org="
+ oldDim.getLength()
+ " for "
+ v
+ " "
+ location);
return;
}
newDims.add(dim);
}
v.setDimensions(newDims);
if (showWork) System.out.printf(" set shared dimensions for %s %n", v.getNameAndDimensions());
}
/**
* Build the configuration from the dataset
*
* @param ncd NetcdfDataset
* @return the trajectory configuration
*/
private static Config buildConfig(NetcdfDataset ncd) {
// already did this in isValid, but we'll keep here for later refactor
Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center");
if (attrib == null) {
return null;
}
if (!attrib.isString()) {
return null;
}
if (!attrib.getStringValue().equals("UCAR/CDAAC")) {
return null;
}
// Check for start_time, stop_time
attrib = ncd.findGlobalAttributeIgnoreCase("start_time");
if (attrib == null) {
return null;
}
if (attrib.isString()) {
return null;
}
double startTime = attrib.getNumericValue().doubleValue();
attrib = ncd.findGlobalAttributeIgnoreCase("stop_time");
if (attrib == null) {
return null;
}
if (attrib.isString()) {
return null;
}
double endTime = attrib.getNumericValue().doubleValue();
// Check that only one dimension and that it is the alt dimension.
List list = ncd.getRootGroup().getDimensions();
if (list.size() != 1) {
return null;
}
Dimension d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
Config trajConfig = new Config();
trajConfig.setTimeDim(d);
// Check for latitude variable with time dimension and units convertable to "degrees_north".
Variable var = ncd.getRootGroup().findVariable(latVarName);
if (var == null) {
return null;
}
list = var.getDimensions();
if (list.size() != 1) {
return null;
}
d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
String units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "degrees_north")) {
return null;
}
trajConfig.setLatVar(var);
// Make the time Variable
int numTimes = d.getLength();
double[] times = new double[numTimes];
// Variable timeVar = new Variable(var);
// timeVar.setName(timeVarName);
VariableDS timeVar =
new VariableDS(
ncd,
ncd.getRootGroup(),
null,
timeVarName,
DataType.DOUBLE,
timeDimName,
"seconds since 1980-01-06 00:00:00",
"Time coordinate");
// Variable timeVar = new Variable(ncd, ncd.getRootGroup(), null,
// timeVarName);
// timeVar.setDataType(DataType.DOUBLE);
// timeVar.setDimensions(list);
// Attribute newUnits =
// new Attribute("units", "seconds since 1980-01-06 00:00:00");
// timeVar.addAttribute(newUnits);
timeVar.setCachedData(
Array.makeArray(DataType.DOUBLE, numTimes, endTime, ((startTime - endTime) / numTimes)),
true);
ncd.addVariable(ncd.getRootGroup(), timeVar);
trajConfig.setTimeVar(timeVar);
// Check for longitude variable with time dimension and units convertable to "degrees_east".
var = ncd.getRootGroup().findVariable(lonVarName);
if (var == null) {
return null;
}
list = var.getDimensions();
if (list.size() != 1) {
return null;
}
d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "degrees_east")) {
return null;
}
trajConfig.setLonVar(var);
// Check for altitude variable with time dimension and units convertable to "m".
var = ncd.getRootGroup().findVariable(elevVarName);
if (var == null) {
return null;
}
list = var.getDimensions();
if (list.size() != 1) {
return null;
}
d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "meters")) {
return null;
}
trajConfig.setElevVar(var);
trajConfig.setTrajectoryId(trajId);
return trajConfig;
}
/**
* Parses a variable recursively into appropriate ViewVariable implementations
*
* @throws IOException
* @throws
*/
private static AbstractViewVariable parseVariableRecursive(Variable var) throws IOException {
List<Dimension> dimensions = var.getDimensions();
// A single dimension means we can parse a SimpleAxis
if (dimensions.size() == 1) {
SimpleAxis axis =
new SimpleAxis(var.getName(), var.getDataType().name(), var.getUnitsString(), null, null);
Dimension d = dimensions.get(0);
axis.setDimensionBounds(new SimpleBounds(0, d.getLength()));
// Read our first and last values
Array first = null, last = null;
try {
first = var.read(new int[] {0}, new int[] {1});
last = var.read(new int[] {d.getLength() - 1}, new int[] {1});
} catch (InvalidRangeException ex) {
throw new IllegalArgumentException(
String.format("Unable to read variable ranges '%1$s'", var), ex);
}
axis.setValueBounds(new SimpleBounds(first.getDouble(0), last.getDouble(0)));
return axis;
// Otherwise we have a multi dimensional variable that we can parse as a grid
} else if (dimensions.size() > 0) {
SimpleGrid grid =
new SimpleGrid(var.getName(), var.getDataType().name(), var.getUnitsString(), null);
List<AbstractViewVariable> childAxes = new ArrayList<>();
// Recursively parse each dimension (which should map to a variable in the parent group)
for (Dimension d : dimensions) {
Variable mappedVariable = d.getGroup().findVariable(d.getName());
if (mappedVariable == null) {
// If the dimension doesn't map to a variable, we can't pull much information out of it
// So instead we'll have to introduce an axis that only includes dimension bounds
log.warn(
String.format(
"Dimension '%1$s' has no matching variable in parent group '%2$s'",
d, d.getGroup()));
SimpleAxis axis = new SimpleAxis(d.getName(), DataType.FLOAT.name(), "????", null, null);
axis.setDimensionBounds(new SimpleBounds(0, d.getLength() - 1));
childAxes.add(axis);
} else {
AbstractViewVariable parsedVar = parseVariableRecursive(mappedVariable);
if (parsedVar != null) childAxes.add(parsedVar);
}
}
if (childAxes.size() > 0) {
grid.setAxes(childAxes.toArray(new AbstractViewVariable[childAxes.size()]));
return grid;
} else {
return null;
}
} else {
// Currently unsupported...
log.warn(
String.format("Variables with 0 dimensions are currently unsupported. var='%1$s'", var));
return null;
}
}
Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
throws IOException, InvalidRangeException {
NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
if (debug) {
System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
}
// global attributes
List<Attribute> glist = bufr.getGlobalAttributes();
for (Attribute att : glist) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
Attribute useAtt;
if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
if (debug) System.out.println("add gatt= " + useAtt);
}
// global dimensions
Dimension recordDim = null;
Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
for (Dimension oldD : bufr.getDimensions()) {
String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
boolean isRecord = useName.equals("record");
Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
dimHash.put(newD.getName(), newD);
if (isRecord) recordDim = newD;
if (debug) System.out.println("add dim= " + newD);
}
// Variables
Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
for (Variable oldVar : recordStruct.getVariables()) {
if (oldVar.getDataType() == DataType.STRUCTURE) continue;
String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
DataType newType = oldVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
for (Dimension dim : oldVar.getDimensions()) {
newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = oldVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
// int max_seq = countSeq(recordStruct);
// Dimension seqD = ncfile.addDimension("level", max_seq);
for (Variable v : recordStruct.getVariables()) {
if (v.getDataType() != DataType.STRUCTURE) continue;
String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
int shape[] = v.getShape();
Dimension structDim = ncfile.addDimension(structName, shape[0]);
Structure struct = (Structure) v;
for (Variable seqVar : struct.getVariables()) {
String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
DataType newType = seqVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
newDims.add(structDim);
for (Dimension dim : seqVar.getDimensions()) {
newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = seqVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
}
// create the file
ncfile.create();
if (debug) System.out.println("File Out= " + ncfile.toString());
// boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
double total = copyVarData(ncfile, recordStruct);
ncfile.flush();
System.out.println("FileWriter done total bytes = " + total);
ncfile.close();
}
public static boolean isValidFile(NetcdfDataset ds) {
// Check that has a time dimension and a trajectory dimension.
List list = ds.getRootGroup().getDimensions();
if (list.size() != 2) return (false);
Dimension d;
for (int i = 0; i < 2; i++) {
d = (Dimension) list.get(i);
if (!d.getShortName().equals(timeDimNameDefault)
&& !d.getShortName().equals(trajDimNameDefault)) return (false);
}
// Check that has a trajectory coordinate variable.
Variable var = ds.getRootGroup().findVariable(trajVarNameDefault);
if (var == null) return (false);
list = var.getDimensions();
if (list.size() != 1) return (false);
d = (Dimension) list.get(0);
if (!d.getShortName().equals(trajDimNameDefault)) return (false);
// Check that has a time coordinate variable with units that are udunits time
var = ds.getRootGroup().findVariable(timeVarNameDefault);
if (var == null) return (false);
list = var.getDimensions();
if (list.size() != 1) return (false);
d = (Dimension) list.get(0);
if (!d.getShortName().equals(timeDimNameDefault)) return (false);
String units = var.findAttribute("units").getStringValue();
Date date = DateUnit.getStandardDate("0 " + units);
if (date == null) return (false);
// Check for variable latitude(time) with units of "deg".
var = ds.getRootGroup().findVariable(latVarNameDefault);
if (var == null) return (false);
list = var.getDimensions();
if (list.size() != 2) return (false);
for (int i = 0; i < 2; i++) {
d = (Dimension) list.get(i);
if (!d.getShortName().equals(timeDimNameDefault)
&& !d.getShortName().equals(trajDimNameDefault)) return (false);
}
// units = var.findAttribute( "units").getStringValue();
// if ( ! SimpleUnit.isCompatible( units, "degrees_north")) return( false);
// Check for variable longitude(time) with units of "deg".
var = ds.getRootGroup().findVariable(lonVarNameDefault);
if (var == null) return (false);
list = var.getDimensions();
if (list.size() != 2) return (false);
for (int i = 0; i < 2; i++) {
d = (Dimension) list.get(i);
if (!d.getShortName().equals(timeDimNameDefault)
&& !d.getShortName().equals(trajDimNameDefault)) return (false);
}
// units = var.findAttribute( "units").getStringValue();
// if ( ! SimpleUnit.isCompatible( units, "degrees_east")) return( false);
// Check for variable altitude(time) with units of "m".
var = ds.getRootGroup().findVariable(elevVarNameDefault);
if (var == null) return (false);
list = var.getDimensions();
if (list.size() != 2) return (false);
for (int i = 0; i < 2; i++) {
d = (Dimension) list.get(i);
if (!d.getShortName().equals(timeDimNameDefault)
&& !d.getShortName().equals(trajDimNameDefault)) return (false);
}
units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "m")) return (false);
return (true);
}