public boolean compareVariables(NetcdfFile org, NetcdfFile copy) {
f.format("Original = %s%n", org.getLocation());
f.format("CompareTo= %s%n", copy.getLocation());
boolean ok = true;
for (Variable orgV : org.getVariables()) {
if (orgV.isCoordinateVariable()) continue;
Variable copyVar = copy.findVariable(orgV.getShortName());
if (copyVar == null) {
f.format(" MISSING '%s' in 2nd file%n", orgV.getFullName());
ok = false;
} else {
List<Dimension> dims1 = orgV.getDimensions();
List<Dimension> dims2 = copyVar.getDimensions();
if (!compare(dims1, dims2)) {
f.format(" %s != %s%n", orgV.getNameAndDimensions(), copyVar.getNameAndDimensions());
} else {
// f.format(" ok %s%n", orgV.getName());
}
}
}
f.format("%n");
for (Variable orgV : copy.getVariables()) {
if (orgV.isCoordinateVariable()) continue;
Variable copyVar = org.findVariable(orgV.getShortName());
if (copyVar == null) {
f.format(" MISSING '%s' in 1st file%n", orgV.getFullName());
ok = false;
}
}
return ok;
}
public boolean compare(
NetcdfFile org, NetcdfFile copy, boolean showCompare, boolean showEach, boolean compareData) {
this.compareData = compareData;
this.showCompare = showCompare;
this.showEach = showEach;
f.format("First file = %s%n", org.getLocation());
f.format("Second file= %s%n", copy.getLocation());
long start = System.currentTimeMillis();
boolean ok = compareGroups(org.getRootGroup(), copy.getRootGroup());
f.format(" Files are the same = %s%n", ok);
long took = System.currentTimeMillis() - start;
f.format(" Time to compare = %d msecs%n", took);
return ok;
}
private FeatureType amendSwath(NetcdfFile ncfile, Element swathElem, Group parent) {
FeatureType featureType = FeatureType.SWATH;
List<Dimension> unknownDims = new ArrayList<>();
// Dimensions
Element d = swathElem.getChild("Dimension");
List<Element> dims = d.getChildren();
for (Element elem : dims) {
String name = elem.getChild("DimensionName").getText().trim();
name = NetcdfFile.makeValidCdmObjectName(name);
if (name.equalsIgnoreCase("scalar")) continue;
String sizeS = elem.getChild("Size").getText().trim();
int length = Integer.parseInt(sizeS);
if (length > 0) {
Dimension dim = parent.findDimensionLocal(name);
if (dim != null) { // already added - may be dimension scale ?
if (dim.getLength() != length) { // ok as long as it matches
log.error("Conflicting Dimensions = {} {}", dim, ncfile.getLocation());
throw new IllegalStateException("Conflicting Dimensions = " + name);
}
} else {
dim = new Dimension(name, length);
if (parent.addDimensionIfNotExists(dim) && showWork)
System.out.printf(" Add dimension %s %n", dim);
}
} else {
log.warn("Dimension " + name + " has size " + sizeS, ncfile.getLocation());
Dimension udim = new Dimension(name, 1);
udim.setGroup(parent);
unknownDims.add(udim);
if (showWork) System.out.printf(" Add dimension %s %n", udim);
}
}
// Dimension Maps
Element dmap = swathElem.getChild("DimensionMap");
List<Element> dimMaps = dmap.getChildren();
for (Element elem : dimMaps) {
String geoDimName = elem.getChild("GeoDimension").getText().trim();
geoDimName = NetcdfFile.makeValidCdmObjectName(geoDimName);
String dataDimName = elem.getChild("DataDimension").getText().trim();
dataDimName = NetcdfFile.makeValidCdmObjectName(dataDimName);
String offsetS = elem.getChild("Offset").getText().trim();
String incrS = elem.getChild("Increment").getText().trim();
int offset = Integer.parseInt(offsetS);
int incr = Integer.parseInt(incrS);
// make new variable for this dimension map
Variable v = new Variable(ncfile, parent, null, dataDimName);
v.setDimensions(geoDimName);
v.setDataType(DataType.INT);
int npts = (int) v.getSize();
Array data = Array.makeArray(v.getDataType(), npts, offset, incr);
v.setCachedData(data, true);
v.addAttribute(new Attribute("_DimensionMap", ""));
parent.addVariable(v);
if (showWork) System.out.printf(" Add dimensionMap %s %n", v);
}
// Geolocation Variables
Group geoFieldsG = parent.findGroup(GEOLOC_FIELDS);
if (geoFieldsG == null) geoFieldsG = parent.findGroup(GEOLOC_FIELDS2);
if (geoFieldsG != null) {
Variable latAxis = null, lonAxis = null;
Element floc = swathElem.getChild("GeoField");
List<Element> varsLoc = floc.getChildren();
for (Element elem : varsLoc) {
String varname = elem.getChild("GeoFieldName").getText().trim();
Variable v = geoFieldsG.findVariable(varname);
// if (v == null)
// v = geoFieldsG.findVariable( H4header.createValidObjectName(varname));
assert v != null : varname;
AxisType axis = addAxisType(ncfile, v);
if (axis == AxisType.Lat) latAxis = v;
if (axis == AxisType.Lon) lonAxis = v;
Element dimList = elem.getChild("DimList");
List<Element> values = dimList.getChildren("value");
setSharedDimensions(v, values, unknownDims, ncfile.getLocation());
if (showWork) System.out.printf(" set coordinate %s %n", v);
}
if ((latAxis != null) && (lonAxis != null)) {
List<Dimension> xyDomain = CoordinateSystem.makeDomain(new Variable[] {latAxis, lonAxis});
if (xyDomain.size() < 2) featureType = FeatureType.PROFILE; // ??
}
}
// Data Variables
Group dataG = parent.findGroup(DATA_FIELDS);
if (dataG == null) dataG = parent.findGroup(DATA_FIELDS2);
if (dataG != null) {
Element f = swathElem.getChild("DataField");
List<Element> vars = f.getChildren();
for (Element elem : vars) {
Element dataFieldNameElem = elem.getChild("DataFieldName");
if (dataFieldNameElem == null) continue;
String varname = NetcdfFile.makeValidCdmObjectName(dataFieldNameElem.getText().trim());
Variable v = dataG.findVariable(varname);
// if (v == null)
// v = dataG.findVariable( H4header.createValidObjectName(varname));
if (v == null) {
log.error("Cant find variable {} {}", varname, ncfile.getLocation());
continue;
}
Element dimList = elem.getChild("DimList");
List<Element> values = dimList.getChildren("value");
setSharedDimensions(v, values, unknownDims, ncfile.getLocation());
}
}
return featureType;
}
/**
* Amend the given NetcdfFile with metadata from HDF-EOS structMetadata
*
* @param ncfile Amend this file
* @param structMetadata structMetadata as String
* @throws IOException on read error
*/
private void amendFromODL(NetcdfFile ncfile, String structMetadata) throws IOException {
Group rootg = ncfile.getRootGroup();
ODLparser parser = new ODLparser();
Element root = parser.parseFromString(structMetadata); // now we have the ODL in JDOM elements
FeatureType featureType = null;
// SWATH
Element swathStructure = root.getChild("SwathStructure");
if (swathStructure != null) {
List<Element> swaths = swathStructure.getChildren();
for (Element elemSwath : swaths) {
Element swathNameElem = elemSwath.getChild("SwathName");
if (swathNameElem == null) {
log.warn("No SwathName element in {} {} ", elemSwath.getName(), ncfile.getLocation());
continue;
}
String swathName = NetcdfFile.makeValidCdmObjectName(swathNameElem.getText().trim());
Group swathGroup = findGroupNested(rootg, swathName);
// if (swathGroup == null)
// swathGroup = findGroupNested(rootg, H4header.createValidObjectName(swathName));
if (swathGroup != null) {
featureType = amendSwath(ncfile, elemSwath, swathGroup);
} else {
log.warn("Cant find swath group {} {}", swathName, ncfile.getLocation());
}
}
}
// GRID
Element gridStructure = root.getChild("GridStructure");
if (gridStructure != null) {
List<Element> grids = gridStructure.getChildren();
for (Element elemGrid : grids) {
Element gridNameElem = elemGrid.getChild("GridName");
if (gridNameElem == null) {
log.warn("No GridName element in {} {} ", elemGrid.getName(), ncfile.getLocation());
continue;
}
String gridName = NetcdfFile.makeValidCdmObjectName(gridNameElem.getText().trim());
Group gridGroup = findGroupNested(rootg, gridName);
// if (gridGroup == null)
// gridGroup = findGroupNested(rootg, H4header.createValidObjectName(gridName));
if (gridGroup != null) {
featureType = amendGrid(elemGrid, ncfile, gridGroup, ncfile.getLocation());
} else {
log.warn("Cant find Grid group {} {}", gridName, ncfile.getLocation());
}
}
}
// POINT - NOT DONE YET
Element pointStructure = root.getChild("PointStructure");
if (pointStructure != null) {
List<Element> pts = pointStructure.getChildren();
for (Element elem : pts) {
Element nameElem = elem.getChild("PointName");
if (nameElem == null) {
log.warn("No PointName element in {} {}", elem.getName(), ncfile.getLocation());
continue;
}
String name = nameElem.getText().trim();
Group ptGroup = findGroupNested(rootg, name);
// if (ptGroup == null)
// ptGroup = findGroupNested(rootg, H4header.createValidObjectName(name));
if (ptGroup != null) {
featureType = FeatureType.POINT;
} else {
log.warn("Cant find Point group {} {}", name, ncfile.getLocation());
}
}
}
if (featureType != null) {
if (showWork) System.out.println("***EOS featureType= " + featureType.toString());
rootg.addAttribute(new Attribute(CF.FEATURE_TYPE, featureType.toString()));
// rootg.addAttribute(new Attribute(CDM.CONVENTIONS, "HDFEOS"));
}
}
Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
throws IOException, InvalidRangeException {
NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
if (debug) {
System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
}
// global attributes
List<Attribute> glist = bufr.getGlobalAttributes();
for (Attribute att : glist) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
Attribute useAtt;
if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
if (debug) System.out.println("add gatt= " + useAtt);
}
// global dimensions
Dimension recordDim = null;
Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
for (Dimension oldD : bufr.getDimensions()) {
String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
boolean isRecord = useName.equals("record");
Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
dimHash.put(newD.getName(), newD);
if (isRecord) recordDim = newD;
if (debug) System.out.println("add dim= " + newD);
}
// Variables
Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
for (Variable oldVar : recordStruct.getVariables()) {
if (oldVar.getDataType() == DataType.STRUCTURE) continue;
String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
DataType newType = oldVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
for (Dimension dim : oldVar.getDimensions()) {
newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = oldVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
// int max_seq = countSeq(recordStruct);
// Dimension seqD = ncfile.addDimension("level", max_seq);
for (Variable v : recordStruct.getVariables()) {
if (v.getDataType() != DataType.STRUCTURE) continue;
String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
int shape[] = v.getShape();
Dimension structDim = ncfile.addDimension(structName, shape[0]);
Structure struct = (Structure) v;
for (Variable seqVar : struct.getVariables()) {
String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
DataType newType = seqVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
newDims.add(structDim);
for (Dimension dim : seqVar.getDimensions()) {
newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = seqVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
}
// create the file
ncfile.create();
if (debug) System.out.println("File Out= " + ncfile.toString());
// boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
double total = copyVarData(ncfile, recordStruct);
ncfile.flush();
System.out.println("FileWriter done total bytes = " + total);
ncfile.close();
}
