private void handleMetadataGroup(Group group, MetadataElement metadataElement)
throws ProductIOException {
List<Variable> variables = group.getVariables();
for (Variable variable : variables) {
final String name = variable.getShortName();
final int dataType = getProductDataType(variable);
Array array;
try {
array = variable.read();
} catch (IOException e) {
throw new ProductIOException(e.getMessage());
}
final ProductData data = ProductData.createInstance(dataType, array.getStorage());
final MetadataAttribute attribute = new MetadataAttribute("data", data, true);
final MetadataElement sdsElement = new MetadataElement(name);
sdsElement.addAttribute(attribute);
metadataElement.addElement(sdsElement);
final List<Attribute> list = variable.getAttributes();
for (Attribute hdfAttribute : list) {
final String attribName = hdfAttribute.getShortName();
if ("units".equals(attribName)) {
attribute.setUnit(hdfAttribute.getStringValue());
} else if ("long_name".equals(attribName)) {
attribute.setDescription(hdfAttribute.getStringValue());
} else {
addAttributeToElement(sdsElement, hdfAttribute);
}
}
}
}
private int addAttributes(opendap.dap.AttributeTable table, Variable v, Iterator iter) {
int count = 0;
// add attribute table for this variable
while (iter.hasNext()) {
Attribute att = (Attribute) iter.next();
int dods_type = DODSNetcdfFile.convertToDODSType(att.getDataType(), false);
try {
String attName = NcDDS.escapeName(att.getName());
if (att.isString()) {
/* FIX String value = escapeAttributeStringValues(att.getStringValue());
table.appendAttribute(attName, dods_type, "\""+value+"\"");
*/
table.appendAttribute(attName, dods_type, att.getStringValue());
} else {
// cant send signed bytes
if (att.getDataType() == DataType.BYTE) {
boolean signed = false;
for (int i = 0; i < att.getLength(); i++) {
if (att.getNumericValue(i).byteValue() < 0) signed = true;
}
if (signed) // promote to signed short
dods_type = opendap.dap.Attribute.INT16;
}
for (int i = 0; i < att.getLength(); i++)
table.appendAttribute(attName, dods_type, att.getNumericValue(i).toString());
}
count++;
} catch (Exception e) {
log.error(
"Error appending attribute " + att.getName() + " = " + att.getStringValue() + "\n" + e);
}
} // loop over variable attributes
// kludgy thing to map char arrays to DODS Strings
if ((v != null) && (v.getDataType().getPrimitiveClassType() == char.class)) {
int rank = v.getRank();
int strlen = (rank == 0) ? 0 : v.getShape(rank - 1);
Dimension dim = (rank == 0) ? null : v.getDimension(rank - 1);
try {
opendap.dap.AttributeTable dodsTable = table.appendContainer("DODS");
dodsTable.appendAttribute("strlen", opendap.dap.Attribute.INT32, Integer.toString(strlen));
if ((dim != null) && dim.isShared())
dodsTable.appendAttribute("dimName", opendap.dap.Attribute.STRING, dim.getName());
count++;
} catch (Exception e) {
log.error("Error appending attribute strlen\n" + e);
}
}
return count;
}
/**
* Returns the phenomenon that the given variable represents.
*
* <p>This name will be, in order of preference:
*
* <p>The standard name
*
* <p>The long name
*
* <p>The variable name
*/
private static String getVariableName(Variable var) {
Attribute stdNameAtt = var.findAttributeIgnoreCase("standard_name");
if (stdNameAtt == null || stdNameAtt.getStringValue().trim().equals("")) {
Attribute longNameAtt = var.findAttributeIgnoreCase("long_name");
if (longNameAtt == null || longNameAtt.getStringValue().trim().equals("")) {
return var.getFullName();
} else {
return longNameAtt.getStringValue();
}
} else {
return stdNameAtt.getStringValue();
}
}
private ProductData.UTC getUTCAttribute(String key, List<Attribute> globalAttributes) {
Attribute attribute = findAttribute(key, globalAttributes);
Boolean isModis = false;
try {
isModis = findAttribute("MODIS_Resolution", globalAttributes).isString();
} catch (Exception ignored) {
}
if (attribute != null) {
String timeString = attribute.getStringValue().trim();
final DateFormat dateFormat = ProductData.UTC.createDateFormat("yyyyDDDHHmmssSSS");
final DateFormat dateFormatModis =
ProductData.UTC.createDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS");
final DateFormat dateFormatOcts =
ProductData.UTC.createDateFormat("yyyyMMdd HH:mm:ss.SSSSSS");
try {
if (isModis) {
final Date date = dateFormatModis.parse(timeString);
String milliSeconds = timeString.substring(timeString.length() - 3);
return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000);
} else if (productReader.getProductType() == SeadasProductReader.ProductType.Level1A_OCTS) {
final Date date = dateFormatOcts.parse(timeString);
String milliSeconds = timeString.substring(timeString.length() - 3);
return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000);
} else {
final Date date = dateFormat.parse(timeString);
String milliSeconds = timeString.substring(timeString.length() - 3);
return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000);
}
} catch (ParseException ignored) {
}
}
return null;
}
public static String getRasterName(Variable variable) {
Attribute attribute = variable.findAttribute(Constants.ORIG_NAME_ATT_NAME);
if (attribute != null) {
return attribute.getStringValue();
} else {
return variable.getName();
}
}
public String getStringAttribute(String key, List<Attribute> attributeList)
throws ProductIOException {
Attribute attribute = findAttribute(key, attributeList);
if (attribute == null || attribute.getLength() != 1) {
throw new ProductIOException("Global attribute '" + key + "' is missing.");
} else {
return attribute.getStringValue().trim();
}
}
protected Band addNewBand(Product product, Variable variable) {
final int sceneRasterWidth = product.getSceneRasterWidth();
final int sceneRasterHeight = product.getSceneRasterHeight();
Band band = null;
int variableRank = variable.getRank();
if (variableRank == 2) {
final int[] dimensions = variable.getShape();
final int height = dimensions[0] - leadLineSkip - tailLineSkip;
final int width = dimensions[1];
if (height == sceneRasterHeight && width == sceneRasterWidth) {
final String name = variable.getShortName();
final int dataType = getProductDataType(variable);
band = new Band(name, dataType, width, height);
final String validExpression = bandInfoMap.get(name);
if (validExpression != null && !validExpression.equals("")) {
band.setValidPixelExpression(validExpression);
}
product.addBand(band);
try {
band.setNoDataValue(
(double) variable.findAttribute("bad_value_scaled").getNumericValue().floatValue());
band.setNoDataValueUsed(true);
} catch (Exception ignored) {
}
final List<Attribute> list = variable.getAttributes();
for (Attribute hdfAttribute : list) {
final String attribName = hdfAttribute.getShortName();
if ("units".equals(attribName)) {
band.setUnit(hdfAttribute.getStringValue());
} else if ("long_name".equals(attribName)) {
band.setDescription(hdfAttribute.getStringValue());
} else if ("slope".equals(attribName)) {
band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
} else if ("intercept".equals(attribName)) {
band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
}
}
}
}
return band;
}
private static IndexCoding readIndexCoding(Variable variable, String indexCodingName)
throws ProductIOException {
final IndexCoding indexCoding = CfIndexCodingPart.readIndexCoding(variable, indexCodingName);
if (indexCoding != null) {
final Attribute descriptionsAtt = variable.findAttributeIgnoreCase(INDEX_DESCRIPTIONS);
if (descriptionsAtt != null) {
final String[] descriptions = descriptionsAtt.getStringValue().split(DESCRIPTION_SEPARATOR);
if (indexCoding.getNumAttributes() == descriptions.length) {
for (int i = 0; i < descriptions.length; i++) {
indexCoding.getAttributeAt(i).setDescription(descriptions[i]);
}
}
}
final Attribute nameAtt = variable.findAttributeIgnoreCase(INDEX_CODING_NAME);
if (nameAtt != null) {
indexCoding.setName(nameAtt.getStringValue());
}
}
return indexCoding;
}
public static boolean isValidFile(NetcdfFile ds) {
Attribute cdmDtAtt = ds.findGlobalAttributeIgnoreCase("cdm_data_type");
if (cdmDtAtt == null) cdmDtAtt = ds.findGlobalAttributeIgnoreCase("cdm_datatype");
if (cdmDtAtt == null) return false;
if (!cdmDtAtt.isString()) return false;
String cdmDtString = cdmDtAtt.getStringValue();
if (cdmDtString == null) return false;
if (!cdmDtString.equalsIgnoreCase(FeatureType.TRAJECTORY.toString())) return false;
Attribute conventionsAtt = ds.findGlobalAttributeIgnoreCase("Conventions");
if (conventionsAtt == null) return (false);
if (!conventionsAtt.isString()) return (false);
String convString = conventionsAtt.getStringValue();
StringTokenizer stoke = new StringTokenizer(convString, ",");
while (stoke.hasMoreTokens()) {
String toke = stoke.nextToken().trim();
if (toke.equalsIgnoreCase("Unidata Observation Dataset v1.0")) return true;
}
return false;
}
/** Instances which have same content are equal. */
@Override
public boolean equals(Object o) {
if (this == o) return true;
if ((o == null) || !(o instanceof Attribute)) return false;
final Attribute att = (Attribute) o;
if (!name.equals(att.name)) return false;
if (nelems != att.nelems) return false;
if (!dataType.equals(att.dataType)) return false;
if (svalue != null) return svalue.equals(att.getStringValue());
if (values != null) {
for (int i = 0; i < getLength(); i++) {
int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
int r2 =
att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
if (r1 != r2) return false;
}
}
return true;
}
/**
* Is this the right format for this adapter
*
* @param ncd NetcdfDataset to check
* @return true if the right format
*/
public static boolean isValidFile(NetcdfDataset ncd) {
// return (buildConfig(ncd) != null);
// Check for "center" attribute w/ value of "UCAR/CDAAC".
Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center");
if (attrib == null) {
return false;
}
if (!attrib.isString()) {
return false;
}
if (!attrib.getStringValue().equals("UCAR/CDAAC")) {
return false;
}
return true;
}
public static ThreddsMetadata.Variables extractVariables(
InvDatasetImpl threddsDataset, GridDataset gridDataset) {
thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType();
if ((fileFormat != null)
&& (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) {
boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1);
ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString());
for (GridDatatype grid : gridDataset.getGrids()) {
ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
v.setName(grid.getName());
v.setDescription(grid.getDescription());
v.setUnits(grid.getUnitsString());
// ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number");
// String paramNumber = (att != null) ? att.getNumericValue().toString() : null;
if (isGrib1) {
v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR"));
v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
} else {
String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", "");
String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", "");
String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", "");
v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName);
v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id"));
}
vars.addVariable(v);
}
vars.sort();
return vars;
} else { // GRID but not GRIB
ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0");
for (GridDatatype grid : gridDataset.getGrids()) {
ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
vars.addVariable(v);
v.setName(grid.getName());
v.setDescription(grid.getDescription());
v.setUnits(grid.getUnitsString());
ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name");
v.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
}
vars.sort();
return vars;
}
}
MetadataAttribute attributeToMetadata(Attribute attribute) {
final int productDataType = getProductDataType(attribute.getDataType(), false, false);
if (productDataType != -1) {
ProductData productData;
if (attribute.isString()) {
productData = ProductData.createInstance(attribute.getStringValue());
} else if (attribute.isArray()) {
productData = ProductData.createInstance(productDataType, attribute.getLength());
productData.setElems(attribute.getValues().getStorage());
} else {
productData = ProductData.createInstance(productDataType, 1);
productData.setElems(attribute.getValues().getStorage());
}
return new MetadataAttribute(attribute.getShortName(), productData, true);
}
return null;
}
/**
* Extract a list of data variables (and their canonical names if possible) from the dataset.
*
* @param threddsDataset open this dataset
* @return ThreddsMetadata.Variables, or null if unable.
* @throws IOException on read error
*/
public static ThreddsMetadata.Variables extractVariables(InvDatasetImpl threddsDataset)
throws IOException {
ThreddsDataFactory.Result result = null;
try {
result = new ThreddsDataFactory().openFeatureDataset(threddsDataset, null);
if (result.fatalError) {
System.out.println(" openDatatype errs=" + result.errLog);
return null;
}
if (result.featureType == FeatureType.GRID) {
// System.out.println(" extractVariables GRID=" + result.location);
GridDataset gridDataset = (GridDataset) result.featureDataset;
return extractVariables(threddsDataset, gridDataset);
} else if ((result.featureType == FeatureType.STATION)
|| (result.featureType == FeatureType.POINT)) {
PointObsDataset pobsDataset = (PointObsDataset) result.featureDataset;
ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0");
for (VariableSimpleIF vs : pobsDataset.getDataVariables()) {
ThreddsMetadata.Variable v = new ThreddsMetadata.Variable();
vars.addVariable(v);
v.setName(vs.getName());
v.setDescription(vs.getDescription());
v.setUnits(vs.getUnitsString());
ucar.nc2.Attribute att = vs.findAttributeIgnoreCase("standard_name");
v.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
}
vars.sort();
return vars;
}
} finally {
try {
if ((result != null) && (result.featureDataset != null)) result.featureDataset.close();
} catch (IOException ioe) {
logger.error("Closing dataset " + result.featureDataset, ioe);
}
}
return null;
}
public static ThreddsMetadata.Variables extractVariables(FeatureDatasetPoint fd) {
ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.5");
List<VariableSimpleIF> dataVars = fd.getDataVariables();
if (dataVars == null) return vars;
for (VariableSimpleIF v : dataVars) {
ThreddsMetadata.Variable tv = new ThreddsMetadata.Variable();
vars.addVariable(tv);
tv.setName(v.getName());
tv.setDescription(v.getDescription());
tv.setUnits(v.getUnitsString());
ucar.nc2.Attribute att = v.findAttributeIgnoreCase("standard_name");
tv.setVocabularyName((att != null) ? att.getStringValue() : "N/A");
}
vars.sort();
return vars;
}
public void testCoordVar(NetcdfFile ncfile) {
Variable lat = ncfile.findVariable("lat");
assert null != lat;
assert lat.getShortName().equals("lat");
assert lat.getRank() == 1;
assert lat.getSize() == 3;
assert lat.getShape()[0] == 3;
assert lat.getDataType() == DataType.FLOAT;
assert !lat.isUnlimited();
assert lat.getDimension(0).equals(ncfile.findDimension("lat"));
Attribute att = lat.findAttribute("units");
assert null != att;
assert !att.isArray();
assert att.isString();
assert att.getDataType() == DataType.STRING;
assert att.getStringValue().equals("degrees_north");
assert att.getNumericValue() == null;
assert att.getNumericValue(3) == null;
try {
Array data = lat.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == float.class;
IndexIterator dataI = data.getIndexIterator();
assert TestUtils.close(dataI.getDoubleNext(), 41.0);
assert TestUtils.close(dataI.getDoubleNext(), 40.0);
assert TestUtils.close(dataI.getDoubleNext(), 39.0);
} catch (IOException io) {
}
}
Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
throws IOException, InvalidRangeException {
NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
if (debug) {
System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
}
// global attributes
List<Attribute> glist = bufr.getGlobalAttributes();
for (Attribute att : glist) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
Attribute useAtt;
if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
if (debug) System.out.println("add gatt= " + useAtt);
}
// global dimensions
Dimension recordDim = null;
Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
for (Dimension oldD : bufr.getDimensions()) {
String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
boolean isRecord = useName.equals("record");
Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
dimHash.put(newD.getName(), newD);
if (isRecord) recordDim = newD;
if (debug) System.out.println("add dim= " + newD);
}
// Variables
Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
for (Variable oldVar : recordStruct.getVariables()) {
if (oldVar.getDataType() == DataType.STRUCTURE) continue;
String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
DataType newType = oldVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
for (Dimension dim : oldVar.getDimensions()) {
newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = oldVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
// int max_seq = countSeq(recordStruct);
// Dimension seqD = ncfile.addDimension("level", max_seq);
for (Variable v : recordStruct.getVariables()) {
if (v.getDataType() != DataType.STRUCTURE) continue;
String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
int shape[] = v.getShape();
Dimension structDim = ncfile.addDimension(structName, shape[0]);
Structure struct = (Structure) v;
for (Variable seqVar : struct.getVariables()) {
String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
DataType newType = seqVar.getDataType();
List<Dimension> newDims = new ArrayList<Dimension>();
newDims.add(recordDim);
newDims.add(structDim);
for (Dimension dim : seqVar.getDimensions()) {
newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
}
Variable newVar = ncfile.addVariable(varName, newType, newDims);
if (debug) System.out.println("add var= " + newVar);
// attributes
List<Attribute> attList = seqVar.getAttributes();
for (Attribute att : attList) {
String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
else if (att.isString())
ncfile.addVariableAttribute(varName, useName, att.getStringValue());
else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
}
}
}
// create the file
ncfile.create();
if (debug) System.out.println("File Out= " + ncfile.toString());
// boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
double total = copyVarData(ncfile, recordStruct);
ncfile.flush();
System.out.println("FileWriter done total bytes = " + total);
ncfile.close();
}
/**
* Build the configuration from the dataset
*
* @param ncd NetcdfDataset
* @return the trajectory configuration
*/
private static Config buildConfig(NetcdfDataset ncd) {
// already did this in isValid, but we'll keep here for later refactor
Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center");
if (attrib == null) {
return null;
}
if (!attrib.isString()) {
return null;
}
if (!attrib.getStringValue().equals("UCAR/CDAAC")) {
return null;
}
// Check for start_time, stop_time
attrib = ncd.findGlobalAttributeIgnoreCase("start_time");
if (attrib == null) {
return null;
}
if (attrib.isString()) {
return null;
}
double startTime = attrib.getNumericValue().doubleValue();
attrib = ncd.findGlobalAttributeIgnoreCase("stop_time");
if (attrib == null) {
return null;
}
if (attrib.isString()) {
return null;
}
double endTime = attrib.getNumericValue().doubleValue();
// Check that only one dimension and that it is the alt dimension.
List list = ncd.getRootGroup().getDimensions();
if (list.size() != 1) {
return null;
}
Dimension d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
Config trajConfig = new Config();
trajConfig.setTimeDim(d);
// Check for latitude variable with time dimension and units convertable to "degrees_north".
Variable var = ncd.getRootGroup().findVariable(latVarName);
if (var == null) {
return null;
}
list = var.getDimensions();
if (list.size() != 1) {
return null;
}
d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
String units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "degrees_north")) {
return null;
}
trajConfig.setLatVar(var);
// Make the time Variable
int numTimes = d.getLength();
double[] times = new double[numTimes];
// Variable timeVar = new Variable(var);
// timeVar.setName(timeVarName);
VariableDS timeVar =
new VariableDS(
ncd,
ncd.getRootGroup(),
null,
timeVarName,
DataType.DOUBLE,
timeDimName,
"seconds since 1980-01-06 00:00:00",
"Time coordinate");
// Variable timeVar = new Variable(ncd, ncd.getRootGroup(), null,
// timeVarName);
// timeVar.setDataType(DataType.DOUBLE);
// timeVar.setDimensions(list);
// Attribute newUnits =
// new Attribute("units", "seconds since 1980-01-06 00:00:00");
// timeVar.addAttribute(newUnits);
timeVar.setCachedData(
Array.makeArray(DataType.DOUBLE, numTimes, endTime, ((startTime - endTime) / numTimes)),
true);
ncd.addVariable(ncd.getRootGroup(), timeVar);
trajConfig.setTimeVar(timeVar);
// Check for longitude variable with time dimension and units convertable to "degrees_east".
var = ncd.getRootGroup().findVariable(lonVarName);
if (var == null) {
return null;
}
list = var.getDimensions();
if (list.size() != 1) {
return null;
}
d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "degrees_east")) {
return null;
}
trajConfig.setLonVar(var);
// Check for altitude variable with time dimension and units convertable to "m".
var = ncd.getRootGroup().findVariable(elevVarName);
if (var == null) {
return null;
}
list = var.getDimensions();
if (list.size() != 1) {
return null;
}
d = (Dimension) list.get(0);
if (!d.getName().equals(timeDimName)) {
return null;
}
units = var.findAttribute("units").getStringValue();
if (!SimpleUnit.isCompatible(units, "meters")) {
return null;
}
trajConfig.setElevVar(var);
trajConfig.setTrajectoryId(trajId);
return trajConfig;
}
// create from a dataset
public ObsBean(Structure obs, StructureData sdata) {
// first choice
for (Variable v : obs.getVariables()) {
Attribute att = v.findAttribute("BUFR:TableB_descriptor");
if (att == null) continue;
String val = att.getStringValue();
if (val.equals("0-5-1") && Double.isNaN(lat)) {
lat = sdata.convertScalarDouble(v.getShortName());
} else if (val.equals("0-6-1") && Double.isNaN(lon)) {
lon = sdata.convertScalarDouble(v.getShortName());
} else if (val.equals("0-7-30") && Double.isNaN(alt)) {
alt = sdata.convertScalarDouble(v.getShortName());
} else if (val.equals("0-4-1") && (year < 0)) {
year = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-4-2") && (month < 0)) {
month = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-4-3") && (day < 0)) {
day = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-4-4") && (hour < 0)) {
hour = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-4-5") && (minute < 0)) {
minute = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-4-6") && (sec < 0)) {
sec = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-1-1") && (wmo_block < 0)) {
wmo_block = sdata.convertScalarInt(v.getShortName());
} else if (val.equals("0-1-2") && (wmo_id < 0)) {
wmo_id = sdata.convertScalarInt(v.getShortName());
} else if ((stn == null)
&& (val.equals("0-1-7")
|| val.equals("0-1-194")
|| val.equals("0-1-11")
|| val.equals("0-1-18"))) {
if (v.getDataType().isString()) stn = sdata.getScalarString(v.getShortName());
else stn = Integer.toString(sdata.convertScalarInt(v.getShortName()));
}
}
// second choice
for (Variable v : obs.getVariables()) {
Attribute att = v.findAttribute("BUFR:TableB_descriptor");
if (att == null) continue;
String val = att.getStringValue();
if (val.equals("0-5-2") && Double.isNaN(lat)) {
lat = sdata.convertScalarDouble(v.getShortName());
} else if (val.equals("0-6-2") && Double.isNaN(lon)) {
lon = sdata.convertScalarDouble(v.getShortName());
} else if (val.equals("0-7-1") && Double.isNaN(alt)) {
alt = sdata.convertScalarDouble(v.getShortName());
} else if ((val.equals("0-4-7")) && (sec < 0)) {
sec = sdata.convertScalarInt(v.getShortName());
}
}
// third choice
for (Variable v : obs.getVariables()) {
Attribute att = v.findAttribute("BUFR:TableB_descriptor");
if (att == null) continue;
String val = att.getStringValue();
if (val.equals("0-7-10") && Double.isNaN(alt)) {
alt = sdata.convertScalarDouble(v.getShortName());
} else if (val.equals("0-7-2") && Double.isNaN(alt)) {
alt = sdata.convertScalarDouble(v.getShortName());
}
}
}
public String getAbbrev() {
Attribute att = ve.findAttributeIgnoreCase(CDM.ABBREV);
return (att == null) ? null : att.getStringValue();
}
private String getCalendarAttribute(VariableEnhanced vds) {
Attribute cal = vds.findAttribute("calendar");
return (cal == null) ? null : cal.getStringValue();
}
protected Map<Band, Variable> addSmiBands(Product product, List<Variable> variables) {
final int sceneRasterWidth = product.getSceneRasterWidth();
final int sceneRasterHeight = product.getSceneRasterHeight();
Map<Band, Variable> bandToVariableMap = new HashMap<Band, Variable>();
for (Variable variable : variables) {
int variableRank = variable.getRank();
if (variableRank == 2) {
final int[] dimensions = variable.getShape();
final int height = dimensions[0];
final int width = dimensions[1];
if (height == sceneRasterHeight && width == sceneRasterWidth) {
String name = variable.getShortName();
if (name.equals("l3m_data")) {
try {
name = getStringAttribute("Parameter") + " " + getStringAttribute("Measure");
} catch (Exception e) {
e.printStackTrace();
}
}
final int dataType = getProductDataType(variable);
final Band band = new Band(name, dataType, width, height);
// band = new Band(name, dataType, width, height);
product.addBand(band);
try {
Attribute fillvalue = variable.findAttribute("_FillValue");
if (fillvalue == null) {
fillvalue = variable.findAttribute("Fill");
}
if (fillvalue != null) {
band.setNoDataValue((double) fillvalue.getNumericValue().floatValue());
band.setNoDataValueUsed(true);
}
} catch (Exception ignored) {
}
bandToVariableMap.put(band, variable);
// Set units, if defined
try {
band.setUnit(getStringAttribute("Units"));
} catch (Exception ignored) {
}
final List<Attribute> list = variable.getAttributes();
double[] validMinMax = {0.0, 0.0};
for (Attribute hdfAttribute : list) {
final String attribName = hdfAttribute.getShortName();
if ("units".equals(attribName)) {
band.setUnit(hdfAttribute.getStringValue());
} else if ("long_name".equalsIgnoreCase(attribName)) {
band.setDescription(hdfAttribute.getStringValue());
} else if ("slope".equalsIgnoreCase(attribName)) {
band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
} else if ("intercept".equalsIgnoreCase(attribName)) {
band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
} else if ("scale_factor".equals(attribName)) {
band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
} else if ("add_offset".equals(attribName)) {
band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
} else if (attribName.startsWith("valid_")) {
if ("valid_min".equals(attribName)) {
validMinMax[0] = hdfAttribute.getNumericValue(0).doubleValue();
} else if ("valid_max".equals(attribName)) {
validMinMax[1] = hdfAttribute.getNumericValue(0).doubleValue();
} else if ("valid_range".equals(attribName)) {
validMinMax[0] = hdfAttribute.getNumericValue(0).doubleValue();
validMinMax[1] = hdfAttribute.getNumericValue(1).doubleValue();
}
}
}
if (validMinMax[0] != validMinMax[1]) {
double[] minmax = {0.0, 0.0};
minmax[0] = validMinMax[0];
minmax[1] = validMinMax[1];
if (band.getScalingFactor() != 1.0) {
minmax[0] *= band.getScalingFactor();
minmax[1] *= band.getScalingFactor();
}
if (band.getScalingOffset() != 0.0) {
minmax[0] += band.getScalingOffset();
minmax[1] += band.getScalingOffset();
}
String validExp = format("%s >= %.2f && %s <= %.2f", name, minmax[0], name, minmax[1]);
band.setValidPixelExpression(
validExp); // .format(name, validMinMax[0], name, validMinMax[1]));
}
}
} else if (variableRank == 4) {
final int[] dimensions = variable.getShape();
final int height = dimensions[2];
final int width = dimensions[3];
if (height == sceneRasterHeight && width == sceneRasterWidth) {
String name = variable.getShortName();
final int dataType = getProductDataType(variable);
final Band band = new Band(name, dataType, width, height);
// band = new Band(name, dataType, width, height);
Variable sliced = null;
try {
sliced = variable.slice(0, 0).slice(0, 0);
} catch (InvalidRangeException e) {
e.printStackTrace(); // Todo change body of catch statement.
}
bandToVariableMap.put(band, sliced);
product.addBand(band);
try {
Attribute fillvalue = variable.findAttribute("_FillValue");
if (fillvalue != null) {
band.setNoDataValue((double) fillvalue.getNumericValue().floatValue());
band.setNoDataValueUsed(true);
}
} catch (Exception ignored) {
}
// Set units, if defined
try {
band.setUnit(getStringAttribute("units"));
} catch (Exception ignored) {
}
final List<Attribute> list = variable.getAttributes();
for (Attribute hdfAttribute : list) {
final String attribName = hdfAttribute.getShortName();
if ("scale_factor".equals(attribName)) {
band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue());
} else if ("add_offset".equals(attribName)) {
band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue());
}
}
}
}
}
return bandToVariableMap;
}
@Override
public AbstractGridDataset createDataset(String id, String location)
throws IOException, EdalException {
NetcdfDataset nc = null;
try {
/*
* Open the dataset, using the cache for NcML aggregations
*/
nc = openAndAggregateDataset(location);
/*-
* We may in future be able to use forecast model run collection aggregations for
* dealing with the case of overlapping time axes. To do this the code will look
* something like this:
*
* StringBuilder sb = new StringBuilder();
* Formatter formatter = new Formatter(sb, Locale.UK);
* Fmrc f = Fmrc.open(location, formatter);
*
* in openAndAggregateDataset. It will need to build up an NcML document which
* does this. It should look something like:
*
* <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="true">
* <aggregation dimName="run" type="forecastModelRunCollection" timeUnitsChange="true">
* <!-- scanFmrc actually works, but what we want is something like the following bit -->
* <scanFmrc location="/home/guy/Data/POLCOMS_IRISH/" regExp=".*\.nc"/>
* <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090320.nc" coordValue="2009-03-20T00:00:00Z" enhance="true" />
* <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090321.nc" coordValue="2009-03-21T00:00:00Z" enhance="true" />
* <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090322.nc" coordValue="2009-03-22T00:00:00Z" enhance="true" />
* </aggregation>
* </netcdf>
*
* For more documentation see:
* http://mailman.unidata.ucar.edu/software/thredds/current/netcdf-java/ncml/FmrcAggregation.html
*
* We then can do stuff like:
*
* ucar.nc2.dt.GridDataset gridDataset = f.getDatasetBest();
*
* To get the single best aggregation of the overlapping time axis
*
* Then we need to work with GridDatasets in place of NetcdfDatasets. Stuff like:
*
* for(Variable variable : gridDataset.getNetcdfFile().getVariables()) {
* // blah blah
* }
*
* will be necessary. We need to check that that works with remote datasets too
*/
/*
* We look for NetCDF-U variables to group mean/standard-deviation.
*
* We need to do this here because we want to subsequently ignore
* parent variables
*/
Map<String, String[]> varId2AncillaryVars = new HashMap<String, String[]>();
for (Variable variable : nc.getVariables()) {
/*
* Just look for parent variables, since these may not have a
* grid directly associated with them
*/
for (Attribute attr : variable.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("ancillary_variables")) {
varId2AncillaryVars.put(variable.getFullName(), attr.getStringValue().split(" "));
continue;
}
}
}
ucar.nc2.dt.GridDataset gridDataset = CdmUtils.getGridDataset(nc);
List<GridVariableMetadata> vars = new ArrayList<GridVariableMetadata>();
/*
* Store a map of component names. Key is the compound name, value
* is a 2-element String array with x, y component IDs
*
* Also store a map of whether these components are really
* eastward/northward, or whether they are locally u/v
*/
Map<String, String[]> xyComponentPairs = new HashMap<String, String[]>();
Map<String, Boolean> xyNameToTrueEN = new HashMap<String, Boolean>();
/*
* Store a map of variable IDs to UncertML URLs. This will be used
* to determine which components are mean/std/etc.
*
* TODO implement more than just Mean/SD
*/
Map<String, String> varId2UncertMLRefs = new HashMap<String, String>();
/*
* Here we store the parent variable IDs and their corresponding
* title.
*/
Map<String, String> parentVarId2Title = new HashMap<String, String>();
for (Gridset gridset : gridDataset.getGridsets()) {
GridCoordSystem coordSys = gridset.getGeoCoordSystem();
HorizontalGrid hDomain = CdmUtils.createHorizontalGrid(coordSys);
VerticalAxis zDomain = CdmUtils.createVerticalAxis(coordSys);
TimeAxis tDomain = CdmUtils.createTimeAxis(coordSys);
/*
* Create a VariableMetadata object for each GridDatatype
*/
for (GridDatatype grid : gridset.getGrids()) {
VariableDS variable = grid.getVariable();
String varId = variable.getFullName();
String name = getVariableName(variable);
/*
* If this is a parent variable for a stats collection, we
* don't want it to be a normal variable as well.
*/
if (varId2AncillaryVars.containsKey(varId)) {
parentVarId2Title.put(varId, name);
continue;
}
/*
* If it is a child variable is (potentially) referenced by
* UncertML, store its ID and the (possible) UncertML URI
*/
for (Attribute attr : variable.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("ref")) {
varId2UncertMLRefs.put(varId, attr.getStringValue());
}
}
Parameter parameter =
new Parameter(
varId,
variable.getShortName(),
variable.getDescription(),
variable.getUnitsString(),
name);
GridVariableMetadata metadata =
new GridVariableMetadata(
variable.getFullName(), parameter, hDomain, zDomain, tDomain, true);
vars.add(metadata);
if (name != null) {
/*
* Check for vector components
*/
if (name.contains("eastward_")) {
String compoundName = name.replaceFirst("eastward_", "");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, true);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[0] = varId;
} else if (name.contains("northward_")) {
String compoundName = name.replaceFirst("northward_", "");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, true);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[1] = varId;
} else if (name.matches("u-.*component")) {
String compoundName = name.replaceFirst("u-(.*)component", "$1");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, false);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[0] = varId;
} else if (name.matches("v-.*component")) {
String compoundName = name.replaceFirst("v-(.*)component", "$1");
String[] cData;
if (!xyComponentPairs.containsKey(compoundName)) {
cData = new String[2];
xyComponentPairs.put(compoundName, cData);
xyNameToTrueEN.put(compoundName, false);
}
cData = xyComponentPairs.get(compoundName);
/*
* By doing this, we will end up with the merged
* coverage
*/
cData[1] = varId;
}
/*
* We could potentially add a check for zonal/meridional
* here if required.
*/
}
}
}
CdmGridDataset cdmGridDataset =
new CdmGridDataset(id, location, vars, CdmUtils.getOptimumDataReadingStrategy(nc));
for (Entry<String, String[]> componentData : xyComponentPairs.entrySet()) {
String commonName = componentData.getKey();
String[] comps = componentData.getValue();
if (comps[0] != null && comps[1] != null) {
cdmGridDataset.addVariablePlugin(
new VectorPlugin(comps[0], comps[1], commonName, xyNameToTrueEN.get(commonName)));
}
}
for (String statsCollectionId : varId2AncillaryVars.keySet()) {
String[] ids = varId2AncillaryVars.get(statsCollectionId);
String meanId = null;
String stddevId = null;
for (String statsVarIds : ids) {
String uncertRef = varId2UncertMLRefs.get(statsVarIds);
if (uncertRef != null
&& uncertRef.equalsIgnoreCase("http://www.uncertml.org/statistics/mean")) {
meanId = statsVarIds;
}
if (uncertRef != null
&& uncertRef.equalsIgnoreCase(
"http://www.uncertml.org/statistics/standard-deviation")) {
stddevId = statsVarIds;
}
}
if (meanId != null && stddevId != null) {
MeanSDPlugin meanSDPlugin =
new MeanSDPlugin(meanId, stddevId, parentVarId2Title.get(statsCollectionId));
cdmGridDataset.addVariablePlugin(meanSDPlugin);
}
}
return cdmGridDataset;
} finally {
CdmUtils.closeDataset(nc);
}
}
/**
* Opens the NetCDF dataset at the given location, using the dataset cache if {@code location}
* represents an NcML aggregation. We cannot use the cache for OPeNDAP or single NetCDF files
* because the underlying data may have changed and the NetcdfDataset cache may cache a dataset
* forever. In the case of NcML we rely on the fact that server administrators ought to have set a
* "recheckEvery" parameter for NcML aggregations that may change with time. It is desirable to
* use the dataset cache for NcML aggregations because they can be time-consuming to assemble and
* we don't want to do this every time a map is drawn.
*
* @param location The location of the data: a local NetCDF file, an NcML aggregation file or an
* OPeNDAP location, {@literal i.e.} anything that can be passed to
* NetcdfDataset.openDataset(location).
* @return a {@link NetcdfDataset} object for accessing the data at the given location.
* @throws IOException if there was an error reading from the data source.
*/
private NetcdfDataset openAndAggregateDataset(String location) throws IOException, EdalException {
NetcdfDataset nc;
if (location.startsWith("dods://") || location.startsWith("http://")) {
/*
* We have a remote dataset
*/
nc = CdmUtils.openDataset(location);
} else {
/*
* We have a local dataset
*/
List<File> files = null;
try {
files = CdmUtils.expandGlobExpression(location);
} catch (NullPointerException e) {
System.out.println("NPE processing location: " + location);
throw e;
}
if (files.size() == 0) {
throw new EdalException(
"The location " + location + " doesn't refer to any existing files.");
}
if (files.size() == 1) {
location = files.get(0).getAbsolutePath();
nc = CdmUtils.openDataset(location);
} else {
/*
* We have multiple files in a glob expression. We write some
* NcML and use the NetCDF aggregation libs to parse this into
* an aggregated dataset.
*
* If we have already generated the ncML on a previous call,
* just use that.
*/
if (ncmlString == null) {
/*
* Find the name of the time dimension
*/
NetcdfDataset first = openAndAggregateDataset(files.get(0).getAbsolutePath());
String timeDimName = null;
for (Variable var : first.getVariables()) {
if (var.isCoordinateVariable()) {
for (Attribute attr : var.getAttributes()) {
if (attr.getFullName().equalsIgnoreCase("units")
&& attr.getStringValue().contains(" since ")) {
/*
* This is the time dimension. Since this is
* a co-ordinate variable, there is only 1
* dimension
*/
Dimension timeDimension = var.getDimension(0);
timeDimName = timeDimension.getFullName();
}
}
}
}
first.close();
if (timeDimName == null) {
throw new EdalException("Cannot join multiple files without time dimensions");
}
/*
* We can't assume that the glob expression will have
* returned the files in time order.
*
* We could assume that alphabetical == time ordered (and
* for properly named files it will - but let's not rely on
* our users having sensible naming conventions...
*
* Sort the list using a comparator which opens the file and
* gets the first value of the time dimension
*/
final String aggDimName = timeDimName;
Collections.sort(
files,
new Comparator<File>() {
@Override
public int compare(File ncFile1, File ncFile2) {
NetcdfFile nc1 = null;
NetcdfFile nc2 = null;
try {
nc1 = NetcdfFile.open(ncFile1.getAbsolutePath());
nc2 = NetcdfFile.open(ncFile2.getAbsolutePath());
Variable timeVar1 = nc1.findVariable(aggDimName);
Variable timeVar2 = nc2.findVariable(aggDimName);
long time1 = timeVar1.read().getLong(0);
long time2 = timeVar2.read().getLong(0);
return Long.compare(time1, time2);
} catch (Exception e) {
/*
* There was a problem reading the data. Sort
* alphanumerically by filename and hope for the
* best...
*
* This catches all exceptions because however
* it fails this is still our best option.
*
* If the error is a genuine problem, it'll show
* up as soon as we try and aggregate.
*/
return ncFile1.getAbsolutePath().compareTo(ncFile2.getAbsolutePath());
} finally {
if (nc1 != null) {
try {
nc1.close();
} catch (IOException e) {
log.error("Problem closing netcdf file", e);
}
}
if (nc2 != null) {
try {
nc2.close();
} catch (IOException e) {
log.error("Problem closing netcdf file", e);
}
}
}
}
});
/*
* Now create the NcML string and use it to create an
* aggregated dataset
*/
StringBuffer ncmlStringBuffer = new StringBuffer();
ncmlStringBuffer.append(
"<netcdf xmlns=\"http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2\">");
ncmlStringBuffer.append(
"<aggregation dimName=\"" + timeDimName + "\" type=\"joinExisting\">");
for (File file : files) {
ncmlStringBuffer.append("<netcdf location=\"" + file.getAbsolutePath() + "\"/>");
}
ncmlStringBuffer.append("</aggregation>");
ncmlStringBuffer.append("</netcdf>");
ncmlString = ncmlStringBuffer.toString();
}
nc = NcMLReader.readNcML(new StringReader(ncmlString), null);
}
}
return nc;
}
@Test
public void testDoradeGround() throws IOException {
System.out.println("**** Open " + groundDoradeFile);
try (NetcdfFile ncfile = NetcdfFile.open(groundDoradeFile)) {
for (Variable v : ncfile.getVariables()) {
System.out.println(v.getFullName());
}
/* test both gate and radial dimension */
Dimension gateDim = ncfile.getRootGroup().findDimension("gate_1");
assert (gateDim.getLength() == 1008);
Dimension radialDim = ncfile.getRootGroup().findDimension("radial");
assert (radialDim.getLength() == 439);
/* test some att */
Attribute testAtt = ncfile.getRootGroup().findAttribute("Conventions");
assert (testAtt.getStringValue().equals(_Coordinate.Convention));
testAtt = ncfile.getRootGroup().findAttribute("format");
assert (testAtt.getStringValue().equals("Unidata/netCDF/Dorade"));
testAtt = ncfile.getRootGroup().findAttribute("Project_name");
assert (testAtt.getStringValue().equalsIgnoreCase("IHOP_2002"));
testAtt = ncfile.getRootGroup().findAttribute("Radar_Name");
assert (testAtt.getStringValue().equalsIgnoreCase("SPOL"));
testAtt = ncfile.getRootGroup().findAttribute("VolumeCoveragePatternName");
assert (testAtt.getStringValue().equalsIgnoreCase("SUR"));
testAtt = ncfile.getRootGroup().findAttribute("Volume_Number");
assert (testAtt.getStringValue().equalsIgnoreCase("1"));
testAtt = ncfile.getRootGroup().findAttribute("Sweep_Number");
assert (testAtt.getStringValue().equalsIgnoreCase("2"));
testAtt = ncfile.getRootGroup().findAttribute("Sweep_Date");
assert (testAtt.getStringValue().equalsIgnoreCase("2002-05-11 01:58:15Z"));
Variable var;
var = ncfile.findVariable("elevation");
testReadData(var);
var = ncfile.findVariable("azimuth");
testReadData(var);
var = ncfile.findVariable("distance_1");
testReadData(var);
var = ncfile.findVariable("latitudes_1");
testReadData(var);
var = ncfile.findVariable("longitudes_1");
testReadData(var);
var = ncfile.findVariable("altitudes_1");
testReadData(var);
var = ncfile.findVariable("rays_time");
testReadData(var);
var = ncfile.findVariable("Range_to_First_Cell");
float t = testReadScalar(var);
assert (t == (float) 150.0);
var = ncfile.findVariable("Cell_Spacing");
t = testReadScalar(var);
assert (t == (float) 149.89624);
var = ncfile.findVariable("Fixed_Angle");
t = testReadScalar(var);
assert (t == (float) 1.1975098);
var = ncfile.findVariable("Nyquist_Velocity");
t = testReadScalar(var);
assert (t == (float) 25.618269);
var = ncfile.findVariable("Unambiguous_Range");
t = testReadScalar(var);
assert (t == (float) 156.11694);
var = ncfile.findVariable("Radar_Constant");
t = testReadScalar(var);
assert (t == (float) 70.325195);
var = ncfile.findVariable("rcvr_gain");
t = testReadScalar(var);
assert (t == (float) 46.95);
var = ncfile.findVariable("ant_gain");
t = testReadScalar(var);
assert (t == (float) 45.58);
var = ncfile.findVariable("sys_gain");
t = testReadScalar(var);
assert (t == (float) 46.95);
var = ncfile.findVariable("bm_width");
t = testReadScalar(var);
assert (t == (float) 0.92);
var = ncfile.findVariable("VE");
testReadData(var);
var = ncfile.findVariable("DM");
testReadData(var);
var = ncfile.findVariable("NCP");
testReadData(var);
var = ncfile.findVariable("SW");
testReadData(var);
var = ncfile.findVariable("DZ");
testReadData(var);
var = ncfile.findVariable("DCZ");
testReadData(var);
var = ncfile.findVariable("LVDR");
testReadData(var);
var = ncfile.findVariable("NIQ");
testReadData(var);
var = ncfile.findVariable("AIQ");
testReadData(var);
var = ncfile.findVariable("CH");
testReadData(var);
var = ncfile.findVariable("AH");
testReadData(var);
var = ncfile.findVariable("CV");
testReadData(var);
var = ncfile.findVariable("AV");
testReadData(var);
var = ncfile.findVariable("RHOHV");
testReadData(var);
var = ncfile.findVariable("LDR");
testReadData(var);
var = ncfile.findVariable("DL");
testReadData(var);
var = ncfile.findVariable("DX");
testReadData(var);
var = ncfile.findVariable("ZDR");
testReadData(var);
var = ncfile.findVariable("PHI");
testReadData(var);
var = ncfile.findVariable("KDP");
testReadData(var);
assert (0 == var.findDimensionIndex("radial"));
assert (1 == var.findDimensionIndex("gate_1"));
}
}
/**
* Constructor. If scale/offset attributes are found, remove them from the decorated variable.
*
* @param forVar the Variable to decorate.
* @param useNaNs pre-fill isMissing() data with NaNs
* @param fillValueIsMissing use _FillValue for isMissing()
* @param invalidDataIsMissing use valid_range for isMissing()
* @param missingDataIsMissing use missing_value for isMissing()
*/
EnhanceScaleMissingImpl(
VariableDS forVar,
boolean useNaNs,
boolean fillValueIsMissing,
boolean invalidDataIsMissing,
boolean missingDataIsMissing) {
this.fillValueIsMissing = fillValueIsMissing;
this.invalidDataIsMissing = invalidDataIsMissing;
this.missingDataIsMissing = missingDataIsMissing;
// see if underlying variable has scale/offset already applied
Variable orgVar = forVar.getOriginalVariable();
if (orgVar instanceof VariableDS) {
VariableDS orgVarDS = (VariableDS) orgVar;
EnumSet<NetcdfDataset.Enhance> orgEnhanceMode = orgVarDS.getEnhanceMode();
if ((orgEnhanceMode != null) && orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing))
return;
}
// the other possibility is that you want to apply scale and offset to a signed value, then
// declare the result unsigned
// this.isUnsigned = (orgVar != null) ? orgVar.isUnsigned() : forVar.isUnsigned();
this.isUnsigned = forVar.isUnsigned();
this.convertedDataType = forVar.getDataType();
DataType scaleType = null, missType = null, validType = null, fillType = null;
if (debug) System.out.println("EnhancementsImpl for Variable = " + forVar.getFullName());
Attribute att;
// scale and offset
if (null != (att = forVar.findAttribute(CDM.SCALE_FACTOR))) {
if (!att.isString()) {
scale = att.getNumericValue().doubleValue();
hasScaleOffset = true;
scaleType = att.getDataType();
forVar.remove(att);
if (debug) System.out.println("scale = " + scale + " type " + scaleType);
}
}
if (null != (att = forVar.findAttribute(CDM.ADD_OFFSET))) {
if (!att.isString()) {
offset = att.getNumericValue().doubleValue();
hasScaleOffset = true;
DataType offType = att.getDataType();
if (rank(offType) > rank(scaleType)) scaleType = offType;
forVar.remove(att);
if (debug) System.out.println("offset = " + offset);
}
}
////// missing data : valid_range. assume here its in units of unpacked data. correct this below
Attribute validRangeAtt;
if (null != (validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE))) {
if (!validRangeAtt.isString() && validRangeAtt.getLength() > 1) {
valid_min = validRangeAtt.getNumericValue(0).doubleValue();
valid_max = validRangeAtt.getNumericValue(1).doubleValue();
hasValidRange = true;
validType = validRangeAtt.getDataType();
if (hasScaleOffset) forVar.remove(validRangeAtt);
if (debug) System.out.println("valid_range = " + valid_min + " " + valid_max);
}
}
Attribute validMinAtt = null, validMaxAtt = null;
if (!hasValidRange) {
if (null != (validMinAtt = forVar.findAttribute("valid_min"))) {
if (!validMinAtt.isString()) {
valid_min = validMinAtt.getNumericValue().doubleValue();
hasValidMin = true;
validType = validMinAtt.getDataType();
if (hasScaleOffset) forVar.remove(validMinAtt);
if (debug) System.out.println("valid_min = " + valid_min);
}
}
if (null != (validMaxAtt = forVar.findAttribute("valid_max"))) {
if (!validMaxAtt.isString()) {
valid_max = validMaxAtt.getNumericValue().doubleValue();
hasValidMax = true;
DataType t = validMaxAtt.getDataType();
if (rank(t) > rank(validType)) validType = t;
if (hasScaleOffset) forVar.remove(validMaxAtt);
if (debug) System.out.println("valid_min = " + valid_max);
}
}
}
boolean hasValidData = hasValidMin || hasValidMax || hasValidRange;
if (hasValidMin && hasValidMax) hasValidRange = true;
/// _FillValue
if ((null != (att = forVar.findAttribute(CDM.FILL_VALUE))) && !att.isString()) {
double[] values = getValueAsDouble(att); // LOOK double WTF ??
if (values.length > 0) {
fillValue = values[0];
hasFillValue = true;
fillType = att.getDataType();
if (hasScaleOffset) forVar.remove(att);
if (debug) System.out.println("missing_datum from _FillValue = " + fillValue);
}
}
/// missing_value
if (null != (att = forVar.findAttribute(CDM.MISSING_VALUE))) {
if (att.isString()) {
String svalue = att.getStringValue();
if (forVar.getDataType() == DataType.CHAR) {
missingValue = new double[1];
if (svalue.length() == 0) missingValue[0] = 0;
else missingValue[0] = svalue.charAt(0);
missType = DataType.CHAR;
hasMissingValue = true;
} else { // not a CHAR - try to fix problem where they use a numeric value as a String
// attribute
try {
missingValue = new double[1];
missingValue[0] = Double.parseDouble(svalue);
missType = att.getDataType();
hasMissingValue = true;
} catch (NumberFormatException ex) {
if (debug)
System.out.println(
"String missing_value not parsable as double= " + att.getStringValue());
}
}
} else { // not a string
missingValue = getValueAsDouble(att);
missType = att.getDataType();
for (double mv : missingValue)
if (!Double.isNaN(mv))
hasMissingValue = true; // dont need to do anything if its already a NaN
}
if (hasScaleOffset) forVar.remove(att);
}
// missing
boolean hasMissing =
(invalidDataIsMissing && hasValidData)
|| (fillValueIsMissing && hasFillValue)
|| (missingDataIsMissing && hasMissingValue);
/// assign convertedDataType if needed
if (hasScaleOffset) {
convertedDataType = forVar.getDataType();
if (hasMissing) {
// has missing data : must be float or double
if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
if (missingDataIsMissing && rank(missType) > rank(convertedDataType))
convertedDataType = missType;
if (fillValueIsMissing && rank(fillType) > rank(convertedDataType))
convertedDataType = fillType;
if (invalidDataIsMissing && rank(validType) > rank(convertedDataType))
convertedDataType = validType;
if (rank(convertedDataType) < rank(DataType.DOUBLE)) convertedDataType = DataType.FLOAT;
} else {
// no missing data; can use wider of data and scale
if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType;
}
if (debug) System.out.println("assign dataType = " + convertedDataType);
// validData may be external or internal
if (hasValidData) {
DataType orgType = forVar.getDataType();
// If valid_range is the same type as scale_factor (actually the wider of
// scale_factor and add_offset) and this is wider than the external data, then it
// will be interpreted as being in the units of the internal (unpacked) data.
// Otherwise it is in the units of the external (unpacked) data.
// we assumed unpacked data above, redo if its really packed data
if (!((rank(validType) == rank(scaleType)) && (rank(scaleType) >= rank(orgType)))) {
if (validRangeAtt != null) {
double[] values = getValueAsDouble(validRangeAtt);
valid_min = values[0];
valid_max = values[1];
} else {
if (validMinAtt != null) {
double[] values = getValueAsDouble(validMinAtt);
valid_min = values[0];
}
if (validMaxAtt != null) {
double[] values = getValueAsDouble(validMaxAtt);
valid_max = values[0];
}
}
}
}
}
if (hasMissing
&& ((convertedDataType == DataType.DOUBLE) || (convertedDataType == DataType.FLOAT)))
this.useNaNs = useNaNs;
if (debug) System.out.println("this.useNaNs = " + this.useNaNs);
}
