private void handleMetadataGroup(Group group, MetadataElement metadataElement) throws ProductIOException { List<Variable> variables = group.getVariables(); for (Variable variable : variables) { final String name = variable.getShortName(); final int dataType = getProductDataType(variable); Array array; try { array = variable.read(); } catch (IOException e) { throw new ProductIOException(e.getMessage()); } final ProductData data = ProductData.createInstance(dataType, array.getStorage()); final MetadataAttribute attribute = new MetadataAttribute("data", data, true); final MetadataElement sdsElement = new MetadataElement(name); sdsElement.addAttribute(attribute); metadataElement.addElement(sdsElement); final List<Attribute> list = variable.getAttributes(); for (Attribute hdfAttribute : list) { final String attribName = hdfAttribute.getShortName(); if ("units".equals(attribName)) { attribute.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { attribute.setDescription(hdfAttribute.getStringValue()); } else { addAttributeToElement(sdsElement, hdfAttribute); } } } }
private int addAttributes(opendap.dap.AttributeTable table, Variable v, Iterator iter) { int count = 0; // add attribute table for this variable while (iter.hasNext()) { Attribute att = (Attribute) iter.next(); int dods_type = DODSNetcdfFile.convertToDODSType(att.getDataType(), false); try { String attName = NcDDS.escapeName(att.getName()); if (att.isString()) { /* FIX String value = escapeAttributeStringValues(att.getStringValue()); table.appendAttribute(attName, dods_type, "\""+value+"\""); */ table.appendAttribute(attName, dods_type, att.getStringValue()); } else { // cant send signed bytes if (att.getDataType() == DataType.BYTE) { boolean signed = false; for (int i = 0; i < att.getLength(); i++) { if (att.getNumericValue(i).byteValue() < 0) signed = true; } if (signed) // promote to signed short dods_type = opendap.dap.Attribute.INT16; } for (int i = 0; i < att.getLength(); i++) table.appendAttribute(attName, dods_type, att.getNumericValue(i).toString()); } count++; } catch (Exception e) { log.error( "Error appending attribute " + att.getName() + " = " + att.getStringValue() + "\n" + e); } } // loop over variable attributes // kludgy thing to map char arrays to DODS Strings if ((v != null) && (v.getDataType().getPrimitiveClassType() == char.class)) { int rank = v.getRank(); int strlen = (rank == 0) ? 0 : v.getShape(rank - 1); Dimension dim = (rank == 0) ? null : v.getDimension(rank - 1); try { opendap.dap.AttributeTable dodsTable = table.appendContainer("DODS"); dodsTable.appendAttribute("strlen", opendap.dap.Attribute.INT32, Integer.toString(strlen)); if ((dim != null) && dim.isShared()) dodsTable.appendAttribute("dimName", opendap.dap.Attribute.STRING, dim.getName()); count++; } catch (Exception e) { log.error("Error appending attribute strlen\n" + e); } } return count; }
/** * Returns the phenomenon that the given variable represents. * * <p>This name will be, in order of preference: * * <p>The standard name * * <p>The long name * * <p>The variable name */ private static String getVariableName(Variable var) { Attribute stdNameAtt = var.findAttributeIgnoreCase("standard_name"); if (stdNameAtt == null || stdNameAtt.getStringValue().trim().equals("")) { Attribute longNameAtt = var.findAttributeIgnoreCase("long_name"); if (longNameAtt == null || longNameAtt.getStringValue().trim().equals("")) { return var.getFullName(); } else { return longNameAtt.getStringValue(); } } else { return stdNameAtt.getStringValue(); } }
private ProductData.UTC getUTCAttribute(String key, List<Attribute> globalAttributes) { Attribute attribute = findAttribute(key, globalAttributes); Boolean isModis = false; try { isModis = findAttribute("MODIS_Resolution", globalAttributes).isString(); } catch (Exception ignored) { } if (attribute != null) { String timeString = attribute.getStringValue().trim(); final DateFormat dateFormat = ProductData.UTC.createDateFormat("yyyyDDDHHmmssSSS"); final DateFormat dateFormatModis = ProductData.UTC.createDateFormat("yyyy-MM-dd HH:mm:ss.SSSSSS"); final DateFormat dateFormatOcts = ProductData.UTC.createDateFormat("yyyyMMdd HH:mm:ss.SSSSSS"); try { if (isModis) { final Date date = dateFormatModis.parse(timeString); String milliSeconds = timeString.substring(timeString.length() - 3); return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000); } else if (productReader.getProductType() == SeadasProductReader.ProductType.Level1A_OCTS) { final Date date = dateFormatOcts.parse(timeString); String milliSeconds = timeString.substring(timeString.length() - 3); return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000); } else { final Date date = dateFormat.parse(timeString); String milliSeconds = timeString.substring(timeString.length() - 3); return ProductData.UTC.create(date, Long.parseLong(milliSeconds) * 1000); } } catch (ParseException ignored) { } } return null; }
public static String getRasterName(Variable variable) { Attribute attribute = variable.findAttribute(Constants.ORIG_NAME_ATT_NAME); if (attribute != null) { return attribute.getStringValue(); } else { return variable.getName(); } }
public String getStringAttribute(String key, List<Attribute> attributeList) throws ProductIOException { Attribute attribute = findAttribute(key, attributeList); if (attribute == null || attribute.getLength() != 1) { throw new ProductIOException("Global attribute '" + key + "' is missing."); } else { return attribute.getStringValue().trim(); } }
protected Band addNewBand(Product product, Variable variable) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Band band = null; int variableRank = variable.getRank(); if (variableRank == 2) { final int[] dimensions = variable.getShape(); final int height = dimensions[0] - leadLineSkip - tailLineSkip; final int width = dimensions[1]; if (height == sceneRasterHeight && width == sceneRasterWidth) { final String name = variable.getShortName(); final int dataType = getProductDataType(variable); band = new Band(name, dataType, width, height); final String validExpression = bandInfoMap.get(name); if (validExpression != null && !validExpression.equals("")) { band.setValidPixelExpression(validExpression); } product.addBand(band); try { band.setNoDataValue( (double) variable.findAttribute("bad_value_scaled").getNumericValue().floatValue()); band.setNoDataValueUsed(true); } catch (Exception ignored) { } final List<Attribute> list = variable.getAttributes(); for (Attribute hdfAttribute : list) { final String attribName = hdfAttribute.getShortName(); if ("units".equals(attribName)) { band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equals(attribName)) { band.setDescription(hdfAttribute.getStringValue()); } else if ("slope".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); } else if ("intercept".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); } } } } return band; }
private static IndexCoding readIndexCoding(Variable variable, String indexCodingName) throws ProductIOException { final IndexCoding indexCoding = CfIndexCodingPart.readIndexCoding(variable, indexCodingName); if (indexCoding != null) { final Attribute descriptionsAtt = variable.findAttributeIgnoreCase(INDEX_DESCRIPTIONS); if (descriptionsAtt != null) { final String[] descriptions = descriptionsAtt.getStringValue().split(DESCRIPTION_SEPARATOR); if (indexCoding.getNumAttributes() == descriptions.length) { for (int i = 0; i < descriptions.length; i++) { indexCoding.getAttributeAt(i).setDescription(descriptions[i]); } } } final Attribute nameAtt = variable.findAttributeIgnoreCase(INDEX_CODING_NAME); if (nameAtt != null) { indexCoding.setName(nameAtt.getStringValue()); } } return indexCoding; }
public static boolean isValidFile(NetcdfFile ds) { Attribute cdmDtAtt = ds.findGlobalAttributeIgnoreCase("cdm_data_type"); if (cdmDtAtt == null) cdmDtAtt = ds.findGlobalAttributeIgnoreCase("cdm_datatype"); if (cdmDtAtt == null) return false; if (!cdmDtAtt.isString()) return false; String cdmDtString = cdmDtAtt.getStringValue(); if (cdmDtString == null) return false; if (!cdmDtString.equalsIgnoreCase(FeatureType.TRAJECTORY.toString())) return false; Attribute conventionsAtt = ds.findGlobalAttributeIgnoreCase("Conventions"); if (conventionsAtt == null) return (false); if (!conventionsAtt.isString()) return (false); String convString = conventionsAtt.getStringValue(); StringTokenizer stoke = new StringTokenizer(convString, ","); while (stoke.hasMoreTokens()) { String toke = stoke.nextToken().trim(); if (toke.equalsIgnoreCase("Unidata Observation Dataset v1.0")) return true; } return false; }
/** Instances which have same content are equal. */ @Override public boolean equals(Object o) { if (this == o) return true; if ((o == null) || !(o instanceof Attribute)) return false; final Attribute att = (Attribute) o; if (!name.equals(att.name)) return false; if (nelems != att.nelems) return false; if (!dataType.equals(att.dataType)) return false; if (svalue != null) return svalue.equals(att.getStringValue()); if (values != null) { for (int i = 0; i < getLength(); i++) { int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode(); int r2 = att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode(); if (r1 != r2) return false; } } return true; }
/** * Is this the right format for this adapter * * @param ncd NetcdfDataset to check * @return true if the right format */ public static boolean isValidFile(NetcdfDataset ncd) { // return (buildConfig(ncd) != null); // Check for "center" attribute w/ value of "UCAR/CDAAC". Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center"); if (attrib == null) { return false; } if (!attrib.isString()) { return false; } if (!attrib.getStringValue().equals("UCAR/CDAAC")) { return false; } return true; }
public static ThreddsMetadata.Variables extractVariables( InvDatasetImpl threddsDataset, GridDataset gridDataset) { thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType(); if ((fileFormat != null) && (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) { boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1); ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString()); for (GridDatatype grid : gridDataset.getGrids()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); v.setName(grid.getName()); v.setDescription(grid.getDescription()); v.setUnits(grid.getUnitsString()); // ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number"); // String paramNumber = (att != null) ? att.getNumericValue().toString() : null; if (isGrib1) { v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR")); v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id")); } else { String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", ""); String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", ""); String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", ""); v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName); v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id")); } vars.addVariable(v); } vars.sort(); return vars; } else { // GRID but not GRIB ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0"); for (GridDatatype grid : gridDataset.getGrids()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); vars.addVariable(v); v.setName(grid.getName()); v.setDescription(grid.getDescription()); v.setUnits(grid.getUnitsString()); ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name"); v.setVocabularyName((att != null) ? att.getStringValue() : "N/A"); } vars.sort(); return vars; } }
MetadataAttribute attributeToMetadata(Attribute attribute) { final int productDataType = getProductDataType(attribute.getDataType(), false, false); if (productDataType != -1) { ProductData productData; if (attribute.isString()) { productData = ProductData.createInstance(attribute.getStringValue()); } else if (attribute.isArray()) { productData = ProductData.createInstance(productDataType, attribute.getLength()); productData.setElems(attribute.getValues().getStorage()); } else { productData = ProductData.createInstance(productDataType, 1); productData.setElems(attribute.getValues().getStorage()); } return new MetadataAttribute(attribute.getShortName(), productData, true); } return null; }
/** * Extract a list of data variables (and their canonical names if possible) from the dataset. * * @param threddsDataset open this dataset * @return ThreddsMetadata.Variables, or null if unable. * @throws IOException on read error */ public static ThreddsMetadata.Variables extractVariables(InvDatasetImpl threddsDataset) throws IOException { ThreddsDataFactory.Result result = null; try { result = new ThreddsDataFactory().openFeatureDataset(threddsDataset, null); if (result.fatalError) { System.out.println(" openDatatype errs=" + result.errLog); return null; } if (result.featureType == FeatureType.GRID) { // System.out.println(" extractVariables GRID=" + result.location); GridDataset gridDataset = (GridDataset) result.featureDataset; return extractVariables(threddsDataset, gridDataset); } else if ((result.featureType == FeatureType.STATION) || (result.featureType == FeatureType.POINT)) { PointObsDataset pobsDataset = (PointObsDataset) result.featureDataset; ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0"); for (VariableSimpleIF vs : pobsDataset.getDataVariables()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); vars.addVariable(v); v.setName(vs.getName()); v.setDescription(vs.getDescription()); v.setUnits(vs.getUnitsString()); ucar.nc2.Attribute att = vs.findAttributeIgnoreCase("standard_name"); v.setVocabularyName((att != null) ? att.getStringValue() : "N/A"); } vars.sort(); return vars; } } finally { try { if ((result != null) && (result.featureDataset != null)) result.featureDataset.close(); } catch (IOException ioe) { logger.error("Closing dataset " + result.featureDataset, ioe); } } return null; }
public static ThreddsMetadata.Variables extractVariables(FeatureDatasetPoint fd) { ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.5"); List<VariableSimpleIF> dataVars = fd.getDataVariables(); if (dataVars == null) return vars; for (VariableSimpleIF v : dataVars) { ThreddsMetadata.Variable tv = new ThreddsMetadata.Variable(); vars.addVariable(tv); tv.setName(v.getName()); tv.setDescription(v.getDescription()); tv.setUnits(v.getUnitsString()); ucar.nc2.Attribute att = v.findAttributeIgnoreCase("standard_name"); tv.setVocabularyName((att != null) ? att.getStringValue() : "N/A"); } vars.sort(); return vars; }
public void testCoordVar(NetcdfFile ncfile) { Variable lat = ncfile.findVariable("lat"); assert null != lat; assert lat.getShortName().equals("lat"); assert lat.getRank() == 1; assert lat.getSize() == 3; assert lat.getShape()[0] == 3; assert lat.getDataType() == DataType.FLOAT; assert !lat.isUnlimited(); assert lat.getDimension(0).equals(ncfile.findDimension("lat")); Attribute att = lat.findAttribute("units"); assert null != att; assert !att.isArray(); assert att.isString(); assert att.getDataType() == DataType.STRING; assert att.getStringValue().equals("degrees_north"); assert att.getNumericValue() == null; assert att.getNumericValue(3) == null; try { Array data = lat.read(); assert data.getRank() == 1; assert data.getSize() == 3; assert data.getShape()[0] == 3; assert data.getElementType() == float.class; IndexIterator dataI = data.getIndexIterator(); assert TestUtils.close(dataI.getDoubleNext(), 41.0); assert TestUtils.close(dataI.getDoubleNext(), 40.0); assert TestUtils.close(dataI.getDoubleNext(), 39.0); } catch (IOException io) { } }
Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill) throws IOException, InvalidRangeException { NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill); if (debug) { System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName); } // global attributes List<Attribute> glist = bufr.getGlobalAttributes(); for (Attribute att : glist) { String useName = N3iosp.makeValidNetcdfObjectName(att.getName()); Attribute useAtt; if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues()); else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue()); else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue()); if (debug) System.out.println("add gatt= " + useAtt); } // global dimensions Dimension recordDim = null; Map<String, Dimension> dimHash = new HashMap<String, Dimension>(); for (Dimension oldD : bufr.getDimensions()) { String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName()); boolean isRecord = useName.equals("record"); Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false); dimHash.put(newD.getName(), newD); if (isRecord) recordDim = newD; if (debug) System.out.println("add dim= " + newD); } // Variables Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord); for (Variable oldVar : recordStruct.getVariables()) { if (oldVar.getDataType() == DataType.STRUCTURE) continue; String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName()); DataType newType = oldVar.getDataType(); List<Dimension> newDims = new ArrayList<Dimension>(); newDims.add(recordDim); for (Dimension dim : oldVar.getDimensions()) { newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength())); } Variable newVar = ncfile.addVariable(varName, newType, newDims); if (debug) System.out.println("add var= " + newVar); // attributes List<Attribute> attList = oldVar.getAttributes(); for (Attribute att : attList) { String useName = N3iosp.makeValidNetcdfObjectName(att.getName()); if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues()); else if (att.isString()) ncfile.addVariableAttribute(varName, useName, att.getStringValue()); else ncfile.addVariableAttribute(varName, useName, att.getNumericValue()); } } // int max_seq = countSeq(recordStruct); // Dimension seqD = ncfile.addDimension("level", max_seq); for (Variable v : recordStruct.getVariables()) { if (v.getDataType() != DataType.STRUCTURE) continue; String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName()); int shape[] = v.getShape(); Dimension structDim = ncfile.addDimension(structName, shape[0]); Structure struct = (Structure) v; for (Variable seqVar : struct.getVariables()) { String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName); DataType newType = seqVar.getDataType(); List<Dimension> newDims = new ArrayList<Dimension>(); newDims.add(recordDim); newDims.add(structDim); for (Dimension dim : seqVar.getDimensions()) { newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength())); } Variable newVar = ncfile.addVariable(varName, newType, newDims); if (debug) System.out.println("add var= " + newVar); // attributes List<Attribute> attList = seqVar.getAttributes(); for (Attribute att : attList) { String useName = N3iosp.makeValidNetcdfObjectName(att.getName()); if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues()); else if (att.isString()) ncfile.addVariableAttribute(varName, useName, att.getStringValue()); else ncfile.addVariableAttribute(varName, useName, att.getNumericValue()); } } } // create the file ncfile.create(); if (debug) System.out.println("File Out= " + ncfile.toString()); // boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE); double total = copyVarData(ncfile, recordStruct); ncfile.flush(); System.out.println("FileWriter done total bytes = " + total); ncfile.close(); }
/** * Build the configuration from the dataset * * @param ncd NetcdfDataset * @return the trajectory configuration */ private static Config buildConfig(NetcdfDataset ncd) { // already did this in isValid, but we'll keep here for later refactor Attribute attrib = ncd.findGlobalAttributeIgnoreCase("center"); if (attrib == null) { return null; } if (!attrib.isString()) { return null; } if (!attrib.getStringValue().equals("UCAR/CDAAC")) { return null; } // Check for start_time, stop_time attrib = ncd.findGlobalAttributeIgnoreCase("start_time"); if (attrib == null) { return null; } if (attrib.isString()) { return null; } double startTime = attrib.getNumericValue().doubleValue(); attrib = ncd.findGlobalAttributeIgnoreCase("stop_time"); if (attrib == null) { return null; } if (attrib.isString()) { return null; } double endTime = attrib.getNumericValue().doubleValue(); // Check that only one dimension and that it is the alt dimension. List list = ncd.getRootGroup().getDimensions(); if (list.size() != 1) { return null; } Dimension d = (Dimension) list.get(0); if (!d.getName().equals(timeDimName)) { return null; } Config trajConfig = new Config(); trajConfig.setTimeDim(d); // Check for latitude variable with time dimension and units convertable to "degrees_north". Variable var = ncd.getRootGroup().findVariable(latVarName); if (var == null) { return null; } list = var.getDimensions(); if (list.size() != 1) { return null; } d = (Dimension) list.get(0); if (!d.getName().equals(timeDimName)) { return null; } String units = var.findAttribute("units").getStringValue(); if (!SimpleUnit.isCompatible(units, "degrees_north")) { return null; } trajConfig.setLatVar(var); // Make the time Variable int numTimes = d.getLength(); double[] times = new double[numTimes]; // Variable timeVar = new Variable(var); // timeVar.setName(timeVarName); VariableDS timeVar = new VariableDS( ncd, ncd.getRootGroup(), null, timeVarName, DataType.DOUBLE, timeDimName, "seconds since 1980-01-06 00:00:00", "Time coordinate"); // Variable timeVar = new Variable(ncd, ncd.getRootGroup(), null, // timeVarName); // timeVar.setDataType(DataType.DOUBLE); // timeVar.setDimensions(list); // Attribute newUnits = // new Attribute("units", "seconds since 1980-01-06 00:00:00"); // timeVar.addAttribute(newUnits); timeVar.setCachedData( Array.makeArray(DataType.DOUBLE, numTimes, endTime, ((startTime - endTime) / numTimes)), true); ncd.addVariable(ncd.getRootGroup(), timeVar); trajConfig.setTimeVar(timeVar); // Check for longitude variable with time dimension and units convertable to "degrees_east". var = ncd.getRootGroup().findVariable(lonVarName); if (var == null) { return null; } list = var.getDimensions(); if (list.size() != 1) { return null; } d = (Dimension) list.get(0); if (!d.getName().equals(timeDimName)) { return null; } units = var.findAttribute("units").getStringValue(); if (!SimpleUnit.isCompatible(units, "degrees_east")) { return null; } trajConfig.setLonVar(var); // Check for altitude variable with time dimension and units convertable to "m". var = ncd.getRootGroup().findVariable(elevVarName); if (var == null) { return null; } list = var.getDimensions(); if (list.size() != 1) { return null; } d = (Dimension) list.get(0); if (!d.getName().equals(timeDimName)) { return null; } units = var.findAttribute("units").getStringValue(); if (!SimpleUnit.isCompatible(units, "meters")) { return null; } trajConfig.setElevVar(var); trajConfig.setTrajectoryId(trajId); return trajConfig; }
// create from a dataset public ObsBean(Structure obs, StructureData sdata) { // first choice for (Variable v : obs.getVariables()) { Attribute att = v.findAttribute("BUFR:TableB_descriptor"); if (att == null) continue; String val = att.getStringValue(); if (val.equals("0-5-1") && Double.isNaN(lat)) { lat = sdata.convertScalarDouble(v.getShortName()); } else if (val.equals("0-6-1") && Double.isNaN(lon)) { lon = sdata.convertScalarDouble(v.getShortName()); } else if (val.equals("0-7-30") && Double.isNaN(alt)) { alt = sdata.convertScalarDouble(v.getShortName()); } else if (val.equals("0-4-1") && (year < 0)) { year = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-4-2") && (month < 0)) { month = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-4-3") && (day < 0)) { day = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-4-4") && (hour < 0)) { hour = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-4-5") && (minute < 0)) { minute = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-4-6") && (sec < 0)) { sec = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-1-1") && (wmo_block < 0)) { wmo_block = sdata.convertScalarInt(v.getShortName()); } else if (val.equals("0-1-2") && (wmo_id < 0)) { wmo_id = sdata.convertScalarInt(v.getShortName()); } else if ((stn == null) && (val.equals("0-1-7") || val.equals("0-1-194") || val.equals("0-1-11") || val.equals("0-1-18"))) { if (v.getDataType().isString()) stn = sdata.getScalarString(v.getShortName()); else stn = Integer.toString(sdata.convertScalarInt(v.getShortName())); } } // second choice for (Variable v : obs.getVariables()) { Attribute att = v.findAttribute("BUFR:TableB_descriptor"); if (att == null) continue; String val = att.getStringValue(); if (val.equals("0-5-2") && Double.isNaN(lat)) { lat = sdata.convertScalarDouble(v.getShortName()); } else if (val.equals("0-6-2") && Double.isNaN(lon)) { lon = sdata.convertScalarDouble(v.getShortName()); } else if (val.equals("0-7-1") && Double.isNaN(alt)) { alt = sdata.convertScalarDouble(v.getShortName()); } else if ((val.equals("0-4-7")) && (sec < 0)) { sec = sdata.convertScalarInt(v.getShortName()); } } // third choice for (Variable v : obs.getVariables()) { Attribute att = v.findAttribute("BUFR:TableB_descriptor"); if (att == null) continue; String val = att.getStringValue(); if (val.equals("0-7-10") && Double.isNaN(alt)) { alt = sdata.convertScalarDouble(v.getShortName()); } else if (val.equals("0-7-2") && Double.isNaN(alt)) { alt = sdata.convertScalarDouble(v.getShortName()); } } }
public String getAbbrev() { Attribute att = ve.findAttributeIgnoreCase(CDM.ABBREV); return (att == null) ? null : att.getStringValue(); }
private String getCalendarAttribute(VariableEnhanced vds) { Attribute cal = vds.findAttribute("calendar"); return (cal == null) ? null : cal.getStringValue(); }
protected Map<Band, Variable> addSmiBands(Product product, List<Variable> variables) { final int sceneRasterWidth = product.getSceneRasterWidth(); final int sceneRasterHeight = product.getSceneRasterHeight(); Map<Band, Variable> bandToVariableMap = new HashMap<Band, Variable>(); for (Variable variable : variables) { int variableRank = variable.getRank(); if (variableRank == 2) { final int[] dimensions = variable.getShape(); final int height = dimensions[0]; final int width = dimensions[1]; if (height == sceneRasterHeight && width == sceneRasterWidth) { String name = variable.getShortName(); if (name.equals("l3m_data")) { try { name = getStringAttribute("Parameter") + " " + getStringAttribute("Measure"); } catch (Exception e) { e.printStackTrace(); } } final int dataType = getProductDataType(variable); final Band band = new Band(name, dataType, width, height); // band = new Band(name, dataType, width, height); product.addBand(band); try { Attribute fillvalue = variable.findAttribute("_FillValue"); if (fillvalue == null) { fillvalue = variable.findAttribute("Fill"); } if (fillvalue != null) { band.setNoDataValue((double) fillvalue.getNumericValue().floatValue()); band.setNoDataValueUsed(true); } } catch (Exception ignored) { } bandToVariableMap.put(band, variable); // Set units, if defined try { band.setUnit(getStringAttribute("Units")); } catch (Exception ignored) { } final List<Attribute> list = variable.getAttributes(); double[] validMinMax = {0.0, 0.0}; for (Attribute hdfAttribute : list) { final String attribName = hdfAttribute.getShortName(); if ("units".equals(attribName)) { band.setUnit(hdfAttribute.getStringValue()); } else if ("long_name".equalsIgnoreCase(attribName)) { band.setDescription(hdfAttribute.getStringValue()); } else if ("slope".equalsIgnoreCase(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); } else if ("intercept".equalsIgnoreCase(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); } else if ("scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); } else if ("add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); } else if (attribName.startsWith("valid_")) { if ("valid_min".equals(attribName)) { validMinMax[0] = hdfAttribute.getNumericValue(0).doubleValue(); } else if ("valid_max".equals(attribName)) { validMinMax[1] = hdfAttribute.getNumericValue(0).doubleValue(); } else if ("valid_range".equals(attribName)) { validMinMax[0] = hdfAttribute.getNumericValue(0).doubleValue(); validMinMax[1] = hdfAttribute.getNumericValue(1).doubleValue(); } } } if (validMinMax[0] != validMinMax[1]) { double[] minmax = {0.0, 0.0}; minmax[0] = validMinMax[0]; minmax[1] = validMinMax[1]; if (band.getScalingFactor() != 1.0) { minmax[0] *= band.getScalingFactor(); minmax[1] *= band.getScalingFactor(); } if (band.getScalingOffset() != 0.0) { minmax[0] += band.getScalingOffset(); minmax[1] += band.getScalingOffset(); } String validExp = format("%s >= %.2f && %s <= %.2f", name, minmax[0], name, minmax[1]); band.setValidPixelExpression( validExp); // .format(name, validMinMax[0], name, validMinMax[1])); } } } else if (variableRank == 4) { final int[] dimensions = variable.getShape(); final int height = dimensions[2]; final int width = dimensions[3]; if (height == sceneRasterHeight && width == sceneRasterWidth) { String name = variable.getShortName(); final int dataType = getProductDataType(variable); final Band band = new Band(name, dataType, width, height); // band = new Band(name, dataType, width, height); Variable sliced = null; try { sliced = variable.slice(0, 0).slice(0, 0); } catch (InvalidRangeException e) { e.printStackTrace(); // Todo change body of catch statement. } bandToVariableMap.put(band, sliced); product.addBand(band); try { Attribute fillvalue = variable.findAttribute("_FillValue"); if (fillvalue != null) { band.setNoDataValue((double) fillvalue.getNumericValue().floatValue()); band.setNoDataValueUsed(true); } } catch (Exception ignored) { } // Set units, if defined try { band.setUnit(getStringAttribute("units")); } catch (Exception ignored) { } final List<Attribute> list = variable.getAttributes(); for (Attribute hdfAttribute : list) { final String attribName = hdfAttribute.getShortName(); if ("scale_factor".equals(attribName)) { band.setScalingFactor(hdfAttribute.getNumericValue(0).doubleValue()); } else if ("add_offset".equals(attribName)) { band.setScalingOffset(hdfAttribute.getNumericValue(0).doubleValue()); } } } } } return bandToVariableMap; }
@Override public AbstractGridDataset createDataset(String id, String location) throws IOException, EdalException { NetcdfDataset nc = null; try { /* * Open the dataset, using the cache for NcML aggregations */ nc = openAndAggregateDataset(location); /*- * We may in future be able to use forecast model run collection aggregations for * dealing with the case of overlapping time axes. To do this the code will look * something like this: * * StringBuilder sb = new StringBuilder(); * Formatter formatter = new Formatter(sb, Locale.UK); * Fmrc f = Fmrc.open(location, formatter); * * in openAndAggregateDataset. It will need to build up an NcML document which * does this. It should look something like: * * <netcdf xmlns="http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2" enhance="true"> * <aggregation dimName="run" type="forecastModelRunCollection" timeUnitsChange="true"> * <!-- scanFmrc actually works, but what we want is something like the following bit --> * <scanFmrc location="/home/guy/Data/POLCOMS_IRISH/" regExp=".*\.nc"/> * <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090320.nc" coordValue="2009-03-20T00:00:00Z" enhance="true" /> * <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090321.nc" coordValue="2009-03-21T00:00:00Z" enhance="true" /> * <netcdf location="/home/guy/Data/POLCOMS_IRISH/polcoms_irish_hourly_20090322.nc" coordValue="2009-03-22T00:00:00Z" enhance="true" /> * </aggregation> * </netcdf> * * For more documentation see: * http://mailman.unidata.ucar.edu/software/thredds/current/netcdf-java/ncml/FmrcAggregation.html * * We then can do stuff like: * * ucar.nc2.dt.GridDataset gridDataset = f.getDatasetBest(); * * To get the single best aggregation of the overlapping time axis * * Then we need to work with GridDatasets in place of NetcdfDatasets. Stuff like: * * for(Variable variable : gridDataset.getNetcdfFile().getVariables()) { * // blah blah * } * * will be necessary. We need to check that that works with remote datasets too */ /* * We look for NetCDF-U variables to group mean/standard-deviation. * * We need to do this here because we want to subsequently ignore * parent variables */ Map<String, String[]> varId2AncillaryVars = new HashMap<String, String[]>(); for (Variable variable : nc.getVariables()) { /* * Just look for parent variables, since these may not have a * grid directly associated with them */ for (Attribute attr : variable.getAttributes()) { if (attr.getFullName().equalsIgnoreCase("ancillary_variables")) { varId2AncillaryVars.put(variable.getFullName(), attr.getStringValue().split(" ")); continue; } } } ucar.nc2.dt.GridDataset gridDataset = CdmUtils.getGridDataset(nc); List<GridVariableMetadata> vars = new ArrayList<GridVariableMetadata>(); /* * Store a map of component names. Key is the compound name, value * is a 2-element String array with x, y component IDs * * Also store a map of whether these components are really * eastward/northward, or whether they are locally u/v */ Map<String, String[]> xyComponentPairs = new HashMap<String, String[]>(); Map<String, Boolean> xyNameToTrueEN = new HashMap<String, Boolean>(); /* * Store a map of variable IDs to UncertML URLs. This will be used * to determine which components are mean/std/etc. * * TODO implement more than just Mean/SD */ Map<String, String> varId2UncertMLRefs = new HashMap<String, String>(); /* * Here we store the parent variable IDs and their corresponding * title. */ Map<String, String> parentVarId2Title = new HashMap<String, String>(); for (Gridset gridset : gridDataset.getGridsets()) { GridCoordSystem coordSys = gridset.getGeoCoordSystem(); HorizontalGrid hDomain = CdmUtils.createHorizontalGrid(coordSys); VerticalAxis zDomain = CdmUtils.createVerticalAxis(coordSys); TimeAxis tDomain = CdmUtils.createTimeAxis(coordSys); /* * Create a VariableMetadata object for each GridDatatype */ for (GridDatatype grid : gridset.getGrids()) { VariableDS variable = grid.getVariable(); String varId = variable.getFullName(); String name = getVariableName(variable); /* * If this is a parent variable for a stats collection, we * don't want it to be a normal variable as well. */ if (varId2AncillaryVars.containsKey(varId)) { parentVarId2Title.put(varId, name); continue; } /* * If it is a child variable is (potentially) referenced by * UncertML, store its ID and the (possible) UncertML URI */ for (Attribute attr : variable.getAttributes()) { if (attr.getFullName().equalsIgnoreCase("ref")) { varId2UncertMLRefs.put(varId, attr.getStringValue()); } } Parameter parameter = new Parameter( varId, variable.getShortName(), variable.getDescription(), variable.getUnitsString(), name); GridVariableMetadata metadata = new GridVariableMetadata( variable.getFullName(), parameter, hDomain, zDomain, tDomain, true); vars.add(metadata); if (name != null) { /* * Check for vector components */ if (name.contains("eastward_")) { String compoundName = name.replaceFirst("eastward_", ""); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, true); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[0] = varId; } else if (name.contains("northward_")) { String compoundName = name.replaceFirst("northward_", ""); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, true); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[1] = varId; } else if (name.matches("u-.*component")) { String compoundName = name.replaceFirst("u-(.*)component", "$1"); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, false); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[0] = varId; } else if (name.matches("v-.*component")) { String compoundName = name.replaceFirst("v-(.*)component", "$1"); String[] cData; if (!xyComponentPairs.containsKey(compoundName)) { cData = new String[2]; xyComponentPairs.put(compoundName, cData); xyNameToTrueEN.put(compoundName, false); } cData = xyComponentPairs.get(compoundName); /* * By doing this, we will end up with the merged * coverage */ cData[1] = varId; } /* * We could potentially add a check for zonal/meridional * here if required. */ } } } CdmGridDataset cdmGridDataset = new CdmGridDataset(id, location, vars, CdmUtils.getOptimumDataReadingStrategy(nc)); for (Entry<String, String[]> componentData : xyComponentPairs.entrySet()) { String commonName = componentData.getKey(); String[] comps = componentData.getValue(); if (comps[0] != null && comps[1] != null) { cdmGridDataset.addVariablePlugin( new VectorPlugin(comps[0], comps[1], commonName, xyNameToTrueEN.get(commonName))); } } for (String statsCollectionId : varId2AncillaryVars.keySet()) { String[] ids = varId2AncillaryVars.get(statsCollectionId); String meanId = null; String stddevId = null; for (String statsVarIds : ids) { String uncertRef = varId2UncertMLRefs.get(statsVarIds); if (uncertRef != null && uncertRef.equalsIgnoreCase("http://www.uncertml.org/statistics/mean")) { meanId = statsVarIds; } if (uncertRef != null && uncertRef.equalsIgnoreCase( "http://www.uncertml.org/statistics/standard-deviation")) { stddevId = statsVarIds; } } if (meanId != null && stddevId != null) { MeanSDPlugin meanSDPlugin = new MeanSDPlugin(meanId, stddevId, parentVarId2Title.get(statsCollectionId)); cdmGridDataset.addVariablePlugin(meanSDPlugin); } } return cdmGridDataset; } finally { CdmUtils.closeDataset(nc); } }
/** * Opens the NetCDF dataset at the given location, using the dataset cache if {@code location} * represents an NcML aggregation. We cannot use the cache for OPeNDAP or single NetCDF files * because the underlying data may have changed and the NetcdfDataset cache may cache a dataset * forever. In the case of NcML we rely on the fact that server administrators ought to have set a * "recheckEvery" parameter for NcML aggregations that may change with time. It is desirable to * use the dataset cache for NcML aggregations because they can be time-consuming to assemble and * we don't want to do this every time a map is drawn. * * @param location The location of the data: a local NetCDF file, an NcML aggregation file or an * OPeNDAP location, {@literal i.e.} anything that can be passed to * NetcdfDataset.openDataset(location). * @return a {@link NetcdfDataset} object for accessing the data at the given location. * @throws IOException if there was an error reading from the data source. */ private NetcdfDataset openAndAggregateDataset(String location) throws IOException, EdalException { NetcdfDataset nc; if (location.startsWith("dods://") || location.startsWith("http://")) { /* * We have a remote dataset */ nc = CdmUtils.openDataset(location); } else { /* * We have a local dataset */ List<File> files = null; try { files = CdmUtils.expandGlobExpression(location); } catch (NullPointerException e) { System.out.println("NPE processing location: " + location); throw e; } if (files.size() == 0) { throw new EdalException( "The location " + location + " doesn't refer to any existing files."); } if (files.size() == 1) { location = files.get(0).getAbsolutePath(); nc = CdmUtils.openDataset(location); } else { /* * We have multiple files in a glob expression. We write some * NcML and use the NetCDF aggregation libs to parse this into * an aggregated dataset. * * If we have already generated the ncML on a previous call, * just use that. */ if (ncmlString == null) { /* * Find the name of the time dimension */ NetcdfDataset first = openAndAggregateDataset(files.get(0).getAbsolutePath()); String timeDimName = null; for (Variable var : first.getVariables()) { if (var.isCoordinateVariable()) { for (Attribute attr : var.getAttributes()) { if (attr.getFullName().equalsIgnoreCase("units") && attr.getStringValue().contains(" since ")) { /* * This is the time dimension. Since this is * a co-ordinate variable, there is only 1 * dimension */ Dimension timeDimension = var.getDimension(0); timeDimName = timeDimension.getFullName(); } } } } first.close(); if (timeDimName == null) { throw new EdalException("Cannot join multiple files without time dimensions"); } /* * We can't assume that the glob expression will have * returned the files in time order. * * We could assume that alphabetical == time ordered (and * for properly named files it will - but let's not rely on * our users having sensible naming conventions... * * Sort the list using a comparator which opens the file and * gets the first value of the time dimension */ final String aggDimName = timeDimName; Collections.sort( files, new Comparator<File>() { @Override public int compare(File ncFile1, File ncFile2) { NetcdfFile nc1 = null; NetcdfFile nc2 = null; try { nc1 = NetcdfFile.open(ncFile1.getAbsolutePath()); nc2 = NetcdfFile.open(ncFile2.getAbsolutePath()); Variable timeVar1 = nc1.findVariable(aggDimName); Variable timeVar2 = nc2.findVariable(aggDimName); long time1 = timeVar1.read().getLong(0); long time2 = timeVar2.read().getLong(0); return Long.compare(time1, time2); } catch (Exception e) { /* * There was a problem reading the data. Sort * alphanumerically by filename and hope for the * best... * * This catches all exceptions because however * it fails this is still our best option. * * If the error is a genuine problem, it'll show * up as soon as we try and aggregate. */ return ncFile1.getAbsolutePath().compareTo(ncFile2.getAbsolutePath()); } finally { if (nc1 != null) { try { nc1.close(); } catch (IOException e) { log.error("Problem closing netcdf file", e); } } if (nc2 != null) { try { nc2.close(); } catch (IOException e) { log.error("Problem closing netcdf file", e); } } } } }); /* * Now create the NcML string and use it to create an * aggregated dataset */ StringBuffer ncmlStringBuffer = new StringBuffer(); ncmlStringBuffer.append( "<netcdf xmlns=\"http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2\">"); ncmlStringBuffer.append( "<aggregation dimName=\"" + timeDimName + "\" type=\"joinExisting\">"); for (File file : files) { ncmlStringBuffer.append("<netcdf location=\"" + file.getAbsolutePath() + "\"/>"); } ncmlStringBuffer.append("</aggregation>"); ncmlStringBuffer.append("</netcdf>"); ncmlString = ncmlStringBuffer.toString(); } nc = NcMLReader.readNcML(new StringReader(ncmlString), null); } } return nc; }
@Test public void testDoradeGround() throws IOException { System.out.println("**** Open " + groundDoradeFile); try (NetcdfFile ncfile = NetcdfFile.open(groundDoradeFile)) { for (Variable v : ncfile.getVariables()) { System.out.println(v.getFullName()); } /* test both gate and radial dimension */ Dimension gateDim = ncfile.getRootGroup().findDimension("gate_1"); assert (gateDim.getLength() == 1008); Dimension radialDim = ncfile.getRootGroup().findDimension("radial"); assert (radialDim.getLength() == 439); /* test some att */ Attribute testAtt = ncfile.getRootGroup().findAttribute("Conventions"); assert (testAtt.getStringValue().equals(_Coordinate.Convention)); testAtt = ncfile.getRootGroup().findAttribute("format"); assert (testAtt.getStringValue().equals("Unidata/netCDF/Dorade")); testAtt = ncfile.getRootGroup().findAttribute("Project_name"); assert (testAtt.getStringValue().equalsIgnoreCase("IHOP_2002")); testAtt = ncfile.getRootGroup().findAttribute("Radar_Name"); assert (testAtt.getStringValue().equalsIgnoreCase("SPOL")); testAtt = ncfile.getRootGroup().findAttribute("VolumeCoveragePatternName"); assert (testAtt.getStringValue().equalsIgnoreCase("SUR")); testAtt = ncfile.getRootGroup().findAttribute("Volume_Number"); assert (testAtt.getStringValue().equalsIgnoreCase("1")); testAtt = ncfile.getRootGroup().findAttribute("Sweep_Number"); assert (testAtt.getStringValue().equalsIgnoreCase("2")); testAtt = ncfile.getRootGroup().findAttribute("Sweep_Date"); assert (testAtt.getStringValue().equalsIgnoreCase("2002-05-11 01:58:15Z")); Variable var; var = ncfile.findVariable("elevation"); testReadData(var); var = ncfile.findVariable("azimuth"); testReadData(var); var = ncfile.findVariable("distance_1"); testReadData(var); var = ncfile.findVariable("latitudes_1"); testReadData(var); var = ncfile.findVariable("longitudes_1"); testReadData(var); var = ncfile.findVariable("altitudes_1"); testReadData(var); var = ncfile.findVariable("rays_time"); testReadData(var); var = ncfile.findVariable("Range_to_First_Cell"); float t = testReadScalar(var); assert (t == (float) 150.0); var = ncfile.findVariable("Cell_Spacing"); t = testReadScalar(var); assert (t == (float) 149.89624); var = ncfile.findVariable("Fixed_Angle"); t = testReadScalar(var); assert (t == (float) 1.1975098); var = ncfile.findVariable("Nyquist_Velocity"); t = testReadScalar(var); assert (t == (float) 25.618269); var = ncfile.findVariable("Unambiguous_Range"); t = testReadScalar(var); assert (t == (float) 156.11694); var = ncfile.findVariable("Radar_Constant"); t = testReadScalar(var); assert (t == (float) 70.325195); var = ncfile.findVariable("rcvr_gain"); t = testReadScalar(var); assert (t == (float) 46.95); var = ncfile.findVariable("ant_gain"); t = testReadScalar(var); assert (t == (float) 45.58); var = ncfile.findVariable("sys_gain"); t = testReadScalar(var); assert (t == (float) 46.95); var = ncfile.findVariable("bm_width"); t = testReadScalar(var); assert (t == (float) 0.92); var = ncfile.findVariable("VE"); testReadData(var); var = ncfile.findVariable("DM"); testReadData(var); var = ncfile.findVariable("NCP"); testReadData(var); var = ncfile.findVariable("SW"); testReadData(var); var = ncfile.findVariable("DZ"); testReadData(var); var = ncfile.findVariable("DCZ"); testReadData(var); var = ncfile.findVariable("LVDR"); testReadData(var); var = ncfile.findVariable("NIQ"); testReadData(var); var = ncfile.findVariable("AIQ"); testReadData(var); var = ncfile.findVariable("CH"); testReadData(var); var = ncfile.findVariable("AH"); testReadData(var); var = ncfile.findVariable("CV"); testReadData(var); var = ncfile.findVariable("AV"); testReadData(var); var = ncfile.findVariable("RHOHV"); testReadData(var); var = ncfile.findVariable("LDR"); testReadData(var); var = ncfile.findVariable("DL"); testReadData(var); var = ncfile.findVariable("DX"); testReadData(var); var = ncfile.findVariable("ZDR"); testReadData(var); var = ncfile.findVariable("PHI"); testReadData(var); var = ncfile.findVariable("KDP"); testReadData(var); assert (0 == var.findDimensionIndex("radial")); assert (1 == var.findDimensionIndex("gate_1")); } }
/** * Constructor. If scale/offset attributes are found, remove them from the decorated variable. * * @param forVar the Variable to decorate. * @param useNaNs pre-fill isMissing() data with NaNs * @param fillValueIsMissing use _FillValue for isMissing() * @param invalidDataIsMissing use valid_range for isMissing() * @param missingDataIsMissing use missing_value for isMissing() */ EnhanceScaleMissingImpl( VariableDS forVar, boolean useNaNs, boolean fillValueIsMissing, boolean invalidDataIsMissing, boolean missingDataIsMissing) { this.fillValueIsMissing = fillValueIsMissing; this.invalidDataIsMissing = invalidDataIsMissing; this.missingDataIsMissing = missingDataIsMissing; // see if underlying variable has scale/offset already applied Variable orgVar = forVar.getOriginalVariable(); if (orgVar instanceof VariableDS) { VariableDS orgVarDS = (VariableDS) orgVar; EnumSet<NetcdfDataset.Enhance> orgEnhanceMode = orgVarDS.getEnhanceMode(); if ((orgEnhanceMode != null) && orgEnhanceMode.contains(NetcdfDataset.Enhance.ScaleMissing)) return; } // the other possibility is that you want to apply scale and offset to a signed value, then // declare the result unsigned // this.isUnsigned = (orgVar != null) ? orgVar.isUnsigned() : forVar.isUnsigned(); this.isUnsigned = forVar.isUnsigned(); this.convertedDataType = forVar.getDataType(); DataType scaleType = null, missType = null, validType = null, fillType = null; if (debug) System.out.println("EnhancementsImpl for Variable = " + forVar.getFullName()); Attribute att; // scale and offset if (null != (att = forVar.findAttribute(CDM.SCALE_FACTOR))) { if (!att.isString()) { scale = att.getNumericValue().doubleValue(); hasScaleOffset = true; scaleType = att.getDataType(); forVar.remove(att); if (debug) System.out.println("scale = " + scale + " type " + scaleType); } } if (null != (att = forVar.findAttribute(CDM.ADD_OFFSET))) { if (!att.isString()) { offset = att.getNumericValue().doubleValue(); hasScaleOffset = true; DataType offType = att.getDataType(); if (rank(offType) > rank(scaleType)) scaleType = offType; forVar.remove(att); if (debug) System.out.println("offset = " + offset); } } ////// missing data : valid_range. assume here its in units of unpacked data. correct this below Attribute validRangeAtt; if (null != (validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE))) { if (!validRangeAtt.isString() && validRangeAtt.getLength() > 1) { valid_min = validRangeAtt.getNumericValue(0).doubleValue(); valid_max = validRangeAtt.getNumericValue(1).doubleValue(); hasValidRange = true; validType = validRangeAtt.getDataType(); if (hasScaleOffset) forVar.remove(validRangeAtt); if (debug) System.out.println("valid_range = " + valid_min + " " + valid_max); } } Attribute validMinAtt = null, validMaxAtt = null; if (!hasValidRange) { if (null != (validMinAtt = forVar.findAttribute("valid_min"))) { if (!validMinAtt.isString()) { valid_min = validMinAtt.getNumericValue().doubleValue(); hasValidMin = true; validType = validMinAtt.getDataType(); if (hasScaleOffset) forVar.remove(validMinAtt); if (debug) System.out.println("valid_min = " + valid_min); } } if (null != (validMaxAtt = forVar.findAttribute("valid_max"))) { if (!validMaxAtt.isString()) { valid_max = validMaxAtt.getNumericValue().doubleValue(); hasValidMax = true; DataType t = validMaxAtt.getDataType(); if (rank(t) > rank(validType)) validType = t; if (hasScaleOffset) forVar.remove(validMaxAtt); if (debug) System.out.println("valid_min = " + valid_max); } } } boolean hasValidData = hasValidMin || hasValidMax || hasValidRange; if (hasValidMin && hasValidMax) hasValidRange = true; /// _FillValue if ((null != (att = forVar.findAttribute(CDM.FILL_VALUE))) && !att.isString()) { double[] values = getValueAsDouble(att); // LOOK double WTF ?? if (values.length > 0) { fillValue = values[0]; hasFillValue = true; fillType = att.getDataType(); if (hasScaleOffset) forVar.remove(att); if (debug) System.out.println("missing_datum from _FillValue = " + fillValue); } } /// missing_value if (null != (att = forVar.findAttribute(CDM.MISSING_VALUE))) { if (att.isString()) { String svalue = att.getStringValue(); if (forVar.getDataType() == DataType.CHAR) { missingValue = new double[1]; if (svalue.length() == 0) missingValue[0] = 0; else missingValue[0] = svalue.charAt(0); missType = DataType.CHAR; hasMissingValue = true; } else { // not a CHAR - try to fix problem where they use a numeric value as a String // attribute try { missingValue = new double[1]; missingValue[0] = Double.parseDouble(svalue); missType = att.getDataType(); hasMissingValue = true; } catch (NumberFormatException ex) { if (debug) System.out.println( "String missing_value not parsable as double= " + att.getStringValue()); } } } else { // not a string missingValue = getValueAsDouble(att); missType = att.getDataType(); for (double mv : missingValue) if (!Double.isNaN(mv)) hasMissingValue = true; // dont need to do anything if its already a NaN } if (hasScaleOffset) forVar.remove(att); } // missing boolean hasMissing = (invalidDataIsMissing && hasValidData) || (fillValueIsMissing && hasFillValue) || (missingDataIsMissing && hasMissingValue); /// assign convertedDataType if needed if (hasScaleOffset) { convertedDataType = forVar.getDataType(); if (hasMissing) { // has missing data : must be float or double if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType; if (missingDataIsMissing && rank(missType) > rank(convertedDataType)) convertedDataType = missType; if (fillValueIsMissing && rank(fillType) > rank(convertedDataType)) convertedDataType = fillType; if (invalidDataIsMissing && rank(validType) > rank(convertedDataType)) convertedDataType = validType; if (rank(convertedDataType) < rank(DataType.DOUBLE)) convertedDataType = DataType.FLOAT; } else { // no missing data; can use wider of data and scale if (rank(scaleType) > rank(convertedDataType)) convertedDataType = scaleType; } if (debug) System.out.println("assign dataType = " + convertedDataType); // validData may be external or internal if (hasValidData) { DataType orgType = forVar.getDataType(); // If valid_range is the same type as scale_factor (actually the wider of // scale_factor and add_offset) and this is wider than the external data, then it // will be interpreted as being in the units of the internal (unpacked) data. // Otherwise it is in the units of the external (unpacked) data. // we assumed unpacked data above, redo if its really packed data if (!((rank(validType) == rank(scaleType)) && (rank(scaleType) >= rank(orgType)))) { if (validRangeAtt != null) { double[] values = getValueAsDouble(validRangeAtt); valid_min = values[0]; valid_max = values[1]; } else { if (validMinAtt != null) { double[] values = getValueAsDouble(validMinAtt); valid_min = values[0]; } if (validMaxAtt != null) { double[] values = getValueAsDouble(validMaxAtt); valid_max = values[0]; } } } } } if (hasMissing && ((convertedDataType == DataType.DOUBLE) || (convertedDataType == DataType.FLOAT))) this.useNaNs = useNaNs; if (debug) System.out.println("this.useNaNs = " + this.useNaNs); }