public void init() throws javax.servlet.ServletException {
super.init();
org.slf4j.Logger logServerStartup = org.slf4j.LoggerFactory.getLogger("serverStartup");
logServerStartup.info(
getClass().getName() + " initialization start - " + UsageLog.setupNonRequestContext());
this.ascLimit = ThreddsConfig.getInt("Opendap.ascLimit", ascLimit);
this.binLimit = ThreddsConfig.getInt("Opendap.binLimit", binLimit);
this.odapVersionString = ThreddsConfig.get("Opendap.serverVersion", odapVersionString);
logServerStartup.info(
getClass().getName()
+ " version= "
+ odapVersionString
+ " ascLimit = "
+ ascLimit
+ " binLimit = "
+ binLimit);
// debugging actions
makeDebugActions();
logServerStartup.info(
getClass().getName()
+ " initialization done - "
+ UsageLog.closingMessageNonRequestContext());
}
/** by Mr Skip on 09.02.2016. */
@Service
public class BusInfoService implements BusInfoLogic {
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BusInfoService.class);
@Autowired private BusInfoRepo busInfoRepo;
@Override
public BusInfo getByDriverName(String name) {
BusInfo busInfo;
if ((busInfo = busInfoRepo.getBusByDriverName(name)) == null) {
LOGGER.warn(
"Not find any Bus for driver name:" + name + ". In class: " + getClass().getName());
busInfo = new BusInfo();
}
return busInfo;
}
@Override
public BusInfo getByDriverPhone(String phone) {
BusInfo busInfo;
if ((busInfo = busInfoRepo.getBusByDriverPhone(phone)) == null) {
LOGGER.warn(
"Not find any Bus for driver phone:" + phone + ". In class: " + getClass().getName());
busInfo = new BusInfo();
}
return busInfo;
}
@Override
public List<BusInfo> getAll() {
List<BusInfo> list;
if ((list = busInfoRepo.findAll()).isEmpty()) {
LOGGER.warn("Cant find any Bus in class: " + getClass().getName());
}
return list;
}
@Override
public Map<Integer, String> getAllBusNumber() {
Map<Integer, String> list = new HashMap<>();
for (String s : busInfoRepo.getAllBusNumber()) {
list.put(list.size() + 1, s);
}
return list;
}
}
/** @author pepijn */
public class WPPluginManager {
private WPPluginManager(UUID uuid, String logPrefix, ClassLoader classLoader) {
allPlugins = org.pepsoft.util.PluginManager.findPlugins(Plugin.class, FILENAME, classLoader);
Set<String> namesEncountered = new HashSet<>();
for (Iterator<Plugin> i = allPlugins.iterator(); i.hasNext(); ) {
Plugin plugin = i.next();
if ((plugin.getUUIDs() != null) && (uuid != null) && (!plugin.getUUIDs().contains(uuid))) {
logger.error(
logPrefix
+ plugin.getName()
+ " plugin is not authorised for this installation; not loading it");
i.remove();
continue;
}
String name = plugin.getName();
if (namesEncountered.contains(name)) {
throw new RuntimeException("Multiple plugins with the same name (" + name + ") detected!");
} else {
namesEncountered.add(name);
}
logger.info(logPrefix + "Loaded plugin: " + name + " (version " + plugin.getVersion() + ")");
}
}
public List<Plugin> getAllPlugins() {
return Collections.unmodifiableList(allPlugins);
}
@SuppressWarnings("unchecked") // Guaranteed by Java
public <T extends Plugin> List<T> getPlugins(Class<T> type) {
List<T> plugins = new ArrayList<>(allPlugins.size());
plugins.addAll(
allPlugins
.stream()
.filter(plugin -> type.isAssignableFrom(plugin.getClass()))
.map(plugin -> (T) plugin)
.collect(Collectors.toList()));
return plugins;
}
public static synchronized void initialise(UUID uuid) {
initialise(uuid, "", ClassLoader.getSystemClassLoader());
}
public static synchronized void initialise(UUID uuid, String logPrefix, ClassLoader classLoader) {
if (instance != null) {
throw new IllegalStateException("Already initialised");
}
instance = new WPPluginManager(uuid, logPrefix, classLoader);
}
public static synchronized WPPluginManager getInstance() {
return instance;
}
private final List<Plugin> allPlugins;
private static WPPluginManager instance;
private static final String FILENAME = "org.pepsoft.worldpainter.plugins";
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(WPPluginManager.class);
}
public class HTMLDriver extends PanelDriver implements Exportable {
private static transient org.slf4j.Logger log =
org.slf4j.LoggerFactory.getLogger(HTMLDriver.class.getName());
public static final String PARAMETER_HTML = "html_code";
public static final String PARAMETER_EDITING_LANG = "edit_lang";
private static final String PAGE_SHOW = "show";
private static final String PAGE_EDIT = "edit";
private static final String PAGE_CHOOSE_IMAGE = "imageSelect";
private static final String PAGE_CHOOSE_LINK = "linkSelect";
public static final String PARAMETER_USE_DEFAULTS = "useDefaultLanguage";
private static final String ATTR_TEXT = "text";
private static final String ATTR_EDITING_LANGUAGE = "lang";
/** The locale manager. */
protected LocaleManager localeManager;
public HTMLDriver() {
localeManager = LocaleManager.lookup();
}
public void init(PanelProvider provider) throws Exception {
super.init(provider);
addParameter(new BooleanParameter(provider, PARAMETER_USE_DEFAULTS, false, true));
String[] methodsForEditMode = new String[] {"actionChangeEditingLanguage", "actionSaveChanges"};
for (int i = 0; i < methodsForEditMode.length; i++) {
String method = methodsForEditMode[i];
addMethodPermission(method, PanelPermission.class, PanelPermission.ACTION_EDIT);
}
}
public void initPanelSession(PanelSession status, HttpSession session) {
status.setCurrentPageId(PAGE_SHOW);
}
protected void beforePanelInstanceRemove(final PanelInstance instance) throws Exception {
super.beforePanelInstanceRemove(instance);
new HibernateTxFragment() {
protected void txFragment(Session session) throws Exception {
HTMLText htmlText = load(instance);
if (htmlText != null) htmlText.delete();
}
}.execute();
}
/** Returns if this driver defines support to activate edit mode. */
public boolean supportsEditMode(Panel panel) {
return true;
}
public int getEditWidth(Panel panel, CommandRequest request) {
return 1000;
}
public int getEditHeight(Panel panel, CommandRequest request) {
return 695;
}
/** Defines the action to be taken when activating edit mode */
public void activateEditMode(Panel panel, CommandRequest request) throws Exception {
super.activateEditMode(panel, request);
HTMLText text = load(panel.getInstance());
SessionManager.getPanelSession(panel).setAttribute(ATTR_TEXT, toEditableObject(text));
}
protected Map toEditableObject(HTMLText text) {
Map m = new HashMap();
for (Iterator it = text.getText().keySet().iterator(); it.hasNext(); ) {
String lang = (String) it.next();
String val = text.getText(lang);
m.put(lang, val);
}
return m;
}
/** Defines the action to be taken when activating edit mode */
public void activateNormalMode(Panel panel, CommandRequest request) throws Exception {
super.activateNormalMode(panel, request);
SessionManager.getPanelSession(panel).removeAttribute(ATTR_TEXT);
SessionManager.getPanelSession(panel).removeAttribute(ATTR_EDITING_LANGUAGE);
}
/**
* Returns if this driver is using default language
*
* @param panel
* @return if this driver is using default language
*/
public boolean isUsingDefaultLanguage(Panel panel) {
return Boolean.valueOf(panel.getParameterValue(PARAMETER_USE_DEFAULTS)).booleanValue();
}
/**
* Returns if this driver is using default language
*
* @param panel
* @return if this driver is using default language
*/
public boolean isUsingDefaultLanguage(PanelInstance panel) {
return Boolean.valueOf(panel.getParameterValue(PARAMETER_USE_DEFAULTS)).booleanValue();
}
/**
* Determine the text being shown for given panel.
*
* @param panel
* @return The text shown, i18n.
*/
public Map getHtmlCode(Panel panel) {
PanelSession pSession = SessionManager.getPanelSession(panel);
Map m = (Map) pSession.getAttribute(ATTR_TEXT);
if (m != null) return m;
HTMLText text = load(panel.getInstance());
if (text != null) return text.getText();
try {
HTMLText textToCreate = new HTMLText();
textToCreate.setPanelInstance(panel.getInstance());
Locale[] locales = LocaleManager.lookup().getPlatformAvailableLocales();
for (int i = 0; i < locales.length; i++) {
Locale locale = locales[i];
ResourceBundle i18n =
localeManager.getBundle("org.jboss.dashboard.ui.panel.advancedHTML.messages", locale);
textToCreate.setText(locale.getLanguage(), i18n.getString("defaultContent"));
}
textToCreate.save();
} catch (Exception e) {
log.error("Error creating empty text for panel: ", e);
}
text = load(panel.getInstance());
if (text != null) return text.getText();
log.error("Current HTML code is null for panel " + panel);
return null;
}
/**
* Determine the editing language.
*
* @return The text shown, i18n.
*/
public String getEditingLanguage(Panel panel) {
String lang =
(String) SessionManager.getPanelSession(panel).getAttribute(ATTR_EDITING_LANGUAGE);
return lang == null ? LocaleManager.lookup().getDefaultLang() : lang;
}
public CommandResponse actionChangeEditingLanguage(Panel panel, CommandRequest request)
throws Exception {
String currentText = request.getRequestObject().getParameter(PARAMETER_HTML);
Map text = (Map) SessionManager.getPanelSession(panel).getAttribute(ATTR_TEXT);
if (text == null) {
text = toEditableObject(load(panel.getInstance()));
SessionManager.getPanelSession(panel).setAttribute(ATTR_TEXT, text);
}
text.put(getEditingLanguage(panel), currentText);
SessionManager.getPanelSession(panel).setAttribute(ATTR_TEXT, text);
String paramLang = request.getRequestObject().getParameter(PARAMETER_EDITING_LANG);
SessionManager.getPanelSession(panel).setAttribute(ATTR_EDITING_LANGUAGE, paramLang);
return new ShowPanelPage();
}
public CommandResponse actionSaveChanges(Panel panel, CommandRequest request) throws Exception {
String currentText = request.getRequestObject().getParameter(PARAMETER_HTML);
Map m = (Map) SessionManager.getPanelSession(panel).getAttribute(ATTR_TEXT);
HTMLText text = load(panel.getInstance());
for (Iterator it = m.keySet().iterator(); it.hasNext(); ) {
String lang = (String) it.next();
String val = (String) m.get(lang);
text.setText(lang, val);
}
text.setText(getEditingLanguage(panel), currentText);
text.save();
activateNormalMode(panel, request);
return new ShowPanelPage();
}
public CommandResponse actionSelectImage(Panel panel, CommandRequest request) throws Exception {
return new ShowPopupPanelPage(panel, PAGE_CHOOSE_IMAGE);
}
public CommandResponse actionSelectLink(Panel panel, CommandRequest request) throws Exception {
return new ShowPopupPanelPage(panel, PAGE_CHOOSE_LINK);
}
public HTMLText load(final PanelInstance instance) {
final List results = new ArrayList();
try {
new HibernateTxFragment() {
protected void txFragment(Session session) throws Exception {
FlushMode oldFlushMode = session.getFlushMode();
session.setFlushMode(FlushMode.NEVER);
Query query =
session.createQuery(
" from "
+ HTMLText.class.getName()
+ " as text where text.panelInstance = :instance");
query.setParameter("instance", instance);
query.setCacheable(true);
results.addAll(query.list());
session.setFlushMode(oldFlushMode);
}
}.execute();
HTMLText text = null;
if (results.size() > 0) text = (HTMLText) results.get(0);
else log.debug("Does not exist a html_text for HTML panel");
return text;
} catch (Exception e) {
log.error("Can't retrive a data for HTML panel ", e);
return null;
}
}
/**
* Replicates panel data.
*
* @param src Source PanelInstance
* @param dest Destinaton PanelInstance
*/
public void replicateData(final PanelInstance src, PanelInstance dest) throws Exception {
super.replicateData(src, dest);
log.debug(
"HTMLDriver replicating Data from PanelInstance "
+ src.getDbid()
+ " to "
+ dest.getDbid()
+ ").");
if (src.equals(dest)) {
log.debug("Ignoring replication, panel instance is the same.");
return;
}
final HTMLText[] textArray = new HTMLText[1];
try {
new HibernateTxFragment() {
protected void txFragment(Session session) throws Exception {
log.debug("Getting text to duplicate for instance " + src.getDbid());
FlushMode oldMode = session.getFlushMode();
session.setFlushMode(
FlushMode
.COMMIT); // Avoids flushing, as we know the text was not modified in this
// transaction.
textArray[0] = load(src);
session.setFlushMode(oldMode);
log.debug("Got text to duplicate for instance " + src.getDbid());
}
}.execute();
} catch (Exception e) {
log.error("Error loading text for instance. ", e);
}
HTMLText text = textArray[0];
if (text == null) {
log.debug(
"Nothing to replicate from PanelInstance " + src.getDbid() + " to " + dest.getDbid());
return;
}
Map htmlSrc = text.getText();
log.debug("htmlCode to replicate = " + htmlSrc);
HTMLText htmlDest = new HTMLText();
htmlDest.setPanelInstance(dest.getInstance());
for (Iterator it = htmlSrc.keySet().iterator(); it.hasNext(); ) {
String key = (String) it.next();
String val = (String) htmlSrc.get(key);
htmlDest.setText(key, val);
}
try {
log.debug("Updating HTMLText: IDText " + htmlDest.getDbid() + " Text " + htmlDest.getText());
htmlDest.save();
} catch (Exception e) {
log.error("Replicating panel data", e);
}
}
/** Write instance content to given OutputStream, which must not be closed. */
public void exportContent(PanelInstance instance, OutputStream os) throws Exception {
HTMLText text = load(instance);
if (text == null) {
try {
text = new HTMLText();
text.setPanelInstance(instance);
text.save();
} catch (Exception e) {
log.error("Error creating empty HTMLText object", e);
}
}
ObjectOutputStream oos = new ObjectOutputStream(os);
if (log.isDebugEnabled()) log.debug("Exporting content: " + text.getText());
HashMap h = new HashMap(); // Avoids serializing a hibernate map
h.putAll(text.getText());
oos.writeObject(h);
}
/** Read instance content from given InputStream, which must not be closed. */
public void importContent(PanelInstance instance, InputStream is) throws Exception {
HTMLText currentText = new HTMLText();
currentText.setPanelInstance(instance);
ObjectInputStream ois = new ObjectInputStream(is);
Map text = (Map) ois.readObject();
if (log.isDebugEnabled()) log.debug("Importing content: " + text);
for (Iterator it = text.keySet().iterator(); it.hasNext(); ) {
String lang = (String) it.next();
String value = (String) text.get(lang);
currentText.setText(lang, value);
}
currentText.save();
}
protected String getPanelHTMLContent(PanelInstance instance, String lang) {
HTMLText text = load(instance);
if (text != null) {
String val = text.getText(lang);
if (StringUtils.isEmpty(val) && isUsingDefaultLanguage(instance)) {
LocaleManager localeManager = LocaleManager.lookup();
val = text.getText(localeManager.getDefaultLang());
}
return val;
}
return null;
}
}
/**
* Choice list initializer that looks up direct sub types from the specified type If no param is
* specified, get the list of nodeTypes
*/
public class SubNodeTypesChoiceListInitializerImpl implements ChoiceListInitializer {
private static transient Logger logger =
org.slf4j.LoggerFactory.getLogger(SubNodeTypesChoiceListInitializerImpl.class);
@SuppressWarnings("unchecked")
public List<ChoiceListValue> getChoiceListValues(
ExtendedPropertyDefinition epd,
String param,
List<ChoiceListValue> values,
Locale locale,
Map<String, Object> context) {
final SortedSet<ChoiceListValue> listValues = new TreeSet<ChoiceListValue>();
if (StringUtils.isEmpty(param)) {
param = "jmix:editorialContent";
}
try {
String includedTypes = StringUtils.substringBefore(param, ";");
Set<String> excludedTypes = new HashSet<String>();
String exclusion = StringUtils.substringAfter(param, ";");
if (StringUtils.isNotBlank(exclusion)) {
excludedTypes.addAll(
CollectionUtils.collect(
Arrays.asList(StringUtils.substringAfter(param, ";").split(",")),
new Transformer() {
public Object transform(Object input) {
return ((String) input).trim();
}
}));
}
for (String nodeTypeName : includedTypes.split(",")) {
nodeTypeName = nodeTypeName.trim();
ExtendedNodeType nodeType = NodeTypeRegistry.getInstance().getNodeType(nodeTypeName);
if (!isExcludedType(nodeType, excludedTypes)) {
listValues.add(new ChoiceListValue(nodeType.getLabel(locale), nodeType.getName()));
}
NodeTypeIterator nti = nodeType.getSubtypes();
while (nti.hasNext()) {
ExtendedNodeType type = (ExtendedNodeType) nti.next();
if (!isExcludedType(type, excludedTypes)) {
listValues.add(new ChoiceListValue(type.getLabel(locale), type.getName()));
}
}
}
} catch (NoSuchNodeTypeException e) {
logger.error("Cannot get type", e);
}
return new LinkedList<ChoiceListValue>(listValues);
}
private boolean isExcludedType(ExtendedNodeType nodeType, Set<String> excludedTypes) {
if (excludedTypes.contains(nodeType.getName())) {
return true;
}
boolean isExcluded = false;
for (String excludedType : excludedTypes) {
if (nodeType.isNodeType(excludedType)) {
isExcluded = true;
break;
}
}
return isExcluded;
}
}
/**
* _more_
*
* @author edavis
* @since 4.0
*/
public class WcsCoverage {
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(WcsCoverage.class);
private GridDataset.Gridset coverage;
private WcsDataset dataset;
private String name;
private String label;
private String description;
private GridCoordSystem coordSys;
private String nativeCRS;
private String defaultRequestCrs;
private List<WcsRequest.Format> supportedCoverageFormatList;
private HashMap<String, WcsRangeField> range;
public WcsCoverage(GridDataset.Gridset coverage, WcsDataset dataset) {
this.dataset = dataset;
if (this.dataset == null) {
log.error("WcsCoverage(): non-null dataset required.");
throw new IllegalArgumentException("Non-null dataset required.");
}
this.coverage = coverage;
if (this.coverage == null) {
log.error("WcsCoverage(): non-null coverage required.");
throw new IllegalArgumentException("Non-null coverage required.");
}
this.coordSys = coverage.getGeoCoordSystem();
if (this.coordSys == null) {
log.error("WcsCoverage(): Coverage must have non-null coordinate system.");
throw new IllegalArgumentException("Non-null coordinate system required.");
}
this.name = this.coordSys.getName();
this.label = this.coordSys.getName();
this.range = new HashMap<String, WcsRangeField>();
StringBuilder descripSB =
new StringBuilder("All parameters on the \"")
.append(this.name)
.append("\" coordinate system: ");
for (GridDatatype curField : this.coverage.getGrids()) {
String stdName = curField.findAttValueIgnoreCase("standard_name", "");
descripSB.append(stdName.length() == 0 ? curField.getFullName() : stdName).append(",");
WcsRangeField field = new WcsRangeField(curField);
range.put(field.getName(), field);
}
descripSB.setCharAt(descripSB.length() - 1, '.');
this.description = descripSB.toString();
this.nativeCRS = EPSG_OGC_CF_Helper.getWcs1_0CrsId(this.coordSys.getProjection());
this.defaultRequestCrs = "OGC:CRS84";
this.supportedCoverageFormatList = new ArrayList<WcsRequest.Format>();
// this.supportedCoverageFormatList = "application/x-netcdf";
this.supportedCoverageFormatList.add(WcsRequest.Format.GeoTIFF);
this.supportedCoverageFormatList.add(WcsRequest.Format.GeoTIFF_Float);
this.supportedCoverageFormatList.add(WcsRequest.Format.NetCDF3);
}
GridDataset.Gridset getGridset() {
return coverage;
}
public String getName() {
return this.name;
}
public String getLabel() {
return this.label;
}
public String getDescription() {
return this.description;
}
public GridCoordSystem getCoordinateSystem() {
return coordSys;
}
public String getDefaultRequestCrs() {
return defaultRequestCrs;
}
public String getNativeCrs() {
return nativeCRS;
}
public List<WcsRequest.Format> getSupportedCoverageFormatList() {
return Collections.unmodifiableList(supportedCoverageFormatList);
}
public boolean isSupportedCoverageFormat(WcsRequest.Format covFormat) {
return this.supportedCoverageFormatList.contains(covFormat);
}
public boolean isRangeFieldName(String fieldName) {
return range.containsKey(fieldName);
}
public Set<String> getRangeFieldNames() {
return range.keySet();
}
public Collection<WcsRangeField> getRange() {
return range.values();
}
private static DiskCache2 diskCache = null;
public static void setDiskCache(DiskCache2 _diskCache) {
diskCache = _diskCache;
}
private static DiskCache2 getDiskCache() {
if (diskCache == null) {
log.error("getDiskCache(): Disk cache has not been set.");
throw new IllegalStateException(
"Disk cache must be set before calling GetCoverage.getDiskCache().");
}
return diskCache;
}
public File writeCoverageDataToFile(
WcsRequest.Format format,
LatLonRect bboxLatLonRect,
AxisSubset vertSubset,
List<String> rangeSubset,
CalendarDateRange timeRange)
throws WcsException {
boolean zRangeDone = false;
boolean tRangeDone = false;
try {
// Get the height range.
Range zRange = vertSubset != null ? vertSubset.getRange() : null;
zRangeDone = true;
// Get the time range.
Range tRange = null;
if (timeRange != null) {
CoordinateAxis1DTime timeAxis = this.coordSys.getTimeAxis1D();
int startIndex = timeAxis.findTimeIndexFromCalendarDate(timeRange.getStart());
int endIndex = timeAxis.findTimeIndexFromCalendarDate(timeRange.getEnd());
tRange = new Range(startIndex, endIndex);
tRangeDone = true;
}
if (format == WcsRequest.Format.GeoTIFF || format == WcsRequest.Format.GeoTIFF_Float) {
if (rangeSubset.size() != 1) {
String msg =
"GeoTIFF response encoding only available for single range field selection ["
+ rangeSubset
+ "].";
log.error("writeCoverageDataToFile(): " + msg);
throw new WcsException(WcsException.Code.InvalidParameterValue, "RangeSubset", msg);
}
String reqRangeFieldName = rangeSubset.get(0);
File dir = new File(getDiskCache().getRootDirectory());
File tifFile = File.createTempFile("WCS", ".tif", dir);
if (log.isDebugEnabled())
log.debug("writeCoverageDataToFile(): tifFile=" + tifFile.getPath());
WcsRangeField rangeField = this.range.get(reqRangeFieldName);
GridDatatype subset =
rangeField.getGridDatatype().makeSubset(tRange, zRange, bboxLatLonRect, 1, 1, 1);
Array data = subset.readDataSlice(0, 0, -1, -1);
GeotiffWriter writer = new GeotiffWriter(tifFile.getPath());
writer.writeGrid(
this.dataset.getDataset(), subset, data, format == WcsRequest.Format.GeoTIFF);
writer.close();
return tifFile;
} else if (format == WcsRequest.Format.NetCDF3) {
File dir = new File(getDiskCache().getRootDirectory());
File outFile = File.createTempFile("WCS", ".nc", dir);
if (log.isDebugEnabled())
log.debug("writeCoverageDataToFile(): ncFile=" + outFile.getPath());
// WTF ?? this.coordSys.getVerticalAxis().isNumeric();
NetcdfFileWriter writer =
NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outFile.getAbsolutePath());
CFGridWriter2.writeFile(
this.dataset.getDataset(),
rangeSubset,
bboxLatLonRect,
null,
1,
zRange,
timeRange,
1,
true,
writer);
return outFile;
} else {
log.error(
"writeCoverageDataToFile(): Unsupported response encoding format [" + format + "].");
throw new WcsException(
WcsException.Code.InvalidFormat,
"Format",
"Unsupported response encoding format [" + format + "].");
}
} catch (InvalidRangeException e) {
String msg = "Failed to subset coverage [" + this.getName();
if (!zRangeDone) msg += "] along vertical axis [" + vertSubset + "]. ";
else if (!tRangeDone) msg += "] along time axis [" + timeRange + "]. ";
else msg += "] in horizontal plane [" + bboxLatLonRect + "]. ";
log.error("writeCoverageDataToFile(): " + msg + e.getMessage());
throw new WcsException(WcsException.Code.CoverageNotDefined, "", msg);
} catch (IOException e) {
log.error(
"writeCoverageDataToFile(): Failed to write file for requested coverage <"
+ this.getName()
+ ">: "
+ e.getMessage());
throw new WcsException(
WcsException.Code.UNKNOWN, "", "Problem creating coverage [" + this.getName() + "].");
}
}
}
/**
* THREDDS opendap server.
*
* @author jcaron
* @author Nathan David Potter
* @since Apr 27, 2009 (branched)
*/
public class OpendapServlet extends AbstractServlet {
static final String DEFAULTCONTEXTPATH = "/thredds";
static final String GDATASET = "guarded_dataset";
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(OpendapServlet.class);
private boolean allowSessions = false;
private boolean allowDeflate = false; // handled by Tomcat
private String odapVersionString = "opendap/3.7";
private URI baseURI = null;
private int ascLimit = 50;
private int binLimit = 500;
private boolean debugSession = false;
public String getDefaultContextPath() {
return DEFAULTCONTEXTPATH;
}
public void init() throws javax.servlet.ServletException {
super.init();
org.slf4j.Logger logServerStartup = org.slf4j.LoggerFactory.getLogger("serverStartup");
logServerStartup.info(
getClass().getName() + " initialization start - " + UsageLog.setupNonRequestContext());
this.ascLimit = ThreddsConfig.getInt("Opendap.ascLimit", ascLimit);
this.binLimit = ThreddsConfig.getInt("Opendap.binLimit", binLimit);
this.odapVersionString = ThreddsConfig.get("Opendap.serverVersion", odapVersionString);
logServerStartup.info(
getClass().getName()
+ " version= "
+ odapVersionString
+ " ascLimit = "
+ ascLimit
+ " binLimit = "
+ binLimit);
// debugging actions
makeDebugActions();
logServerStartup.info(
getClass().getName()
+ " initialization done - "
+ UsageLog.closingMessageNonRequestContext());
}
public String getServerVersion() {
return this.odapVersionString;
}
// Servlets that support HTTP GET requests and can quickly determine their last modification time
// should
// override this method. This makes browser and proxy caches work more effectively, reducing the
// load on
// server and network resources.
protected long getLastModified(HttpServletRequest req) {
String query = req.getQueryString();
if (query != null) return -1;
String path = req.getPathInfo();
if (path == null) return -1;
if (path.endsWith(".asc")) path = path.substring(0, path.length() - 4);
else if (path.endsWith(".ascii")) path = path.substring(0, path.length() - 6);
else if (path.endsWith(".das")) path = path.substring(0, path.length() - 4);
else if (path.endsWith(".dds")) path = path.substring(0, path.length() - 4);
else if (path.endsWith(".ddx")) path = path.substring(0, path.length() - 4);
else if (path.endsWith(".dods")) path = path.substring(0, path.length() - 5);
else if (path.endsWith(".html")) path = path.substring(0, path.length() - 5);
else if (path.endsWith(".info")) path = path.substring(0, path.length() - 5);
else if (path.endsWith(".opendap")) path = path.substring(0, path.length() - 5);
else return -1;
// if (null != DatasetHandler.findResourceControl( path)) return -1; // LOOK weird Firefox
// beahviour?
File file = DataRootHandler.getInstance().getCrawlableDatasetAsFile(path);
if ((file != null) && file.exists()) return file.lastModified();
return -1;
}
/////////////////////////////////////////////////////////////////////////////
public void doGet(HttpServletRequest request, HttpServletResponse response) {
log.info("doGet(): " + UsageLog.setupRequestContext(request));
// System.out.printf("opendap doGet: req=%s%n%s%n", ServletUtil.getRequest(request),
// ServletUtil.showRequestDetail(this, request));
String path = null;
ReqState rs = getRequestState(request, response);
try {
path = request.getPathInfo();
log.debug("doGet path={}", path);
if (thredds.servlet.Debug.isSet("showRequestDetail"))
log.debug(ServletUtil.showRequestDetail(this, request));
if (path == null) {
log.info(
"doGet(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
response.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (baseURI == null) { // first time, set baseURI
URI reqURI = ServletUtil.getRequestURI(request);
// Build base URI from request (rather than hard-coding "/thredds/dodsC/").
String baseUriString = request.getContextPath() + request.getServletPath() + "/";
baseURI = reqURI.resolve(baseUriString);
log.debug("doGet(): baseURI was set = {}", baseURI);
}
if (path.endsWith("latest.xml")) {
DataRootHandler.getInstance().processReqForLatestDataset(this, request, response);
return;
}
// Redirect all catalog requests at the root level.
if (path.equals("/") || path.equals("/catalog.html") || path.equals("/catalog.xml")) {
ServletUtil.sendPermanentRedirect(ServletUtil.getContextPath() + path, request, response);
return;
}
// Make sure catalog requests match a dataRoot before trying to handle.
if (path.endsWith("/") || path.endsWith("/catalog.html") || path.endsWith("/catalog.xml")) {
if (!DataRootHandler.getInstance().hasDataRootMatch(path)) {
log.info(
"doGet(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
response.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
if (!DataRootHandler.getInstance().processReqForCatalog(request, response))
log.error(
"doGet(): "
+ UsageLog.closingMessageForRequestContext(
ServletUtil.STATUS_FORWARD_FAILURE, -1));
return;
}
if (rs != null) {
String dataSet = rs.getDataSet();
String requestSuffix = rs.getRequestSuffix();
if ((dataSet == null) || dataSet.equals("/") || dataSet.equals("")) {
doGetDIR(rs);
} else if (requestSuffix.equalsIgnoreCase("blob")) {
doGetBLOB(rs);
} else if (requestSuffix.equalsIgnoreCase("close")) {
doClose(rs);
} else if (requestSuffix.equalsIgnoreCase("dds")) {
doGetDDS(rs);
} else if (requestSuffix.equalsIgnoreCase("das")) {
doGetDAS(rs);
} else if (requestSuffix.equalsIgnoreCase("ddx")) {
doGetDDX(rs);
} else if (requestSuffix.equalsIgnoreCase("dods")) {
doGetDAP2Data(rs);
} else if (requestSuffix.equalsIgnoreCase("asc")
|| requestSuffix.equalsIgnoreCase("ascii")) {
doGetASC(rs);
} else if (requestSuffix.equalsIgnoreCase("info")) {
doGetINFO(rs);
} else if (requestSuffix.equalsIgnoreCase("html")
|| requestSuffix.equalsIgnoreCase("htm")) {
doGetHTML(rs);
} else if (requestSuffix.equalsIgnoreCase("ver")
|| requestSuffix.equalsIgnoreCase("version")
|| dataSet.equalsIgnoreCase("/version")
|| dataSet.equalsIgnoreCase("/version/")) {
doGetVER(rs);
} else if (dataSet.equalsIgnoreCase("/help")
|| dataSet.equalsIgnoreCase("/help/")
|| dataSet.equalsIgnoreCase("/" + requestSuffix)
|| requestSuffix.equalsIgnoreCase("help")) {
doGetHELP(rs);
} else {
sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
return;
}
} else {
sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
return;
}
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, -1));
// plain ol' 404
} catch (FileNotFoundException e) {
sendErrorResponse(response, HttpServletResponse.SC_NOT_FOUND, e.getMessage());
// DAP2Exception bad url
} catch (BadURLException e) {
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
dap2ExceptionHandler(e, rs);
// all other DAP2Exception
} catch (DAP2Exception de) {
int status =
(de.getErrorCode() == DAP2Exception.NO_SUCH_FILE)
? HttpServletResponse.SC_NOT_FOUND
: HttpServletResponse.SC_BAD_REQUEST;
if ((de.getErrorCode() != DAP2Exception.NO_SUCH_FILE) && (de.getErrorMessage() != null))
log.debug(de.getErrorMessage());
log.info(UsageLog.closingMessageForRequestContext(status, -1));
response.setStatus(status);
dap2ExceptionHandler(de, rs);
// parsing, usually the CE
} catch (ParseException pe) {
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
parseExceptionHandler(pe, response);
// 403 - request too big
} catch (UnsupportedOperationException e) {
sendErrorResponse(response, HttpServletResponse.SC_FORBIDDEN, e.getMessage());
} catch (java.net.SocketException e) {
log.info("SocketException: " + e.getMessage(), e);
log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));
} catch (IOException e) {
String eName =
e.getClass().getName(); // dont want compile time dependency on ClientAbortException
if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
log.debug("ClientAbortException: " + e.getMessage());
log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));
return;
}
log.error("path= " + path, e);
sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());
// everything else
} catch (Throwable t) {
log.error("path= " + path, t);
t.printStackTrace();
sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, t.getMessage());
}
}
public void doGetASC(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (ds == null) return;
response.setHeader("XDODS-Server", getServerVersion());
response.setContentType("text/plain");
response.setHeader("Content-Description", "dods-ascii");
log.debug(
"Sending OPeNDAP ASCII Data For: " + rs + " CE: '" + rs.getConstraintExpression() + "'");
ServerDDS dds = ds.getDDS();
CEEvaluator ce = new CEEvaluator(dds);
ce.parseConstraint(rs);
checkSize(dds, true);
PrintWriter pw = new PrintWriter(response.getOutputStream());
dds.printConstrained(pw);
pw.println("---------------------------------------------");
AsciiWriter writer = new AsciiWriter(); // could be static
writer.toASCII(pw, dds, ds);
// the way that getDAP2Data works
// DataOutputStream sink = new DataOutputStream(bOut);
// ce.send(myDDS.getName(), sink, ds);
pw.flush();
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGetDAS(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (ds == null) return;
response.setContentType("text/plain");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-das");
OutputStream Out = new BufferedOutputStream(response.getOutputStream());
DAS myDAS = ds.getDAS();
myDAS.print(Out);
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGetDDS(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (null == ds) return;
response.setContentType("text/plain");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-dds");
OutputStream out = new BufferedOutputStream(response.getOutputStream());
ServerDDS myDDS = ds.getDDS();
if (rs.getConstraintExpression().equals("")) { // No Constraint Expression?
// Send the whole DDS
myDDS.print(out);
out.flush();
} else { // Otherwise, send the constrained DDS
// Instantiate the CEEvaluator and parse the constraint expression
CEEvaluator ce = new CEEvaluator(myDDS);
ce.parseConstraint(rs);
// Send the constrained DDS back to the client
PrintWriter pw = new PrintWriter(new OutputStreamWriter(out));
myDDS.printConstrained(pw);
pw.flush();
}
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGetDDX(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (null == ds) return;
response.setContentType("text/plain");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-ddx");
OutputStream out = new BufferedOutputStream(response.getOutputStream());
ServerDDS myDDS = ds.getDDS();
myDDS.ingestDAS(ds.getDAS());
if (rs.getConstraintExpression().equals("")) { // No Constraint Expression?
// Send the whole DDS
myDDS.printXML(out);
out.flush();
} else { // Otherwise, send the constrained DDS
// Instantiate the CEEvaluator and parse the constraint expression
CEEvaluator ce = new CEEvaluator(myDDS);
ce.parseConstraint(rs);
// Send the constrained DDS back to the client
PrintWriter pw = new PrintWriter(new OutputStreamWriter(out));
myDDS.printConstrainedXML(pw);
pw.flush();
}
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGetBLOB(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (null == ds) return;
response.setContentType("application/octet-stream");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-blob");
ServletOutputStream sOut = response.getOutputStream();
OutputStream bOut;
DeflaterOutputStream dOut = null;
if (rs.getAcceptsCompressed() && allowDeflate) {
response.setHeader("Content-Encoding", "deflate");
dOut = new DeflaterOutputStream(sOut);
bOut = new BufferedOutputStream(dOut);
} else {
bOut = new BufferedOutputStream(sOut);
}
ServerDDS myDDS = ds.getDDS();
CEEvaluator ce = new CEEvaluator(myDDS);
ce.parseConstraint(rs);
checkSize(myDDS, false);
// Send the binary data back to the client
DataOutputStream sink = new DataOutputStream(bOut);
ce.send(myDDS.getEncodedName(), sink, ds);
sink.flush();
// Finish up sending the compressed stuff, but don't
// close the stream (who knows what the Servlet may expect!)
if (null != dOut) dOut.finish();
bOut.flush();
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
private void doClose(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
HttpServletRequest request = rs.getRequest();
String reqPath = rs.getDataSet();
HttpSession session = request.getSession();
session.removeAttribute(reqPath); // work done in the listener
response.setHeader("XDODS-Server", getServerVersion()); // needed by client
/* if (path.endsWith(".close")) {
closeSession(request, response);
response.setContentLength(0);
log.info("doGet(): " + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, 0));
return;
}
// so we need to worry about deleting sessions?
session.invalidate(); */
}
public void doGetDAP2Data(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (null == ds) return;
response.setContentType("application/octet-stream");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-data");
ServletOutputStream sOut = response.getOutputStream();
OutputStream bOut;
DeflaterOutputStream dOut = null;
if (rs.getAcceptsCompressed() && allowDeflate) {
response.setHeader("Content-Encoding", "deflate");
dOut = new DeflaterOutputStream(sOut);
bOut = new BufferedOutputStream(dOut);
} else {
bOut = new BufferedOutputStream(sOut);
}
ServerDDS myDDS = ds.getDDS();
CEEvaluator ce = new CEEvaluator(myDDS);
ce.parseConstraint(rs);
checkSize(myDDS, false);
// Send the constrained DDS back to the client
PrintWriter pw = new PrintWriter(new OutputStreamWriter(bOut));
myDDS.printConstrained(pw);
// Send the Data delimiter back to the client
pw.flush();
bOut.write("\nData:\n".getBytes());
bOut.flush();
// Send the binary data back to the client
DataOutputStream sink = new DataOutputStream(bOut);
ce.send(myDDS.getEncodedName(), sink, ds);
sink.flush();
// Finish up sending the compressed stuff, but don't
// close the stream (who knows what the Servlet may expect!)
if (null != dOut) dOut.finish();
bOut.flush();
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGetVER(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
response.setContentType("text/plain");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-version");
PrintWriter pw = new PrintWriter(new OutputStreamWriter(response.getOutputStream()));
pw.println("Server Version: " + getServerVersion());
pw.flush();
}
public void doGetHELP(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
response.setContentType("text/html");
response.setHeader("XDODS-Server", getServerVersion());
response.setHeader("Content-Description", "dods-help");
PrintWriter pw = new PrintWriter(new OutputStreamWriter(response.getOutputStream()));
printHelpPage(pw);
pw.flush();
}
public void doGetDIR(ReqState rs) throws Exception {
// rather dangerous here, since you can go into an infinite loop
// so we're going to insist that there's no suffix
HttpServletResponse response = rs.getResponse();
HttpServletRequest request = rs.getRequest();
if ((rs.getRequestSuffix() == null) || (rs.getRequestSuffix().length() == 0)) {
ServletUtil.forwardToCatalogServices(request, response);
return;
}
sendErrorResponse(response, 0, "Unrecognized request");
}
public void doGetINFO(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (null == ds) return;
PrintStream pw = new PrintStream(response.getOutputStream());
response.setHeader("XDODS-Server", getServerVersion());
response.setContentType("text/html");
response.setHeader("Content-Description", "dods-description");
GetInfoHandler di = new GetInfoHandler();
di.sendINFO(pw, ds, rs);
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
public void doGetHTML(ReqState rs) throws Exception {
HttpServletResponse response = rs.getResponse();
HttpServletRequest request = rs.getRequest();
GuardedDataset ds = null;
try {
ds = getDataset(rs);
if (ds == null) return;
response.setHeader("XDODS-Server", getServerVersion());
response.setContentType("text/html");
response.setHeader("Content-Description", "dods-form");
// Utilize the getDDS() method to get a parsed and populated DDS
// for this server.
ServerDDS myDDS = ds.getDDS();
DAS das = ds.getDAS();
GetHTMLInterfaceHandler2 di = new GetHTMLInterfaceHandler2();
di.sendDataRequestForm(request, response, rs.getDataSet(), myDDS, das);
} finally { // release lock if needed
if (ds != null) ds.release();
}
}
///////////////////////////////////////////////////////////////////////////////////////////////
// debugging
private void makeDebugActions() {
DebugHandler debugHandler = DebugHandler.get("ncdodsServer");
DebugHandler.Action act;
act =
new DebugHandler.Action("help", "Show help page") {
public void doAction(DebugHandler.Event e) {
try {
doGetHELP(getRequestState(e.req, e.res));
} catch (Exception ioe) {
log.error("ShowHelp", ioe);
}
}
};
debugHandler.addAction(act);
act =
new DebugHandler.Action("version", "Show server version") {
public void doAction(DebugHandler.Event e) {
e.pw.println(" version= " + getServerVersion());
}
};
debugHandler.addAction(act);
}
public String getServerName() {
return this.getClass().getName();
}
/*protected ReqState getRequestState(HttpServletRequest request, HttpServletResponse response) {
ReqState rs = null;
// The url and query strings will come to us in encoded form
// (see HTTPmethod.newMethod())
String baseurl = request.getRequestURL().toString();
baseurl = EscapeStrings.escapeURL(baseurl);
log.debug("doGet baseurl={}", baseurl);
String query = request.getQueryString();
query = EscapeStrings.unescapeURLQuery(query);
log.debug("doGet query={}", query);
try {
rs = new ReqState(request, response, getServletConfig(), getServerName(), baseurl, query);
} catch (BadURLException bue) {
rs = null;
}
return rs;
}*/
/**
* ************************************************************************ Prints the OPeNDAP
* Server help page to the passed PrintWriter
*
* @param pw PrintWriter stream to which to dump the help page.
*/
private void printHelpPage(PrintWriter pw) {
pw.println("<h3>OPeNDAP Server Help</h3>");
pw.println("To access most of the features of this OPeNDAP server, append");
pw.println(
"one of the following a eight suffixes to a URL: .das, .dds, .dods, .ddx, .blob, .info,");
pw.println(".ver or .help. Using these suffixes, you can ask this server for:");
pw.println("<dl>");
pw.println("<dt> das </dt> <dd> Dataset Attribute Structure (DAS)</dd>");
pw.println("<dt> dds </dt> <dd> Dataset Descriptor Structure (DDS)</dd>");
pw.println("<dt> dods </dt> <dd> DataDDS object (A constrained DDS populated with data)</dd>");
pw.println("<dt> ddx </dt> <dd> XML version of the DDS/DAS</dd>");
pw.println(
"<dt> blob </dt> <dd> Serialized binary data content for requested data set, "
+ "with the constraint expression applied.</dd>");
pw.println("<dt> info </dt> <dd> info object (attributes, types and other information)</dd>");
pw.println("<dt> html </dt> <dd> html form for this dataset</dd>");
pw.println("<dt> ver </dt> <dd> return the version number of the server</dd>");
pw.println("<dt> help </dt> <dd> help information (this text)</dd>");
pw.println("</dl>");
pw.println("For example, to request the DAS object from the FNOC1 dataset at URI/GSO (a");
pw.println("test dataset) you would appand `.das' to the URL:");
pw.println("http://opendap.gso.url.edu/cgi-bin/nph-nc/data/fnoc1.nc.das.");
pw.println("<p><b>Note</b>: Many OPeNDAP clients supply these extensions for you so you don't");
pw.println("need to append them (for example when using interfaces supplied by us or");
pw.println("software re-linked with a OPeNDAP client-library). Generally, you only need to");
pw.println("add these if you are typing a URL directly into a WWW browser.");
pw.println("<p><b>Note</b>: If you would like version information for this server but");
pw.println("don't know a specific data file or data set name, use `/version' for the");
pw.println("filename. For example: http://opendap.gso.url.edu/cgi-bin/nph-nc/version will");
pw.println("return the version number for the netCDF server used in the first example. ");
pw.println("<p><b>Suggestion</b>: If you're typing this URL into a WWW browser and");
pw.println("would like information about the dataset, use the `.info' extension.");
pw.println("<p>If you'd like to see a data values, use the `.html' extension and submit a");
pw.println("query using the customized form.");
}
// **************************************************************************
/**
* ************************************************************************ Prints the Bad URL
* Page page to the passed PrintWriter
*
* @param pw PrintWriter stream to which to dump the bad URL page.
*/
private void printBadURLPage(PrintWriter pw) {
String serverContactName = ThreddsConfig.get("serverInformation.contact.name", "UNKNOWN");
String serverContactEmail = ThreddsConfig.get("serverInformation.contact.email", "UNKNOWN");
pw.println("<h3>Error in URL</h3>");
pw.println("The URL extension did not match any that are known by this");
pw.println("server. Below is a list of the five extensions that are be recognized by");
pw.println("all OPeNDAP servers. If you think that the server is broken (that the URL you");
pw.println("submitted should have worked), then please contact the");
pw.println("administrator of this server [" + serverContactName + "] at: ");
pw.println("<a href='mailto:" + serverContactEmail + "'>" + serverContactEmail + "</a><p>");
}
///////////////////////////////////////////////////////
// utils
/**
* @param request
* @param response
* @return the request state
*/
protected ReqState getRequestState(HttpServletRequest request, HttpServletResponse response) {
ReqState rs = null;
// Assume url was encoded
String baseurl = request.getRequestURL().toString();
baseurl = EscapeStrings.unescapeURL(baseurl);
// Assume query was encoded
String query = request.getQueryString();
query = EscapeStrings.unescapeURLQuery(query);
log.info(String.format("OpendapServlet: nominal url: %s?%s", baseurl, query));
try {
rs = new ReqState(request, response, this, getServerName(), baseurl, query);
} catch (Exception bue) {
rs = null;
}
return rs;
}
private void checkSize(ServerDDS dds, boolean isAscii) throws Exception {
long size = computeSize(dds, isAscii);
// System.err.printf("total (constrained) size=%s\n", size);
log.debug("total (constrained) size={}", size);
double dsize = size / (1000 * 1000);
double maxSize = isAscii ? ascLimit : binLimit; // Mbytes
if (dsize > maxSize) {
log.info("Reject request size = {} Mbytes", dsize);
throw new UnsupportedOperationException(
"Request too big=" + dsize + " Mbytes, max=" + maxSize);
}
}
// Recursively compute size of the dds to be returned
private long computeSize(DConstructor ctor, boolean isAscii) throws Exception {
long projectsize = 0; // accumulate size of projected variables
long othersize = 0; // accumulate size of non-projected variables
long fieldsize = 0;
int projectedcount = 0;
int fieldcount = 0;
Enumeration vars = ctor.getVariables();
while (vars.hasMoreElements()) {
fieldcount++;
BaseType field = (BaseType) vars.nextElement();
fieldsize = computeFieldSize(field, isAscii);
// accumulate the field sizes
if (field.isProject()) {
projectsize += fieldsize;
projectedcount++;
} else {
othersize += fieldsize;
}
}
// Cases to consider:
// 1. If all of the fields of this ctor are projected,
// then return projectsize
// 2. If none of the fields of this ctor are projected,
// then return othersize
// 3. otherwise, at least one field, but not all, is projected,
// => return projectsize;
if (projectedcount == fieldcount) return projectsize;
else if (projectedcount == 0) return othersize;
else {
assert (projectedcount > 0 && projectedcount < fieldcount);
return projectsize;
}
}
long computeFieldSize(BaseType bt, boolean isAscii) throws Exception {
long fieldsize = 0;
// Figure out what this field is (e.g. primitive or not)
// Somewhat convoluted.
if (bt instanceof DConstructor) {
// simple struct, seq, or grid => recurse
fieldsize = computeSize((DConstructor) bt, isAscii);
} else if (bt instanceof DArray) {
SDArray da = (SDArray) bt;
// Separate structure arrays from primitive arrays
if (da.getContainerVar() instanceof DPrimitive) {
fieldsize = computeArraySize(da);
} else if (da.getContainerVar() instanceof DStructure) {
fieldsize = computeSize((DStructure) da.getContainerVar(), isAscii); // recurse
} else { // Some kind of problem
throw new NoSuchTypeException("Computesize: unexpected type for " + bt.getLongName());
}
} else if (bt instanceof DPrimitive) {
DPrimitive dp = (DPrimitive) bt;
if (dp instanceof DString) {
String v = ((DString) dp).getValue();
fieldsize = (v == null ? 0 : v.length());
} else {
DataType dtype = DODSNetcdfFile.convertToNCType(bt);
fieldsize = dtype.getSize();
}
} else { // Some kind of problem
throw new NoSuchTypeException("Computesize: unknown type for " + bt.getLongName());
}
return fieldsize;
}
long computeArraySize(SDArray da) throws Exception {
assert (da.getContainerVar() instanceof DPrimitive);
BaseType base = da.getPrimitiveVector().getTemplate();
DataType dtype = DODSNetcdfFile.convertToNCType(base);
int elemSize = dtype.getSize();
int n = da.numDimensions();
List<Range> ranges = new ArrayList<Range>(n);
long size = 0;
for (int i = 0; i < n; i++) {
ranges.add(new Range(da.getStart(i), da.getStop(i), da.getStride(i)));
Section s = new Section(ranges);
size += s.computeSize() * elemSize;
}
return size;
}
/*
* *********************** dataset caching ***********************************************
*/
// any time the server needs access to the dataset, it gets a "GuardedDataset" which allows us to
// add caching
// optionally, a session may be established, which allows us to reserve the dataset for that
// session.
protected GuardedDataset getDataset(ReqState preq) throws Exception {
HttpServletRequest req = preq.getRequest();
String reqPath = preq.getDataSet();
// see if the client wants sessions
boolean acceptSession = false;
String s = req.getHeader("X-Accept-Session");
if (s != null && s.equalsIgnoreCase("true") && allowSessions) acceptSession = true;
HttpSession session = null;
if (acceptSession) {
// see if theres already a session established, create one if not
session = req.getSession();
if (!session.isNew()) {
GuardedDataset gdataset = (GuardedDataset) session.getAttribute(reqPath);
if (null != gdataset) {
if (debugSession)
System.out.printf(" found gdataset %s in session %s %n", reqPath, session.getId());
if (log.isDebugEnabled())
log.debug(" found gdataset " + gdataset + " in session " + session.getId());
return gdataset;
}
}
}
NetcdfFile ncd = DatasetHandler.getNetcdfFile(req, preq.getResponse(), reqPath);
if (null == ncd) return null;
GuardedDataset gdataset = new GuardedDatasetCacheAndClone(reqPath, ncd, acceptSession);
// GuardedDataset gdataset = new GuardedDatasetImpl(reqPath, ncd, acceptSession);
if (acceptSession) {
String cookiePath = req.getRequestURI();
String suffix = "." + preq.getRequestSuffix();
if (cookiePath.endsWith(suffix)) // snip off the suffix
cookiePath = cookiePath.substring(0, cookiePath.length() - suffix.length());
session.setAttribute(reqPath, gdataset);
session.setAttribute(CookieFilter.SESSION_PATH, cookiePath);
// session.setAttribute("dataset", ncd.getLocation()); // for UsageValve
// session.setMaxInactiveInterval(30); // 30 second timeout !!
if (debugSession)
System.out.printf(
" added gdataset %s in session %s cookiePath %s %n",
reqPath, session.getId(), cookiePath);
if (log.isDebugEnabled())
log.debug(" added gdataset " + gdataset + " in session " + session.getId());
} /* else {
session = req.getSession();
session.setAttribute("dataset", ncd.getLocation()); // for UsageValve
} */
return gdataset;
}
//////////////////////////////////////////////////////////////////////////////
public void parseExceptionHandler(ParseException pe, HttpServletResponse response) {
try {
BufferedOutputStream eOut = new BufferedOutputStream(response.getOutputStream());
response.setHeader("Content-Description", "dods-error");
response.setContentType("text/plain");
String msg = pe.getMessage().replace('\"', '\'');
DAP2Exception de2 = new DAP2Exception(opendap.dap.DAP2Exception.CANNOT_READ_FILE, msg);
de2.print(eOut);
} catch (Exception e) {
System.err.println("parseExceptionHandler: " + e);
}
}
public void dap2ExceptionHandler(DAP2Exception de, ReqState rs) {
rs.getResponse().setHeader("Content-Description", "dods-error");
rs.getResponse().setContentType("text/plain");
try {
de.print(rs.getResponse().getOutputStream());
} catch (Exception e) {
System.err.println("dap2ExceptionHandler: " + e);
}
}
private void sendErrorResponse(HttpServletResponse response, int errorCode, String errorMessage) {
try {
log.info(UsageLog.closingMessageForRequestContext(errorCode, -1));
response.setStatus(errorCode);
response.setHeader("Content-Description", "dods-error");
response.setContentType("text/plain");
PrintWriter pw = new PrintWriter(response.getOutputStream());
pw.println("Error {");
pw.println(" code = " + errorCode + ";");
pw.println(" message = \"" + errorMessage + "\";");
pw.println("};");
pw.flush();
} catch (Exception e) {
System.err.println("sendErrorResponse: " + e);
}
}
}
/** Created by Martin on 26.6.2015. */
public class ProjectsClient {
private static final Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectsClient.class);
private GerritApi api;
private IProjectsRepository repo;
private ProjectsCache cache;
private Boolean downloadParents;
private WaitCaller caller;
public ProjectsClient(
GerritApi api, WaitCaller caller, IProjectsRepository repository, Boolean downloadParents) {
this.api = api;
this.repo = repository;
this.caller = caller;
this.cache = new ProjectsCache();
this.downloadParents = downloadParents;
}
public void prepare() {
cache.restore(repo.getAllProjects());
}
public ProjectDto getProject(String id) {
if (cache.isCached(id)) {
return cache.tryGetCached(id);
}
ProjectInfo info = null;
try {
info = caller.waitOrCall(() -> api.projects().name(id).get());
} catch (Exception e) {
logger.error(Logging.prepare("getProject", id), e);
}
if (info != null) {
ProjectDto project = new ProjectDto(info.id, info.name);
if (info.parent != null && downloadParents) {
ProjectDto parent = getProject(info.parent);
project.parentId = Optional.of(parent.id);
}
project = repo.add(project);
// getProjectBranches(project);
// getApprovals(project);
cache.cache(project);
return project;
}
return null;
}
//
// private void getApprovals(ProjectDto project) {
// int start = 0;
// int limit = 1;
//
// boolean getCommands = true;
//
// List<ApprovalTypeDto> approvals = new ArrayList<>();
// while (approvals.size() <= 0) {
// List<ChangeInfo> changes = null;
//
// try {
// Changes.QueryRequest request = api.changes().query()
// .withStart(start)
// .withLimit(limit)
// .withOptions(ListChangesOption.DETAILED_LABELS);
// if (getCommands) {
// request.withOption(ListChangesOption.DOWNLOAD_COMMANDS);
// }
//
// changes = caller.waitOrCall(() -> request.get());
// } catch (Exception e) {
// e.printStackTrace();
// getCommands = false;
// }
//
// if (changes != null && changes.size() > 0) {
// ChangeInfo change = changes.get(0);
//
// if (change.labels != null && change.labels.size() > 0) {
// approvals.addAll(createApprovalTypes(project, change.labels));
// }
//
// }
//
// start += limit;
// }
//
// approvals.forEach(repo::addApprovalType);
//
// approvals.forEach(a -> project.approvals.put(a.name, a));
// }
private List<ApprovalTypeDto> createApprovalTypes(
ProjectDto project, Map<String, LabelInfo> labels) {
return labels
.entrySet()
.stream()
.filter(e -> project.approvals.containsKey(e.getKey()) == false)
.map(e -> createApprovalType(project, e.getKey(), e.getValue()))
.collect(Collectors.toList());
}
private ApprovalTypeDto createApprovalType(ProjectDto project, String type, LabelInfo label) {
short defaultValue = 0;
if (label != null && label.defaultValue != null) {
defaultValue = label.defaultValue;
}
return new ApprovalTypeDto(
project,
type,
defaultValue,
label
.values
.entrySet()
.stream()
.map(ev -> new ApprovalValueDto(ev.getKey(), ev.getValue()))
.toArray(ApprovalValueDto[]::new));
}
public void addApprovals(ProjectDto project, Map<String, LabelInfo> labels) {
if (labels.size() > 0) {
labels.entrySet().forEach(e -> addApproval(project, e.getKey(), e.getValue()));
}
}
public void addApproval(ProjectDto project, String key, LabelInfo label) {
if (project.approvals.containsKey(key) == false) {
ApprovalTypeDto approval = createApprovalType(project, key, label);
repo.addApprovalType(approval);
project.approvals.put(approval.name, approval);
}
}
private void getProjectBranches(ProjectDto project) {
List<BranchInfo> branches = null;
try {
ProjectApi.ListBranchesRequest request = api.projects().name(project.projectId).branches();
branches = caller.waitOrCall(() -> request.get());
} catch (Exception e) {
logger.error(Logging.prepare("getProjectBranches", project.projectId), e);
}
if (branches != null) {
List<BranchDto> projectBranches =
branches
.stream()
.filter(b -> project.hasBranch(b.ref) == false)
.map(b -> repo.addBranch(project, b.ref, b.revision))
.filter(b -> b != null)
.collect(Collectors.toList());
projectBranches.forEach(project.branches::add);
}
}
public BranchDto getBranch(ProjectDto project, String name) {
if (project.hasBranch(name) == false) {
repo.loadProjectBranches(project);
}
if (project.hasBranch(name) == false) {
getProjectBranches(project);
}
return project.getBranch(name);
}
}
/**
* @author : rincevent
* @since JAHIA 6.5 Created : 14 sept. 2010
*/
public class JBPMMailProducer extends MailProducerImpl {
private static final long serialVersionUID = -5084848266010688683L;
private static transient Logger logger =
org.slf4j.LoggerFactory.getLogger(JBPMMailProducer.class);
ScriptEngine scriptEngine;
private Bindings bindings;
private String templateKey;
public String getTemplateKey() {
return templateKey;
}
public void setTemplateKey(String templateKey) {
this.templateKey = templateKey;
}
public Collection<Message> produce(final Execution execution) {
final Map<String, Object> vars = ((ExecutionImpl) execution).getVariables();
Locale locale = (Locale) vars.get("locale");
if (templateKey != null) {
MailTemplate template = null;
MailTemplateRegistry templateRegistry =
((ProcessEngine) SpringContextSingleton.getBean("processEngine"))
.get(MailTemplateRegistry.class);
if (locale != null) {
template = (templateRegistry.getTemplate(templateKey + "." + locale.toString()));
if (template == null) {
template = (templateRegistry.getTemplate(templateKey + "." + locale.getLanguage()));
}
}
if (template == null) {
template = templateRegistry.getTemplate(templateKey);
}
setTemplate(template);
}
if (ServicesRegistry.getInstance().getMailService().isEnabled() && getTemplate() != null) {
try {
return JCRTemplate.getInstance()
.doExecuteWithSystemSession(
null,
"default",
locale,
new JCRCallback<Collection<Message>>() {
public Collection<Message> doInJCR(JCRSessionWrapper session)
throws RepositoryException {
try {
scriptEngine =
ScriptEngineUtils.getInstance()
.getEngineByName(getTemplate().getLanguage());
bindings = null;
Message email = instantiateEmail();
fillFrom(email, execution, session);
fillRecipients(email, execution, session);
fillSubject(email, execution, session);
fillContent(email, execution, session);
Address[] addresses = email.getRecipients(Message.RecipientType.TO);
if (addresses != null && addresses.length > 0) {
return Collections.singleton(email);
} else {
return Collections.emptyList();
}
} catch (MessagingException e) {
logger.error(e.getMessage(), e);
} catch (ScriptException e) {
logger.error(e.getMessage(), e);
}
return Collections.emptyList();
}
});
} catch (RepositoryException e) {
logger.error(e.getMessage(), e);
}
}
return Collections.emptyList();
}
/**
* Fills the <code>from</code> attribute of the given email. The sender addresses are an optional
* element in the mail template. If absent, each mail server supplies the current user's email
* address.
*
* @see {@link InternetAddress#getLocalAddress(Session)}
*/
protected void fillFrom(Message email, Execution execution, JCRSessionWrapper session)
throws MessagingException {
try {
AddressTemplate fromTemplate = getTemplate().getFrom();
// "from" attribute is optional
if (fromTemplate == null) return;
// resolve and parse addresses
String addresses = fromTemplate.getAddresses();
if (addresses != null) {
addresses = evaluateExpression(execution, addresses, session);
// non-strict parsing applies to a list of mail addresses entered by a human
email.addFrom(InternetAddress.parse(addresses, false));
}
EnvironmentImpl environment = EnvironmentImpl.getCurrent();
IdentitySession identitySession = environment.get(IdentitySession.class);
AddressResolver addressResolver = environment.get(AddressResolver.class);
// resolve and tokenize users
String userList = fromTemplate.getUsers();
if (userList != null) {
String[] userIds = tokenizeActors(userList, execution, session);
List<User> users = identitySession.findUsersById(userIds);
email.addFrom(resolveAddresses(users, addressResolver));
}
// resolve and tokenize groups
String groupList = fromTemplate.getGroups();
if (groupList != null) {
for (String groupId : tokenizeActors(groupList, execution, session)) {
Group group = identitySession.findGroupById(groupId);
email.addFrom(addressResolver.resolveAddresses(group));
}
}
} catch (ScriptException e) {
logger.error(e.getMessage(), e);
} catch (RepositoryException e) {
logger.error(e.getMessage(), e);
}
}
protected void fillRecipients(Message email, Execution execution, JCRSessionWrapper session)
throws MessagingException {
try {
// to
AddressTemplate to = getTemplate().getTo();
if (to != null) {
fillRecipients(to, email, Message.RecipientType.TO, execution, session);
}
// cc
AddressTemplate cc = getTemplate().getCc();
if (cc != null) {
fillRecipients(cc, email, Message.RecipientType.CC, execution, session);
}
// bcc
AddressTemplate bcc = getTemplate().getBcc();
if (bcc != null) {
fillRecipients(bcc, email, Message.RecipientType.BCC, execution, session);
}
} catch (ScriptException e) {
logger.error(e.getMessage(), e);
} catch (RepositoryException e) {
logger.error(e.getMessage(), e);
}
}
private void fillRecipients(
AddressTemplate addressTemplate,
Message email,
Message.RecipientType recipientType,
Execution execution,
JCRSessionWrapper session)
throws MessagingException, RepositoryException, ScriptException {
// resolve and parse addresses
String addresses = addressTemplate.getAddresses();
if (addresses != null) {
addresses = evaluateExpression(execution, addresses, session);
// non-strict parsing applies to a list of mail addresses entered by a human
email.addRecipients(recipientType, InternetAddress.parse(addresses, false));
}
EnvironmentImpl environment = EnvironmentImpl.getCurrent();
IdentitySession identitySession = environment.get(IdentitySession.class);
AddressResolver addressResolver = environment.get(AddressResolver.class);
// resolve and tokenize users
String userList = addressTemplate.getUsers();
if (userList != null) {
String[] userIds = tokenizeActors(userList, execution, session);
List<User> users = identitySession.findUsersById(userIds);
email.addRecipients(recipientType, resolveAddresses(users, addressResolver));
}
// resolve and tokenize groups
String groupList = addressTemplate.getGroups();
if (groupList != null) {
for (String groupId : tokenizeActors(groupList, execution, session)) {
Group group = identitySession.findGroupById(groupId);
email.addRecipients(recipientType, addressResolver.resolveAddresses(group));
}
}
}
private String[] tokenizeActors(String recipients, Execution execution, JCRSessionWrapper session)
throws RepositoryException, ScriptException {
String[] actors = evaluateExpression(execution, recipients, session).split("[,;\\s]+");
if (actors.length == 0) throw new JbpmException("recipient list is empty: " + recipients);
return actors;
}
/** construct recipient addresses from user entities */
private Address[] resolveAddresses(List<User> users, AddressResolver addressResolver) {
int userCount = users.size();
List<Address> addresses = new ArrayList<Address>();
for (int i = 0; i < userCount; i++) {
if (!StringUtils.isEmpty(users.get(i).getBusinessEmail())) {
addresses.add(addressResolver.resolveAddress(users.get(i)));
}
}
return addresses.toArray(new Address[addresses.size()]);
}
protected void fillSubject(Message email, Execution execution, JCRSessionWrapper session)
throws MessagingException {
String subject = getTemplate().getSubject();
if (subject != null) {
try {
String evaluatedSubject =
evaluateExpression(execution, subject, session).replaceAll("[\r\n]", "");
email.setSubject(WordUtils.abbreviate(evaluatedSubject, 60, 74, "..."));
} catch (RepositoryException e) {
logger.error(e.getMessage(), e);
} catch (ScriptException e) {
logger.error(e.getMessage(), e);
}
}
}
protected void fillContent(Message email, Execution execution, JCRSessionWrapper session)
throws MessagingException {
String text = getTemplate().getText();
String html = getTemplate().getHtml();
List<AttachmentTemplate> attachmentTemplates = getTemplate().getAttachmentTemplates();
try {
if (html != null || !attachmentTemplates.isEmpty()) {
// multipart
MimeMultipart multipart = new MimeMultipart("related");
BodyPart p = new MimeBodyPart();
Multipart alternatives = new MimeMultipart("alternative");
p.setContent(alternatives, "multipart/alternative");
multipart.addBodyPart(p);
// html
if (html != null) {
BodyPart htmlPart = new MimeBodyPart();
html = evaluateExpression(execution, html, session);
htmlPart.setContent(html, "text/html; charset=UTF-8");
alternatives.addBodyPart(htmlPart);
}
// text
if (text != null) {
BodyPart textPart = new MimeBodyPart();
text = evaluateExpression(execution, text, session);
textPart.setContent(text, "text/plain; charset=UTF-8");
alternatives.addBodyPart(textPart);
}
// attachments
if (!attachmentTemplates.isEmpty()) {
addAttachments(execution, multipart);
}
email.setContent(multipart);
} else if (text != null) {
// unipart
text = evaluateExpression(execution, text, session);
email.setText(text);
}
} catch (RepositoryException e) {
logger.error(e.getMessage(), e);
} catch (ScriptException e) {
logger.error(e.getMessage(), e);
}
}
private String evaluateExpression(
Execution execution, String scriptToExecute, JCRSessionWrapper session)
throws RepositoryException, ScriptException {
ScriptContext scriptContext = scriptEngine.getContext();
if (bindings == null) {
bindings = getBindings(execution, session);
}
scriptContext.setWriter(new StringWriter());
scriptContext.setErrorWriter(new StringWriter());
scriptEngine.eval(scriptToExecute, bindings);
String error = scriptContext.getErrorWriter().toString();
if (error.length() > 0) {
logger.error("Scripting error : " + error);
}
return scriptContext.getWriter().toString().trim();
}
private Bindings getBindings(Execution execution, JCRSessionWrapper session)
throws RepositoryException {
EnvironmentImpl environment = EnvironmentImpl.getCurrent();
final Map<String, Object> vars = ((ExecutionImpl) execution).getVariables();
Locale locale = (Locale) vars.get("locale");
final Bindings bindings = new MyBindings(environment);
ResourceBundle resourceBundle =
JahiaResourceBundle.lookupBundle(
"org.jahia.services.workflow."
+ ((ExecutionImpl) execution).getProcessDefinition().getKey(),
locale);
bindings.put("bundle", resourceBundle);
JahiaUser jahiaUser =
ServicesRegistry.getInstance()
.getJahiaUserManagerService()
.lookupUserByKey((String) vars.get("user"));
bindings.put("user", jahiaUser);
bindings.put("date", new DateTool());
bindings.put("submissionDate", Calendar.getInstance());
bindings.put("locale", locale);
bindings.put("workspace", vars.get("workspace"));
List<JCRNodeWrapper> nodes = new LinkedList<JCRNodeWrapper>();
@SuppressWarnings("unchecked")
List<String> stringList = (List<String>) vars.get("nodeIds");
for (String s : stringList) {
JCRNodeWrapper nodeByUUID = session.getNodeByUUID(s);
if (!nodeByUUID.isNodeType("jnt:translation")) {
nodes.add(nodeByUUID);
}
}
bindings.put("nodes", nodes);
return bindings;
}
public class MyBindings extends SimpleBindings {
private final Environment environment;
public MyBindings(Environment environment) {
super();
this.environment = environment;
}
@Override
public boolean containsKey(Object key) {
return super.containsKey(key) || environment.get((String) key) != null;
}
@Override
public Object get(Object key) {
return super.containsKey(key) ? super.get(key) : environment.get((String) key);
}
}
}
public class RC {
static boolean showlog = false; /* do not do any logging */
public static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(RC.class);
//////////////////////////////////////////////////
// Predefined flags
// To add a new flag:
// 1. choose a name for the flag
// 2. Define the protected static field with default value
// 3. Define a get function
// 4. Add an arm to the set function
// 5. Add any usefull utilities like booleanize()
public static final String USEGROUPSKEY = "ucar.nc2.cdm.usegroups";
public static final String VERIFYSERVERKEY = "ucar.nc2.net.verifyserver";
public static final String ALLOWSELFSIGNEDKEY = "ucar.nc2.net.allowselfsigned";
protected static boolean useGroups = true;
protected static boolean verifyServer = false;
protected static boolean allowSelfSigned = true;
public static boolean getUseGroups() {
if (!initialized) RC.initialize();
return useGroups;
}
public static boolean getVerifyServer() {
if (!initialized) RC.initialize();
return verifyServer;
}
public static boolean getAllowSelfSigned() {
if (!initialized) RC.initialize();
return allowSelfSigned;
}
public static void set(String key, String value) {
// TODO: think about the rc properties naming hierarchy
assert (key != null);
if (USEGROUPSKEY.equals(key)) {
useGroups = booleanize(value);
} else if (VERIFYSERVERKEY.equals(key)) {
verifyServer = booleanize(value);
} else if (ALLOWSELFSIGNEDKEY.equals(key)) {
allowSelfSigned = booleanize(value);
}
}
static boolean booleanize(String value) {
// canonical boolean values
if (value == null || "0".equals(value) || "false".equalsIgnoreCase(value)) return false;
if (value.length() == 0 || "1".equals(value) || "true".equalsIgnoreCase(value)) return true;
return value != null; // any non-null value?
}
//////////////////////////////////////////////////
static final String DFALTRCFILE = ".threddsrc";
static final char LTAG = '[';
static final char RTAG = ']';
static final String[] rcfilelist = new String[] {".dodsrc", ".tdsrc"};
static int urlCompare(URL u1, URL u2) {
int relation;
if (u1 == null && u2 == null) return 0;
if (u1 == null) return -1;
if (u2 == null) return +1;
// 1. host test
String host1 = (new StringBuilder(u1.getHost())).reverse().toString();
String host2 = (new StringBuilder(u2.getHost())).reverse().toString();
// Use lexical order on the reversed host names
relation = host1.compareTo(host2);
if (relation != 0) return relation;
// 2. path test
relation = (u1.getPath().compareTo(u2.getPath()));
if (relation != 0) return relation;
// 3. port number
relation = (u1.getPort() - u2.getPort());
if (relation != 0) return relation;
// 4. note: all other fields are ignored
return 0;
}
// Match has different semantics than urlCompare
static boolean urlMatch(URL pattern, URL url) {
int relation;
if (pattern == null) return (url == null);
if (!(url.getHost().endsWith(pattern.getHost())))
return false; // e.g. pattern=x.y.org url=y.org
if (!(url.getPath().startsWith(pattern.getPath())))
return false; // e.g. pattern=y.org/a/b url=y.org/a
if (pattern.getPort() > 0 && pattern.getPort() != url.getPort()) return false;
// note: all other fields are ignored
return true;
}
public static class Triple implements Comparable {
public String key; // also sort key
public String value;
public URL url;
public Triple(String key, String value, String url) {
URL u = null;
if (url != null && url.length() > 0)
try {
u = new URL(url);
} catch (MalformedURLException e) {
u = null;
}
set(key, value, u);
}
public Triple(String key, String value, URL url) {
set(key, value, url);
}
void set(String key, String value, URL url) {
this.key = key.trim().toLowerCase();
this.url = url;
this.value = value;
if (this.value == null) this.value = "";
}
public boolean equals(Object o) {
if (o == null || !(o instanceof Triple)) return false;
return (compareTo((Triple) o) == 0);
}
public int compareTo(Object o) {
if (o == null) throw new NullPointerException();
Triple t = (Triple) o;
int relation = key.compareTo(t.key);
if (relation != 0) return relation;
relation = urlCompare(this.url, t.url);
return relation;
}
// toString produces an rc line
public String toString() {
StringBuilder line = new StringBuilder();
if (url != null) {
line.append("[");
line.append(url.toString());
line.append("]");
}
line.append(key);
line.append("=");
line.append(value);
return line.toString();
}
}
// Define a singlton RC instance for general global use
static RC dfaltRC = null;
private static boolean initialized = false;
static {
RC.initialize();
}
public static synchronized void initialize() {
if (!initialized) {
initialized = true;
RC.loadDefaults();
RC.setWellKnown();
RC.loadFromJava();
}
}
/**
* Allow users to add to the default rc
*
* @param key
* @param value
* @param url null => not url specific
*/
public static synchronized void add(String key, String value, String url) {
if (key == null) return;
if (!initialized) RC.initialize();
Triple t = new Triple(key, value, url);
dfaltRC.insert(t);
// recompute well-knowns
setWellKnown();
}
/**
* Allow users to search the default rc
*
* @param key
* @param url null => not url specific
* @return value corresponding to key+url, or null if does not exist
*/
public static synchronized String find(String key, String url) {
if (key == null) return null;
if (!initialized) RC.initialize();
Triple t = dfaltRC.lookup(key, url);
return (t == null ? null : t.value);
}
/** Record some well known parameters */
static void setWellKnown() {
if (dfaltRC.triplestore.size() == 0) return;
// Walk the set of triples looking for those that have no url
for (String key : dfaltRC.keySet()) {
Triple triple = dfaltRC.lookup(key);
if (triple.url == null) {
RC.set(key, triple.value); // let set sort it out
}
}
}
static void loadDefaults() {
RC rc0 = new RC();
String[] locations =
new String[] {
System.getProperty("user.home"), System.getProperty("user.dir"),
};
boolean found1 = false;
for (String loc : locations) {
if (loc == null) continue;
String dir = loc.replace('\\', '/');
if (dir.endsWith("/")) dir = dir.substring(0, dir.length() - 1);
for (String rcpath : rcfilelist) {
String filepath = loc + "/" + rcpath;
if (rc0.load(filepath)) found1 = true;
}
}
if (!found1) if (showlog) log.debug("No .rc file found");
dfaltRC = rc0;
}
static void loadFromJava() {
String[] flags = new String[] {USEGROUPSKEY, VERIFYSERVERKEY, ALLOWSELFSIGNEDKEY};
for (String flag : flags) {
String value = System.getProperty(flag);
if (value != null) {
set(flag, value);
}
}
}
static RC getDefault() {
return dfaltRC;
}
//////////////////////////////////////////////////
// Instance Data
Map<String, List<Triple>> triplestore;
//////////////////////////////////////////////////
// constructors
public RC() {
triplestore = new HashMap<String, List<Triple>>();
}
//////////////////////////////////////////////////
// Loaders
// Load this triple store from an rc file
// overwrite existing entries
public boolean load(String abspath) {
abspath = abspath.replace('\\', '/');
File rcFile = new File(abspath);
if (!rcFile.exists() || !rcFile.canRead()) {
return false;
}
if (showlog) log.debug("Loading rc file: " + abspath);
try (BufferedReader rdr =
new BufferedReader(new InputStreamReader(new FileInputStream(rcFile), CDM.UTF8))) {
for (int lineno = 1; ; lineno++) {
URL url = null;
String line = rdr.readLine();
if (line == null) break;
// trim leading blanks
line = line.trim();
if (line.length() == 0) continue; // empty line
if (line.charAt(0) == '#') continue; // check for comment
// parse the line
if (line.charAt(0) == LTAG) {
int rindex = line.indexOf(RTAG);
if (rindex < 0) return false;
if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
String surl = line.substring(1, rindex);
try {
url = new URL(surl);
} catch (MalformedURLException mue) {
if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
}
line = line.substring(rindex + 1);
// trim again
line = line.trim();
}
// Get the key,value part
String[] pieces = line.split("\\s*=\\s*");
assert (pieces.length == 1 || pieces.length == 2);
// Create the triple
String value = "1";
if (pieces.length == 2) value = pieces[1].trim();
Triple triple = new Triple(pieces[0].trim(), value, url);
List<Triple> list = triplestore.get(triple.key);
if (list == null) list = new ArrayList<Triple>();
Triple prev = addtriple(list, triple);
triplestore.put(triple.key, list);
}
} catch (FileNotFoundException fe) {
if (showlog) log.debug("Loading rc file: " + abspath);
return false;
} catch (IOException ioe) {
if (showlog) log.error("File " + abspath + ": IO exception: " + ioe.getMessage());
return false;
}
return true;
}
public Set<String> keySet() {
return triplestore.keySet();
}
public List<Triple> getTriples(String key) {
List<Triple> list = triplestore.get(key);
if (list == null) list = new ArrayList<Triple>();
return list;
}
public Triple lookup(String key) {
return lookup(key, (URL) null);
}
public Triple lookup(String key, String url) {
if (url == null || url.length() == 0) return lookup(key);
try {
URL u = new URL(url);
return lookup(key, u);
} catch (MalformedURLException m) {
}
return null;
}
public Triple lookup(String key, URL url) {
List<Triple> list = triplestore.get(key);
if (list == null) return null;
if (url == null) {
if (list.size() == 0) return null;
return list.get(0);
} else
for (Triple t : list) {
if (urlMatch(t.url, url)) return t;
}
return null;
}
Triple addtriple(List<Triple> list, Triple triple) {
Triple prev = null;
assert (list != null);
// Look for duplicates
int i = list.indexOf(triple);
if (i >= 0) {
prev = list.remove(i);
}
list.add(triple);
Collections.sort(list);
return prev;
}
// Allow for external loading
public Triple insert(Triple t) {
if (t.key == null) return null;
List<Triple> list = triplestore.get(t.key);
if (list == null) list = new ArrayList<Triple>();
Triple prev = addtriple(list, t);
triplestore.put(t.key, list);
return prev;
}
// Output in .rc form
public String toString() {
StringBuilder rc = new StringBuilder();
for (String key : keySet()) {
List<Triple> list = getTriples(key);
for (Triple triple : list) {
String line = triple.toString();
rc.append(line);
rc.append("\n");
}
}
return rc.toString();
}
} // class RC
/**
* An IOServiceProvider for CINRAD level II files.
*
* @author caron
*/
public class Cinrad2IOServiceProvider extends AbstractIOServiceProvider {
private static org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(Cinrad2IOServiceProvider.class);
private static final int MISSING_INT = -9999;
private static final float MISSING_FLOAT = Float.NaN;
public boolean isValidFileOld(RandomAccessFile raf) {
try {
String loc = raf.getLocation();
int posFirst = loc.lastIndexOf('/') + 1;
if (posFirst < 0) posFirst = 0;
String stationId = loc.substring(posFirst, posFirst + 4);
NexradStationDB.init();
NexradStationDB.Station station = NexradStationDB.get("K" + stationId);
if (station != null) return true;
else return false;
} catch (IOException ioe) {
return false;
}
}
public boolean isValidFile(RandomAccessFile raf) {
int data_msecs = 0;
short data_julian_date = 0;
try {
raf.order(RandomAccessFile.LITTLE_ENDIAN);
raf.seek(0);
raf.skipBytes(14);
short message_type = raf.readShort();
if (message_type != 1) return false;
raf.skipBytes(12);
// data header
byte[] b4 = raf.readBytes(4);
data_msecs = bytesToInt(b4, true);
byte[] b2 = raf.readBytes(2);
data_julian_date = (short) bytesToShort(b2, true);
java.util.Date dd = Cinrad2Record.getDate(data_julian_date, data_msecs);
Calendar cal = new GregorianCalendar(new SimpleTimeZone(0, "GMT"));
cal.clear();
cal.setTime(dd);
int year = cal.get(Calendar.YEAR);
cal.setTime(new Date());
int cyear = cal.get(Calendar.YEAR);
if (year < 1990 || year > cyear) return false;
return true;
} catch (IOException ioe) {
return false;
}
}
public static int bytesToInt(byte[] bytes, boolean swapBytes) {
byte a = bytes[0];
byte b = bytes[1];
byte c = bytes[2];
byte d = bytes[3];
if (swapBytes) {
return ((a & 0xff)) + ((b & 0xff) << 8) + ((c & 0xff) << 16) + ((d & 0xff) << 24);
} else {
return ((a & 0xff) << 24) + ((b & 0xff) << 16) + ((c & 0xff) << 8) + ((d & 0xff));
}
}
public static int bytesToShort(byte[] bytes, boolean swapBytes) {
byte a = bytes[0];
byte b = bytes[1];
if (swapBytes) {
return ((a & 0xff)) + ((b & 0xff) << 8);
} else {
return ((a & 0xff) << 24) + ((b & 0xff) << 16);
}
}
public String getFileTypeId() {
return "CINRAD";
}
public String getFileTypeDescription() {
return "Chinese Level-II Base Data";
}
private Cinrad2VolumeScan volScan;
private Dimension radialDim;
private double radarRadius;
private DateFormatter formatter = new DateFormatter();
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask)
throws IOException {
NexradStationDB.init();
volScan = new Cinrad2VolumeScan(raf, cancelTask);
if (volScan.hasDifferentDopplarResolutions())
throw new IllegalStateException("volScan.hasDifferentDopplarResolutions");
radialDim = new Dimension("radial", volScan.getMaxRadials());
ncfile.addDimension(null, radialDim);
makeVariable(
ncfile,
Cinrad2Record.REFLECTIVITY,
"Reflectivity",
"Reflectivity",
"R",
volScan.getReflectivityGroups());
int velocity_type =
(volScan.getDopplarResolution() == Cinrad2Record.DOPPLER_RESOLUTION_HIGH_CODE)
? Cinrad2Record.VELOCITY_HI
: Cinrad2Record.VELOCITY_LOW;
Variable v =
makeVariable(
ncfile,
velocity_type,
"RadialVelocity",
"Radial Velocity",
"V",
volScan.getVelocityGroups());
makeVariableNoCoords(
ncfile, Cinrad2Record.SPECTRUM_WIDTH, "SpectrumWidth", "Spectrum Width", v);
if (volScan.getStationId() != null) {
ncfile.addAttribute(null, new Attribute("Station", volScan.getStationId()));
ncfile.addAttribute(null, new Attribute("StationName", volScan.getStationName()));
ncfile.addAttribute(
null, new Attribute("StationLatitude", new Double(volScan.getStationLatitude())));
ncfile.addAttribute(
null, new Attribute("StationLongitude", new Double(volScan.getStationLongitude())));
ncfile.addAttribute(
null,
new Attribute("StationElevationInMeters", new Double(volScan.getStationElevation())));
double latRadiusDegrees = Math.toDegrees(radarRadius / ucar.unidata.geoloc.Earth.getRadius());
ncfile.addAttribute(
null,
new Attribute(
"geospatial_lat_min", new Double(volScan.getStationLatitude() - latRadiusDegrees)));
ncfile.addAttribute(
null,
new Attribute(
"geospatial_lat_max", new Double(volScan.getStationLatitude() + latRadiusDegrees)));
double cosLat = Math.cos(Math.toRadians(volScan.getStationLatitude()));
double lonRadiusDegrees =
Math.toDegrees(radarRadius / cosLat / ucar.unidata.geoloc.Earth.getRadius());
ncfile.addAttribute(
null,
new Attribute(
"geospatial_lon_min", new Double(volScan.getStationLongitude() - lonRadiusDegrees)));
ncfile.addAttribute(
null,
new Attribute(
"geospatial_lon_max", new Double(volScan.getStationLongitude() + lonRadiusDegrees)));
// add a radial coordinate transform (experimental)
Variable ct = new Variable(ncfile, null, null, "radialCoordinateTransform");
ct.setDataType(DataType.CHAR);
ct.setDimensions(""); // scalar
ct.addAttribute(new Attribute("transform_name", "Radial"));
ct.addAttribute(new Attribute("center_latitude", new Double(volScan.getStationLatitude())));
ct.addAttribute(new Attribute("center_longitude", new Double(volScan.getStationLongitude())));
ct.addAttribute(new Attribute("center_elevation", new Double(volScan.getStationElevation())));
ct.addAttribute(new Attribute(_Coordinate.TransformType, "Radial"));
ct.addAttribute(
new Attribute(_Coordinate.AxisTypes, "RadialElevation RadialAzimuth RadialDistance"));
Array data =
Array.factory(DataType.CHAR.getPrimitiveClassType(), new int[0], new char[] {' '});
ct.setCachedData(data, true);
ncfile.addVariable(null, ct);
}
DateFormatter formatter = new DateFormatter();
ncfile.addAttribute(null, new Attribute(CDM.CONVENTIONS, _Coordinate.Convention));
ncfile.addAttribute(null, new Attribute("format", volScan.getDataFormat()));
ncfile.addAttribute(null, new Attribute(CF.FEATURE_TYPE, FeatureType.RADIAL.toString()));
// Date d = Cinrad2Record.getDate(volScan.getTitleJulianDays(), volScan.getTitleMsecs());
// ncfile.addAttribute(null, new Attribute("base_date", formatter.toDateOnlyString(d)));
ncfile.addAttribute(
null,
new Attribute(
"time_coverage_start", formatter.toDateTimeStringISO(volScan.getStartDate())));
; // .toDateTimeStringISO(d)));
ncfile.addAttribute(
null,
new Attribute("time_coverage_end", formatter.toDateTimeStringISO(volScan.getEndDate())));
ncfile.addAttribute(
null,
new Attribute(CDM.HISTORY, "Direct read of Nexrad Level 2 file into NetCDF-Java 2.2 API"));
ncfile.addAttribute(null, new Attribute("DataType", "Radial"));
ncfile.addAttribute(
null,
new Attribute(
"Title",
"Nexrad Level 2 Station "
+ volScan.getStationId()
+ " from "
+ formatter.toDateTimeStringISO(volScan.getStartDate())
+ " to "
+ formatter.toDateTimeStringISO(volScan.getEndDate())));
ncfile.addAttribute(
null,
new Attribute(
"Summary",
"Weather Surveillance Radar-1988 Doppler (WSR-88D) "
+ "Level II data are the three meteorological base data quantities: reflectivity, mean radial velocity, and "
+ "spectrum width."));
ncfile.addAttribute(
null,
new Attribute(
"keywords",
"WSR-88D; NEXRAD; Radar Level II; reflectivity; mean radial velocity; spectrum width"));
ncfile.addAttribute(
null,
new Attribute(
"VolumeCoveragePatternName",
Cinrad2Record.getVolumeCoveragePatternName(volScan.getVCP())));
ncfile.addAttribute(
null, new Attribute("VolumeCoveragePattern", new Integer(volScan.getVCP())));
ncfile.addAttribute(
null,
new Attribute(
"HorizonatalBeamWidthInDegrees", new Double(Cinrad2Record.HORIZONTAL_BEAM_WIDTH)));
ncfile.finish();
}
public Variable makeVariable(
NetcdfFile ncfile,
int datatype,
String shortName,
String longName,
String abbrev,
List groups)
throws IOException {
int nscans = groups.size();
if (nscans == 0) {
throw new IllegalStateException("No data for " + shortName);
}
// get representative record
List firstGroup = (List) groups.get(0);
Cinrad2Record firstRecord = (Cinrad2Record) firstGroup.get(0);
int ngates = firstRecord.getGateCount(datatype);
String scanDimName = "scan" + abbrev;
String gateDimName = "gate" + abbrev;
Dimension scanDim = new Dimension(scanDimName, nscans);
Dimension gateDim = new Dimension(gateDimName, ngates);
ncfile.addDimension(null, scanDim);
ncfile.addDimension(null, gateDim);
ArrayList dims = new ArrayList();
dims.add(scanDim);
dims.add(radialDim);
dims.add(gateDim);
Variable v = new Variable(ncfile, null, null, shortName);
v.setDataType(DataType.BYTE);
v.setDimensions(dims);
ncfile.addVariable(null, v);
v.addAttribute(new Attribute(CDM.UNITS, Cinrad2Record.getDatatypeUnits(datatype)));
v.addAttribute(new Attribute(CDM.LONG_NAME, longName));
byte[] b = new byte[2];
b[0] = Cinrad2Record.MISSING_DATA;
b[1] = Cinrad2Record.BELOW_THRESHOLD;
Array missingArray = Array.factory(DataType.BYTE.getPrimitiveClassType(), new int[] {2}, b);
v.addAttribute(new Attribute(CDM.MISSING_VALUE, missingArray));
v.addAttribute(
new Attribute("signal_below_threshold", new Byte(Cinrad2Record.BELOW_THRESHOLD)));
v.addAttribute(
new Attribute(CDM.SCALE_FACTOR, new Float(Cinrad2Record.getDatatypeScaleFactor(datatype))));
v.addAttribute(
new Attribute(CDM.ADD_OFFSET, new Float(Cinrad2Record.getDatatypeAddOffset(datatype))));
v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));
ArrayList dim2 = new ArrayList();
dim2.add(scanDim);
dim2.add(radialDim);
// add time coordinate variable
String timeCoordName = "time" + abbrev;
Variable timeVar = new Variable(ncfile, null, null, timeCoordName);
timeVar.setDataType(DataType.INT);
timeVar.setDimensions(dim2);
ncfile.addVariable(null, timeVar);
// int julianDays = volScan.getTitleJulianDays();
// Date d = Cinrad2Record.getDate( julianDays, 0);
// Date d = Cinrad2Record.getDate(volScan.getTitleJulianDays(), volScan.getTitleMsecs());
Date d = volScan.getStartDate();
String units = "msecs since " + formatter.toDateTimeStringISO(d);
timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "time since base date"));
timeVar.addAttribute(new Attribute(CDM.UNITS, units));
timeVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Integer(MISSING_INT)));
timeVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Time.toString()));
// add elevation coordinate variable
String elevCoordName = "elevation" + abbrev;
Variable elevVar = new Variable(ncfile, null, null, elevCoordName);
elevVar.setDataType(DataType.FLOAT);
elevVar.setDimensions(dim2);
ncfile.addVariable(null, elevVar);
elevVar.addAttribute(new Attribute(CDM.UNITS, "degrees"));
elevVar.addAttribute(
new Attribute(
CDM.LONG_NAME,
"elevation angle in degres: 0 = parallel to pedestal base, 90 = perpendicular"));
elevVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(MISSING_FLOAT)));
elevVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialElevation.toString()));
// add azimuth coordinate variable
String aziCoordName = "azimuth" + abbrev;
Variable aziVar = new Variable(ncfile, null, null, aziCoordName);
aziVar.setDataType(DataType.FLOAT);
aziVar.setDimensions(dim2);
ncfile.addVariable(null, aziVar);
aziVar.addAttribute(new Attribute(CDM.UNITS, "degrees"));
aziVar.addAttribute(
new Attribute(CDM.LONG_NAME, "azimuth angle in degrees: 0 = true north, 90 = east"));
aziVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(MISSING_FLOAT)));
aziVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialAzimuth.toString()));
// add gate coordinate variable
String gateCoordName = "distance" + abbrev;
Variable gateVar = new Variable(ncfile, null, null, gateCoordName);
gateVar.setDataType(DataType.FLOAT);
gateVar.setDimensions(gateDimName);
Array data =
Array.makeArray(
DataType.FLOAT,
ngates,
(double) firstRecord.getGateStart(datatype),
(double) firstRecord.getGateSize(datatype));
gateVar.setCachedData(data, false);
ncfile.addVariable(null, gateVar);
radarRadius = firstRecord.getGateStart(datatype) + ngates * firstRecord.getGateSize(datatype);
gateVar.addAttribute(new Attribute(CDM.UNITS, "m"));
gateVar.addAttribute(new Attribute(CDM.LONG_NAME, "radial distance to start of gate"));
gateVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialDistance.toString()));
// add number of radials variable
String nradialsName = "numRadials" + abbrev;
Variable nradialsVar = new Variable(ncfile, null, null, nradialsName);
nradialsVar.setDataType(DataType.INT);
nradialsVar.setDimensions(scanDim.getName());
nradialsVar.addAttribute(new Attribute(CDM.LONG_NAME, "number of valid radials in this scan"));
ncfile.addVariable(null, nradialsVar);
// add number of gates variable
String ngateName = "numGates" + abbrev;
Variable ngateVar = new Variable(ncfile, null, null, ngateName);
ngateVar.setDataType(DataType.INT);
ngateVar.setDimensions(scanDim.getName());
ngateVar.addAttribute(new Attribute(CDM.LONG_NAME, "number of valid gates in this scan"));
ncfile.addVariable(null, ngateVar);
makeCoordinateDataWithMissing(
datatype, timeVar, elevVar, aziVar, nradialsVar, ngateVar, groups);
// back to the data variable
String coordinates =
timeCoordName + " " + elevCoordName + " " + aziCoordName + " " + gateCoordName;
v.addAttribute(new Attribute(_Coordinate.Axes, coordinates));
// make the record map
int nradials = radialDim.getLength();
Cinrad2Record[][] map = new Cinrad2Record[nscans][nradials];
for (int i = 0; i < groups.size(); i++) {
Cinrad2Record[] mapScan = map[i];
List group = (List) groups.get(i);
for (int j = 0; j < group.size(); j++) {
Cinrad2Record r = (Cinrad2Record) group.get(j);
int radial = r.radial_num - 1;
mapScan[radial] = r;
}
}
Vgroup vg = new Vgroup(datatype, map);
v.setSPobject(vg);
return v;
}
private void makeVariableNoCoords(
NetcdfFile ncfile, int datatype, String shortName, String longName, Variable from) {
Variable v = new Variable(ncfile, null, null, shortName);
v.setDataType(DataType.BYTE);
v.setDimensions(from.getDimensions());
ncfile.addVariable(null, v);
v.addAttribute(new Attribute(CDM.UNITS, Cinrad2Record.getDatatypeUnits(datatype)));
v.addAttribute(new Attribute(CDM.LONG_NAME, longName));
byte[] b = new byte[2];
b[0] = Cinrad2Record.MISSING_DATA;
b[1] = Cinrad2Record.BELOW_THRESHOLD;
Array missingArray = Array.factory(DataType.BYTE.getPrimitiveClassType(), new int[] {2}, b);
v.addAttribute(new Attribute(CDM.MISSING_VALUE, missingArray));
v.addAttribute(
new Attribute("signal_below_threshold", new Byte(Cinrad2Record.BELOW_THRESHOLD)));
v.addAttribute(
new Attribute(CDM.SCALE_FACTOR, new Float(Cinrad2Record.getDatatypeScaleFactor(datatype))));
v.addAttribute(
new Attribute(CDM.ADD_OFFSET, new Float(Cinrad2Record.getDatatypeAddOffset(datatype))));
v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));
Attribute fromAtt = from.findAttribute(_Coordinate.Axes);
v.addAttribute(new Attribute(_Coordinate.Axes, fromAtt));
Vgroup vgFrom = (Vgroup) from.getSPobject();
Vgroup vg = new Vgroup(datatype, vgFrom.map);
v.setSPobject(vg);
}
private void makeCoordinateData(
int datatype,
Variable time,
Variable elev,
Variable azi,
Variable nradialsVar,
Variable ngatesVar,
List groups) {
Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape());
IndexIterator timeDataIter = timeData.getIndexIterator();
Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape());
IndexIterator elevDataIter = elevData.getIndexIterator();
Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape());
IndexIterator aziDataIter = aziData.getIndexIterator();
Array nradialsData =
Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape());
IndexIterator nradialsIter = nradialsData.getIndexIterator();
Array ngatesData =
Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape());
IndexIterator ngatesIter = ngatesData.getIndexIterator();
int last_msecs = Integer.MIN_VALUE;
int nscans = groups.size();
int maxRadials = volScan.getMaxRadials();
for (int i = 0; i < nscans; i++) {
List scanGroup = (List) groups.get(i);
int nradials = scanGroup.size();
Cinrad2Record first = null;
for (int j = 0; j < nradials; j++) {
Cinrad2Record r = (Cinrad2Record) scanGroup.get(j);
if (first == null) first = r;
timeDataIter.setIntNext(r.data_msecs);
elevDataIter.setFloatNext(r.getElevation());
aziDataIter.setFloatNext(r.getAzimuth());
if (r.data_msecs < last_msecs)
logger.warn("makeCoordinateData time out of order " + r.data_msecs);
last_msecs = r.data_msecs;
}
for (int j = nradials; j < maxRadials; j++) {
timeDataIter.setIntNext(MISSING_INT);
elevDataIter.setFloatNext(MISSING_FLOAT);
aziDataIter.setFloatNext(MISSING_FLOAT);
}
nradialsIter.setIntNext(nradials);
ngatesIter.setIntNext(first.getGateCount(datatype));
}
time.setCachedData(timeData, false);
elev.setCachedData(elevData, false);
azi.setCachedData(aziData, false);
nradialsVar.setCachedData(nradialsData, false);
ngatesVar.setCachedData(ngatesData, false);
}
private void makeCoordinateDataWithMissing(
int datatype,
Variable time,
Variable elev,
Variable azi,
Variable nradialsVar,
Variable ngatesVar,
List groups) {
Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape());
Index timeIndex = timeData.getIndex();
Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape());
Index elevIndex = elevData.getIndex();
Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape());
Index aziIndex = aziData.getIndex();
Array nradialsData =
Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape());
IndexIterator nradialsIter = nradialsData.getIndexIterator();
Array ngatesData =
Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape());
IndexIterator ngatesIter = ngatesData.getIndexIterator();
// first fill with missing data
IndexIterator ii = timeData.getIndexIterator();
while (ii.hasNext()) ii.setIntNext(MISSING_INT);
ii = elevData.getIndexIterator();
while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT);
ii = aziData.getIndexIterator();
while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT);
// now set the coordinate variables from the Cinrad2Record radial
int last_msecs = Integer.MIN_VALUE;
int nscans = groups.size();
try {
for (int scan = 0; scan < nscans; scan++) {
List scanGroup = (List) groups.get(scan);
int nradials = scanGroup.size();
Cinrad2Record first = null;
for (int j = 0; j < nradials; j++) {
Cinrad2Record r = (Cinrad2Record) scanGroup.get(j);
if (first == null) first = r;
int radial = r.radial_num - 1;
timeData.setInt(timeIndex.set(scan, radial), r.data_msecs);
elevData.setFloat(elevIndex.set(scan, radial), r.getElevation());
aziData.setFloat(aziIndex.set(scan, radial), r.getAzimuth());
if (r.data_msecs < last_msecs)
logger.warn("makeCoordinateData time out of order " + r.data_msecs);
last_msecs = r.data_msecs;
}
nradialsIter.setIntNext(nradials);
ngatesIter.setIntNext(first.getGateCount(datatype));
}
} catch (java.lang.ArrayIndexOutOfBoundsException ae) {
logger.debug("Cinrad2IOSP.uncompress ", ae);
}
time.setCachedData(timeData, false);
elev.setCachedData(elevData, false);
azi.setCachedData(aziData, false);
nradialsVar.setCachedData(nradialsData, false);
ngatesVar.setCachedData(ngatesData, false);
}
public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
Vgroup vgroup = (Vgroup) v2.getSPobject();
Range scanRange = section.getRange(0);
Range radialRange = section.getRange(1);
Range gateRange = section.getRange(2);
Array data = Array.factory(v2.getDataType().getPrimitiveClassType(), section.getShape());
IndexIterator ii = data.getIndexIterator();
for (int i = scanRange.first(); i <= scanRange.last(); i += scanRange.stride()) {
Cinrad2Record[] mapScan = vgroup.map[i];
readOneScan(mapScan, radialRange, gateRange, vgroup.datatype, ii);
}
return data;
}
private void readOneScan(
Cinrad2Record[] mapScan, Range radialRange, Range gateRange, int datatype, IndexIterator ii)
throws IOException {
for (int i = radialRange.first(); i <= radialRange.last(); i += radialRange.stride()) {
Cinrad2Record r = mapScan[i];
readOneRadial(r, datatype, gateRange, ii);
}
}
private void readOneRadial(Cinrad2Record r, int datatype, Range gateRange, IndexIterator ii)
throws IOException {
if (r == null) {
for (int i = gateRange.first(); i <= gateRange.last(); i += gateRange.stride())
ii.setByteNext(Cinrad2Record.MISSING_DATA);
return;
}
r.readData(volScan.raf, datatype, gateRange, ii);
}
private class Vgroup {
Cinrad2Record[][] map;
int datatype;
Vgroup(int datatype, Cinrad2Record[][] map) {
this.datatype = datatype;
this.map = map;
}
}
/////////////////////////////////////////////////////////////////////
public void close() throws IOException {
volScan.raf.close();
}
}
/**
* A session is encapsulated in an instance of the class HTTPSession. The encapsulation is with
* respect to a specific HttpHost "realm", where the important part is is host+port. This means that
* once a session is specified, it is tied permanently to that realm.
*
* <p>A Session encapsulate a number of other objects:
*
* <ul>
* <li>An instance of an Apache HttpClient.
* <li>A http session id
* <li>A RequestContext object; this also includes authentication: specifically a credential and a
* credentials provider.
* </ul>
*
* <p>Currently, it is assumed that only one set of credentials is needed, whether directly for
* server X or for server Y. This may change in the future.
*
* <p>As a rule, if the client gives an HTTPSession object to the "create method" procedures of
* HTTPFactory (e.g. HTTPFactory.Get or HTTPFactory.Post) then that creation call must specify a url
* that is "compatible" with the scope of the session. The method url is <it>compatible</i> if its
* host+port is the same as the session's host+port (=scope) and its scheme is compatible, where
* e.g. http is compatible with https (see HTTPAuthUtil.httphostCompatible)
*
* <p>If the HTTPFactory method creation call does not specify a session object, then one is created
* (and destroyed) behind the scenes along with the method.
*
* <p>Note that the term legalurl in the following code means that the url has reserved characters
* within identifieers in escaped form. This is particularly and issue for queries. Especially:
* ?x[0:5] is legal and the square brackets need not be encoded.
*
* <p>As of the move to Apache Httpclient 4.4 and later, the underlying HttpClient objects are
* generally immutable. This means that at least this class (HTTPSession) and the HTTPMethod class
* must store the relevant info and create the HttpClient and HttpMethod objects dynamically. This
* also means that when a parameter is changed (Agent, for example), any existing cached HttpClient
* must be thrown away and reconstructed using the change. As a rule, the HttpClient object will be
* created at the last minute so that multiple parameter changes can be effected without have to
* re-create the HttpClient for each parameter change. Also note that the immutable objects will be
* cached and reused if no parameters are changed.
*
* <p><em>Authorization</em> We assume that the session supports two CredentialsProvider instances:
* one global to all HTTPSession objects and one specific to each HTTPSession object.
*
* <p>As an aside, authentication is a bit tricky because some authorization schemes use
* redirection. That is, the initial request is made to server X, but X says: goto to server Y" to
* get, say, and authorization token. Then Y says: return to X with this token and proceed.
*
* <p><em>SSL</em> TBD.
*/
public class HTTPSession implements Closeable {
//////////////////////////////////////////////////
// Constants
// Define all the legal properties
// Previously taken from class AllClientPNames, but that is now
// deprecated, so just use an enum
static /*package*/ enum Prop {
ALLOW_CIRCULAR_REDIRECTS,
HANDLE_REDIRECTS,
HANDLE_AUTHENTICATION,
MAX_REDIRECTS,
MAX_THREADS,
SO_TIMEOUT,
CONN_TIMEOUT,
CONN_REQ_TIMEOUT,
USER_AGENT,
COOKIE_STORE,
RETRIES,
UNAVAILRETRIES,
COMPRESSION,
CREDENTIALS,
USESESSIONS,
}
// Header names
// from: http://en.wikipedia.org/wiki/List_of_HTTP_header_fields
public static final String HEADER_USERAGENT = "User-Agent";
public static final String ACCEPT_ENCODING = "Accept-Encoding";
static final int DFALTTHREADCOUNT = 50;
static final int DFALTREDIRECTS = 25;
static final int DFALTCONNTIMEOUT = 1 * 60 * 1000; // 1 minutes (60000 milliseconds)
static final int DFALTCONNREQTIMEOUT = DFALTCONNTIMEOUT;
static final int DFALTSOTIMEOUT = 5 * 60 * 1000; // 5 minutes (300000 milliseconds)
static final int DFALTRETRIES = 3;
static final int DFALTUNAVAILRETRIES = 3;
static final int DFALTUNAVAILINTERVAL = 3000; // 3 seconds
static final String DFALTUSERAGENT = "/NetcdfJava/HttpClient4.4";
static final String[] KNOWNCOMPRESSORS = {"gzip", "deflate"};
//////////////////////////////////////////////////////////////////////////
static final boolean IGNORECERTS = false;
//////////////////////////////////////////////////////////////////////////
// Type Declaration(s)
// Support loose certificate acceptance
static class LooseTrustStrategy extends TrustSelfSignedStrategy {
@Override
public boolean isTrusted(final X509Certificate[] chain, String authType)
throws CertificateException {
try {
if (super.isTrusted(chain, authType)) return true;
// check expiration dates
for (X509Certificate x5 : chain) {
try {
x5.checkValidity();
} catch (CertificateExpiredException | CertificateNotYetValidException ce) {
return true;
}
}
} catch (CertificateException e) {
return true; // temporary
}
return false;
}
}
/** Sub-class HashTable<String,Object> for mnemonic convenience and for synchronized access. */
static class Settings extends Hashtable<Prop, Object> {
public Settings() {}
public Set<Prop> getKeys() {
return keySet();
}
public Object getParameter(Prop param) {
return super.get(param);
}
public long getIntParameter(Prop param) {
return (Long) super.get(param);
}
public Settings setParameter(Prop param, Object value) {
super.put(param, value);
return this;
}
public Object removeParameter(Prop param) {
return super.remove(param);
}
}
// For communication between HTTPSession.execute and HTTPMethod.execute.
static /*package*/ class ExecState {
public HttpRequestBase request = null;
public CloseableHttpResponse response = null;
}
static /*package*/ enum Methods {
Get("get"),
Head("head"),
Put("put"),
Post("post"),
Options("options");
private final String name;
Methods(String name) {
this.name = name;
}
public String getName() {
return name;
}
}
static class GZIPResponseInterceptor implements HttpResponseInterceptor {
public void process(final HttpResponse response, final HttpContext context)
throws HttpException, IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement h : codecs) {
if (h.getName().equalsIgnoreCase("gzip")) {
response.setEntity(new GzipDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
}
static class DeflateResponseInterceptor implements HttpResponseInterceptor {
public void process(final HttpResponse response, final HttpContext context)
throws HttpException, IOException {
HttpEntity entity = response.getEntity();
if (entity != null) {
Header ceheader = entity.getContentEncoding();
if (ceheader != null) {
HeaderElement[] codecs = ceheader.getElements();
for (HeaderElement h : codecs) {
if (h.getName().equalsIgnoreCase("deflate")) {
response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
return;
}
}
}
}
}
}
static class ZipStreamFactory implements InputStreamFactory {
// InputStreamFactory methods
@Override
public InputStream create(InputStream instream) throws IOException {
return new ZipInputStream(instream, HTTPUtil.UTF8);
}
}
static class GZIPStreamFactory implements InputStreamFactory {
// InputStreamFactory methods
@Override
public InputStream create(InputStream instream) throws IOException {
return new GZIPInputStream(instream);
}
}
////////////////////////////////////////////////////////////////////////
// Static variables
public static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(HTTPSession.class);
protected static PoolingHttpClientConnectionManager connmgr;
// Define a settings object to hold all the
// settable values; there will be one
// instance for global and one for local.
static Settings globalsettings;
// Define interceptor instances
static List<HttpRequestInterceptor> reqintercepts = new ArrayList<HttpRequestInterceptor>();
static List<HttpResponseInterceptor> rspintercepts = new ArrayList<HttpResponseInterceptor>();
// This is a hack to suppress content-encoding headers from request
protected static HttpResponseInterceptor CEKILL;
// Debug Header interceptors
protected static List<HttpRequestInterceptor> dbgreq = new ArrayList<>();
protected static List<HttpResponseInterceptor> dbgrsp = new ArrayList<>();
protected static Map<String, InputStreamFactory> contentDecoderMap;
// public final HttpClientBuilder setContentDecoderRegistry(Map<String,InputStreamFactory>
// contentDecoderMap)
// Since can't access CredentialsProvider map, mimic
protected static Map<AuthScope, CredentialsProvider> globalcreds = new HashMap<>();
// As taken from the command line, usually
protected static KeyStore keystore = null;
protected static KeyStore truststore = null;
protected static String keypath = null;
protected static String keypassword = null;
protected static String trustpath = null;
protected static String trustpassword = null;
protected static SSLConnectionSocketFactory globalsslfactory = null;
protected static Registry<ConnectionSocketFactory> sslregistry = null;
protected static HttpHost httpproxy = null;
protected static HttpHost httpsproxy = null;
protected static String proxyuser = null;
protected static String proxypwd = null;
// For debugging
protected static Boolean globaldebugheaders = null;
static { // watch out: order is important for these initializers
CEKILL = new HTTPUtil.ContentEncodingInterceptor();
contentDecoderMap = new HashMap<String, InputStreamFactory>();
contentDecoderMap.put("zip", new ZipStreamFactory());
contentDecoderMap.put("gzip", new GZIPStreamFactory());
globalsettings = new Settings();
setDefaults(globalsettings);
processDFlags(); // Process all -D flags
connmgr = new PoolingHttpClientConnectionManager(sslregistry);
setGlobalUserAgent(DFALTUSERAGENT);
// does not work setGlobalThreadCount(DFALTTHREADCOUNT);
setGlobalConnectionTimeout(DFALTCONNTIMEOUT);
setGlobalSoTimeout(DFALTSOTIMEOUT);
}
//////////////////////////////////////////////////////////////////////////
// Static Initialization
// Provide defaults for a settings map
protected static void setDefaults(Settings props) {
if (false) { // turn off for now
props.setParameter(Prop.HANDLE_AUTHENTICATION, Boolean.TRUE);
}
props.setParameter(Prop.HANDLE_REDIRECTS, Boolean.TRUE);
props.setParameter(Prop.ALLOW_CIRCULAR_REDIRECTS, Boolean.TRUE);
props.setParameter(Prop.MAX_REDIRECTS, (Integer) DFALTREDIRECTS);
props.setParameter(Prop.SO_TIMEOUT, (Integer) DFALTSOTIMEOUT);
props.setParameter(Prop.CONN_TIMEOUT, (Integer) DFALTCONNTIMEOUT);
props.setParameter(Prop.CONN_REQ_TIMEOUT, (Integer) DFALTCONNREQTIMEOUT);
props.setParameter(Prop.USER_AGENT, DFALTUSERAGENT);
}
static synchronized void processDFlags() {
// SSL flags
keypath = cleanproperty("keystore");
keypassword = cleanproperty("keystorepassword");
trustpath = cleanproperty("truststore");
trustpassword = cleanproperty("truststorepassword");
setGlobalSSLAuth(keypath, keypassword, trustpath, trustpassword);
// Proxy flags
String proxyurl = cleanproperty("proxyurl");
if (proxyurl != null) setGlobalProxy(proxyurl);
else { // Check the java.net flags
String proxyhost = cleanproperty("https.proxyHost");
if (proxyhost != null) {
StringBuilder buf = new StringBuilder();
buf.append("https://");
buf.append(proxyhost);
String proxyport = cleanproperty("https.proxyPort");
if (proxyport != null) {
buf.append(":");
buf.append(proxyport);
}
setGlobalProxy(buf.toString());
}
}
}
//////////////////////////////////////////////////////////////////////////
// Static Methods (Mostly global accessors)
public static synchronized void setGlobalUserAgent(String userAgent) {
globalsettings.setParameter(Prop.USER_AGENT, userAgent);
}
public static synchronized String getGlobalUserAgent() {
return (String) globalsettings.getParameter(Prop.USER_AGENT);
}
public static synchronized void setGlobalThreadCount(int nthreads) {
// globalsettings.setParameter(Prop.MAX_THREADS,nthreads);
throw new UnsupportedOperationException(
"HTTPSession.setGlobalThreadCount is currently not working");
}
// Alias
public static void setGlobalMaxConnections(int nthreads) {
setGlobalThreadCount(nthreads);
}
public static synchronized int getGlobalThreadCount() {
return connmgr.getMaxTotal();
}
// Timeouts
public static synchronized void setGlobalConnectionTimeout(int timeout) {
if (timeout >= 0) {
globalsettings.setParameter(Prop.CONN_TIMEOUT, (Integer) timeout);
globalsettings.setParameter(Prop.CONN_REQ_TIMEOUT, (Integer) timeout);
}
}
public static synchronized void setGlobalSoTimeout(int timeout) {
if (timeout >= 0) globalsettings.setParameter(Prop.SO_TIMEOUT, (Integer) timeout);
}
/** Enable/disable redirection following Default is yes. */
public static synchronized void setGlobalFollowRedirects(boolean tf) {
globalsettings.setParameter(Prop.HANDLE_REDIRECTS, (Boolean) tf);
}
/**
* Set the max number of redirects to follow
*
* @param n
*/
public static synchronized void setGlobalMaxRedirects(int n) {
if (n < 0) // validate
throw new IllegalArgumentException("setMaxRedirects");
globalsettings.setParameter(Prop.MAX_REDIRECTS, n);
}
public static synchronized Object getGlobalSetting(String key) {
return globalsettings.get(key);
}
//////////////////////////////////////////////////
// Compression
public static synchronized void setGlobalCompression(String compressors) {
if (globalsettings.getParameter(Prop.COMPRESSION) != null) removeGlobalCompression();
String compresslist = checkCompressors(compressors);
if (HTTPUtil.nullify(compresslist) == null)
throw new IllegalArgumentException("Bad compressors: " + compressors);
globalsettings.setParameter(Prop.COMPRESSION, compresslist);
HttpResponseInterceptor hrsi;
if (compresslist.contains("gzip")) {
hrsi = new GZIPResponseInterceptor();
rspintercepts.add(hrsi);
}
if (compresslist.contains("deflate")) {
hrsi = new DeflateResponseInterceptor();
rspintercepts.add(hrsi);
}
}
public static void removeGlobalCompression() {
if (globalsettings.removeParameter(Prop.COMPRESSION) != null) {
for (int i = rspintercepts.size() - 1; i >= 0; i--) { // walk backwards
HttpResponseInterceptor hrsi = rspintercepts.get(i);
if (hrsi instanceof GZIPResponseInterceptor || hrsi instanceof DeflateResponseInterceptor)
rspintercepts.remove(i);
}
}
}
protected static synchronized String checkCompressors(String compressors) {
// Syntactic check of compressors
Set<String> cset = new HashSet<>();
compressors = compressors.replace(',', ' ');
compressors = compressors.replace('\t', ' ');
String[] pieces = compressors.split("[ ]+");
for (String p : pieces) {
for (String c : KNOWNCOMPRESSORS) {
if (p.equalsIgnoreCase(c)) {
cset.add(c);
break;
}
}
}
StringBuilder buf = new StringBuilder();
for (String s : cset) {
if (buf.length() > 0) buf.append(",");
buf.append(s);
}
return buf.toString();
}
//////////////////////////////////////////////////
// Authorization
/**
* @param provider
* @throws HTTPException
*/
public static void setGlobalCredentialsProvider(CredentialsProvider provider)
throws HTTPException {
setGlobalCredentialsProvider(provider, (AuthScope) null);
}
/**
* This is the most general case
*
* @param provider the credentials provider
* @param scope where to use it (i.e. on what host)
* @throws HTTPException
*/
public static void setGlobalCredentialsProvider(CredentialsProvider provider, AuthScope scope)
throws HTTPException {
mapcreds(provider, scope, globalcreds);
}
/**
* It is convenient to be able to directly set the Credentials (not the provider) when those
* credentials are fixed. Scope defaults to ANY
*
* @param creds
* @throws HTTPException
*/
public static void setGlobalCredentials(Credentials creds) throws HTTPException {
setGlobalCredentials(creds, null);
}
/**
* It is convenient to be able to directly set the Credentials (not the provider) when those
* credentials are fixed.
*
* @param creds
* @param scope where to use it (i.e. on what host)
* @throws HTTPException
*/
public static void setGlobalCredentials(Credentials creds, AuthScope scope) throws HTTPException {
assert (creds != null);
if (scope == null) scope = AuthScope.ANY;
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(scope, creds);
setGlobalCredentialsProvider(provider, scope);
}
/* Make this externally accessible primarily for testing */
public static synchronized void setGlobalSSLAuth(
String keypath, String keypassword, String trustpath, String trustpassword) {
// load the stores if defined
try {
if (trustpath != null && trustpassword != null) {
truststore = KeyStore.getInstance(KeyStore.getDefaultType());
try (FileInputStream instream = new FileInputStream(new File(trustpath))) {
truststore.load(instream, trustpassword.toCharArray());
}
} else truststore = null;
if (keypath != null && keypassword != null) {
keystore = KeyStore.getInstance(KeyStore.getDefaultType());
try (FileInputStream instream = new FileInputStream(new File(keypath))) {
keystore.load(instream, keypassword.toCharArray());
}
} else keystore = null;
} catch (IOException | NoSuchAlgorithmException | CertificateException | KeyStoreException ex) {
log.error("Illegal -D keystore parameters: " + ex.getMessage());
truststore = null;
keystore = null;
}
try {
// set up the context
SSLContext scxt = null;
if (IGNORECERTS) {
scxt = SSLContext.getInstance("TLS");
TrustManager[] trust_mgr =
new TrustManager[] {
new X509TrustManager() {
public X509Certificate[] getAcceptedIssuers() {
return null;
}
public void checkClientTrusted(X509Certificate[] certs, String t) {}
public void checkServerTrusted(X509Certificate[] certs, String t) {}
}
};
scxt.init(
null, // key manager
trust_mgr, // trust manager
new SecureRandom()); // random number generator
} else {
SSLContextBuilder sslbuilder = SSLContexts.custom();
TrustStrategy strat = new LooseTrustStrategy();
if (truststore != null) sslbuilder.loadTrustMaterial(truststore, strat);
else sslbuilder.loadTrustMaterial(strat);
sslbuilder.loadTrustMaterial(truststore, new LooseTrustStrategy());
if (keystore != null) sslbuilder.loadKeyMaterial(keystore, keypassword.toCharArray());
scxt = sslbuilder.build();
}
globalsslfactory = new SSLConnectionSocketFactory(scxt, new NoopHostnameVerifier());
RegistryBuilder rb = RegistryBuilder.<ConnectionSocketFactory>create();
rb.register("https", globalsslfactory);
sslregistry = rb.build();
} catch (KeyStoreException
| NoSuchAlgorithmException
| KeyManagementException
| UnrecoverableEntryException e) {
log.error("Failed to set key/trust store(s): " + e.getMessage());
sslregistry = null;
globalsslfactory = null;
}
}
public static synchronized void setGlobalProxy(String proxyurl) {
if (proxyurl == null) throw new IllegalArgumentException("Bad proxy URL: " + proxyurl);
URI uri;
try {
uri = HTTPUtil.parseToURI(proxyurl);
} catch (URISyntaxException e) {
throw new IllegalArgumentException("Bad proxy URL: " + proxyurl);
}
if (uri.getScheme().equals("http"))
httpproxy = new HttpHost(uri.getHost(), uri.getPort(), "http");
else if (uri.getScheme().equals("https"))
httpsproxy = new HttpHost(uri.getHost(), uri.getPort(), "https");
String upw = uri.getUserInfo();
if (upw != null) {
String[] pieces = upw.split("[:]");
if (pieces.length != 2
|| HTTPUtil.nullify(pieces[0]) == null
|| HTTPUtil.nullify(pieces[1]) == null)
throw new IllegalArgumentException("Bad userinfo: " + proxyurl);
proxyuser = pieces[0];
proxypwd = pieces[1];
}
}
//////////////////////////////////////////////////
// Instance variables
// Currently, the granularity of authorization is host+port.
protected String sessionURI = null; // This is either a real url
// or one constructed from an AuthScope
protected URI scopeURI = null; // constructed
protected AuthScope scope = null;
protected boolean closed = false;
// Since can't access CredentialsProvider map, mimic
protected Map<AuthScope, CredentialsProvider> localcreds = new HashMap<>();
protected List<ucar.httpservices.HTTPMethod> methodList = new Vector<HTTPMethod>();
protected String identifier = "Session";
protected Settings localsettings = new Settings();
// We currently only allow the use of global interceptors
// protected List<Object> intercepts = new ArrayList<Object>(); // current set of interceptors;
// This context is re-used over all method executions so that we maintain
// cookies, credentials, etc.
// But we do need away to clear so that e.g. we can clear credentials cache
protected HttpClientContext sessioncontext = HttpClientContext.create();
// cached and recreated as needed
protected boolean cachevalid = false; // Are cached items up-to-date?
protected CloseableHttpClient cachedclient = null;
protected RequestConfig cachedconfig = null;
protected URI requestURI = null; // full uri from the HTTPMethod call
protected ExecState execution = new ExecState();
//////////////////////////////////////////////////
// Constructor(s)
// All are package level so that only HTTPFactory can be used externally
protected HTTPSession() throws HTTPException {}
HTTPSession(String host, int port) throws HTTPException {
init(new AuthScope(host, port, null, null), null);
}
HTTPSession(String uri) throws HTTPException {
init(HTTPAuthUtil.uriToAuthScope(uri), uri);
}
HTTPSession(HttpHost httphost) throws HTTPException {
init(HTTPAuthUtil.hostToAuthScope(httphost), null);
}
protected void init(AuthScope scope, String actualurl) throws HTTPException {
assert (scope != null);
if (actualurl != null) this.sessionURI = actualurl;
else this.sessionURI = HTTPAuthUtil.authscopeToURI(scope).toString();
this.scope = scope;
this.scopeURI = HTTPAuthUtil.authscopeToURI(scope);
this.cachevalid = false; // Force build on first use
this.sessioncontext.setCookieStore(new BasicCookieStore());
this.sessioncontext.setAttribute(HttpClientContext.AUTH_CACHE, new BasicAuthCache());
}
//////////////////////////////////////////////////
// Interceptors: Only supported at global level
protected static void setInterceptors(HttpClientBuilder cb) {
for (HttpRequestInterceptor hrq : reqintercepts) {
cb.addInterceptorLast(hrq);
}
for (HttpResponseInterceptor hrs : rspintercepts) {
cb.addInterceptorLast(hrs);
}
// Add debug interceptors
for (HttpRequestInterceptor hrq : dbgreq) {
cb.addInterceptorFirst(hrq);
}
for (HttpResponseInterceptor hrs : dbgrsp) {
cb.addInterceptorFirst(hrs);
}
// Hack: add Content-Encoding suppressor
cb.addInterceptorFirst(CEKILL);
}
//////////////////////////////////////////////////
// Accessor(s)
public AuthScope getAuthScope() {
return this.scope;
}
public String getSessionURI() {
return this.sessionURI;
}
/**
* Extract the sessionid cookie value
*
* @return sessionid string
*/
public String getSessionID() {
String sid = null;
String jsid = null;
List<Cookie> cookies = this.sessioncontext.getCookieStore().getCookies();
for (Cookie cookie : cookies) {
if (cookie.getName().equalsIgnoreCase("sessionid")) sid = cookie.getValue();
if (cookie.getName().equalsIgnoreCase("jsessionid")) jsid = cookie.getValue();
}
return (sid == null ? jsid : sid);
}
public HTTPSession setUserAgent(String agent) {
if (agent == null || agent.length() == 0) throw new IllegalArgumentException("null argument");
localsettings.setParameter(Prop.USER_AGENT, agent);
this.cachevalid = false;
return this;
}
public HTTPSession setSoTimeout(int timeout) {
if (timeout <= 0) throw new IllegalArgumentException("setSoTimeout");
localsettings.setParameter(Prop.SO_TIMEOUT, timeout);
this.cachevalid = false;
return this;
}
public HTTPSession setConnectionTimeout(int timeout) {
if (timeout <= 0) throw new IllegalArgumentException("setConnectionTImeout");
localsettings.setParameter(Prop.CONN_TIMEOUT, timeout);
localsettings.setParameter(Prop.CONN_REQ_TIMEOUT, timeout);
this.cachevalid = false;
return this;
}
/**
* Set the max number of redirects to follow
*
* @param n
*/
public HTTPSession setMaxRedirects(int n) {
if (n < 0) // validate
throw new IllegalArgumentException("setMaxRedirects");
localsettings.setParameter(Prop.MAX_REDIRECTS, n);
this.cachevalid = false;
return this;
}
/** Enable/disable redirection following Default is yes. */
public HTTPSession setFollowRedirects(boolean tf) {
localsettings.setParameter(Prop.HANDLE_REDIRECTS, (Boolean) tf);
this.cachevalid = false;
return this;
}
/**
* Should we use sessionid's?
*
* @param tf
*/
public HTTPSession setUseSessions(boolean tf) {
localsettings.setParameter(Prop.USESESSIONS, (Boolean) tf);
this.cachevalid = false;
return this;
}
public List<Cookie> getCookies() {
if (this.sessioncontext == null) return null;
List<Cookie> cookies = this.sessioncontext.getCookieStore().getCookies();
return cookies;
}
public HTTPSession clearCookies() {
BasicCookieStore cookies = (BasicCookieStore) this.sessioncontext.getCookieStore();
if (cookies != null) cookies.clear();
return this;
}
public HTTPSession clearCredentialsCache() {
BasicAuthCache ac =
(BasicAuthCache) this.sessioncontext.getAttribute(HttpClientContext.AUTH_CACHE);
if (ac != null) ac.clear();
return this;
}
// make package specific
HttpClient getClient() {
return this.cachedclient;
}
HttpClientContext getExecutionContext() {
return this.sessioncontext;
}
public Object getSetting(String key) {
return localsettings.get(key);
}
//////////////////////////////////////////////////
/** Close the session. This implies closing any open methods. */
public synchronized void close() {
if (this.closed) return; // multiple calls ok
while (methodList.size() > 0) {
HTTPMethod m = methodList.get(0);
m.close(); // forcibly close; will invoke removemethod().
}
closed = true;
}
synchronized HTTPSession addMethod(HTTPMethod m) {
if (!methodList.contains(m)) methodList.add(m);
return this;
}
synchronized HTTPSession removeMethod(HTTPMethod m) {
methodList.remove(m);
return this;
}
//////////////////////////////////////////////////
// Authorization
// per-session versions of the global accessors
/**
* @param provider
* @throws HTTPException
*/
public HTTPSession setCredentialsProvider(CredentialsProvider provider) throws HTTPException {
setCredentialsProvider(provider, null);
return this;
}
/**
* This is the most general case
*
* @param provider the credentials provider
* @param scope where to use it (i.e. on what host+port)
* @throws HTTPException
*/
public HTTPSession setCredentialsProvider(CredentialsProvider provider, AuthScope scope)
throws HTTPException {
mapcreds(provider, scope, localcreds);
return this;
}
/**
* It is convenient to be able to directly set the Credentials (not the provider) when those
* credentials are fixed. Scope defaults to ANY
*
* @param creds
* @throws HTTPException
*/
public HTTPSession setCredentials(Credentials creds) throws HTTPException {
setCredentials(creds, null);
return this;
}
/**
* It is convenient to be able to directly set the Credentials (not the provider) when those
* credentials are fixed.
*
* @param creds
* @param scope where to use it (i.e. on what host)
* @throws HTTPException
*/
public HTTPSession setCredentials(Credentials creds, AuthScope scope) throws HTTPException {
assert (creds != null);
if (scope == null) scope = AuthScope.ANY;
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(scope, creds);
setCredentialsProvider(provider, scope);
return this;
}
//////////////////////////////////////////////////
// Execution (do an actual execution)
// Package visible
/**
* Called primarily from HTTPMethod to do the bulk of the execution. Assumes HTTPMethod has
* inserted its headers into request.
*
* @param method
* @param methoduri
* @param rb
* @return Request+Response pair
* @throws HTTPException
*/
ExecState execute(HTTPMethod method, URI methoduri, RequestBuilder rb) throws HTTPException {
this.execution = new ExecState();
this.requestURI = methoduri;
AuthScope methodscope = HTTPAuthUtil.uriToAuthScope(methoduri);
AuthScope target = HTTPAuthUtil.authscopeUpgrade(this.scope, methodscope);
synchronized (this) { // keep coverity happy
// Merge Settings;
Settings merged = HTTPUtil.merge(globalsettings, localsettings);
if (!this.cachevalid) {
RequestConfig.Builder rcb = RequestConfig.custom();
this.cachedconfig = configureRequest(rcb, merged);
HttpClientBuilder cb = HttpClients.custom();
configClient(cb, merged);
setAuthenticationAndProxy(cb);
this.cachedclient = cb.build();
rb.setConfig(this.cachedconfig);
this.cachevalid = true;
}
}
this.execution.request = (HttpRequestBase) rb.build();
try {
HttpHost targethost = HTTPAuthUtil.authscopeToHost(target);
this.execution.response =
cachedclient.execute(targethost, this.execution.request, this.sessioncontext);
} catch (IOException ioe) {
throw new HTTPException(ioe);
}
return this.execution;
}
protected RequestConfig configureRequest(RequestConfig.Builder rcb, Settings settings)
throws HTTPException {
// Configure the RequestConfig
for (Prop key : settings.getKeys()) {
Object value = settings.getParameter(key);
boolean tf = (value instanceof Boolean ? (Boolean) value : false);
if (key == Prop.ALLOW_CIRCULAR_REDIRECTS) {
rcb.setCircularRedirectsAllowed(tf);
} else if (key == Prop.HANDLE_REDIRECTS) {
rcb.setRedirectsEnabled(tf);
rcb.setRelativeRedirectsAllowed(tf);
} else if (key == Prop.MAX_REDIRECTS) {
rcb.setMaxRedirects((Integer) value);
} else if (key == Prop.SO_TIMEOUT) {
rcb.setSocketTimeout((Integer) value);
} else if (key == Prop.CONN_TIMEOUT) {
rcb.setConnectTimeout((Integer) value);
} else if (key == Prop.CONN_REQ_TIMEOUT) {
rcb.setConnectionRequestTimeout((Integer) value);
} else if (key == Prop.MAX_THREADS) {
connmgr.setMaxTotal((Integer) value);
connmgr.setDefaultMaxPerRoute((Integer) value);
} /* else ignore */
}
RequestConfig cfg = rcb.build();
return cfg;
}
protected void configClient(HttpClientBuilder cb, Settings settings) throws HTTPException {
cb.useSystemProperties();
String agent = (String) settings.get(Prop.USER_AGENT);
if (agent != null) cb.setUserAgent(agent);
setInterceptors(cb);
cb.setContentDecoderRegistry(contentDecoderMap);
}
/**
* Handle authentication and Proxy'ing
*
* @param cb
* @throws HTTPException
*/
protected synchronized void setAuthenticationAndProxy(HttpClientBuilder cb) throws HTTPException {
// First, setup the ssl factory
cb.setSSLSocketFactory(globalsslfactory);
// Second, Construct a CredentialsProvider that is
// the union of the Proxy credentials plus
// either the global local credentials; local overrides global
// Unfortunately, we cannot either clone or extract the contents
// of the client supplied provider, so we are forced (for now)
// to modify the client supplied provider.
// Look in the local authcreds for best scope match
AuthScope bestMatch = HTTPAuthUtil.bestmatch(scope, localcreds.keySet());
CredentialsProvider cp = null;
if (bestMatch != null) {
cp = localcreds.get(bestMatch);
} else {
bestMatch = HTTPAuthUtil.bestmatch(scope, globalcreds.keySet());
if (bestMatch != null) cp = globalcreds.get(bestMatch);
}
// Build the proxy credentials and AuthScope
Credentials proxycreds = null;
AuthScope proxyscope = null;
if (proxyuser != null && (httpproxy != null || httpsproxy != null)) {
if (httpproxy != null) proxyscope = HTTPAuthUtil.hostToAuthScope(httpproxy);
else // httpsproxy != null
proxyscope = HTTPAuthUtil.hostToAuthScope(httpsproxy);
proxycreds = new UsernamePasswordCredentials(proxyuser, proxypwd);
}
if (cp == null && proxycreds != null && proxyscope != null) {
// If client provider is null and proxycreds are not,
// then use proxycreds alone
cp = new BasicCredentialsProvider();
cp.setCredentials(proxyscope, proxycreds);
} else if (cp != null && proxycreds != null && proxyscope != null) {
// If client provider is not null and proxycreds are not,
// then add proxycreds to the client provider
cp.setCredentials(proxyscope, proxycreds);
}
if (cp != null) this.sessioncontext.setCredentialsProvider(cp);
}
//////////////////////////////////////////////////
// Utilities Static and Per-Instance
static String getCanonicalURL(String legalurl) {
if (legalurl == null) return null;
int index = legalurl.indexOf('?');
if (index >= 0) legalurl = legalurl.substring(0, index);
// remove any trailing extension
// index = legalurl.lastIndexOf('.');
// if(index >= 0) legalurl = legalurl.substring(0,index);
return HTTPUtil.canonicalpath(legalurl);
}
static String getUrlAsString(String url) throws HTTPException {
try (HTTPMethod m = HTTPFactory.Get(url); ) {
int status = m.execute();
String content = null;
if (status == 200) {
content = m.getResponseAsString();
}
return content;
}
}
static int putUrlAsString(String content, String url) throws HTTPException {
int status = 0;
try {
try (HTTPMethod m = HTTPFactory.Put(url)) {
m.setRequestContent(
new StringEntity(content, ContentType.create("application/text", "UTF-8")));
status = m.execute();
}
} catch (UnsupportedCharsetException uce) {
throw new HTTPException(uce);
}
return status;
}
static String getstorepath(String prefix) {
String path = System.getProperty(prefix + "store");
if (path != null) {
path = path.trim();
if (path.length() == 0) path = null;
}
return path;
}
static String getpassword(String prefix) {
String password = System.getProperty(prefix + "storepassword");
if (password != null) {
password = password.trim();
if (password.length() == 0) password = null;
}
return password;
}
static String cleanproperty(String property) {
String value = System.getProperty(property);
if (value != null) {
value = value.trim();
if (value.length() == 0) value = null;
}
return value;
}
static void mapcreds(
CredentialsProvider provider,
AuthScope scope,
Map<AuthScope, CredentialsProvider> authcreds) {
assert (provider != null);
if (scope == null) scope = AuthScope.ANY;
authcreds.put(scope, provider);
}
//////////////////////////////////////////////////
// Testing support
// Expose the state for testing purposes
public synchronized boolean isClosed() {
return this.closed;
}
public synchronized int getMethodcount() {
return methodList.size();
}
public RequestConfig getDebugConfig() {
return (this.cachevalid ? this.cachedconfig : null);
}
public Header[] getRequestHeaders() {
if (!this.cachevalid) return null;
Header[] hdrs = null;
if (this.execution.request != null) hdrs = this.execution.request.getAllHeaders();
return hdrs;
}
//////////////////////////////////////////////////
// Debug interface
// Provide a way to kill everything at the end of a Test
// When testing, we need to be able to clean up
// all existing sessions because JUnit can run all
// test within a single jvm.
static List<HTTPSession> sessionList = null; // List of all HTTPSession instances
// only used when testing flag is set
public static boolean TESTING = false; // set to true during testing, should be false otherwise
protected static synchronized void kill() {
if (sessionList != null) {
for (HTTPSession session : sessionList) {
session.close();
}
sessionList.clear();
// Rebuild the connection manager
connmgr.shutdown();
connmgr = new PoolingHttpClientConnectionManager(sslregistry);
setGlobalThreadCount(DFALTTHREADCOUNT);
}
}
// If we are testing, then track the sessions for kill
protected static synchronized void track(HTTPSession session) {
if (sessionList == null) sessionList = new ArrayList<HTTPSession>();
sessionList.add(session);
}
public static synchronized void debugHeaders(boolean print) {
HTTPUtil.InterceptRequest rq = new HTTPUtil.InterceptRequest();
HTTPUtil.InterceptResponse rs = new HTTPUtil.InterceptResponse();
rq.setPrint(print);
rs.setPrint(print);
/* remove any previous */
for (int i = reqintercepts.size() - 1; i >= 0; i--) {
HttpRequestInterceptor hr = reqintercepts.get(i);
if (hr instanceof HTTPUtil.InterceptCommon) reqintercepts.remove(i);
}
for (int i = rspintercepts.size() - 1; i >= 0; i--) {
HttpResponseInterceptor hr = rspintercepts.get(i);
if (hr instanceof HTTPUtil.InterceptCommon) rspintercepts.remove(i);
}
reqintercepts.add(rq);
rspintercepts.add(rs);
}
public static void debugReset() {
for (HttpRequestInterceptor hri : reqintercepts) {
if (hri instanceof HTTPUtil.InterceptCommon) ((HTTPUtil.InterceptCommon) hri).clear();
}
}
public static HTTPUtil.InterceptRequest debugRequestInterceptor() {
for (HttpRequestInterceptor hri : reqintercepts) {
if (hri instanceof HTTPUtil.InterceptRequest) return ((HTTPUtil.InterceptRequest) hri);
}
return null;
}
public static HTTPUtil.InterceptResponse debugResponseInterceptor() {
for (HttpResponseInterceptor hri : rspintercepts) {
if (hri instanceof HTTPUtil.InterceptResponse) return ((HTTPUtil.InterceptResponse) hri);
}
return null;
}
//////////////////////////////////////////////////
// Deprecated, but here for back compatibility
@Deprecated
public static void setGlobalCredentialsProvider(AuthScope scope, CredentialsProvider provider)
throws HTTPException {
setGlobalCredentialsProvider(provider, scope);
}
@Deprecated
public static void setGlobalCredentialsProvider(String url, CredentialsProvider provider)
throws HTTPException {
assert (url != null && provider != null);
AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
setGlobalCredentialsProvider(provider, scope);
}
@Deprecated
public static void setGlobalCredentials(String url, Credentials creds) throws HTTPException {
assert (url != null && creds != null);
AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(scope, creds);
setGlobalCredentialsProvider(provider, scope);
}
@Deprecated
public void setCredentials(String url, Credentials creds) throws HTTPException {
assert (creds != null);
AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
setCredentials(creds, scope);
}
@Deprecated
public void setCredentialsProvider(String url, CredentialsProvider provider)
throws HTTPException {
assert (url != null && provider != null);
AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
setCredentialsProvider(provider, scope);
}
@Deprecated
public void setCredentialsProvider(AuthScope scope, CredentialsProvider provider)
throws HTTPException {
setCredentialsProvider(provider, scope);
}
@Deprecated
public static int getRetryCount() {
throw new UnsupportedOperationException();
}
@Deprecated
public static void setGlobalCompression() {
setGlobalCompression("gzip,deflate");
}
@Deprecated
public static void setGlobalProxy(String host, int port) {
try {
URL u = new URL("http", host, port, null);
setGlobalProxy(u.toString());
} catch (MalformedURLException e) {
throw new IllegalArgumentException(e);
}
}
@Deprecated
public void setProxy(String host, int port) {
setGlobalProxy(host, port);
}
@Deprecated
public static void setGlobalCredentialsProvider(CredentialsProvider provider, String scheme)
throws HTTPException {
setGlobalCredentialsProvider(provider);
}
@Deprecated
public static void setRetryCount(int count) {
throw new UnsupportedOperationException();
}
@Deprecated
public void clearState() {
// no-op
}
@Deprecated
public String getSessionURL() {
return getSessionURI();
}
}
/**
* This class reads a NEXRAD level II data file. It can handle NCDC archives (ARCHIVE2), as well as
* CRAFT/IDD compressed files (AR2V0001).
*
* <p>Adapted with permission from the Java Iras software developed by David Priegnitz at NSSL.
*
* <p>
*
* <p>Documentation on Archive Level II data format can be found at: <a
* href="http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html">
* http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html</a>
*
* @author caron
* @author David Priegnitz
*/
public class Level2VolumeScan {
// data formats
public static final String ARCHIVE2 = "ARCHIVE2";
public static final String AR2V0001 = "AR2V0001";
public static final String AR2V0002 = "AR2V0002";
public static final String AR2V0003 = "AR2V0003";
public static final String AR2V0004 = "AR2V0004";
public static final String AR2V0006 = "AR2V0006";
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Level2VolumeScan.class);
////////////////////////////////////////////////////////////////////////////////////
// Data file
RandomAccessFile raf;
private String dataFormat = null; // ARCHIVE2 or AR2V0001
private int title_julianDay; // days since 1/1/70
private int title_msecs; // milliseconds since midnight
private String stationId; // 4 letter station assigned by ICAO
private NexradStationDB.Station station; // from lookup table, may be null
private Level2Record first, last;
private int vcp = 0; // Volume coverage pattern
private int max_radials = 0;
private int min_radials = Integer.MAX_VALUE;
private int max_radials_hr = 0;
private int min_radials_hr = Integer.MAX_VALUE;
private int dopplarResolution;
private boolean hasDifferentDopplarResolutions;
private boolean hasHighResolutionData;
private boolean hasHighResolutionREF;
private boolean hasHighResolutionVEL;
private boolean hasHighResolutionSW;
private boolean hasHighResolutionZDR;
private boolean hasHighResolutionPHI;
private boolean hasHighResolutionRHO;
// List of List of Level2Record
private List<List<Level2Record>> reflectivityGroups, dopplerGroups;
// private ArrayList reflectivityGroups, dopplerGroups;
private List<List<Level2Record>> reflectivityHighResGroups;
private List<List<Level2Record>> velocityHighResGroups;
private List<List<Level2Record>> spectrumHighResGroups;
private ArrayList diffReflectHighResGroups;
private ArrayList diffPhaseHighResGroups;
private ArrayList coefficientHighResGroups;
private boolean showMessages = false,
showData = false,
debugScans = false,
debugGroups2 = false,
debugRadials = false,
debugStats = false;
private boolean runCheck = false;
Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
this.raf = orgRaf;
if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());
raf.seek(0);
raf.order(RandomAccessFile.BIG_ENDIAN);
// volume scan header
dataFormat = raf.readString(8);
raf.skipBytes(1);
String volumeNo = raf.readString(3);
title_julianDay = raf.readInt(); // since 1/1/70
title_msecs = raf.readInt();
stationId = raf.readString(4).trim(); // only in AR2V0001
if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);
if (stationId.length() == 0) {
// try to get it from the filename LOOK
stationId = null;
}
// try to find the station
if (stationId != null) {
if (!stationId.startsWith("K") && stationId.length() == 4) {
String _stationId = "K" + stationId;
station = NexradStationDB.get(_stationId);
} else station = NexradStationDB.get(stationId);
}
// see if we have to uncompress
if (dataFormat.equals(AR2V0001)
|| dataFormat.equals(AR2V0003)
|| dataFormat.equals(AR2V0004)
|| dataFormat.equals(AR2V0006)) {
raf.skipBytes(4);
String BZ = raf.readString(2);
if (BZ.equals("BZ")) {
RandomAccessFile uraf;
File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");
if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
// see if its locked - another thread is writing it
FileInputStream fstream = null;
FileLock lock = null;
try {
fstream = new FileInputStream(uncompressedFile);
// lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
while (true) { // loop waiting for the lock
try {
lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
break;
}
}
}
} finally {
if (lock != null) lock.release();
if (fstream != null) fstream.close();
}
uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");
} else {
// nope, gotta uncompress it
uraf = uncompress(raf, uncompressedFile.getPath());
if (log.isDebugEnabled())
log.debug("made uncompressed file= " + uncompressedFile.getPath());
}
// switch to uncompressed file
raf.close();
raf = uraf;
raf.order(RandomAccessFile.BIG_ENDIAN);
}
raf.seek(Level2Record.FILE_HEADER_SIZE);
}
List<Level2Record> reflectivity = new ArrayList<Level2Record>();
List<Level2Record> doppler = new ArrayList<Level2Record>();
List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highVelocity = new ArrayList<Level2Record>();
List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();
long message_offset31 = 0;
int recno = 0;
while (true) {
Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
if (r == null) break;
if (showData) r.dump2(System.out);
// skip non-data messages
if (r.message_type == 31) {
message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
}
if (r.message_type != 1 && r.message_type != 31) {
if (showMessages) r.dumpMessage(System.out);
continue;
}
// if (showData) r.dump2(System.out);
/* skip bad
if (!r.checkOk()) {
r.dump(System.out);
continue;
} */
// some global params
if (vcp == 0) vcp = r.vcp;
if (first == null) first = r;
last = r;
if (runCheck && !r.checkOk()) {
continue;
}
if (r.hasReflectData) reflectivity.add(r);
if (r.hasDopplerData) doppler.add(r);
if (r.message_type == 31) {
if (r.hasHighResREFData) highReflectivity.add(r);
if (r.hasHighResVELData) highVelocity.add(r);
if (r.hasHighResSWData) highSpectrum.add(r);
if (r.hasHighResZDRData) highDiffReflectivity.add(r);
if (r.hasHighResPHIData) highDiffPhase.add(r);
if (r.hasHighResRHOData) highCorreCoefficient.add(r);
}
if ((cancelTask != null) && cancelTask.isCancel()) return;
}
if (debugRadials)
System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
if (highReflectivity.size() == 0) {
reflectivityGroups = sortScans("reflect", reflectivity, 600);
dopplerGroups = sortScans("doppler", doppler, 600);
}
if (highReflectivity.size() > 0)
reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
if (highVelocity.size() > 0)
velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
if (highSpectrum.size() > 0)
spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
if (highDiffReflectivity.size() > 0)
diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
if (highDiffPhase.size() > 0)
diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
if (highCorreCoefficient.size() > 0)
coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
}
private ArrayList sortScans(String name, List<Level2Record> scans, int siz) {
// now group by elevation_num
Map<Short, List<Level2Record>> groupHash = new HashMap<Short, List<Level2Record>>(siz);
for (Level2Record record : scans) {
List<Level2Record> group = groupHash.get(record.elevation_num);
if (null == group) {
group = new ArrayList<Level2Record>();
groupHash.put(record.elevation_num, group);
}
group.add(record);
}
// sort the groups by elevation_num
ArrayList groups = new ArrayList(groupHash.values());
Collections.sort(groups, new GroupComparator());
// use the maximum radials
for (int i = 0; i < groups.size(); i++) {
ArrayList group = (ArrayList) groups.get(i);
Level2Record r = (Level2Record) group.get(0);
if (runCheck) testScan(name, group);
if (r.getGateCount(REFLECTIVITY_HIGH) > 500 || r.getGateCount(VELOCITY_HIGH) > 1000) {
max_radials_hr = Math.max(max_radials_hr, group.size());
min_radials_hr = Math.min(min_radials_hr, group.size());
} else {
max_radials = Math.max(max_radials, group.size());
min_radials = Math.min(min_radials, group.size());
}
}
if (debugRadials) {
System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials);
for (int i = 0; i < groups.size(); i++) {
ArrayList group = (ArrayList) groups.get(i);
Level2Record lastr = (Level2Record) group.get(0);
for (int j = 1; j < group.size(); j++) {
Level2Record r = (Level2Record) group.get(j);
if (r.data_msecs < lastr.data_msecs) System.out.println(" out of order " + j);
lastr = r;
}
}
}
testVariable(name, groups);
if (debugScans) System.out.println("-----------------------------");
return groups;
}
public int getMaxRadials(int r) {
if (r == 0) return max_radials;
else if (r == 1) return max_radials_hr;
else return 0;
}
public int getMinRadials(int r) {
if (r == 0) return min_radials;
else if (r == 1) return min_radials_hr;
else return 0;
}
public int getDopplarResolution() {
return dopplarResolution;
}
public boolean hasDifferentDopplarResolutions() {
return hasDifferentDopplarResolutions;
}
public boolean hasHighResolutions(int dt) {
if (dt == 0) return hasHighResolutionData;
else if (dt == 1) return hasHighResolutionREF;
else if (dt == 2) return hasHighResolutionVEL;
else if (dt == 3) return hasHighResolutionSW;
else if (dt == 4) return hasHighResolutionZDR;
else if (dt == 5) return hasHighResolutionPHI;
else if (dt == 6) return hasHighResolutionRHO;
else return false;
}
// do we have same characteristics for all records in a scan?
private int MAX_RADIAL = 721;
private int[] radial = new int[MAX_RADIAL];
private boolean testScan(String name, ArrayList group) {
int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
Level2Record first = (Level2Record) group.get(0);
int n = group.size();
if (debugScans) {
boolean hasBoth = first.hasDopplerData && first.hasReflectData;
System.out.println(
name
+ " "
+ first
+ " has "
+ n
+ " radials resolution= "
+ first.resolution
+ " has both = "
+ hasBoth);
}
boolean ok = true;
double sum = 0.0;
double sum2 = 0.0;
for (int i = 0; i < MAX_RADIAL; i++) radial[i] = 0;
for (int i = 0; i < group.size(); i++) {
Level2Record r = (Level2Record) group.get(i);
/* this appears to be common - seems to be ok, we put missing values in
if (r.getGateCount(datatype) != first.getGateCount(datatype)) {
log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+
"!="+first.getGateCount(datatype)+") in record "+name+ " "+r);
ok = false;
} */
if (r.getGateSize(datatype) != first.getGateSize(datatype)) {
log.warn(
raf.getLocation()
+ " different gate size ("
+ r.getGateSize(datatype)
+ ") in record "
+ name
+ " "
+ r);
ok = false;
}
if (r.getGateStart(datatype) != first.getGateStart(datatype)) {
log.warn(
raf.getLocation()
+ " different gate start ("
+ r.getGateStart(datatype)
+ ") in record "
+ name
+ " "
+ r);
ok = false;
}
if (r.resolution != first.resolution) {
log.warn(
raf.getLocation()
+ " different resolution ("
+ r.resolution
+ ") in record "
+ name
+ " "
+ r);
ok = false;
}
if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) {
log.info(
raf.getLocation()
+ " radial out of range= "
+ r.radial_num
+ " in record "
+ name
+ " "
+ r);
continue;
}
if (radial[r.radial_num] > 0) {
log.warn(
raf.getLocation()
+ " duplicate radial = "
+ r.radial_num
+ " in record "
+ name
+ " "
+ r);
ok = false;
}
radial[r.radial_num] = r.recno + 1;
sum += r.getElevation();
sum2 += r.getElevation() * r.getElevation();
// System.out.println(" elev="+r.getElevation()+" azi="+r.getAzimuth());
}
for (int i = 1; i < radial.length; i++) {
if (0 == radial[i]) {
if (n != (i - 1)) {
log.warn(" missing radial(s)");
ok = false;
}
break;
}
}
double avg = sum / n;
double sd = Math.sqrt((n * sum2 - sum * sum) / (n * (n - 1)));
// System.out.println(" avg elev="+avg+" std.dev="+sd);
return ok;
}
// do we have same characteristics for all groups in a variable?
private boolean testVariable(String name, List scans) {
int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
if (scans.size() == 0) {
log.warn(" No data for = " + name);
return false;
}
boolean ok = true;
List firstScan = (List) scans.get(0);
Level2Record firstRecord = (Level2Record) firstScan.get(0);
dopplarResolution = firstRecord.resolution;
if (debugGroups2)
System.out.println(
"Group "
+ Level2Record.getDatatypeName(datatype)
+ " ngates = "
+ firstRecord.getGateCount(datatype)
+ " start = "
+ firstRecord.getGateStart(datatype)
+ " size = "
+ firstRecord.getGateSize(datatype));
for (int i = 1; i < scans.size(); i++) {
List scan = (List) scans.get(i);
Level2Record record = (Level2Record) scan.get(0);
if ((datatype == Level2Record.VELOCITY_HI)
&& (record.resolution
!= firstRecord.resolution)) { // do all velocity resolutions match ??
log.warn(
name
+ " scan "
+ i
+ " diff resolutions = "
+ record.resolution
+ ", "
+ firstRecord.resolution
+ " elev= "
+ record.elevation_num
+ " "
+ record.getElevation());
ok = false;
hasDifferentDopplarResolutions = true;
}
if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) {
log.warn(
name
+ " scan "
+ i
+ " diff gates size = "
+ record.getGateSize(datatype)
+ " "
+ firstRecord.getGateSize(datatype)
+ " elev= "
+ record.elevation_num
+ " "
+ record.getElevation());
ok = false;
} else if (debugGroups2)
System.out.println(
" ok gates size elev= " + record.elevation_num + " " + record.getElevation());
if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) {
log.warn(
name
+ " scan "
+ i
+ " diff gates start = "
+ record.getGateStart(datatype)
+ " "
+ firstRecord.getGateStart(datatype)
+ " elev= "
+ record.elevation_num
+ " "
+ record.getElevation());
ok = false;
} else if (debugGroups2)
System.out.println(
" ok gates start elev= " + record.elevation_num + " " + record.getElevation());
if (record.message_type == 31) {
hasHighResolutionData = true;
// each data type
if (record.hasHighResREFData) hasHighResolutionREF = true;
if (record.hasHighResVELData) hasHighResolutionVEL = true;
if (record.hasHighResSWData) hasHighResolutionSW = true;
if (record.hasHighResZDRData) hasHighResolutionZDR = true;
if (record.hasHighResPHIData) hasHighResolutionPHI = true;
if (record.hasHighResRHOData) hasHighResolutionRHO = true;
}
}
return ok;
}
/**
* Get Reflectivity Groups Groups are all the records for a variable and elevation_num;
*
* @return List of type List of type Level2Record
*/
public List getReflectivityGroups() {
return reflectivityGroups;
}
/**
* Get Velocity Groups Groups are all the records for a variable and elevation_num;
*
* @return List of type List of type Level2Record
*/
public List getVelocityGroups() {
return dopplerGroups;
}
public List getHighResVelocityGroups() {
return velocityHighResGroups;
}
public List getHighResReflectivityGroups() {
return reflectivityHighResGroups;
}
public List getHighResSpectrumGroups() {
return spectrumHighResGroups;
}
public List getHighResDiffReflectGroups() {
return diffReflectHighResGroups;
}
public List getHighResDiffPhaseGroups() {
return diffPhaseHighResGroups;
}
public List getHighResCoeffocientGroups() {
return coefficientHighResGroups;
}
private class GroupComparator implements Comparator<List<Level2Record>> {
public int compare(List<Level2Record> group1, List<Level2Record> group2) {
Level2Record record1 = group1.get(0);
Level2Record record2 = group2.get(0);
// if (record1.elevation_num != record2.elevation_num)
return record1.elevation_num - record2.elevation_num;
// return record1.cut - record2.cut;
}
}
/**
* Get data format (ARCHIVE2, AR2V0001) for this file.
*
* @return data format (ARCHIVE2, AR2V0001) for this file.
*/
public String getDataFormat() {
return dataFormat;
}
/**
* Get the starting Julian day for this volume
*
* @return days since 1/1/70.
*/
public int getTitleJulianDays() {
return title_julianDay;
}
/**
* Get the starting time in seconds since midnight.
*
* @return Generation time of data in milliseconds of day past midnight (UTC).
*/
public int getTitleMsecs() {
return title_msecs;
}
/**
* Get the Volume Coverage Pattern number for this data.
*
* @return VCP
* @see Level2Record#getVolumeCoveragePatternName
*/
public int getVCP() {
return vcp;
}
/**
* Get the 4-char station ID for this data
*
* @return station ID (may be null)
*/
public String getStationId() {
return stationId;
}
public String getStationName() {
return station == null ? "unknown" : station.name;
}
public double getStationLatitude() {
return station == null ? 0.0 : station.lat;
}
public double getStationLongitude() {
return station == null ? 0.0 : station.lon;
}
public double getStationElevation() {
return station == null ? 0.0 : station.elev;
}
public Date getStartDate() {
return first.getDate();
}
public Date getEndDate() {
return last.getDate();
}
/**
* Write equivilent uncompressed version of the file.
*
* @param inputRaf file to uncompress
* @param ufilename write to this file
* @return raf of uncompressed file
* @throws IOException on read error
*/
private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
throws IOException {
RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
FileLock lock = null;
while (true) { // loop waiting for the lock
try {
lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
}
}
}
try {
inputRaf.seek(0);
byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
inputRaf.read(header);
outputRaf.write(header);
boolean eof = false;
int numCompBytes;
byte[] ubuff = new byte[40000];
byte[] obuff = new byte[40000];
try {
CBZip2InputStream cbzip2 = new CBZip2InputStream();
while (!eof) {
try {
numCompBytes = inputRaf.readInt();
if (numCompBytes == -1) {
if (log.isDebugEnabled()) log.debug(" done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
log.warn(" got EOFException ");
break; // assume this is ok
}
if (log.isDebugEnabled()) {
log.debug(
"reading compressed bytes "
+ numCompBytes
+ " input starts at "
+ inputRaf.getFilePointer()
+ "; output starts at "
+ outputRaf.getFilePointer());
}
/*
* For some stupid reason, the last block seems to
* have the number of bytes negated. So, we just
* assume that any negative number (other than -1)
* is the last block and go on our merry little way.
*/
if (numCompBytes < 0) {
if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
numCompBytes = -numCompBytes;
eof = true;
}
byte[] buf = new byte[numCompBytes];
inputRaf.readFully(buf);
ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);
// CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
cbzip2.setStream(bis);
int total = 0;
int nread;
/*
while ((nread = cbzip2.read(ubuff)) != -1) {
dout2.write(ubuff, 0, nread);
total += nread;
}
*/
try {
while ((nread = cbzip2.read(ubuff)) != -1) {
if (total + nread > obuff.length) {
byte[] temp = obuff;
obuff = new byte[temp.length * 2];
System.arraycopy(temp, 0, obuff, 0, temp.length);
}
System.arraycopy(ubuff, 0, obuff, total, nread);
total += nread;
}
if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
} catch (BZip2ReadException ioe) {
log.warn("Nexrad2IOSP.uncompress ", ioe);
}
float nrecords = (float) (total / 2432.0);
if (log.isDebugEnabled())
log.debug(
" unpacked "
+ total
+ " num bytes "
+ nrecords
+ " records; ouput ends at "
+ outputRaf.getFilePointer());
}
} catch (Exception e) {
if (outputRaf != null) outputRaf.close();
outputRaf = null;
// dont leave bad files around
File ufile = new File(ufilename);
if (ufile.exists()) {
if (!ufile.delete())
log.warn("failed to delete uncompressed file (IOException)" + ufilename);
}
if (e instanceof IOException) throw (IOException) e;
else throw new RuntimeException(e);
}
} finally {
if (null != outputRaf) outputRaf.flush();
if (lock != null) lock.release();
}
return outputRaf;
}
// debugging
static void bdiff(String filename) throws IOException {
InputStream in1 = new FileInputStream(filename + ".tmp");
InputStream in2 = new FileInputStream(filename + ".tmp2");
int count = 0;
int bad = 0;
while (true) {
int b1 = in1.read();
int b2 = in2.read();
if (b1 < 0) break;
if (b2 < 0) break;
if (b1 != b2) {
System.out.println(count + " in1=" + b1 + " in2= " + b2);
bad++;
if (bad > 130) break;
}
count++;
}
System.out.println("total read = " + count);
}
// check if compressed file seems ok
public static long testValid(String ufilename) throws IOException {
boolean lookForHeader = false;
// gotta make it
RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
byte[] b = new byte[8];
raf.read(b);
String test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println("--Good header= " + test);
raf.seek(24);
} else {
System.out.println("--No header ");
lookForHeader = true;
raf.seek(0);
}
boolean eof = false;
int numCompBytes;
try {
while (!eof) {
if (lookForHeader) {
raf.read(b);
test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println(" found header= " + test);
raf.skipBytes(16);
lookForHeader = false;
} else {
raf.skipBytes(-8);
}
}
try {
numCompBytes = raf.readInt();
if (numCompBytes == -1) {
System.out.println("\n--done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
System.out.println("\n--got EOFException ");
break; // assume this is ok
}
System.out.print(" " + numCompBytes + ",");
if (numCompBytes < 0) {
System.out.println("\n--last block " + numCompBytes);
numCompBytes = -numCompBytes;
if (!lookForHeader) eof = true;
}
raf.skipBytes(numCompBytes);
}
} catch (EOFException e) {
e.printStackTrace();
}
return raf.getFilePointer();
}
/** test */
public static void main2(String[] args) throws IOException {
File testDir = new File("C:/data/bad/radar2/");
File[] files = testDir.listFiles();
for (int i = 0; i < files.length; i++) {
File file = files[i];
if (!file.getPath().endsWith(".ar2v")) continue;
System.out.println(file.getPath() + " " + file.length());
long pos = testValid(file.getPath());
if (pos == file.length()) {
System.out.println("OK");
try {
NetcdfFile.open(file.getPath());
} catch (Throwable t) {
System.out.println("ERROR= " + t);
}
} else System.out.println("NOT pos=" + pos);
System.out.println();
}
}
public static void main(String args[]) throws IOException {
NexradStationDB.init();
RandomAccessFile raf =
new RandomAccessFile(
"/upc/share/testdata/radar/nexrad/level2/Level2_KFTG_20060818_1814.ar2v.uncompress.missingradials",
"r");
// RandomAccessFile raf = new
// RandomAccessFile("R:/testdata2/radar/nexrad/level2/problem/KCCX_20060627_1701", "r");
new Level2VolumeScan(raf, null);
}
}
/** Handles the Ensemble coordinate dimension. Assumes GribGridRecord */
public class GridEnsembleCoord {
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GridEnsembleCoord.class);
private List<EnsCoord> ensCoords;
private int seq = 0;
/**
* Create a new GridEnsembleCoord with the list of records
*
* @param records records to use
*/
GridEnsembleCoord(List<GridRecord> records) {
Map<Integer, EnsCoord> map = new HashMap<Integer, EnsCoord>();
for (GridRecord record : records) {
GribGridRecord ggr = (GribGridRecord) record;
int ensNumber = ggr.getPds().getPerturbationNumber();
int ensType = ggr.getPds().getPerturbationType();
int h = 1000 * ensNumber + ensType; // unique perturbation number and type
map.put(h, new EnsCoord(ensNumber, ensType));
}
ensCoords = new ArrayList<EnsCoord>(map.values());
Collections.sort(ensCoords);
}
private class EnsCoord implements Comparable<EnsCoord> {
int number, type;
private EnsCoord(int number, int type) {
this.number = number;
this.type = type;
}
@Override
public int compareTo(EnsCoord o) {
int r = number - o.number;
return (r == 0) ? type - o.type : r;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
EnsCoord ensCoord = (EnsCoord) o;
if (number != ensCoord.number) return false;
if (type != ensCoord.type) return false;
return true;
}
@Override
public int hashCode() {
return 1000 * number + type;
}
@Override
public String toString() {
return "number=" + number + ", type=" + type;
}
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
GridEnsembleCoord that = (GridEnsembleCoord) o;
return ensCoords.equals(that.ensCoords);
}
@Override
public int hashCode() {
return ensCoords.hashCode();
}
/**
* Set the sequence number
*
* @param seq the sequence number
*/
void setSequence(int seq) {
this.seq = seq;
}
/**
* Get the name
*
* @return the name
*/
String getName() {
return (seq == 0) ? "ens" : "ens" + seq;
}
/**
* Add this as a dimension to a netCDF file
*
* @param ncfile the netCDF file
* @param g the group in the file
*/
void addDimensionsToNetcdfFile(NetcdfFile ncfile, Group g) {
ncfile.addDimension(g, new Dimension(getName(), getNEnsembles(), true));
}
/**
* Add this as a variable to the netCDF file
*
* @param ncfile the netCDF file
* @param g the group in the file
*/
void addToNetcdfFile(NetcdfFile ncfile, Group g) {
Variable v = new Variable(ncfile, g, null, getName());
v.setDimensions(v.getShortName());
v.setDataType(DataType.STRING);
v.addAttribute(new Attribute("long_name", "ensemble"));
v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Ensemble.toString()));
String[] data = new String[getNEnsembles()];
for (int i = 0; i < getNEnsembles(); i++) {
EnsCoord ec = ensCoords.get(i);
data[i] = Grib2Tables.codeTable4_6(ec.type) + " " + ec.number;
}
Array dataArray = Array.factory(DataType.STRING, new int[] {getNEnsembles()}, data);
v.setCachedData(dataArray, false);
ncfile.addVariable(g, v);
}
/**
* Get the index of a GridRecord
*
* @param ggr the grib record
* @return the index or -1 if not found
*/
int getIndex(GribGridRecord ggr) {
int ensNumber = ggr.getPds().getPerturbationNumber();
int ensType = ggr.getPds().getPerturbationType();
int count = 0;
for (EnsCoord coord : ensCoords) {
if ((coord.number == ensNumber) && (coord.type == ensType)) return count;
count++;
}
return -1;
}
/**
* Get the number of Ensembles
*
* @return the number of Ensembles
*/
int getNEnsembles() {
return ensCoords.size();
}
}
public class DashboardFilterHandler extends UIComponentHandlerFactoryElement {
private static transient org.slf4j.Logger log =
org.slf4j.LoggerFactory.getLogger(DashboardFilterHandler.class.getName());
// Component JSPs
protected String componentIncludeJSP;
protected String componentIncludeJSPshow;
protected String componentIncludeJSPedit;
protected String componentIncludeJSPproperties;
protected String componentIncludeJSPshort_show;
protected String componentIncludeJSPshort_edit;
protected String componentIncludeJSPshort_properties;
public static final String I18N_PREFFIX = "dashboardFilter.";
public static final String PARAM_VALUE = "value";
public static final String PARAM_VALUE_MIN = "minValue";
public static final String PARAM_VALUE_MAX = "maxValue";
public static final String PARAM_CUSTOM_VALUE = "customValue";
public static final String PARAM_LAST_HOUR = "lastHour";
public static final String PARAM_LAST_12HOURS = "last12Hours";
public static final String PARAM_TODAY = "today";
public static final String PARAM_YESTERDAY = "yesterday";
public static final String PARAM_LAST_7DAYS = "last7Days";
public static final String PARAM_THIS_MONTH = "thisMonth";
public static final String PARAM_LAST_MONTH = "lastMonth";
public static final String PARAM_THIS_QUARTER = "thisQuarter";
public static final String PARAM_LAST_QUARTER = "lastQuarter";
public static final String PARAM_LAST_6MONTHS = "last12Months";
public static final String PARAM_THIS_YEAR = "thisYear";
public static final String PARAM_LAST_YEAR = "lastYear";
public static final String PARAM_NULL_VALUE = "---";
public static final String PARAM_VISIBLE = "visible";
public static final String PARAM_DRILLDOWN_DISABLED = "drillDownDisabled";
public static final String PARAM_SECTION = "section";
public static final String PARAM_SHOW_REFRESH_BUTTON = "showRefreshButton";
public static final String PARAM_SHOW_APPLY_BUTTON = "showApplyButton";
public static final String PARAM_SHOW_CLEAR_BUTTON = "showClearButton";
public static final String PARAM_SHOW_PROPERTY_NAMES = "showPropertyNames";
public static final String PARAM_SHOW_SUBMIT_ON_CHANGE = "showSubmitOnChange";
public static final String PARAM_SHOW_AUTO_REFRESH = "showAutoRefresh";
public static final String PARAM_SHORT_MODE = "shortMode";
public static final String PARAM_SHOW_LEGEND = "showLegend";
protected DashboardHandler dashboardHandler;
protected DashboardFilterRequestProcessor requestProcessor;
protected String serializedProperties;
// Component options.
protected boolean isShowMode;
protected boolean isEditMode;
protected boolean isShortMode;
protected boolean showPropertyNames;
protected boolean showRefreshButton;
protected boolean showApplyButton;
protected boolean showClearButton;
protected boolean showSubmitOnChange;
protected boolean showLegend;
protected boolean showAutoRefresh;
// Handle component properties.
protected List
properties; // Properties selected in edit mode. Filter can be executed with these properties.
protected List
notAllowedProperties; // Properties that cannot be displayed because there is another property
// visible with same property identifier.
protected Set filterPropertyErrors; // Errors when data input to filter cannot be parsed.
protected boolean
panelDuplicated; // Flag which indicate if a panel instance is duplicated in different
// sections.
// Auto-Refresh control
protected int autoRefreshTimeout;
protected boolean refreshEnabled;
public DashboardFilterHandler() {
serializedProperties = null;
isShowMode = true;
isEditMode = false;
isShortMode = true;
properties = new ArrayList();
filterPropertyErrors = new HashSet();
notAllowedProperties = new ArrayList();
showPropertyNames = true;
showRefreshButton = true;
showApplyButton = true;
showClearButton = true;
showSubmitOnChange = true;
showLegend = true;
showAutoRefresh = false;
autoRefreshTimeout = 15;
panelDuplicated = false;
}
// -------------- START GETTERS AND SETTERS --------------------- //
public boolean isPanelDuplicated() {
return panelDuplicated;
}
public void setPanelDuplicated(boolean panelDuplicated) {
this.panelDuplicated = panelDuplicated;
}
public boolean isRefreshEnabled() {
return refreshEnabled;
}
public void setRefreshEnabled(boolean refreshEnabled) {
this.refreshEnabled = refreshEnabled;
}
public int getAutoRefreshTimeout() {
return autoRefreshTimeout;
}
public void setAutoRefreshTimeout(int autoRefreshTimeout) {
this.autoRefreshTimeout = autoRefreshTimeout;
}
public boolean isShowAutoRefresh() {
return showAutoRefresh;
}
public void setShowAutoRefresh(boolean showAutoRefresh) {
this.showAutoRefresh = showAutoRefresh;
}
public List getNotAllowedProperties() {
return notAllowedProperties;
}
public void setNotAllowedProperties(List notAllowedProperties) {
this.notAllowedProperties = notAllowedProperties;
}
public boolean isShowSubmitOnChange() {
return showSubmitOnChange;
}
public void setShowSubmitOnChange(boolean showSubmitOnChange) {
this.showSubmitOnChange = showSubmitOnChange;
}
public boolean isShowLegend() {
return showLegend;
}
public void setShowLegend(boolean showLegend) {
this.showLegend = showLegend;
}
public DashboardFilterRequestProcessor getRequestProcessor() {
return requestProcessor;
}
public void setRequestProcessor(DashboardFilterRequestProcessor requestProcessor) {
this.requestProcessor = requestProcessor;
}
public boolean isShowApplyButton() {
return showApplyButton;
}
public void setShowApplyButton(boolean showApplyButton) {
this.showApplyButton = showApplyButton;
}
public boolean isShowClearButton() {
return showClearButton;
}
public void setShowClearButton(boolean showClearButton) {
this.showClearButton = showClearButton;
}
public boolean isShowRefreshButton() {
return showRefreshButton;
}
public void setShowRefreshButton(boolean showRefreshButton) {
this.showRefreshButton = showRefreshButton;
}
public boolean isShowPropertyNames() {
return showPropertyNames;
}
public void setShowPropertyNames(boolean showPropertyNames) {
this.showPropertyNames = showPropertyNames;
}
public List getProperties() {
return properties;
}
public void setProperties(List properties) {
this.properties = properties;
}
public DashboardHandler getDashboardHandler() {
return dashboardHandler;
}
public void setDashboardHandler(DashboardHandler dashboardHandler) {
this.dashboardHandler = dashboardHandler;
}
public boolean isShortMode() {
return isShortMode;
}
public void setShortMode(boolean shortMode) {
isShortMode = shortMode;
}
public boolean isEditMode() {
return isEditMode;
}
public boolean isShowMode() {
return isShowMode;
}
public String getComponentIncludeJSPshort_edit() {
return componentIncludeJSPshort_edit;
}
public void setComponentIncludeJSPshort_edit(String componentIncludeJSPshort_edit) {
this.componentIncludeJSPshort_edit = componentIncludeJSPshort_edit;
}
public String getComponentIncludeJSPshort_properties() {
return componentIncludeJSPshort_properties;
}
public void setComponentIncludeJSPshort_properties(String componentIncludeJSPshort_properties) {
this.componentIncludeJSPshort_properties = componentIncludeJSPshort_properties;
}
public String getComponentIncludeJSPshort_show() {
return componentIncludeJSPshort_show;
}
public void setComponentIncludeJSPshort_show(String componentIncludeJSPshort_show) {
this.componentIncludeJSPshort_show = componentIncludeJSPshort_show;
}
public String getComponentIncludeJSPproperties() {
return componentIncludeJSPproperties;
}
public String getJSPForShowMode() {
if (isShortMode) return componentIncludeJSPshort_show;
else return componentIncludeJSPshow;
}
public String getJSPForEditMode() {
return componentIncludeJSPedit;
}
public String getJSPForProperties() {
if (!isShortMode) return componentIncludeJSPproperties;
else return componentIncludeJSPshort_properties;
}
public String getJSP() {
if (isEditMode) return getJSPForEditMode();
return getJSPForShowMode();
}
public void setComponentIncludeJSPproperties(String componentIncludeJSPproperties) {
this.componentIncludeJSPproperties = componentIncludeJSPproperties;
}
public String getComponentIncludeJSPedit() {
return componentIncludeJSPedit;
}
public void setComponentIncludeJSPedit(String componentIncludeJSPedit) {
this.componentIncludeJSPedit = componentIncludeJSPedit;
}
public String getComponentIncludeJSPshow() {
return componentIncludeJSPshow;
}
public void setComponentIncludeJSPshow(String componentIncludeJSPshow) {
this.componentIncludeJSPshow = componentIncludeJSPshow;
}
public String getComponentIncludeJSP() {
return getJSP();
}
public void setComponentIncludeJSP(String componentIncludeJSP) {
this.componentIncludeJSP = componentIncludeJSP;
}
// -------------- END GETTERS AND SETTERS --------------------- //
// -------------- START HANDLING METHODS ---------------------- //
public static DashboardFilterHandler lookup(String code) {
DashboardFilterHandler handler = null;
// Get handler for code;
if (!StringUtils.isBlank(code))
handler =
(DashboardFilterHandler)
Factory.lookup("org.jboss.dashboard.ui.components.DashboardFilterHandler_" + code);
if (handler == null)
handler =
(DashboardFilterHandler)
Factory.lookup("org.jboss.dashboard.ui.components.DashboardFilterHandler");
return handler;
}
public String getComponentPath() {
return getComponentName();
}
public Dashboard getDashboard() {
return dashboardHandler.getCurrentDashboard();
}
public DashboardFilter getFilter() {
return getDashboard().getDashboardFilter();
}
public void enableEditMode() {
isEditMode = true;
isShowMode = false;
}
public void enableShowMode() {
isShowMode = true;
isEditMode = false;
// Enable or disable autorefresh on show mode.
if (showAutoRefresh) setRefreshEnabled(true);
else setRefreshEnabled(false);
}
public String getSerializedProperties() {
return serializedProperties;
}
public void setSerializedProperties(String serializedProperties) {
this.serializedProperties = serializedProperties;
}
public DashboardFilterProperty getDashboardFilterPropertyForCurrentFilter(
String dataProviderCode, String propertyId) {
Iterator it = properties.iterator();
while (it.hasNext()) {
DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
if (dataProviderCode.equals(dashboardFilterProperty.getDataProviderCode())
&& propertyId.equals(dashboardFilterProperty.getPropertyId()))
return dashboardFilterProperty;
}
return null;
}
public DashboardFilterProperty getDashboardFilterProperty(String propertyId) {
Iterator it = properties.iterator();
while (it.hasNext()) {
DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
if (propertyId.equals(dashboardFilterProperty.getPropertyId()))
return dashboardFilterProperty;
}
return null;
}
// Calls dashboard filter to get static properties but keep properties instance configuration. Is
// propety is found on this instance properties List thsi instance is returned.
public DashboardFilterProperty[] getStaticPropertiesForCurrentFilter() {
DashboardFilterProperty[] staticProperties = getFilter().getStaticProperties();
if (staticProperties == null) return null;
DashboardFilterProperty[] results = new DashboardFilterProperty[staticProperties.length];
for (int i = 0; i < staticProperties.length; i++) {
DashboardFilterProperty staticProperty = staticProperties[i];
DashboardFilterProperty property =
getDashboardFilterPropertyForCurrentFilter(
staticProperty.getDataProviderCode(), staticProperty.getPropertyId());
if (property != null) results[i] = property;
else results[i] = staticProperty;
}
return results;
}
public DashboardFilterProperty[] getAllPropertiesForCurrentFilter() {
List results = new ArrayList();
try {
// Static properties.
DashboardFilterProperty[] staticProps = getStaticPropertiesForCurrentFilter();
if (staticProps != null) results.addAll(Arrays.asList(staticProps));
// Dynamic properties.
Iterator it = getDashboard().getDataProviders().iterator();
while (it.hasNext()) {
DataProvider dataProvider = (DataProvider) it.next();
DataProperty[] allProperties = dataProvider.getDataSet().getProperties();
for (int i = 0; i < allProperties.length; i++) {
DataProperty property = allProperties[i];
DashboardFilterProperty prop =
getDashboardFilterPropertyForCurrentFilter(
dataProvider.getCode(), property.getPropertyId());
if (prop == null)
prop =
new DashboardFilterProperty(
dataProvider.getCode(), property.getPropertyId(), getFilter(), null, false);
results.add(prop);
}
}
} catch (Exception e) {
log.error("Cannot get data provider results.", e);
}
return (DashboardFilterProperty[]) results.toArray(new DashboardFilterProperty[results.size()]);
}
public DashboardFilterProperty[] getBeingFilteredProperties() {
List results = new ArrayList();
Iterator it = properties.iterator();
while (it.hasNext()) {
DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
if (dashboardFilterProperty.isBeingFiltered()) results.add(dashboardFilterProperty);
}
return (DashboardFilterProperty[]) results.toArray(new DashboardFilterProperty[results.size()]);
}
public List<DashboardFilterProperty> getVisibleProperties() {
List<DashboardFilterProperty> results = new ArrayList<DashboardFilterProperty>();
Iterator it = properties.iterator();
while (it.hasNext()) {
DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
if (dashboardFilterProperty.isVisible()) results.add(dashboardFilterProperty);
}
return results;
}
public boolean hasError(String propertyId) {
return filterPropertyErrors.contains(propertyId);
}
// --------------- END HANDLING METHODS ------------------------ //
// --------------- START ACTIONS ------------------------------- //
public void actionStore(CommandRequest request) {
Map parameters = request.getRequestObject().getParameterMap();
// Initialize parameters and properties to default.
showPropertyNames = false;
showRefreshButton = false;
showApplyButton = false;
showClearButton = false;
showSubmitOnChange = false;
isShortMode = false;
showLegend = false;
showAutoRefresh = false;
properties.clear();
notAllowedProperties.clear();
// Component options.
if (parameters.containsKey(PARAM_SHOW_REFRESH_BUTTON)) showRefreshButton = true;
if (parameters.containsKey(PARAM_SHOW_PROPERTY_NAMES)) showPropertyNames = true;
if (parameters.containsKey(PARAM_SHOW_CLEAR_BUTTON)) showClearButton = true;
if (parameters.containsKey(PARAM_SHOW_APPLY_BUTTON)) showApplyButton = true;
if (parameters.containsKey(PARAM_SHOW_SUBMIT_ON_CHANGE)) showSubmitOnChange = true;
if (parameters.containsKey(PARAM_SHORT_MODE)) isShortMode = true;
if (parameters.containsKey(PARAM_SHOW_LEGEND)) showLegend = true;
if (parameters.containsKey(PARAM_SHOW_AUTO_REFRESH)) showAutoRefresh = true;
// Component properties.
DashboardFilterProperty[] allProperties = getAllPropertiesForCurrentFilter();
for (int i = 0; i < allProperties.length; i++) {
DashboardFilterProperty property = allProperties[i];
String dataProviderCode = property.getDataProviderCode();
String propertyId = property.getPropertyId();
String visibleParamKey =
new StringBuffer()
.append(PARAM_VISIBLE)
.append("/")
.append(dataProviderCode)
.append("/")
.append(propertyId)
.toString();
String drillDownParamKey =
new StringBuffer()
.append(PARAM_SECTION)
.append("/")
.append(dataProviderCode)
.append("/")
.append(propertyId)
.toString();
boolean isVisible = parameters.containsKey(visibleParamKey);
Long sectionId = null;
if (parameters.containsKey(drillDownParamKey)) {
String sectionIdStr = ((String[]) parameters.get(drillDownParamKey))[0];
if (!PARAM_DRILLDOWN_DISABLED.equals(sectionIdStr)) sectionId = Long.decode(sectionIdStr);
}
if (!isVisible && sectionId == null) continue;
// Property must be added?
DashboardFilterProperty prop =
getDashboardFilterPropertyForCurrentFilter(dataProviderCode, propertyId);
if (prop == null)
prop = new DashboardFilterProperty(dataProviderCode, propertyId, getFilter(), null, false);
// Check if another property with same identifier.
if (getDashboardFilterProperty(propertyId) != null) {
// Another property with same id is already set to the filter.
// Filter cannot use two properties with same property id., so show warning.
notAllowedProperties.add(prop);
continue;
}
// Add property to this component.
properties.add(prop);
// Set property parameters
prop.setBeignFiltered(false);
prop.setVisible(isVisible);
prop.setSectionId(sectionId);
}
}
public CommandResponse actionFilter(CommandRequest request) {
try {
// Init attributes for start applying the filter.
filterPropertyErrors.clear();
// Parse parameters and set the filter.
Iterator visiblePropertiesIt = properties.iterator();
while (visiblePropertiesIt.hasNext()) {
DashboardFilterProperty dashboardFilterProperty =
(DashboardFilterProperty) visiblePropertiesIt.next();
// Is property already in the dashboard filter?. Then is not possible to filter by this
// property, it's already added to dashboard filter.
if (dashboardFilterProperty.isBeingFiltered()) continue;
if (!dashboardFilterProperty.isPropertyAlive()) {
log.warn(
"Trying to filter by "
+ dashboardFilterProperty.getPropertyId()
+ ". This property is not in any dataset.");
continue;
}
if (!dashboardFilterProperty.isVisible()) continue;
Object[] result;
try {
result =
requestProcessor.parseDashboardProperty(
request.getRequestObject(), dashboardFilterProperty);
} catch (Exception e) {
log.error("Error parsing property " + dashboardFilterProperty.getPropertyId() + ".", e);
continue;
}
if (result.length != 3) {
log.error(
"Error parsing property: '"
+ dashboardFilterProperty.getPropertyId()
+ "' for dataProvider: '"
+ dashboardFilterProperty.getDataProviderCode()
+ "'");
continue;
}
Collection allowedValues = (Collection) result[0];
Object minValue = result[1];
Object maxValue = result[2];
if (allowedValues == null && minValue == null && maxValue == null) continue;
// Set the filter with this property.
Dashboard currentDashboard = DashboardHandler.lookup().getCurrentDashboard();
if (currentDashboard.filter(
dashboardFilterProperty.getPropertyId(),
minValue,
true,
maxValue,
true,
allowedValues,
FilterByCriteria.ALLOW_ANY)) {
return new ShowCurrentScreenResponse();
}
}
} catch (Exception e) {
log.error("Error trying to filter properties for dashboard", e);
}
return null;
}
public void actionRefresh(CommandRequest request) {
try {
String timeOutValue = request.getRequestObject().getParameter("refreshTimeOut");
if (!StringUtils.isBlank(timeOutValue)) {
try {
autoRefreshTimeout = Integer.decode(timeOutValue).intValue();
} catch (NumberFormatException e) {
log.warn("Cannot parse auto refresh value as a number.");
}
}
getDashboard().refresh();
} catch (Exception e) {
log.error("Cannot refresh dashboard.", e);
}
}
public CommandResponse actionClear(CommandRequest request) {
try {
Dashboard dashboard = getDashboard();
if (dashboard.unfilter()) {
return new ShowCurrentScreenResponse();
}
} catch (Exception e) {
log.error("Cannot refresh dashboard.", e);
}
return null;
}
public CommandResponse actionDeleteFilteredProperty(CommandRequest request) {
String propertyToDelete = request.getRequestObject().getParameter("filteredPropertyToDelete");
if (propertyToDelete == null || propertyToDelete.trim().length() == 0) return null;
try {
Dashboard dashboard = getDashboard();
if (dashboard.unfilter(propertyToDelete)) {
return new ShowCurrentScreenResponse();
}
} catch (Exception e) {
log.error("Cannot remove filter property.", e);
}
return null;
}
public void actionPlay(CommandRequest request) {
setRefreshEnabled(true);
}
public void actionStop(CommandRequest request) {
setRefreshEnabled(false);
}
// --------- END ACTIONS ---------------------------------- //
// --------- SERIALIZATION / DESERIALIZATION -------------- //
public String serializeComponentData() throws Exception {
// Serialize visible properties and options.
StringWriter sw = new StringWriter();
PrintWriter out = new PrintWriter(sw);
int indent = 0;
printIndent(out, indent);
out.println("<dashboard_filter>");
Iterator it = properties.iterator();
while (it.hasNext()) {
DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
printIndent(out, indent + 1);
out.println(
"<property id=\""
+ StringEscapeUtils.escapeXml(dashboardFilterProperty.getPropertyId())
+ "\" providerCode =\""
+ StringEscapeUtils.escapeXml(dashboardFilterProperty.getDataProviderCode())
+ "\">");
printIndent(out, indent + 2);
out.println("<visible>" + dashboardFilterProperty.isVisible() + "</visible>");
if (dashboardFilterProperty.getSectionId() != null) {
printIndent(out, indent + 2);
out.println("<section>" + dashboardFilterProperty.getSectionId() + "</section>");
}
printIndent(out, indent + 1);
out.println("</property>");
}
// Serialize options.
printIndent(out, indent + 1);
out.println("<options>");
printIndent(out, indent + 2);
out.println("<shortViewMode>" + isShortMode + "</shortViewMode>");
printIndent(out, indent + 2);
out.println("<showLegend>" + showLegend + "</showLegend>");
printIndent(out, indent + 2);
out.println("<showRefreshButton>" + showRefreshButton + "</showRefreshButton>");
printIndent(out, indent + 2);
out.println("<showApplyhButton>" + showApplyButton + "</showApplyhButton>");
printIndent(out, indent + 2);
out.println("<showClearButton>" + showClearButton + "</showClearButton>");
printIndent(out, indent + 2);
out.println("<showPropertyNames>" + showPropertyNames + "</showPropertyNames>");
printIndent(out, indent + 2);
out.println("<showSubmitOnChange>" + showSubmitOnChange + "</showSubmitOnChange>");
printIndent(out, indent + 1);
out.println("<showAutoRefresh>" + showAutoRefresh + "</showAutoRefresh>");
printIndent(out, indent + 1);
out.println("</options>");
printIndent(out, indent);
out.println("</dashboard_filter>");
serializedProperties = sw.toString();
return sw.toString();
}
protected void printIndent(PrintWriter out, int indent) {
for (int i = 0; i < indent; i++) {
out.print(" ");
}
}
// Return if is needed to serialize and save properties after this call because properties that
// does not exist on current filter or data providers must be deleted from persistence.
// return: must clear serialized trash properties after deserialize process saving this data.
public boolean deserializeComponentData(String serializedData) throws Exception {
// Load options and visible properties
if (serializedData == null || serializedData.trim().length() == 0) {
log.info("No data to deserialize.");
return false;
}
DOMParser parser = new DOMParser();
parser.parse(new InputSource(new StringReader(serializedData)));
Document doc = parser.getDocument();
NodeList nodes = doc.getElementsByTagName("dashboard_filter");
if (nodes.getLength() > 1) {
log.error("Each dashboard filter component just can parse one <dashboard_filter>");
return false;
}
if (nodes.getLength() == 0) {
log.info("No data to deserialize.");
return false;
}
boolean needsToSerializeAfter = false;
serializedProperties = serializedData;
Node rootNode = nodes.item(0);
nodes = rootNode.getChildNodes();
for (int x = 0; x < nodes.getLength(); x++) {
Node node = nodes.item(x);
if (node.getNodeName().equals("property")) {
// Parse visible properties.
String dataProviderCode = node.getAttributes().getNamedItem("providerCode").getNodeValue();
String propertyId = node.getAttributes().getNamedItem("id").getNodeValue();
String sectionId = null;
boolean visible = false;
NodeList subnodes = node.getChildNodes();
for (int i = 0; i < subnodes.getLength(); i++) {
Node subnode = subnodes.item(i);
if (subnode.getNodeName().equals("section")) {
sectionId = subnode.getFirstChild().getNodeValue();
}
if (subnode.getNodeName().equals("visible")) {
visible = Boolean.valueOf(subnode.getFirstChild().getNodeValue()).booleanValue();
}
}
Long lSectionId = sectionId != null ? Long.decode(sectionId) : null;
DashboardFilterProperty filterProp =
new DashboardFilterProperty(
dataProviderCode, propertyId, getFilter(), lSectionId, true);
filterProp.setVisible(visible);
if (filterProp.isPropertyAlive()) properties.add(filterProp);
else needsToSerializeAfter = true;
} else if (node.getNodeName().equals("options")) {
// Parse component options.
NodeList options = node.getChildNodes();
String showRefreshButton = null;
String showPropertyNames = null;
String showClearButton = null;
String showApplyButton = null;
String showSubmitOnChange = null;
String showShortViewMode = null;
String showLegend = null;
String showAutoRefresh = null;
for (int i = 0; i < options.getLength(); i++) {
Node option = options.item(i);
if (option.getNodeName().equals("showRefreshButton"))
showRefreshButton = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("showPropertyNames"))
showPropertyNames = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("showClearButton"))
showClearButton = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("showApplyhButton"))
showApplyButton = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("showSubmitOnChange"))
showSubmitOnChange = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("shortViewMode"))
showShortViewMode = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("showLegend"))
showLegend = option.getFirstChild().getNodeValue();
if (option.getNodeName().equals("showAutoRefresh"))
showAutoRefresh = option.getFirstChild().getNodeValue();
}
this.showPropertyNames = Boolean.valueOf(showPropertyNames).booleanValue();
this.showRefreshButton = Boolean.valueOf(showRefreshButton).booleanValue();
this.showApplyButton = Boolean.valueOf(showApplyButton).booleanValue();
this.showClearButton = Boolean.valueOf(showClearButton).booleanValue();
this.showSubmitOnChange = Boolean.valueOf(showSubmitOnChange).booleanValue();
this.isShortMode = Boolean.valueOf(showShortViewMode).booleanValue();
this.showLegend = Boolean.valueOf(showLegend).booleanValue();
this.showAutoRefresh = Boolean.valueOf(showAutoRefresh).booleanValue();
// Enable auto-refresh if necessary on start.
if (this.showAutoRefresh) setRefreshEnabled(true);
}
}
return needsToSerializeAfter;
}
public void beforeRenderComponent() {
super.beforeRenderComponent();
// Initialize the dashboard (loads all its kpi panels)
DashboardHandler.lookup().getCurrentDashboard();
// Get the filter.
DashboardFilter filter = getFilter();
// Check all visible properties exist.
Iterator props = properties.iterator();
while (props.hasNext()) {
DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) props.next();
if (!dashboardFilterProperty.isPropertyAlive()) props.remove();
}
// Check if filtered properties for this filter component are already in dashboard filter.
DashboardFilterProperty[] beingFilteredProps = getBeingFilteredProperties();
for (int i = 0; i < beingFilteredProps.length; i++) {
List dfProperties = Arrays.asList(filter.getPropertyIds());
DashboardFilterProperty beignFilteredProp = beingFilteredProps[i];
if (!dfProperties.contains(beignFilteredProp.getPropertyId()))
beignFilteredProp.setBeignFiltered(false);
}
// Check filtered properties and hide from available filter properties (set property not
// visible)
String[] propIds = filter.getPropertyIds();
for (int i = 0; i < propIds.length; i++) {
String propId = propIds[i];
DashboardFilterProperty prop = getDashboardFilterProperty(propId);
if (prop == null) {
DashboardFilterProperty parentProperty = filter.getPropertyInParentDashboards(propId);
if (parentProperty != null) {
prop =
new DashboardFilterProperty(
parentProperty.getDataProviderCode(), propId, getFilter(), null, true);
properties.add(prop);
}
} else {
prop.setBeignFiltered(true);
}
}
}
}
/**
* Helper class for using the netcdf-3 record dimension.
*
* @author caron
* @since Feb 29, 2008
*/
public class RecordDatasetHelper {
private static org.slf4j.Logger log =
org.slf4j.LoggerFactory.getLogger(RecordDatasetHelper.class);
protected NetcdfDataset ncfile;
protected String obsTimeVName, nomTimeVName;
protected String latVName, lonVName, zcoordVName, zcoordUnits;
protected String stnIdVName, stnIndexVName, stnDescVName;
protected StationHelper stationHelper;
protected DataType stationIdType;
protected StructureDS recordVar;
protected Dimension obsDim;
protected LatLonRect boundingBox;
protected double minDate, maxDate;
protected DateUnit timeUnit;
protected double altScaleFactor = 1.0;
protected Formatter errs = null;
protected boolean showErrors = true;
/**
* Constructor.
*
* @param ncfile the netccdf file
* @param typedDataVariables list of data variables; all record variables will be added to this
* list, except . You can remove extra
* @param obsTimeVName observation time variable name (required)
* @param nomTimeVName nominal time variable name (may be null)
* @throws IllegalArgumentException if ncfile has no unlimited dimension and recDimName is null.
*/
public RecordDatasetHelper(
NetcdfDataset ncfile,
String obsTimeVName,
String nomTimeVName,
List<VariableSimpleIF> typedDataVariables,
String recDimName,
Formatter errBuffer) {
this.ncfile = ncfile;
this.obsTimeVName = obsTimeVName;
this.nomTimeVName = nomTimeVName;
this.errs = errBuffer;
// check if we already have a structure vs if we have to add it.
if (this.ncfile.hasUnlimitedDimension()) {
this.ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
this.recordVar = (StructureDS) this.ncfile.getRootGroup().findVariable("record");
this.obsDim = ncfile.getUnlimitedDimension();
} else {
if (recDimName == null)
throw new IllegalArgumentException(
"File <"
+ this.ncfile.getLocation()
+ "> has no unlimited dimension, specify psuedo record dimension with observationDimension global attribute.");
this.obsDim = this.ncfile.getRootGroup().findDimension(recDimName);
this.recordVar = new StructurePseudoDS(this.ncfile, null, "record", null, obsDim);
}
// create member variables
List<Variable> recordMembers = ncfile.getVariables();
for (Variable v : recordMembers) {
if (v == recordVar) continue;
if (v.isScalar()) continue;
if (v.getDimension(0) == this.obsDim) typedDataVariables.add(v);
}
// need the time units
Variable timeVar = ncfile.findVariable(obsTimeVName);
String timeUnitString =
ncfile.findAttValueIgnoreCase(timeVar, CDM.UNITS, "seconds since 1970-01-01");
try {
timeUnit = new DateUnit(timeUnitString);
} catch (Exception e) {
if (null != errs) errs.format("Error on string = %s == %s%n", timeUnitString, e.getMessage());
try {
timeUnit = new DateUnit("seconds since 1970-01-01");
} catch (Exception e1) {
// cant happen
}
}
}
/**
* Set extra information used by station obs datasets. Use stnIdVName or stnIndexVName.
*
* @param stnIdVName the obs variable that is used to find the station in the stnHash; may be type
* int or a String (char).
* @param stnDescVName optional station var containing station description
*/
public void setStationInfo(
String stnIdVName, String stnDescVName, String stnIndexVName, StationHelper stationHelper) {
this.stnIdVName = stnIdVName;
this.stnDescVName = stnDescVName;
this.stnIndexVName = stnIndexVName;
this.stationHelper = stationHelper;
if (stnIdVName != null) {
Variable stationVar = ncfile.findVariable(stnIdVName);
stationIdType = stationVar.getDataType();
}
}
public void setLocationInfo(String latVName, String lonVName, String zcoordVName) {
this.latVName = latVName;
this.lonVName = lonVName;
this.zcoordVName = zcoordVName;
// check for meter conversion
if (zcoordVName != null) {
Variable v = ncfile.findVariable(zcoordVName);
zcoordUnits = ncfile.findAttValueIgnoreCase(v, CDM.UNITS, null);
if (zcoordUnits != null)
try {
altScaleFactor = getMetersConversionFactor(zcoordUnits);
} catch (Exception e) {
if (errs != null) errs.format("%s", e.getMessage());
}
}
}
// make structure variable names to shortNames so StructureData sdata can
// access it members
public void setShortNames(
String latVName, String lonVName, String altVName, String obsTimeVName, String nomTimeVName) {
this.latVName = latVName;
this.lonVName = lonVName;
this.zcoordVName = altVName;
this.obsTimeVName = obsTimeVName;
this.nomTimeVName = nomTimeVName;
}
protected static double getMetersConversionFactor(String unitsString) throws Exception {
SimpleUnit unit = SimpleUnit.factoryWithExceptions(unitsString);
return unit.convertTo(1.0, SimpleUnit.meterUnit);
}
public Structure getRecordVar() {
return (this.recordVar);
}
public int getRecordCount() {
Dimension unlimitedDim = ncfile.getUnlimitedDimension();
return unlimitedDim.getLength();
}
public void setTimeUnit(DateUnit timeUnit) {
this.timeUnit = timeUnit;
}
public DateUnit getTimeUnit() {
return this.timeUnit;
}
public LatLonPoint getLocation(StructureData sdata) {
StructureMembers members = sdata.getStructureMembers();
double lat = sdata.convertScalarDouble(members.findMember(latVName));
double lon = sdata.convertScalarDouble(members.findMember(lonVName));
return new LatLonPointImpl(lat, lon);
}
public double getLatitude(StructureData sdata) {
StructureMembers members = sdata.getStructureMembers();
return sdata.convertScalarDouble(members.findMember(latVName));
}
public double getLongitude(StructureData sdata) {
StructureMembers members = sdata.getStructureMembers();
return sdata.convertScalarDouble(members.findMember(lonVName));
}
public double getZcoordinate(StructureData sdata) {
StructureMembers members = sdata.getStructureMembers();
return (zcoordVName == null)
? Double.NaN
: sdata.convertScalarDouble(members.findMember(zcoordVName));
}
public String getZcoordUnits() {
return zcoordUnits;
}
public Date getObservationTimeAsDate(StructureData sdata) {
return timeUnit.makeDate(getObservationTime(sdata));
}
public double getObservationTime(StructureData sdata) {
return getTime(sdata.findMember(obsTimeVName), sdata);
}
private double getTime(StructureMembers.Member timeVar, StructureData sdata) {
if (timeVar == null) return 0.0;
if ((timeVar.getDataType() == DataType.CHAR) || (timeVar.getDataType() == DataType.STRING)) {
String time = sdata.getScalarString(timeVar);
CalendarDate date = CalendarDateFormatter.isoStringToCalendarDate(null, time);
if (date == null) {
log.error("Cant parse date - not ISO formatted, = " + time);
return 0.0;
}
return date.getMillis() / 1000.0;
} else {
return sdata.convertScalarDouble(timeVar);
}
}
/*
* This reads through all the records in the dataset, and constructs a list of
* RecordPointObs or RecordStationObs. It does not cache the data.
* <p>If stnIdVName is not null, its a StationDataset, then construct a Station HashMap of StationImpl
* objects. Add the RecordStationObs into the list of obs for that station.
*
* @param cancel allow user to cancel
* @return List of RecordPointObs or RecordStationObs
* @throws IOException on read error
*
public List<RecordPointObs> readAllCreateObs(CancelTask cancel) throws IOException {
// see if its a station or point dataset
boolean hasStations = stnIdVName != null;
if (hasStations)
stnHash = new HashMap<Object, Station>();
// get min and max date and lat,lon
double minDate = Double.MAX_VALUE;
double maxDate = -Double.MAX_VALUE;
double minLat = Double.MAX_VALUE;
double maxLat = -Double.MAX_VALUE;
double minLon = Double.MAX_VALUE;
double maxLon = -Double.MAX_VALUE;
// read all the data, create a RecordObs
StructureMembers members = null;
List<RecordPointObs> records = new ArrayList<RecordPointObs>();
int recno = 0;
Structure.Iterator ii = recordVar.getStructureIterator();
while (ii.hasNext()) {
StructureData sdata = ii.next();
if (members == null)
members = sdata.getStructureMembers();
Object stationId = null;
if (hasStations) {
if (stationIdType == DataType.INT) {
stationId = sdata.getScalarInt(stnIdVName);
} else
stationId = sdata.getScalarString(stnIdVName).trim();
}
String desc = (stnDescVName == null) ? null : sdata.getScalarString(stnDescVName);
double lat = sdata.getScalarDouble(latVName);
double lon = sdata.getScalarDouble(lonVName);
double alt = (altVName == null) ? 0.0 : altScaleFactor * sdata.getScalarDouble(altVName);
double obsTime = sdata.convertScalarDouble(members.findMember(obsTimeVName));
double nomTime = (nomTimeVName == null) ? obsTime : sdata.convertScalarDouble(members.findMember(nomTimeVName));
//double obsTime = sdata.convertScalarDouble( members.findMember( obsTimeVName) );
//double nomTime = (nomTimeVName == null) ? obsTime : sdata.convertScalarDouble( members.findMember( nomTimeVName));
if (hasStations) {
Station stn = stnHash.get(stationId);
if (stn == null) {
stn = new Station(stationId.toString(), desc, lat, lon, alt);
stnHash.put(stationId, stn);
}
RecordStationObs stnObs = new RecordStationObs(stn, obsTime, nomTime, timeUnit, recno);
records.add(stnObs);
//stn.addObs( stnObs);
} else {
records.add(new RecordPointObs(new EarthLocation(lat, lon, alt), obsTime, nomTime, timeUnit, recno));
}
// track date range and bounding box
minDate = Math.min(minDate, obsTime);
maxDate = Math.max(maxDate, obsTime);
minLat = Math.min(minLat, lat);
maxLat = Math.max(maxLat, lat);
minLon = Math.min(minLon, lon);
maxLon = Math.max(maxLon, lon);
recno++;
if ((cancel != null) && cancel.isCancel()) return null;
}
boundingBox = new LatLonRect(new LatLonPointImpl(minLat, minLon), new LatLonPointImpl(maxLat, maxLon));
return records;
}
/* private boolean debugBB = false;
public List getData(ArrayList records, LatLonRect boundingBox, CancelTask cancel) throws IOException {
if (debugBB) System.out.println("Want bb= "+boundingBox);
ArrayList result = new ArrayList();
for (int i = 0; i < records.size(); i++) {
RecordDatasetHelper.RecordPointObs r = (RecordDatasetHelper.RecordPointObs) records.get(i);
if (boundingBox.contains(r.getLatLon())) {
if (debugBB) System.out.println(" ok latlon= "+r.getLatLon());
result.add( r);
}
if ((cancel != null) && cancel.isCancel()) return null;
}
return result;
}
// return List<PointObsDatatype>
public List getData(ArrayList records, LatLonRect boundingBox, double startTime, double endTime, CancelTask cancel) throws IOException {
if (debugBB) System.out.println("Want bb= "+boundingBox);
ArrayList result = new ArrayList();
for (int i = 0; i < records.size(); i++) {
RecordDatasetHelper.RecordPointObs r = (RecordDatasetHelper.RecordPointObs) records.get(i);
if (boundingBox.contains(r.getLatLon())) {
if (debugBB) System.out.println(" ok latlon= "+r.getLatLon());
double timeValue = r.getObservationTime();
if ((timeValue >= startTime) && (timeValue <= endTime))
result.add( r);
}
if ((cancel != null) && cancel.isCancel()) return null;
}
return result;
} */
//////////////////////////////////////////////////////////////////////////////////////
public PointFeature factory(StationImpl s, StructureData sdata, int recno) {
if (s == null) return new RecordPointObs(sdata, recno);
else return new RecordStationObs(s, sdata, recno);
}
class RecordPointObs extends PointFeatureImpl {
protected int recno;
protected StructureData sdata;
RecordPointObs(int recno) {
super(RecordDatasetHelper.this.timeUnit);
this.recno = recno;
}
// Constructor for the case where you keep track of the location, time of each record, but not
// the data.
protected RecordPointObs(
EarthLocation location, double obsTime, double nomTime, DateUnit timeUnit, int recno) {
super(location, obsTime, nomTime, timeUnit);
this.recno = recno;
}
// Constructor for when you already have the StructureData and want to wrap it in a
// StationObsDatatype
protected RecordPointObs(StructureData sdata, int recno) {
super(RecordDatasetHelper.this.timeUnit);
this.sdata = sdata;
this.recno = recno;
StructureMembers members = sdata.getStructureMembers();
obsTime = getTime(members.findMember(obsTimeVName), sdata);
nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);
// this assumes the lat/lon/alt is stored in the obs record
double lat = sdata.convertScalarDouble(members.findMember(latVName));
double lon = sdata.convertScalarDouble(members.findMember(lonVName));
double alt =
(zcoordVName == null)
? 0.0
: altScaleFactor * sdata.convertScalarDouble(members.findMember(zcoordVName));
location = new EarthLocationImpl(lat, lon, alt);
}
public String getId() {
return Integer.toString(recno);
}
public LatLonPoint getLatLon() {
return new LatLonPointImpl(location.getLatitude(), location.getLongitude());
}
public StructureData getFeatureData() throws IOException {
if (null == sdata) {
try {
// deal with files that are updating // LOOK kludge?
if (recno > getRecordCount()) {
int n = getRecordCount();
ncfile.syncExtend();
log.info(
"RecordPointObs.getData recno="
+ recno
+ " > "
+ n
+ "; after sync= "
+ getRecordCount());
}
sdata = recordVar.readStructure(recno);
} catch (ucar.ma2.InvalidRangeException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
}
return sdata;
}
public ucar.ma2.StructureData getDataAll() throws java.io.IOException {
return getFeatureData();
}
}
//////////////////////////////////////////////////////////////////////////////////////
// a PointObs with the location info stored as a Station
class RecordStationObs extends RecordPointObs {
private Station station;
/**
* Constructor for the case where you keep track of the station, time of each record, but the
* data reading is deferred.
*
* @param station data is for this Station
* @param obsTime observation time
* @param nomTime nominal time (may be NaN)
* @param recno data is at this record number
*/
protected RecordStationObs(
Station station, double obsTime, double nomTime, DateUnit timeUnit, int recno) {
super(station, obsTime, nomTime, timeUnit, recno);
this.station = station;
}
// Constructor for when you have everything
protected RecordStationObs(
Station station, double obsTime, double nomTime, StructureData sdata, int recno) {
super(recno);
this.station = station;
this.location = station;
this.obsTime = obsTime;
this.nomTime = nomTime;
this.sdata = sdata;
}
// Constructor for when you already have the StructureData and Station, and calculate times
protected RecordStationObs(Station station, StructureData sdata, int recno) {
super(recno);
this.station = station;
this.location = station;
this.sdata = sdata;
StructureMembers members = sdata.getStructureMembers();
obsTime = getTime(members.findMember(obsTimeVName), sdata);
nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);
}
// Constructor for when you already have the StructureData, and need to find Station and times
protected RecordStationObs(StructureData sdata, int recno, boolean useId) {
super(recno);
this.recno = recno;
this.sdata = sdata;
this.timeUnit = RecordDatasetHelper.this.timeUnit;
StructureMembers members = sdata.getStructureMembers();
obsTime = getTime(members.findMember(obsTimeVName), sdata);
nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);
if (useId) {
// this assumes the station id/name is stored in the obs record
String stationId;
if (stationIdType == DataType.INT) {
stationId = Integer.toString(sdata.getScalarInt(stnIdVName));
} else stationId = sdata.getScalarString(stnIdVName).trim();
station = stationHelper.getStation(stationId);
if (null != errs)
errs.format(" cant find station id = <%s> when reading record %d%n", stationId, recno);
log.error(" cant find station id = <" + stationId + "> when reading record " + recno);
} else {
// use a station index
List<Station> stations = stationHelper.getStations();
int stationIndex = sdata.getScalarInt(stnIndexVName);
if (stationIndex < 0 || stationIndex >= stations.size()) {
if (null != errs)
errs.format(
" cant find station at index =%d when reading record %d%n", stationIndex, recno);
log.error(
"cant find station at index = " + stationIndex + " when reading record " + recno);
} else station = stations.get(stationIndex);
}
location = station;
}
}
}
/**
* Build a GribCollection object. Rectilyse and manage grib collection index. Covers
* GribCollectionProto.
*
* @author caron
* @since 4/6/11
*/
public class Grib2CollectionBuilder {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(GribCollection.class);
public static final String MAGIC_START = "Grib2CollectionIndex";
protected static final int version = 5;
private static final boolean debug = false;
// from a single file, read in the index, create if it doesnt exist
public static GribCollection createFromSingleFile(
File file, CollectionManager.Force force, Formatter f) throws IOException {
Grib2CollectionBuilder builder = new Grib2CollectionBuilder(file, f);
builder.readOrCreateIndex(force, f);
return builder.gc;
}
// from a collection, read in the index, create if it doesnt exist or is out of date
// assume that the CollectionManager is up to date, eg doesnt need to be scanned
public static GribCollection factory(
CollectionManager dcm, CollectionManager.Force force, Formatter f) throws IOException {
Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm);
builder.readOrCreateIndex(force, f);
return builder.gc;
}
// read in the index, index raf already open
public static GribCollection createFromIndex(String name, File directory, RandomAccessFile raf)
throws IOException {
Grib2CollectionBuilder builder = new Grib2CollectionBuilder(name, directory);
if (builder.readIndex(raf)) return builder.gc;
throw new IOException("Reading index failed");
}
// this writes the index always
public static boolean writeIndexFile(File indexFile, CollectionManager dcm, Formatter f)
throws IOException {
Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm);
return builder.createIndex(indexFile, CollectionManager.Force.always, f);
}
////////////////////////////////////////////////////////////////
private final List<CollectionManager> collections = new ArrayList<CollectionManager>();
protected GribCollection gc;
// single file
private Grib2CollectionBuilder(File file, Formatter f) throws IOException {
try {
// String spec = StringUtil2.substitute(file.getPath(), "\\", "/");
CollectionManager dcm = new DatasetCollectionSingleFile(file);
this.collections.add(dcm);
this.gc = new Grib2Collection(file.getName(), new File(dcm.getRoot()));
} catch (Exception e) {
ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
e.printStackTrace(new PrintStream(bos));
f.format("%s", bos.toString());
throw new IOException(e);
}
}
private Grib2CollectionBuilder(CollectionManager dcm) {
this.collections.add(dcm);
this.gc = new Grib2Collection(dcm.getCollectionName(), new File(dcm.getRoot()));
}
private Grib2CollectionBuilder(String name, File directory) {
this.gc = new Grib2Collection(name, directory);
}
protected Grib2CollectionBuilder() {
this.gc = null;
}
protected int getVersion() {
return version;
}
// read or create index
private void readOrCreateIndex(CollectionManager.Force ff, Formatter f) throws IOException {
// force new index or test for new index needed
boolean force =
((ff == CollectionManager.Force.always)
|| (ff == CollectionManager.Force.test && needsUpdate()));
// otherwise, we're good as long as the index file exists
File idx = gc.getIndexFile();
if (force || !idx.exists() || !readIndex(idx.getPath())) {
logger.info("GribCollection {}: createIndex {}", gc.getName(), idx.getPath());
createIndex(idx, ff, f); // write out index
gc.rafLocation = idx.getPath();
gc.setRaf(new RandomAccessFile(idx.getPath(), "r"));
readIndex(gc.getRaf()); // read back in index
}
}
public boolean needsUpdate() {
File idx = gc.getIndexFile();
return !idx.exists() || needsUpdate(idx.lastModified());
}
private boolean needsUpdate(long idxLastModified) {
CollectionManager.ChangeChecker cc = Grib2Index.getChangeChecker();
for (CollectionManager dcm : collections) {
for (MFile mfile : dcm.getFiles()) {
if (cc.hasChangedSince(mfile, idxLastModified)) return true;
}
}
return false;
}
////////////////////////////////////////////////////////////////////////////////////////////////////
// reading
public boolean readIndex(String filename) throws IOException {
return readIndex(new RandomAccessFile(filename, "r"));
}
public boolean readIndex(RandomAccessFile raf) {
gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
try {
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
//// header message
if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
logger.error("GribCollection {}: invalid index", gc.getName());
return false;
}
int v = raf.readInt();
if (v != getVersion()) {
logger.warn(
"GribCollection {}: index found version={}, want version= {} on file {}",
new Object[] {gc.getName(), v, version, raf.getLocation()});
return false;
}
long skip = raf.readLong();
raf.skipBytes(skip);
int size = NcStream.readVInt(raf);
if ((size < 0) || (size > 100 * 1000 * 1000)) {
logger.warn("GribCollection {}: invalid index ", gc.getName());
return false;
}
byte[] m = new byte[size];
raf.readFully(m);
GribCollectionProto.GribCollectionIndex proto =
GribCollectionProto.GribCollectionIndex.parseFrom(m);
gc.center = proto.getCenter();
gc.subcenter = proto.getSubcenter();
gc.master = proto.getMaster();
gc.local = proto.getLocal();
gc.genProcessType = proto.getGenProcessType();
gc.genProcessId = proto.getGenProcessId();
gc.backProcessId = proto.getBackProcessId();
gc.local = proto.getLocal();
// gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);
gc.filenames = new ArrayList<String>(proto.getFilesCount());
for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));
// error condition on a GribCollection Index
if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
return false;
}
gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
for (int i = 0; i < proto.getGroupsCount(); i++)
gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
Collections.sort(gc.groups);
gc.params = new ArrayList<Parameter>(proto.getParamsCount());
for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));
if (!readPartitions(proto)) {
logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
return false;
}
return true;
} catch (Throwable t) {
logger.error("Error reading index " + raf.getLocation(), t);
return false;
}
}
protected boolean readPartitions(GribCollectionProto.GribCollectionIndex proto) {
return true;
}
protected void readTimePartitions(
GribCollection.GroupHcs group, GribCollectionProto.Group proto) {
// NOOP
}
GribCollection.GroupHcs readGroup(GribCollectionProto.Group p, GribCollection.GroupHcs group)
throws IOException {
Grib2SectionGridDefinition gdss = new Grib2SectionGridDefinition(p.getGds().toByteArray());
Grib2Gds gds = gdss.getGDS();
group.setHorizCoordSystem(gds.makeHorizCoordSys());
group.varIndex = new ArrayList<GribCollection.VariableIndex>();
for (int i = 0; i < p.getVariablesCount(); i++)
group.varIndex.add(readVariable(p.getVariables(i), group));
Collections.sort(group.varIndex);
group.timeCoords = new ArrayList<TimeCoord>(p.getTimeCoordsCount());
for (int i = 0; i < p.getTimeCoordsCount(); i++)
group.timeCoords.add(readTimeCoord(p.getTimeCoords(i)));
group.vertCoords = new ArrayList<VertCoord>(p.getVertCoordsCount());
for (int i = 0; i < p.getVertCoordsCount(); i++)
group.vertCoords.add(readVertCoord(p.getVertCoords(i)));
group.ensCoords = new ArrayList<EnsCoord>(p.getEnsCoordsCount());
for (int i = 0; i < p.getEnsCoordsCount(); i++)
group.ensCoords.add(readEnsCoord(p.getEnsCoords(i)));
group.filenose = new int[p.getFilenoCount()];
for (int i = 0; i < p.getFilenoCount(); i++) group.filenose[i] = p.getFileno(i);
readTimePartitions(group, p);
// finish
for (GribCollection.VariableIndex vi : group.varIndex) {
TimeCoord tc = group.timeCoords.get(vi.timeIdx);
vi.ntimes = tc.getSize();
VertCoord vc = (vi.vertIdx < 0) ? null : group.vertCoords.get(vi.vertIdx);
vi.nverts = (vc == null) ? 0 : vc.getSize();
EnsCoord ec = (vi.ensIdx < 0) ? null : group.ensCoords.get(vi.ensIdx);
vi.nens = (ec == null) ? 0 : ec.getSize();
}
// group.assignVertNames();
return group;
}
private Parameter readParam(GribCollectionProto.Parameter pp) throws IOException {
if (pp.hasSdata()) return new Parameter(pp.getName(), pp.getSdata());
int count = 0;
double[] vals = new double[pp.getDataCount()];
for (double val : pp.getDataList()) vals[count++] = val;
return new Parameter(pp.getName(), vals);
}
private TimeCoord readTimeCoord(GribCollectionProto.Coord pc) throws IOException {
if (pc.getBoundCount() > 0) { // its an interval
List<TimeCoord.Tinv> coords = new ArrayList<TimeCoord.Tinv>(pc.getValuesCount());
for (int i = 0; i < pc.getValuesCount(); i++)
coords.add(new TimeCoord.Tinv((int) pc.getValues(i), (int) pc.getBound(i)));
return new TimeCoord(pc.getCode(), pc.getUnit(), coords);
} else {
List<Integer> coords = new ArrayList<Integer>(pc.getValuesCount());
for (float value : pc.getValuesList()) coords.add((int) value);
return new TimeCoord(pc.getCode(), pc.getUnit(), coords);
}
}
private VertCoord readVertCoord(GribCollectionProto.Coord pc) throws IOException {
boolean isLayer = (pc.getBoundCount() > 0);
List<VertCoord.Level> coords = new ArrayList<VertCoord.Level>(pc.getValuesCount());
for (int i = 0; i < pc.getValuesCount(); i++)
coords.add(new VertCoord.Level(pc.getValues(i), isLayer ? pc.getBound(i) : 0));
return new VertCoord(pc.getCode(), coords, isLayer);
}
private EnsCoord readEnsCoord(GribCollectionProto.Coord pc) throws IOException {
List<EnsCoord.Coord> coords = new ArrayList<EnsCoord.Coord>(pc.getValuesCount());
for (int i = 0; i < pc.getValuesCount(); i += 2)
coords.add(new EnsCoord.Coord((int) pc.getValues(i), (int) pc.getValues(i + 1)));
return new EnsCoord(coords);
}
protected GribCollection.VariableIndex readVariable(
GribCollectionProto.Variable pv, GribCollection.GroupHcs group) {
int discipline = pv.getDiscipline();
int category = pv.getCategory();
int param = pv.getParameter();
int levelType = pv.getLevelType();
int intvType = pv.getIntervalType();
boolean isLayer = pv.getIsLayer();
int ensDerivedType = pv.getEnsDerivedType();
int probType = pv.getProbabilityType();
String probabilityName = pv.getProbabilityName();
int cdmHash = pv.getCdmHash();
long recordsPos = pv.getRecordsPos();
int recordsLen = pv.getRecordsLen();
int timeIdx = pv.getTimeIdx();
int vertIdx = pv.getVertIdx();
int ensIdx = pv.getEnsIdx();
int tableVersion = pv.getTableVersion();
return gc.makeVariableIndex(
group,
tableVersion,
discipline,
category,
param,
levelType,
isLayer,
intvType,
ensDerivedType,
probType,
probabilityName,
cdmHash,
timeIdx,
vertIdx,
ensIdx,
recordsPos,
recordsLen);
}
///////////////////////////////////////////////////////////////////////////////////
// writing
private class Group {
public Grib2SectionGridDefinition gdss;
public int gdsHash; // may have been modified
public Grib2Rectilyser rect;
public List<Grib2Record> records = new ArrayList<Grib2Record>();
public String name;
public Set<Integer>
fileSet; // this is so we can show just the component files that are in this group
private Group(Grib2SectionGridDefinition gdss, int gdsHash) {
this.gdss = gdss;
this.gdsHash = gdsHash;
Grib2Gds gds = gdss.getGDS();
name = gds.getNameShort() + "-" + gds.ny + "X" + gds.nx;
}
}
///////////////////////////////////////////////////
// create the index
private boolean createIndex(File indexFile, CollectionManager.Force ff, Formatter f)
throws IOException {
long start = System.currentTimeMillis();
ArrayList<String> filenames = new ArrayList<String>();
List<Group> groups = makeAggregatedGroups(filenames, ff, f);
createIndex(indexFile, groups, filenames, f);
long took = System.currentTimeMillis() - start;
f.format("That took %d msecs%n", took);
return true;
}
// read all records in all files,
// divide into groups based on GDS hash
// each group has an arraylist of all records that belong to it.
// for each group, run rectlizer to derive the coordinates and variables
public List<Group> makeAggregatedGroups(
ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();
f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
int total = 0;
int fileno = 0;
for (CollectionManager dcm : collections) {
// dcm.scanIfNeeded(); // LOOK ??
f.format(" dcm= %s%n", dcm);
Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash");
for (MFile mfile : dcm.getFiles()) {
// f.format("%3d: %s%n", fileno, mfile.getPath());
filenames.add(mfile.getPath());
Grib2Index index = new Grib2Index();
try {
if (!index.readIndex(
mfile.getPath(),
mfile.getLastModified(),
force)) { // heres where the index date is checked against the data file
index.makeIndex(mfile.getPath(), f);
f.format(
" Index written: %s == %d records %n",
mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
} else if (debug) {
f.format(
" Index read: %s == %d records %n",
mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
}
} catch (IOException ioe) {
f.format(
"GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n skipping %s%n",
ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT);
continue;
}
for (Grib2Record gr : index.getRecords()) {
gr.setFile(fileno); // each record tracks which file it belongs to
int gdsHash =
gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
if (gdsConvert != null
&& gdsConvert.get(gdsHash)
!= null) { // allow external config to muck with gdsHash. Why? because of error in
// encoding
gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
}
Group g = gdsMap.get(gdsHash);
if (g == null) {
g = new Group(gr.getGDSsection(), gdsHash);
gdsMap.put(gdsHash, g);
}
g.records.add(gr);
total++;
}
fileno++;
}
}
f.format(" total grib records= %d%n", total);
Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter();
List<Group> result = new ArrayList<Group>(gdsMap.values());
for (Group g : result) {
g.rect = new Grib2Rectilyser(g.records, g.gdsHash);
f.format(" GDS hash %d == ", g.gdsHash);
g.rect.make(f, c);
}
f.format(
" Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);
return result;
}
/*
MAGIC_START
version
sizeRecords
VariableRecords (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
*/
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}
/* private void createIndexForGroup(Group group, ArrayList<String> filenames) throws IOException {
Grib2Record first = null; // take global metadata from here
File file = new File(gc.getDirectory(), group.name + GribCollection.IDX_EXT);
if (file.exists()) file.delete(); // replace it
RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
String magic = gc.getMagicBytes();
raf.write(magic.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
group.fileSet = new HashSet<Integer>();
for (Rectilyser.VariableBag vb : group.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = makeRecordsProto(vb, group.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
}
countRecords += group.records.size();
if (countRecords == 0) countRecords = 1;
long bytesPerRecord = countBytes / countRecords;
logger.debug("VariableRecords: bytes = {} record = {} bytesPerRecord={}", new Object[] {countBytes, countRecords, bytesPerRecord});
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder = GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(group.name);
for (String fn : filenames)
indexBuilder.addFiles(fn);
indexBuilder.addGroups(makeGroupProto(group));
int count = 0;
for (CollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.toString())));
count++;
}
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
logger.debug("GribCollectionIndex= {} bytes%n", b.length);
} finally {
logger.debug("file size = {} bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
} */
private GribCollectionProto.VariableRecords writeRecordsProto(
Grib2Rectilyser.VariableBag vb, Set<Integer> fileSet) throws IOException {
GribCollectionProto.VariableRecords.Builder b =
GribCollectionProto.VariableRecords.newBuilder();
b.setCdmHash(vb.first.cdmVariableHash(0));
for (Grib2Rectilyser.Record ar : vb.recordMap) {
GribCollectionProto.Record.Builder br = GribCollectionProto.Record.newBuilder();
if (ar == null || ar.gr == null) {
br.setFileno(0);
br.setPos(0); // missing : ok to use 0 since drsPos > 0
} else {
br.setFileno(ar.gr.getFile());
fileSet.add(ar.gr.getFile());
Grib2SectionDataRepresentation drs = ar.gr.getDataRepresentationSection();
br.setPos(drs.getStartingPosition());
}
b.addRecords(br);
}
return b.build();
}
private GribCollectionProto.Group writeGroupProto(Group g) throws IOException {
GribCollectionProto.Group.Builder b = GribCollectionProto.Group.newBuilder();
b.setGds(ByteString.copyFrom(g.gdss.getRawBytes()));
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars())
b.addVariables(writeVariableProto(vb));
List<TimeCoord> timeCoords = g.rect.getTimeCoords();
for (int i = 0; i < timeCoords.size(); i++)
b.addTimeCoords(writeCoordProto(timeCoords.get(i), i));
List<VertCoord> vertCoords = g.rect.getVertCoords();
for (int i = 0; i < vertCoords.size(); i++)
b.addVertCoords(writeCoordProto(vertCoords.get(i), i));
List<EnsCoord> ensCoords = g.rect.getEnsCoords();
for (int i = 0; i < ensCoords.size(); i++) b.addEnsCoords(writeCoordProto(ensCoords.get(i), i));
for (Integer aFileSet : g.fileSet) b.addFileno(aFileSet);
return b.build();
}
private GribCollectionProto.Variable writeVariableProto(Grib2Rectilyser.VariableBag vb)
throws IOException {
GribCollectionProto.Variable.Builder b = GribCollectionProto.Variable.newBuilder();
b.setDiscipline(vb.first.getDiscipline());
Grib2Pds pds = vb.first.getPDS();
b.setCategory(pds.getParameterCategory());
b.setParameter(pds.getParameterNumber());
b.setLevelType(pds.getLevelType1());
b.setIsLayer(Grib2Utils.isLayer(vb.first));
b.setIntervalType(pds.getStatisticalProcessType());
b.setCdmHash(vb.first.cdmVariableHash(0));
b.setRecordsPos(vb.pos);
b.setRecordsLen(vb.length);
b.setTimeIdx(vb.timeCoordIndex);
if (vb.vertCoordIndex >= 0) b.setVertIdx(vb.vertCoordIndex);
if (vb.ensCoordIndex >= 0) b.setEnsIdx(vb.ensCoordIndex);
if (pds.isEnsembleDerived()) {
Grib2Pds.PdsEnsembleDerived pdsDerived = (Grib2Pds.PdsEnsembleDerived) pds;
b.setEnsDerivedType(pdsDerived.getDerivedForecastType()); // derived type (table 4.7)
}
if (pds.isProbability()) {
Grib2Pds.PdsProbability pdsProb = (Grib2Pds.PdsProbability) pds;
b.setProbabilityName(pdsProb.getProbabilityName());
b.setProbabilityType(pdsProb.getProbabilityType());
}
return b.build();
}
protected GribCollectionProto.Parameter writeParamProto(Parameter param) throws IOException {
GribCollectionProto.Parameter.Builder b = GribCollectionProto.Parameter.newBuilder();
b.setName(param.getName());
if (param.isString()) b.setSdata(param.getStringValue());
else {
for (int i = 0; i < param.getLength(); i++) b.addData(param.getNumericValue(i));
}
return b.build();
}
protected GribCollectionProto.Coord writeCoordProto(TimeCoord tc, int index) throws IOException {
GribCollectionProto.Coord.Builder b = GribCollectionProto.Coord.newBuilder();
b.setCode(index);
b.setUnit(tc.getUnits());
float scale =
(float) tc.getTimeUnitScale(); // deal with, eg, "6 hours" by multiplying values by 6
if (tc.isInterval()) {
for (TimeCoord.Tinv tinv : tc.getIntervals()) {
b.addValues(tinv.getBounds1() * scale);
b.addBound(tinv.getBounds2() * scale);
}
} else {
for (int value : tc.getCoords()) b.addValues(value * scale);
}
return b.build();
}
protected GribCollectionProto.Coord writeCoordProto(VertCoord vc, int index) throws IOException {
GribCollectionProto.Coord.Builder b = GribCollectionProto.Coord.newBuilder();
b.setCode(vc.getCode());
b.setUnit(vc.getUnits());
for (VertCoord.Level coord : vc.getCoords()) {
if (vc.isLayer()) {
b.addValues((float) coord.getValue1());
b.addBound((float) coord.getValue2());
} else {
b.addValues((float) coord.getValue1());
}
}
return b.build();
}
protected GribCollectionProto.Coord writeCoordProto(EnsCoord ec, int index) throws IOException {
GribCollectionProto.Coord.Builder b = GribCollectionProto.Coord.newBuilder();
b.setCode(0);
b.setUnit("");
for (EnsCoord.Coord coord : ec.getCoords()) {
b.addValues((float) coord.getCode());
b.addValues((float) coord.getEnsMember());
}
return b.build();
}
}
/*
* Processes Queries for the RadarServer Spring Framework
*/
public class QueryRadarServerController extends AbstractController {
private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(getClass());
private TdsContext tdsContext;
private HtmlWriter htmlWriter;
private boolean htmlView;
private boolean releaseDataset = false;
public void setTdsContext(TdsContext tdsContext) {
this.tdsContext = tdsContext;
}
public void setHtmlWriter(HtmlWriter htmlWriter) {
this.htmlWriter = htmlWriter;
}
public boolean isHtmlView() {
return htmlView;
}
public void setHtmlView(boolean htmlView) {
this.htmlView = htmlView;
}
public boolean isReleaseDataset() {
return releaseDataset;
}
public void setReleaseDataset(boolean releaseDataset) {
this.releaseDataset = releaseDataset;
}
/** The view to forward to in case a bad query. */
private static final String CREATE_VIEW = "forward:badquery.htm";
/** The model key used to retrieve the message from the model. */
private static final String MODEL_KEY = "message";
/** The unique key for retrieving the text associated with this message. */
private static final String MSG_CODE = "message.bad.query";
/*
* why calculate over and over again 1970-01-01T00:00:00
*/
private static DateType epicDateType;
static {
try {
epicDateType = new DateType(RadarServerUtil.epic, null, null);
} catch (java.text.ParseException e) {
}
}
static SimpleDateFormat dateFormat;
static {
dateFormat = new SimpleDateFormat("yyyyMMdd", Locale.US);
dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
}
/**
* Query RadarServer controller for Spring Framework
*
* @param request HttpServletRequest
* @param response HttpServletResponse
* @return ModelAndView
* @throws Exception
*/
protected ModelAndView handleRequestInternal(
HttpServletRequest request, HttpServletResponse response) throws Exception {
try {
// Gather diagnostics for logging request.
log.info("handleRequestInternal(): " + UsageLog.setupRequestContext(request));
// catch rogue invalid request here
if (request.getQueryString() == null) {
log.info("Invalid dataset url reference " + request.getPathInfo());
throw new RadarServerException("Invalid dataset url reference " + request.getPathInfo());
}
// Query results in model
Map<String, Object> model = new HashMap<String, Object>();
radarQuery(request, response, model);
if (model == null || model.size() == 0) {
ModelAndView mav = new ModelAndView(CREATE_VIEW);
mav.addObject(MODEL_KEY, MSG_CODE);
return mav;
} else {
return new ModelAndView("queryXml", model);
}
} catch (RadarServerException e) {
throw e; // pass it onto Spring exceptionResolver
} catch (Throwable e) {
log.error("handleRequestInternal(): Problem handling request.", e);
log.info(
"handleRequestInternal(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
throw new RadarServerException("handleRequestInternal(): Problem handling request.");
}
}
// get/check/process query
public void radarQuery(HttpServletRequest req, HttpServletResponse res, Map<String, Object> model)
throws ServletException, IOException, RadarServerException {
// long startms = System.currentTimeMillis();
// long endms;
DatasetRepository.RadarType radarType = DatasetRepository.RadarType.nexrad;
// need to extract data according to the (dataset) given
String pathInfo = req.getPathInfo();
if (pathInfo == null) pathInfo = "";
if (pathInfo.startsWith("/")) pathInfo = pathInfo.substring(1);
try {
String rt = pathInfo.substring(0, pathInfo.indexOf('/', 1));
radarType = DatasetRepository.RadarType.valueOf(rt);
} catch (Exception e) {
log.info("Invalid dataset url reference " + pathInfo);
throw new RadarServerException("Invalid dataset url reference " + pathInfo);
}
Boolean level2 = pathInfo.contains("level2");
// parse the input
QueryParams qp = new QueryParams();
if (!qp.parseQuery(
req,
res,
new String[] {QueryParams.XML, QueryParams.HTML, QueryParams.RAW, QueryParams.NETCDF})) {
// log.error( "parseQuery Failed "+ qp.errs.toString() + req.getQueryString() );
// throw new RadarServerException( qp.errs.toString() );//+ req.getQueryString() );
return; // TODO: uncomment above 2 lines when QueryParams exception is fixed
}
// endms = System.currentTimeMillis();
// System.out.println( "after QueryParams "+ (endms - startms));
// startms = System.currentTimeMillis();
// check Query Params
if (!checkQueryParms(radarType, qp, level2)) {
log.error("checkQueryParms Failed " + qp.errs.toString() + req.getQueryString());
throw new RadarServerException(qp.errs.toString()); // + req.getQueryString() );
}
// endms = System.currentTimeMillis();
// System.out.println( "after checkQueryParms "+ (endms - startms));
// startms = System.currentTimeMillis();
// check type of output wanted XML html
qp.acceptType = qp.acceptType.replaceFirst(".*/", "");
// creates first part of catalog
if (!createHeader(radarType, qp, pathInfo, model)) {
log.error("Write Header Failed " + qp.errs.toString() + req.getQueryString());
throw new RadarServerException(qp.errs.toString()); // req.getQueryString() );
}
// endms = System.currentTimeMillis();
// System.out.println( "after writeHeader "+ (endms - startms));
// startms = System.currentTimeMillis();
// gets products according to stations, time, and variables
boolean dataFound = false;
List<DatasetEntry> entries = new ArrayList<DatasetEntry>();
if (qp.vars == null) {
dataFound = processQuery(pathInfo, qp, null, entries);
if (releaseDataset) DatasetRepository.removeRadarDatasetCollection(pathInfo, null);
} else {
int count = 0;
for (String var : qp.vars) {
dataFound = processQuery(pathInfo, qp, var, entries);
if (dataFound) count++;
if (releaseDataset) DatasetRepository.removeRadarDatasetCollection(pathInfo, var);
}
if (count > 0) dataFound = true;
}
// save entries
model.put("datasets", entries);
if (dataFound) {
model.put("documentation", Integer.toString(entries.size()) + " datasets found for query");
} else if (qp.errs.length() > 0) {
model.put("documentation", qp.errs.toString());
} else {
model.put("documentation", "No data available for station(s) and time range");
}
// endms = System.currentTimeMillis();
// System.out.println( "after radarQuery "+ (endms - startms));
// startms = System.currentTimeMillis();
} // end radarNexradQuery
// check that parms have valid stations, vars and times
private Boolean checkQueryParms(
DatasetRepository.RadarType radarType, QueryParams qp, Boolean level2) throws IOException {
if (qp.hasBB) {
if (radarType.equals(DatasetRepository.RadarType.nexrad))
qp.stns = RadarServerUtil.getStationNames(qp.getBB(), DatasetRepository.nexradList);
else qp.stns = RadarServerUtil.getStationNames(qp.getBB(), DatasetRepository.terminalList);
if (qp.stns.size() == 0) {
qp.errs.append("Bounding Box contains no stations ");
return false;
}
if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
}
if (qp.hasStns) {
if (RadarServerUtil.isStationListEmpty(qp.stns, radarType)) {
qp.errs.append("No valid stations specified, need 1 ");
return false;
} else if (level2) {
for (String stn : qp.stns) {
if (stn.length() == 3) {
qp.errs.append("Need 4 character station names ");
return false;
}
}
} else if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
}
if (qp.hasLatlonPoint) {
qp.stns = new ArrayList<String>();
if (radarType.equals(DatasetRepository.RadarType.nexrad))
qp.stns.add(
RadarServerUtil.findClosestStation(qp.lat, qp.lon, DatasetRepository.nexradList));
else
qp.stns.add(
RadarServerUtil.findClosestStation(qp.lat, qp.lon, DatasetRepository.terminalList));
if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
} else if (qp.fatal) {
qp.errs.append("No valid stations specified 2 ");
return false;
}
if (qp.stns == null || qp.stns.size() == 0) {
qp.errs.append("No valid stations specified, need 1 ");
return false;
}
boolean useAllStations = (qp.stns.get(0).toUpperCase().equals("ALL"));
if (useAllStations) {
if (radarType.equals(DatasetRepository.RadarType.nexrad))
qp.stns =
RadarServerUtil.getStationNames(DatasetRepository.nexradList); // need station names
else
qp.stns =
RadarServerUtil.getStationNames(DatasetRepository.terminalList); // need station names
if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
}
if (qp.hasTimePoint) {
if (qp.time.isPresent()) {
try {
qp.time_end = new DateType("present", null, null);
qp.time_start = epicDateType;
} catch (java.text.ParseException e) {
qp.errs.append("Illegal param= 'time' must be valid ISO Duration");
return false;
}
} else {
qp.time_end = qp.time;
qp.time_start = qp.time;
}
} else if (qp.hasDateRange) {
DateRange dr = qp.getCalendarDateRange().toDateRange();
qp.time_start = dr.getStart();
qp.time_end = dr.getEnd();
} else { // get all times
qp.time_latest = 1;
// qp.hasDateRange = true;
try {
qp.time = new DateType("present", null, null);
qp.time_end = new DateType("present", null, null);
qp.time_start = epicDateType;
} catch (java.text.ParseException e) {
qp.errs.append("Illegal param= 'time' must be valid ISO Duration ");
return false;
}
}
if (level2) {
qp.vars = null; // level2 can't select vars
} else if (qp.vars == null) { // level 3 with no vars
qp.errs.append("No vars selected ");
return false;
} else if (qp.vars.get(0).contains("/")) { // remove desc from vars
ArrayList<String> tmp = new ArrayList<String>();
for (String var : qp.vars) {
tmp.add(var.replaceFirst("/.*", ""));
}
qp.vars = tmp;
}
return true;
}
// create catalog Header
private Boolean createHeader(
DatasetRepository.RadarType radarType,
QueryParams qp,
String pathInfo,
Map<String, Object> model)
throws IOException {
Boolean level2 = pathInfo.contains("level2");
int level = (level2) ? 2 : 3;
StringBuffer str = new StringBuffer();
str.append("Radar Level").append(level).append(" datasets in near real time");
model.put("name", str.toString());
str.setLength(0);
str.append("/thredds/dodsC/").append(pathInfo).append("/");
model.put("base", str.toString());
str.setLength(0);
str.append("RadarLevel").append(level).append(" datasets for available stations and times");
model.put("dname", str.toString());
str.setLength(0);
str.append("accept=").append(qp.acceptType).append("&");
if (!level2 && qp.vars != null) { // add vars
str.append("var=");
for (int i = 0; i < qp.vars.size(); i++) {
str.append(qp.vars.get(i));
if (i < qp.vars.size() - 1) {
str.append(",");
}
}
str.append("&");
}
// use all stations
if (qp.stns.get(0).toUpperCase().equals("ALL")) {
str.append("stn=ALL&");
} else if (qp.hasStns) {
for (String station : qp.stns) {
str.append("stn=").append(station).append("&");
}
} else if (qp.hasBB) {
str.append("south=").append(qp.south).append("&north=").append(qp.north).append("&");
str.append("west=").append(qp.west).append("&east=").append(qp.east).append("&");
}
// no time given
if (qp.time_latest == 1) {
// str.deleteCharAt( str.length() -1);
str.append("time=present");
} else if (qp.hasDateRange) {
if (qp.time_start.getDate() == null
|| qp.time_start.isBlank()
|| qp.time_end.getDate() == null
|| qp.time_end.isBlank()) {
str.append("time_start=").append(qp.time_start.toString());
str.append("&time_end=").append(qp.time_end.toString());
qp.errs.append("need ISO time format ");
return false;
} else {
str.append("time_start=").append(qp.time_start.toDateTimeStringISO());
str.append("&time_end=").append(qp.time_end.toDateTimeStringISO());
}
} else if (qp.time.isPresent()) {
str.append("time=present");
} else if (qp.hasTimePoint) {
if (qp.time.getDate() == null || qp.time.isBlank()) {
str.append("time=").append(qp.time.toString());
qp.errs.append("need ISO time format ");
return false;
} else {
str.append("time=").append(qp.time.toDateTimeStringISO());
}
}
model.put("ID", str.toString());
if (level2) {
model.put("type", "NEXRAD2");
} else if (radarType.equals(DatasetRepository.RadarType.nexrad)) {
model.put("type", "NIDS");
} else {
model.put("type", "TDWR");
}
// at this point must have stations
if (RadarServerUtil.isStationListEmpty(qp.stns, radarType)) {
qp.errs.append("No station(s) meet query criteria ");
return false;
}
return true;
}
/*
Final Output format, save information in DatasetEntry de
<dataset name="Level2_KFTG_20100121_0000.ar2v" ID="735519521"
urlPath="KFTG/20100121/Level2_KFTG_20100121_0000.ar2v">
<date type="start of ob">2010-01-21T00:00:00</date>
</dataset>
*/
private Boolean processQuery(
String dataset, QueryParams qp, String var, List<DatasetEntry> entries)
throws RadarServerException {
Boolean getAllTimes = true;
String yyyymmddStart = null;
String yyyymmddEnd = null;
String dateStart = null;
String dateEnd = null;
try {
if (!qp.time_start.equals(epicDateType)) {
getAllTimes = false;
yyyymmddStart = qp.time_start.toDateString();
yyyymmddStart = yyyymmddStart.replace("-", "");
yyyymmddEnd = qp.time_end.toDateString();
yyyymmddEnd = yyyymmddEnd.replace("-", "");
dateStart = yyyymmddStart + "_" + RadarServerUtil.hhmm(qp.time_start.toDateTimeString());
dateEnd = yyyymmddEnd + "_" + RadarServerUtil.hhmm(qp.time_end.toDateTimeString());
}
RadarDatasetCollection rdc = DatasetRepository.getRadarDatasetCollection(dataset, var);
if (rdc == null) {
qp.errs.append("Invalid dataset =").append(dataset);
qp.errs.append(" or var =").append(var);
return false;
}
StringBuffer time = new StringBuffer();
StringBuffer product = new StringBuffer();
StringBuffer url = new StringBuffer();
boolean isLevel2 = dataset.contains("level2");
String type = (isLevel2 ? "Level2" : "Level3");
String suffix = (isLevel2 ? ".ar2v" : ".nids");
Calendar cal = Calendar.getInstance(java.util.TimeZone.getTimeZone("GMT"));
Date now = cal.getTime();
String currentDay = dateFormat.format(now);
for (String stn : qp.stns) {
RadarStationCollection rsc = rdc.queryStation(stn, currentDay);
if (rsc == null) continue;
for (String day : rsc.getDays()) {
// check for valid day
if (!getAllTimes && !RadarServerUtil.isValidDay(day, yyyymmddStart, yyyymmddEnd))
continue;
ArrayList<String> tal;
if (rdc.isCaseStudy()) { //
tal = rsc.getHourMinute("all");
for (String prod : tal) {
// check times
if (!getAllTimes && !RadarServerUtil.isValidDate(prod, dateStart, dateEnd)) continue;
// save this entry
DatasetEntry de = new DatasetEntry();
int idx = prod.indexOf('/');
if (idx > 0) {
de.setName(prod.substring(idx + 1));
} else {
de.setName(prod);
}
de.setID(Integer.toString(prod.hashCode()));
url.setLength(0);
url.append(stn).append("/");
if (var != null) {
url.append(var).append("/");
}
url.append(prod);
de.setUrlPath(url.toString());
de.setDate(RadarServerUtil.getObTimeISO(prod));
entries.add(de);
}
continue;
} else {
tal = rsc.getHourMinute(day);
}
if (tal == null) continue;
for (String hm : tal) {
time.setLength(0);
time.append(day).append("_").append(hm);
if (!getAllTimes && !RadarServerUtil.isValidDate(time.toString(), dateStart, dateEnd))
continue;
// save this entry
DatasetEntry de = new DatasetEntry();
product.setLength(0);
product.append(type).append("_").append(rsc.getStnName()).append("_");
if (!isLevel2) product.append(var).append("_");
product.append(day).append("_").append(hm).append(suffix);
de.setName(product.toString());
de.setID(Integer.toString(product.toString().hashCode()));
url.setLength(0);
if (!isLevel2) {
url.append(var).append("/");
}
url.append(rsc.getStnName())
.append("/")
.append(day)
.append("/")
.append(product.toString());
de.setUrlPath(url.toString());
de.setDate(RadarServerUtil.getObTimeISO(product.toString()));
entries.add(de);
if (qp.hasTimePoint) break;
}
if (qp.hasTimePoint) break;
}
}
return true;
} catch (Throwable e) {
log.error("Invalid dataset =" + dataset + " or var =" + var, e);
log.info(
"handleRequestInternal(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
throw new RadarServerException("Invalid dataset =" + dataset + " or var =" + var);
}
}
/*
* Used to store the information about a dataset
*/
public class DatasetEntry {
private String name;
private String ID;
private String urlPath;
private String date;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getID() {
return ID;
}
public void setID(String ID) {
this.ID = ID;
}
public String getUrlPath() {
return urlPath;
}
public void setUrlPath(String urlPath) {
this.urlPath = urlPath;
}
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
}
}
public class ServletUtil {
public static final String CONTENT_TEXT = "text/plain; charset=utf-8";
// bogus status returns for our logging
public static final int STATUS_CLIENT_ABORT = 1000;
public static final int STATUS_FORWARDED = 1001;
public static final int STATUS_FORWARD_FAILURE = 1002;
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(ServletUtil.class);
private static boolean isDebugInit = false;
private static String contextPath = null;
private static String rootPath = null;
private static String contentPath = null;
/**
* @param context the Servlet context.
* @deprecated Now handled in TdsContext.init().
*/
public static void initContext(ServletContext context) {
// setContextPath(context);
if (contextPath == null) {
// Servlet 2.5 allows the following.
// contextPath = servletContext.getContextPath();
String tmpContextPath =
context.getInitParameter("ContextPath"); // cannot be overridden in the ThreddsConfig file
if (tmpContextPath == null) tmpContextPath = "thredds";
contextPath = "/" + tmpContextPath;
}
// setRootPath(context);
if (rootPath == null) {
rootPath = context.getRealPath("/");
rootPath = rootPath.replace('\\', '/');
}
// setContentPath();
if (contentPath == null) {
String tmpContentPath = "../../content" + getContextPath() + "/";
File cf = new File(getRootPath(), tmpContentPath);
try {
contentPath = cf.getCanonicalPath() + "/";
contentPath = contentPath.replace('\\', '/');
} catch (IOException e) {
throw new RuntimeException(e.getMessage());
}
}
// initDebugging(context);
initDebugging(context);
}
public static void setContextPath(String newContextPath) {
contextPath = newContextPath;
}
public static void setRootPath(String newRootPath) {
rootPath = newRootPath;
}
public static void setContentPath(String newContentPath) {
contentPath = newContentPath;
if (!contentPath.endsWith("/")) contentPath = contentPath + "/";
}
public static void initDebugging(ServletContext webapp) {
if (isDebugInit) return;
isDebugInit = true;
String debugOn = webapp.getInitParameter("DebugOn");
if (debugOn != null) {
StringTokenizer toker = new StringTokenizer(debugOn);
while (toker.hasMoreTokens()) Debug.set(toker.nextToken(), true);
}
}
/**
* Return the real path on the servers file system that corresponds to the root document ("/") on
* the given servlet.
*
* @return the real path on the servers file system that corresponds to the root document ("/") on
* the given servlet.
*/
public static String getRootPath() {
return rootPath;
}
/**
* Return the context path for the given servlet. Note - ToDo: Why not just use
* ServletContext.getServletContextName()?
*
* @return the context path for the given servlet.
*/
public static String getContextPath() {
return contextPath;
}
/**
* Return the content path for the given servlet.
*
* @return the content path for the given servlet.
*/
public static String getContentPath() {
return contentPath;
}
/**
* Return the default/initial content path for the given servlet. The content of which is copied
* to the content path when the web app is first installed.
*
* @return the default/initial content path for the given servlet.
*/
public static String getInitialContentPath() {
return getRootPath() + "/WEB-INF/altContent/startup/";
}
/**
* Return the file path dealing with leading and trailing path seperators (which must be a slash
* ("/")) for the given directory and file paths.
*
* <p>Note: Dealing with path strings is fragile. ToDo: Switch from using path strings to
* java.io.Files.
*
* @param dirPath the directory path.
* @param filePath the file path.
* @return a full file path with the given directory and file paths.
*/
public static String formFilename(String dirPath, String filePath) {
if ((dirPath == null) || (filePath == null)) return null;
if (filePath.startsWith("/")) filePath = filePath.substring(1);
return dirPath.endsWith("/") ? dirPath + filePath : dirPath + "/" + filePath;
}
/**
* Handle a request for a raw/static file (i.e., not a catalog or dataset request).
*
* <p>Look in the content (user) directory then the root (distribution) directory for a file that
* matches the given path and, if found, return it as the content of the HttpServletResponse. If
* the file is forbidden (i.e., the path contains a "..", "WEB-INF", or "META-INF" directory),
* send a HttpServletResponse.SC_FORBIDDEN response. If no file matches the request (including an
* "index.html" file if the path ends in "/"), send an HttpServletResponse.SC_NOT_FOUND..
*
* <p>
*
* <ol>
* <li>Make sure the path does not contain ".." directories.
* <li>Make sure the path does not contain "WEB-INF" or "META-INF".
* <li>Check for requested file in the content directory (if the path is a directory, make sure
* the path ends with "/" and check for an "index.html" file).
* <li>Check for requested file in the root directory (if the path is a directory, make sure the
* path ends with "/" and check for an "index.html" file). </ol
*
* @param path the requested path
* @param servlet the servlet handling the request
* @param req the HttpServletRequest
* @param res the HttpServletResponse
* @throws IOException if can't complete request due to IO problems.
*/
public static void handleRequestForRawFile(
String path, HttpServlet servlet, HttpServletRequest req, HttpServletResponse res)
throws IOException {
// Don't allow ".." directories in path.
if (path.indexOf("/../") != -1
|| path.equals("..")
|| path.startsWith("../")
|| path.endsWith("/..")) {
res.sendError(HttpServletResponse.SC_FORBIDDEN, "Path cannot contain \"..\" directory.");
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, -1));
return;
}
// Don't allow access to WEB-INF or META-INF directories.
String upper = path.toUpperCase();
if (upper.indexOf("WEB-INF") != -1 || upper.indexOf("META-INF") != -1) {
res.sendError(
HttpServletResponse.SC_FORBIDDEN, "Path cannot contain \"WEB-INF\" or \"META-INF\".");
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, -1));
return;
}
// Find a regular file
File regFile = null;
// Look in content directory for regular file.
File cFile = new File(ServletUtil.formFilename(getContentPath(), path));
if (cFile.exists()) {
if (cFile.isDirectory()) {
if (!path.endsWith("/")) {
String newPath = req.getRequestURL().append("/").toString();
ServletUtil.sendPermanentRedirect(newPath, req, res);
}
// If request path is a directory, check for index.html file.
cFile = new File(cFile, "index.html");
if (cFile.exists() && !cFile.isDirectory()) regFile = cFile;
}
// If not a directory, use this file.
else regFile = cFile;
}
if (regFile == null) {
// Look in root directory.
File rFile = new File(ServletUtil.formFilename(getRootPath(), path));
if (rFile.exists()) {
if (rFile.isDirectory()) {
if (!path.endsWith("/")) {
String newPath = req.getRequestURL().append("/").toString();
ServletUtil.sendPermanentRedirect(newPath, req, res);
}
rFile = new File(rFile, "index.html");
if (rFile.exists() && !rFile.isDirectory()) regFile = rFile;
} else regFile = rFile;
}
}
if (regFile == null) {
res.sendError(HttpServletResponse.SC_NOT_FOUND); // 404
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
return;
}
ServletUtil.returnFile(servlet, req, res, regFile, null);
}
/**
* Handle an explicit request for a content directory file (path must start with "/content/".
*
* <p>Note: As these requests will show the configuration files for the server, these requests
* should be covered by security constraints.
*
* <p>
*
* <ol>
* <li>Make sure the path does not contain ".." directories.
* <li>Check for the requested file in the content directory. </ol
*
* @param path the requested path (must start with "/content/")
* @param servlet the servlet handling the request
* @param req the HttpServletRequest
* @param res the HttpServletResponse
* @throws IOException if can't complete request due to IO problems.
*/
public static void handleRequestForContentFile(
String path, HttpServlet servlet, HttpServletRequest req, HttpServletResponse res)
throws IOException {
handleRequestForContentOrRootFile("/content/", path, servlet, req, res);
}
/**
* Handle an explicit request for a root directory file (path must start with "/root/".
*
* <p>Note: As these requests will show the configuration files for the server, these requests
* should be covered by security constraints.
*
* <p>
*
* <ol>
* <li>Make sure the path does not contain ".." directories.
* <li>Check for the requested file in the root directory. </ol
*
* @param path the requested path (must start with "/root/")
* @param servlet the servlet handling the request
* @param req the HttpServletRequest
* @param res the HttpServletResponse
* @throws IOException if can't complete request due to IO problems.
*/
public static void handleRequestForRootFile(
String path, HttpServlet servlet, HttpServletRequest req, HttpServletResponse res)
throws IOException {
handleRequestForContentOrRootFile("/root/", path, servlet, req, res);
}
/**
* Convenience routine used by handleRequestForContentFile() and handleRequestForRootFile().
*
* @param pathPrefix
* @param path
* @param servlet
* @param req request
* @param res response
* @throws IOException on IO error
*/
private static void handleRequestForContentOrRootFile(
String pathPrefix,
String path,
HttpServlet servlet,
HttpServletRequest req,
HttpServletResponse res)
throws IOException {
if (!pathPrefix.equals("/content/") && !pathPrefix.equals("/root/")) {
log.error(
"handleRequestForContentFile(): The path prefix <"
+ pathPrefix
+ "> must be \"/content/\" or \"/root/\".");
throw new IllegalArgumentException("Path prefix must be \"/content/\" or \"/root/\".");
}
if (!path.startsWith(pathPrefix)) {
log.error(
"handleRequestForContentFile(): path <"
+ path
+ "> must start with \""
+ pathPrefix
+ "\".");
throw new IllegalArgumentException("Path must start with \"" + pathPrefix + "\".");
}
// Don't allow ".." directories in path.
if (path.indexOf("/../") != -1
|| path.equals("..")
|| path.startsWith("../")
|| path.endsWith("/..")) {
res.sendError(HttpServletResponse.SC_FORBIDDEN, "Path cannot contain \"..\" directory.");
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, -1));
return;
}
// Find the requested file.
File file =
new File(
ServletUtil.formFilename(getContentPath(), path.substring(pathPrefix.length() - 1)));
if (file.exists()) {
// Do not allow request for a directory.
if (file.isDirectory()) {
if (!path.endsWith("/")) {
String redirectPath = req.getRequestURL().append("/").toString();
ServletUtil.sendPermanentRedirect(redirectPath, req, res);
return;
}
int i = HtmlWriter.getInstance().writeDirectory(res, file, path);
int status = i == 0 ? HttpServletResponse.SC_NOT_FOUND : HttpServletResponse.SC_OK;
log.info(UsageLog.closingMessageForRequestContext(status, i));
return;
}
// Return the requested file.
ServletUtil.returnFile(servlet, req, res, file, null);
} else {
// Requested file not found.
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
res.sendError(HttpServletResponse.SC_NOT_FOUND); // 404
}
}
/**
* Send a permanent redirect (HTTP status 301 "Moved Permanently") response with the given target
* path.
*
* <p>The given target path may be relative or absolute. If it is relative, it will be resolved
* against the request URL.
*
* @param targetPath the path to which the client is redirected.
* @param req the HttpServletRequest
* @param res the HttpServletResponse
* @throws IOException if can't write the response.
*/
public static void sendPermanentRedirect(
String targetPath, HttpServletRequest req, HttpServletResponse res) throws IOException {
// Absolute URL needed so resolve the target path against the request URL.
URI uri;
try {
uri = new URI(req.getRequestURL().toString());
} catch (URISyntaxException e) {
log.error(
"sendPermanentRedirect(): Bad syntax on request URL <" + req.getRequestURL() + ">.", e);
log.info(
"sendPermanentRedirect(): "
+ UsageLog.closingMessageForRequestContext(
HttpServletResponse.SC_INTERNAL_SERVER_ERROR, 0));
if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
return;
}
String absolutePath = uri.resolve(targetPath).toString();
absolutePath = res.encodeRedirectURL(absolutePath);
res.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY);
res.addHeader("Location", absolutePath);
String title = "Permanently Moved - 301";
String body =
new StringBuilder()
.append("<p>")
.append("The requested URL <")
.append(req.getRequestURL())
.append("> has been permanently moved (HTTP status code 301).")
.append(" Instead, please use the following URL: <a href=\"")
.append(absolutePath)
.append("\">")
.append(absolutePath)
.append("</a>.")
.append("</p>")
.toString();
String htmlResp =
new StringBuilder()
.append(HtmlWriter.getInstance().getHtmlDoctypeAndOpenTag())
.append("<head><title>")
.append(title)
.append("</title></head><body>")
.append("<h1>")
.append(title)
.append("</h1>")
.append(body)
.append("</body></html>")
.toString();
log.info("sendPermanentRedirect(): redirect to " + absolutePath);
log.info(
"sendPermanentRedirect(): "
+ UsageLog.closingMessageForRequestContext(
HttpServletResponse.SC_MOVED_PERMANENTLY, htmlResp.length()));
// Write the catalog out.
PrintWriter out = res.getWriter();
res.setContentType("text/html");
out.print(htmlResp);
out.flush();
}
/**
* Write a file to the response stream.
*
* @param servlet called from here
* @param contentPath file root path
* @param path file path reletive to the root
* @param req the request
* @param res the response
* @param contentType content type, or null
* @throws IOException on write error
*/
public static void returnFile(
HttpServlet servlet,
String contentPath,
String path,
HttpServletRequest req,
HttpServletResponse res,
String contentType)
throws IOException {
String filename = ServletUtil.formFilename(contentPath, path);
log.debug("returnFile(): returning file <" + filename + ">.");
// No file, nothing to view
if (filename == null) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
res.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
// dontallow ..
if (filename.indexOf("..") != -1) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
res.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
// dont allow access to WEB-INF or META-INF
String upper = filename.toUpperCase();
if (upper.indexOf("WEB-INF") != -1 || upper.indexOf("META-INF") != -1) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
res.sendError(HttpServletResponse.SC_FORBIDDEN);
return;
}
returnFile(servlet, req, res, new File(filename), contentType);
}
private static FileCacheRaf fileCacheRaf;
public static void setFileCache(FileCacheRaf fileCache) {
fileCacheRaf = fileCache;
}
public static FileCacheRaf getFileCache() {
return fileCacheRaf;
}
/**
* Write a file to the response stream. Handles Range requests.
*
* @param servlet called from here
* @param req the request
* @param res the response
* @param file to serve
* @param contentType content type, if null, will try to guess
* @throws IOException on write error
*/
public static void returnFile(
HttpServlet servlet,
HttpServletRequest req,
HttpServletResponse res,
File file,
String contentType)
throws IOException {
// No file, nothing to view
if (file == null) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
res.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
// check that it exists
if (!file.exists()) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
res.sendError(HttpServletResponse.SC_NOT_FOUND);
return;
}
// not a directory
if (!file.isFile()) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, 0));
res.sendError(HttpServletResponse.SC_BAD_REQUEST);
return;
}
// Set the type of the file
String filename = file.getPath();
if (null == contentType) {
if (filename.endsWith(".html")) contentType = "text/html; charset=iso-8859-1";
else if (filename.endsWith(".xml")) contentType = "text/xml; charset=iso-8859-1";
else if (filename.endsWith(".txt") || (filename.endsWith(".log"))) contentType = CONTENT_TEXT;
else if (filename.indexOf(".log.") > 0) contentType = CONTENT_TEXT;
else if (filename.endsWith(".nc")) contentType = "application/x-netcdf";
else contentType = servlet.getServletContext().getMimeType(filename);
if (contentType == null) contentType = "application/octet-stream";
}
returnFile(req, res, file, contentType);
}
/**
* Write a file to the response stream. Handles Range requests.
*
* @param req request
* @param res response
* @param file must exists and not be a directory
* @param contentType must not be null
* @throws IOException or error
*/
public static void returnFile(
HttpServletRequest req, HttpServletResponse res, File file, String contentType)
throws IOException {
res.setContentType(contentType);
// see if its a Range Request
boolean isRangeRequest = false;
long startPos = 0, endPos = Long.MAX_VALUE;
String rangeRequest = req.getHeader("Range");
if (rangeRequest != null) { // bytes=12-34 or bytes=12-
int pos = rangeRequest.indexOf("=");
if (pos > 0) {
int pos2 = rangeRequest.indexOf("-");
if (pos2 > 0) {
String startString = rangeRequest.substring(pos + 1, pos2);
String endString = rangeRequest.substring(pos2 + 1);
startPos = Long.parseLong(startString);
if (endString.length() > 0) endPos = Long.parseLong(endString) + 1;
isRangeRequest = true;
}
}
}
// set content length
long fileSize = file.length();
long contentLength = fileSize;
if (isRangeRequest) {
endPos = Math.min(endPos, fileSize);
contentLength = endPos - startPos;
}
if (contentLength > Integer.MAX_VALUE)
res.addHeader(
"Content-Length", Long.toString(contentLength)); // allow content length > MAX_INT
else res.setContentLength((int) contentLength); // note HEAD only allows this
String filename = file.getPath();
boolean debugRequest = Debug.isSet("returnFile");
if (debugRequest)
log.debug(
"returnFile(): filename = "
+ filename
+ " contentType = "
+ contentType
+ " contentLength = "
+ contentLength);
// indicate we allow Range Requests
res.addHeader("Accept-Ranges", "bytes");
if (req.getMethod().equals("HEAD")) {
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, 0));
return;
}
try {
if (isRangeRequest) {
// set before content is sent
res.addHeader("Content-Range", "bytes " + startPos + "-" + (endPos - 1) + "/" + fileSize);
res.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
FileCacheRaf.Raf craf = null;
try {
craf = fileCacheRaf.acquire(filename);
IO.copyRafB(
craf.getRaf(), startPos, contentLength, res.getOutputStream(), new byte[60000]);
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(
HttpServletResponse.SC_PARTIAL_CONTENT, contentLength));
return;
} finally {
if (craf != null) fileCacheRaf.release(craf);
}
}
// Return the file
ServletOutputStream out = res.getOutputStream();
IO.copyFileB(file, out, 60000);
res.flushBuffer();
out.close();
if (debugRequest) log.debug("returnFile(): returnFile ok = " + filename);
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, contentLength));
}
// @todo Split up this exception handling: those from file access vs those from dealing with
// response
// File access: catch and res.sendError()
// response: don't catch (let bubble up out of doGet() etc)
catch (FileNotFoundException e) {
log.error("returnFile(): FileNotFoundException= " + filename);
log.info(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_NOT_FOUND);
} catch (java.net.SocketException e) {
log.info("returnFile(): SocketException sending file: " + filename + " " + e.getMessage());
log.info("returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
} catch (IOException e) {
String eName =
e.getClass().getName(); // dont want compile time dependency on ClientAbortException
if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
log.info(
"returnFile(): ClientAbortException while sending file: "
+ filename
+ " "
+ e.getMessage());
log.info(
"returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
return;
}
log.error("returnFile(): IOException (" + e.getClass().getName() + ") sending file ", e);
log.error(
"returnFile(): "
+ UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
if (!res.isCommitted())
res.sendError(HttpServletResponse.SC_NOT_FOUND, "Problem sending file: " + e.getMessage());
}
}
/**
* Send given content string as the HTTP response.
*
* @param contents the string to return as the HTTP response.
* @param res the HttpServletResponse
* @throws IOException if an I/O error occurs while writing the response.
*/
public static void returnString(String contents, HttpServletResponse res) throws IOException {
try {
ServletOutputStream out = res.getOutputStream();
IO.copy(new ByteArrayInputStream(contents.getBytes()), out);
log.info(
UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, contents.length()));
} catch (IOException e) {
log.error(" IOException sending string: ", e);
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
res.sendError(HttpServletResponse.SC_NOT_FOUND, "Problem sending string: " + e.getMessage());
}
}
/**
* Return the request URL relative to the server (i.e., starting with the context path).
*
* @param req request
* @return URL relative to the server
*/
public static String getReletiveURL(HttpServletRequest req) {
return req.getContextPath() + req.getServletPath() + req.getPathInfo();
}
/**
* Forward this request to the CatalogServices servlet ("/catalog.html").
*
* @param req request
* @param res response
* @throws IOException on IO error
* @throws ServletException other error
*/
public static void forwardToCatalogServices(HttpServletRequest req, HttpServletResponse res)
throws IOException, ServletException {
String reqs = "catalog=" + getReletiveURL(req);
String query = req.getQueryString();
if (query != null) reqs = reqs + "&" + query;
log.info("forwardToCatalogServices(): request string = \"/catalog.html?" + reqs + "\"");
// dispatch to CatalogHtml servlet
RequestForwardUtils.forwardRequestRelativeToCurrentContext("/catalog.html?" + reqs, req, res);
}
public static boolean saveFile(
HttpServlet servlet,
String contentPath,
String path,
HttpServletRequest req,
HttpServletResponse res) {
// @todo Need to use logServerAccess() below here.
boolean debugRequest = Debug.isSet("SaveFile");
if (debugRequest) log.debug(" saveFile(): path= " + path);
String filename = contentPath + path; // absolute path
File want = new File(filename);
// backup current version if it exists
int version = getBackupVersion(want.getParent(), want.getName());
String fileSave = filename + "~" + version;
File file = new File(filename);
if (file.exists()) {
try {
IO.copyFile(filename, fileSave);
} catch (IOException e) {
log.error(
"saveFile(): Unable to save copy of file "
+ filename
+ " to "
+ fileSave
+ "\n"
+ e.getMessage());
return false;
}
}
// save new file
try {
OutputStream out = new BufferedOutputStream(new FileOutputStream(filename));
IO.copy(req.getInputStream(), out);
out.close();
if (debugRequest) log.debug("saveFile(): ok= " + filename);
res.setStatus(HttpServletResponse.SC_CREATED);
log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_CREATED, -1));
return true;
} catch (IOException e) {
log.error(
"saveFile(): Unable to PUT file " + filename + " to " + fileSave + "\n" + e.getMessage());
return false;
}
}
private static int getBackupVersion(String dirName, String fileName) {
int maxN = 0;
File dir = new File(dirName);
if (!dir.exists()) return -1;
String[] files = dir.list();
if (null == files) return -1;
for (String name : files) {
if (name.indexOf(fileName) < 0) continue;
int pos = name.indexOf('~');
if (pos < 0) continue;
String ver = name.substring(pos + 1);
int n = 0;
try {
n = Integer.parseInt(ver);
} catch (NumberFormatException e) {
log.error("Format Integer error on backup filename= " + ver);
}
maxN = Math.max(n, maxN);
}
return maxN + 1;
}
public static boolean copyDir(String fromDir, String toDir) throws IOException {
File contentFile = new File(toDir + ".INIT");
if (!contentFile.exists()) {
IO.copyDirTree(fromDir, toDir);
contentFile.createNewFile();
return true;
}
return false;
}
/**
* ************************************************************************ Sends an error to the
* client.
*
* @param t The exception that caused the problem.
* @param res The <code>HttpServletResponse</code> for the client.
*/
public static void handleException(Throwable t, HttpServletResponse res) {
try {
String message = t.getMessage();
if (message == null) message = "NULL message " + t.getClass().getName();
if (Debug.isSet("trustedMode")) { // security issue: only show stack if trusted
ByteArrayOutputStream bs = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bs);
t.printStackTrace(ps);
message = new String(bs.toByteArray());
}
log.info(
UsageLog.closingMessageForRequestContext(
HttpServletResponse.SC_BAD_REQUEST, message.length()));
log.error("handleException", t);
t.printStackTrace(); // debugging - log.error not showing stack trace !!
if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_BAD_REQUEST, message);
} catch (Throwable e) {
log.error("handleException() had problem reporting Exception", e);
t.printStackTrace();
}
}
public static void showServerInfo(PrintStream out) {
out.println("Server Info");
out.println(
" getDocumentBuilderFactoryVersion(): "
+ XMLEntityResolver.getDocumentBuilderFactoryVersion());
out.println();
Properties sysp = System.getProperties();
Enumeration e = sysp.propertyNames();
List<String> list = Collections.list(e);
Collections.sort(list);
out.println("System Properties:");
for (String name : list) {
String value = System.getProperty(name);
out.println(" " + name + " = " + value);
}
out.println();
}
public static void showServletInfo(HttpServlet servlet, PrintStream out) {
out.println("Servlet Info");
out.println(" getServletName(): " + servlet.getServletName());
out.println(" getRootPath(): " + getRootPath());
out.println(" Init Parameters:");
Enumeration params = servlet.getInitParameterNames();
while (params.hasMoreElements()) {
String name = (String) params.nextElement();
out.println(" " + name + ": " + servlet.getInitParameter(name));
}
out.println();
ServletContext context = servlet.getServletContext();
out.println("Context Info");
try {
out.println(" context.getResource('/'): " + context.getResource("/"));
} catch (java.net.MalformedURLException e) {
} // cant happen
out.println(" context.getServerInfo(): " + context.getServerInfo());
out.println(" name: " + getServerInfoName(context.getServerInfo()));
out.println(" version: " + getServerInfoVersion(context.getServerInfo()));
out.println(" context.getInitParameterNames():");
params = context.getInitParameterNames();
while (params.hasMoreElements()) {
String name = (String) params.nextElement();
out.println(" " + name + ": " + context.getInitParameter(name));
}
out.println(" context.getAttributeNames():");
params = context.getAttributeNames();
while (params.hasMoreElements()) {
String name = (String) params.nextElement();
out.println(" context.getAttribute(\"" + name + "\"): " + context.getAttribute(name));
}
out.println();
}
/**
* Show the pieces of the request, for debugging
*
* @param req the HttpServletRequest
* @return parsed request
*/
public static String getRequestParsed(HttpServletRequest req) {
return req.getRequestURI()
+ " = "
+ req.getContextPath()
+ "(context), "
+ req.getServletPath()
+ "(servletPath), "
+ req.getPathInfo()
+ "(pathInfo), "
+ req.getQueryString()
+ "(query)";
}
/**
* This is the server part, eg http://motherlode:8080
*
* @param req the HttpServletRequest
* @return request server
*/
public static String getRequestServer(HttpServletRequest req) {
return req.getScheme() + "://" + req.getServerName() + ":" + req.getServerPort();
}
/**
* This is everything except the query string
*
* @param req the HttpServletRequest
* @return parsed request base
*/
public static String getRequestBase(HttpServletRequest req) {
// return "http://"+req.getServerName()+":"+ req.getServerPort()+req.getRequestURI();
return req.getRequestURL().toString();
}
/**
* The request base as a URI
*
* @param req the HttpServletRequest
* @return parsed request as a URI
*/
public static URI getRequestURI(HttpServletRequest req) {
try {
return new URI(getRequestBase(req));
} catch (URISyntaxException e) {
e.printStackTrace();
return null;
}
}
/**
* servletPath + pathInfo
*
* @param req the HttpServletRequest
* @return parsed request servletPath + pathInfo
*/
public static String getRequestPath(HttpServletRequest req) {
StringBuffer buff = new StringBuffer();
if (req.getServletPath() != null) buff.append(req.getServletPath());
if (req.getPathInfo() != null) buff.append(req.getPathInfo());
return buff.toString();
}
/**
* The entire request including query string
*
* @param req the HttpServletRequest
* @return entire parsed request
*/
public static String getRequest(HttpServletRequest req) {
String query = req.getQueryString();
return getRequestBase(req) + (query == null ? "" : "?" + query);
}
/**
* Return the value of the given parameter for the given request. Should only be used if the
* parameter is known to only have one value. If used on a multi-valued parameter, the first value
* is returned.
*
* @param req the HttpServletRequest
* @param paramName the name of the parameter to find.
* @return the value of the given parameter for the given request.
*/
public static String getParameterIgnoreCase(HttpServletRequest req, String paramName) {
Enumeration e = req.getParameterNames();
while (e.hasMoreElements()) {
String s = (String) e.nextElement();
if (s.equalsIgnoreCase(paramName)) return req.getParameter(s);
}
return null;
}
/**
* Return the values of the given parameter (ignoring case) for the given request.
*
* @param req the HttpServletRequest
* @param paramName the name of the parameter to find.
* @return the values of the given parameter for the given request.
*/
public static String[] getParameterValuesIgnoreCase(HttpServletRequest req, String paramName) {
Enumeration e = req.getParameterNames();
while (e.hasMoreElements()) {
String s = (String) e.nextElement();
if (s.equalsIgnoreCase(paramName)) return req.getParameterValues(s);
}
return null;
}
public static String getFileURL(String filename) {
filename = filename.replace('\\', '/');
filename = StringUtil.replace(filename, ' ', "+");
return "file:" + filename;
}
/**
* Show details about the request
*
* @param servlet used to get teh servlet context, may be null
* @param req the request
* @return string showing the details of the request.
*/
public static String showRequestDetail(HttpServlet servlet, HttpServletRequest req) {
StringBuilder sbuff = new StringBuilder();
sbuff.append("Request Info\n");
sbuff.append(" req.getServerName(): ").append(req.getServerName()).append("\n");
sbuff.append(" req.getServerPort(): ").append(req.getServerPort()).append("\n");
sbuff.append(" req.getContextPath:").append(req.getContextPath()).append("\n");
sbuff.append(" req.getServletPath:").append(req.getServletPath()).append("\n");
sbuff.append(" req.getPathInfo:").append(req.getPathInfo()).append("\n");
sbuff.append(" req.getQueryString:").append(req.getQueryString()).append("\n");
sbuff
.append(" getQueryStringDecoded:")
.append(EscapeStrings.urlDecode(req.getQueryString()))
.append("\n");
/*try {
sbuff.append(" getQueryStringDecoded:").append(URLDecoder.decode(req.getQueryString(), "UTF-8")).append("\n");
} catch (UnsupportedEncodingException e1) {
e1.printStackTrace();
}*/
sbuff.append(" req.getRequestURI:").append(req.getRequestURI()).append("\n");
sbuff.append(" getRequestBase:").append(getRequestBase(req)).append("\n");
sbuff.append(" getRequestServer:").append(getRequestServer(req)).append("\n");
sbuff.append(" getRequest:").append(getRequest(req)).append("\n");
sbuff.append("\n");
sbuff.append(" req.getPathTranslated:").append(req.getPathTranslated()).append("\n");
String path = req.getPathTranslated();
if ((path != null) && (servlet != null)) {
ServletContext context = servlet.getServletContext();
sbuff.append(" getMimeType:").append(context.getMimeType(path)).append("\n");
}
sbuff.append("\n");
sbuff.append(" req.getScheme:").append(req.getScheme()).append("\n");
sbuff.append(" req.getProtocol:").append(req.getProtocol()).append("\n");
sbuff.append(" req.getMethod:").append(req.getMethod()).append("\n");
sbuff.append("\n");
sbuff.append(" req.getContentType:").append(req.getContentType()).append("\n");
sbuff.append(" req.getContentLength:").append(req.getContentLength()).append("\n");
sbuff.append(" req.getRemoteAddr():").append(req.getRemoteAddr());
try {
sbuff
.append(" getRemoteHost():")
.append(java.net.InetAddress.getByName(req.getRemoteHost()).getHostName())
.append("\n");
} catch (java.net.UnknownHostException e) {
sbuff.append(" getRemoteHost():").append(e.getMessage()).append("\n");
}
sbuff.append(" getRemoteUser():").append(req.getRemoteUser()).append("\n");
sbuff.append("\n");
sbuff.append("Request Parameters:\n");
Enumeration params = req.getParameterNames();
while (params.hasMoreElements()) {
String name = (String) params.nextElement();
String values[] = req.getParameterValues(name);
if (values != null) {
for (int i = 0; i < values.length; i++) {
sbuff
.append(" ")
.append(name)
.append(" (")
.append(i)
.append("): ")
.append(values[i])
.append("\n");
}
}
}
sbuff.append("\n");
sbuff.append("Request Headers:\n");
Enumeration names = req.getHeaderNames();
while (names.hasMoreElements()) {
String name = (String) names.nextElement();
Enumeration values = req.getHeaders(name); // support multiple values
if (values != null) {
while (values.hasMoreElements()) {
String value = (String) values.nextElement();
sbuff.append(" ").append(name).append(": ").append(value).append("\n");
}
}
}
sbuff.append(" ------------------\n");
return sbuff.toString();
}
public static String showRequestHeaders(HttpServletRequest req) {
StringBuilder sbuff = new StringBuilder();
sbuff.append("Request Headers:\n");
Enumeration names = req.getHeaderNames();
while (names.hasMoreElements()) {
String name = (String) names.nextElement();
Enumeration values = req.getHeaders(name); // support multiple values
if (values != null) {
while (values.hasMoreElements()) {
String value = (String) values.nextElement();
sbuff.append(" ").append(name).append(": ").append(value).append("\n");
}
}
}
return sbuff.toString();
}
public static void showSession(HttpServletRequest req, HttpServletResponse res, PrintStream out) {
// res.setContentType("text/html");
// Get the current session object, create one if necessary
HttpSession session = req.getSession();
// Increment the hit count for this page. The value is saved
// in this client's session under the name "snoop.count".
Integer count = (Integer) session.getAttribute("snoop.count");
if (count == null) {
count = 1;
} else count = count + 1;
session.setAttribute("snoop.count", count);
out.println(HtmlWriter.getInstance().getHtmlDoctypeAndOpenTag());
out.println("<HEAD><TITLE>SessionSnoop</TITLE></HEAD>");
out.println("<BODY><H1>Session Snoop</H1>");
// Display the hit count for this page
out.println(
"You've visited this page " + count + ((!(count.intValue() != 1)) ? " time." : " times."));
out.println("<P>");
out.println("<H3>Here is your saved session data:</H3>");
Enumeration atts = session.getAttributeNames();
while (atts.hasMoreElements()) {
String name = (String) atts.nextElement();
out.println(name + ": " + session.getAttribute(name) + "<BR>");
}
out.println("<H3>Here are some vital stats on your session:</H3>");
out.println("Session id: " + session.getId() + " <I>(keep it secret)</I><BR>");
out.println("New session: " + session.isNew() + "<BR>");
out.println("Timeout: " + session.getMaxInactiveInterval());
out.println("<I>(" + session.getMaxInactiveInterval() / 60 + " minutes)</I><BR>");
out.println("Creation time: " + session.getCreationTime());
out.println("<I>(" + new Date(session.getCreationTime()) + ")</I><BR>");
out.println("Last access time: " + session.getLastAccessedTime());
out.println("<I>(" + new Date(session.getLastAccessedTime()) + ")</I><BR>");
out.println(
"Requested session ID from cookie: " + req.isRequestedSessionIdFromCookie() + "<BR>");
out.println("Requested session ID from URL: " + req.isRequestedSessionIdFromURL() + "<BR>");
out.println("Requested session ID valid: " + req.isRequestedSessionIdValid() + "<BR>");
out.println("<H3>Test URL Rewriting</H3>");
out.println("Click <A HREF=\"" + res.encodeURL(req.getRequestURI()) + "\">here</A>");
out.println("to test that session tracking works via URL");
out.println("rewriting even when cookies aren't supported.");
out.println("</BODY></HTML>");
}
public static void showSession(HttpServletRequest req, PrintStream out) {
// res.setContentType("text/html");
// Get the current session object, create one if necessary
HttpSession session = req.getSession();
out.println("Session id: " + session.getId());
out.println(" session.isNew(): " + session.isNew());
out.println(" session.getMaxInactiveInterval(): " + session.getMaxInactiveInterval() + " secs");
out.println(
" session.getCreationTime(): "
+ session.getCreationTime()
+ " ("
+ new Date(session.getCreationTime())
+ ")");
out.println(
" session.getLastAccessedTime(): "
+ session.getLastAccessedTime()
+ " ("
+ new Date(session.getLastAccessedTime())
+ ")");
out.println(" req.isRequestedSessionIdFromCookie: " + req.isRequestedSessionIdFromCookie());
out.println(" req.isRequestedSessionIdFromURL: " + req.isRequestedSessionIdFromURL());
out.println(" req.isRequestedSessionIdValid: " + req.isRequestedSessionIdValid());
out.println("Saved session Attributes:");
Enumeration atts = session.getAttributeNames();
while (atts.hasMoreElements()) {
String name = (String) atts.nextElement();
out.println(" " + name + ": " + session.getAttribute(name) + "<BR>");
}
}
public static String showSecurity(HttpServletRequest req, String role) {
StringBuilder sbuff = new StringBuilder();
sbuff.append("Security Info\n");
sbuff.append(" req.getRemoteUser(): ").append(req.getRemoteUser()).append("\n");
sbuff.append(" req.getUserPrincipal(): ").append(req.getUserPrincipal()).append("\n");
sbuff
.append(" req.isUserInRole(")
.append(role)
.append("):")
.append(req.isUserInRole(role))
.append("\n");
sbuff.append(" ------------------\n");
return sbuff.toString();
}
/* from luca / ageci code, portResolver, portMapper not known
static public void getSecureRedirect(HttpServletRequest req) {
String pathInfo = req.getPathInfo();
String queryString = req.getQueryString();
String contextPath = req.getContextPath();
String destination = req.getServletPath() + ((pathInfo == null) ? "" : pathInfo)
+ ((queryString == null) ? "" : ("?" + queryString));
String redirectUrl = contextPath;
Integer httpPort = new Integer(portResolver.getServerPort(req));
Integer httpsPort = portMapper.lookupHttpsPort(httpPort);
if (httpsPort != null) {
boolean includePort = true;
if (httpsPort.intValue() == 443) {
includePort = false;
}
redirectUrl = "https://" + req.getServerName() + ((includePort) ? (":" + httpsPort) : "") + contextPath
+ destination;
}
} */
private static String getServerInfoName(String serverInfo) {
int slash = serverInfo.indexOf('/');
if (slash == -1) return serverInfo;
else return serverInfo.substring(0, slash);
}
private static String getServerInfoVersion(String serverInfo) {
// Version info is everything between the slash and the space
int slash = serverInfo.indexOf('/');
if (slash == -1) return null;
int space = serverInfo.indexOf(' ', slash);
if (space == -1) space = serverInfo.length();
return serverInfo.substring(slash + 1, space);
}
public static void showThreads(PrintStream pw) {
Thread current = Thread.currentThread();
ThreadGroup group = current.getThreadGroup();
while (true) {
if (group.getParent() == null) break;
group = group.getParent();
}
showThreads(pw, group, current);
}
private static void showThreads(PrintStream pw, ThreadGroup g, Thread current) {
int nthreads = g.activeCount();
pw.println("\nThread Group = " + g.getName() + " activeCount= " + nthreads);
Thread[] tarray = new Thread[nthreads];
int n = g.enumerate(tarray, false);
for (int i = 0; i < n; i++) {
Thread thread = tarray[i];
ClassLoader loader = thread.getContextClassLoader();
String loaderName = (loader == null) ? "Default" : loader.getClass().getName();
Thread.State state = thread.getState();
long id = thread.getId();
pw.print(" " + id + " " + thread.getName() + " " + state + " " + loaderName);
if (thread == current) pw.println(" **** CURRENT ***");
else pw.println();
}
int ngroups = g.activeGroupCount();
ThreadGroup[] garray = new ThreadGroup[ngroups];
int ng = g.enumerate(garray, false);
for (int i = 0; i < ng; i++) {
ThreadGroup nested = garray[i];
showThreads(pw, nested, current);
}
}
}
/**
* A class containing static methods which deliver descriptions and names of parameters, levels and
* units for byte codes from GRIB records.
*
* <p>Performs operations related to loading parameter tables stored in files. Through a lookup
* table (see readParameterTableLookup) all of the supported Parameter Tables are known. An actual
* table is not loaded until a parameter from that center/subcenter/table is loaded. see <a
* href="../../../Parameters.txt">Parameters.txt</a>
*
* <p>For now, the lookup table name is hard coded to "resources/grib/tables/tablelookup.lst"
*
* @author Capt Richard D. Gonzalez modified by Robb Kambic threadsafe 9/25/08 jcaron see
* http://www.ibm.com/developerworks/java/library/j-hashmap.html
*/
public final class GribPDSParamTable {
private static org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(GribPDSParamTable.class);
private static final String RESOURCE_PATH = "resources/grib/tables";
private static final String TABLE_LIST = "tablelookup.lst";
private static final Pattern valid = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_@:\\.\\-\\+]*$");
private static final Pattern numberFirst = Pattern.compile("^[0-9]");
/**
* Added by Richard D. Gonzalez static Array with parameter tables used by the GRIB file (should
* only be one, but not actually limited to that - this allows GRIB files to be read that have
* more than one center's information in it)
*/
private static volatile GribPDSParamTable[] paramTables = null;
private static Object lock = new Object();
private static boolean debug = false;
private static int wmoTable;
/**
* This is a mapping from (center,subcenter,number)-> Param table for any data that has been
* loaded
*/
private static Map<String, GribPDSParamTable> tableMap =
new ConcurrentHashMap<String, GribPDSParamTable>();
static {
try {
ArrayList<GribPDSParamTable> tables = new ArrayList<GribPDSParamTable>();
String resourceName = RESOURCE_PATH + "/" + TABLE_LIST;
readTableEntries(resourceName, tables);
paramTables = (GribPDSParamTable[]) tables.toArray(new GribPDSParamTable[tables.size()]);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
/**
* _more_
*
* @param aTableList _more_
* @param aTables _more_
* @return Was read successful
* @throws IOException On badness
*/
private static boolean readTableEntries(String aTableList, ArrayList<GribPDSParamTable> aTables)
throws IOException {
InputStream inputStream = GribResourceReader.getInputStream(aTableList);
if (inputStream == null) {
logger.debug("Could not open table file:" + aTableList);
return false;
}
return readTableEntries(inputStream, aTableList, aTables);
}
/**
* Read the table list contained in the input stream
*
* @param is The input stream
* @param aTableList The name of the table list file
* @param aTables The list to add the tables into
* @return Was successful
* @throws IOException On badness
*/
private static boolean readTableEntries(
InputStream is, String aTableList, ArrayList<GribPDSParamTable> aTables) throws IOException {
if (is == null) return false;
InputStreamReader isr = new InputStreamReader(is);
BufferedReader br = new BufferedReader(isr);
String line;
while ((line = br.readLine()) != null) {
line = line.trim();
if ((line.length() == 0) || line.startsWith("#")) {
continue;
}
GribPDSParamTable table = new GribPDSParamTable();
String[] tableDefArr = line.split(":");
table.center_id = Integer.parseInt(tableDefArr[0].trim());
table.subcenter_id = Integer.parseInt(tableDefArr[1].trim());
table.table_number = Integer.parseInt(tableDefArr[2].trim());
table.filename = tableDefArr[3].trim();
if (table.filename.startsWith("/")
|| table.filename.startsWith("\\")
|| table.filename.startsWith("file:")
|| table.filename.startsWith("http://")) {
table.path = table.filename;
} else if (aTableList != null) {
table.path = GribResourceReader.getFileRoot(aTableList);
if (table.path.equals(aTableList)) {
table.path = table.filename;
} else {
table.path = table.path + "/" + table.filename;
}
table.path = table.path.replace('\\', '/');
}
aTables.add(table);
}
is.close();
return true;
}
public static GribPDSParamTable[] getParameterTables() {
return paramTables;
}
/**
* Reads in the list of tables available and stores them. Does not actually open the parameter
* tables files, nor store the list of parameters, but just stores the file names of the parameter
* tables. Parameters for a table are read in when the table is requested (in the
* getParameterTable method).
*
* @param is UserGribTabList as a InputStream
* @throws IOException or read error
*/
public static void addParameterUserLookup(InputStream is) throws IOException {
// leave out of lock since it does IO
ArrayList<GribPDSParamTable> tables = new ArrayList<GribPDSParamTable>();
if (!readTableEntries(is, null, tables)) {
return;
}
synchronized (lock) {
// tmp table stores new user defined tables plus tablelookup.lst table entries
GribPDSParamTable[] tmp = new GribPDSParamTable[paramTables.length + tables.size()];
for (int idx = 0; idx < paramTables.length + tables.size(); idx++) {
if (idx < tables.size()) {
tmp[idx] = (GribPDSParamTable) tables.get(idx);
// System.out.println( "usrlookup tables = " + tmp[ idx ].path );
} else {
tmp[idx] = paramTables[idx - tables.size()]; // tablelookup.lst entries
}
}
paramTables = tmp; // new copy of the data structure
}
}
/**
* Reads in the list of tables available and stores them. Does not actually open the parameter
* tables files, nor store the list of parameters, but just stores the file names of the parameter
* tables. Parameters for a table are read in when the table is requested (in the
* getParameterTable method).
*
* @param userGribTabList string of userlookup file
* @throws IOException if file found but read error
* @return true if read ok, false if file not found
*/
public static boolean addParameterUserLookup(String userGribTabList) throws IOException {
// leave out of lock since it does IO
ArrayList<GribPDSParamTable> tables = new ArrayList<GribPDSParamTable>();
if (!readTableEntries(userGribTabList, tables)) {
// logger.error("could not read:" + userGribTabList);
return false;
}
synchronized (lock) {
// tmp table stores new user defined tables plus tablelookup.lst table entries
GribPDSParamTable[] tmp = new GribPDSParamTable[paramTables.length + tables.size()];
for (int idx = 0; idx < paramTables.length + tables.size(); idx++) {
if (idx < tables.size()) {
tmp[idx] = tables.get(idx); // new stuff first
} else {
tmp[idx] = paramTables[idx - tables.size()]; // old stuff
}
}
paramTables = tmp; // new copy of the data structure
}
return true;
}
/**
* Looks for the parameter table which matches the center, subcenter and table version from the
* tables array. If this is the first time asking for this table, then the parameters for this
* table have not been read in yet, so this is done as well.
*
* @param center - integer from PDS octet 5, representing Center.
* @param subcenter - integer from PDS octet 26, representing Subcenter
* @param tableVersion - integer from PDS octet 4, representing Parameter Table Version
* @return GribPDSParamTable matching center, subcenter, and number
* @throws NotSupportedException no table found
*/
public static GribPDSParamTable getParameterTable(int center, int subcenter, int tableVersion)
throws NotSupportedException {
String key = center + "_" + subcenter + "_" + tableVersion;
if (center == -1) { // non existent table
logger.error("GribPDSParamTable: non existent table for center, subcenter, table = " + key);
return null;
}
GribPDSParamTable table = tableMap.get(key);
if (table != null) return table;
table = readParameterTable(center, subcenter, tableVersion, true);
if (table == null) {
logger.error("GribPDSParamTable: cannot find table for center, subcenter, table " + key);
throw new NotSupportedException(
"Could not find a table entry for GRIB file with center: "
+ center
+ " subCenter: "
+ subcenter
+ " number: "
+ tableVersion);
}
tableMap.put(key, table);
return table;
}
/**
* Looks for the parameter table which matches the center, subcenter and table version from the
* tables array. If this is the first time asking for this table, then the parameters for this
* table have not been read in yet, so this is done as well.
*
* @param center - integer from PDS octet 5, representing Center.
* @param subcenter - integer from PDS octet 26, representing Subcenter
* @param number - integer from PDS octet 4, representing Parameter Table Version
* @param firstTry - Is this the first call or are we trying the wild cards
* @return GribPDSParamTable matching center, subcenter, and number
*/
private static GribPDSParamTable readParameterTable(
int center, int subcenter, int number, boolean firstTry) {
if (firstTry) wmoTable = number;
GribPDSParamTable[] localCopy = paramTables; // thread safe
for (GribPDSParamTable table : localCopy) {
if (center == table.center_id) {
if ((table.subcenter_id == -1) || (subcenter == table.subcenter_id)) {
if (number == table.table_number) {
// now that this table is being used, check to see if the
// parameters for this table have been read in yet.
// If not, initialize table and read them in now.
if (table.parameters == null) {
if (!firstTry) {
logger.warn(
"GribPDSParamTable: Using default table:"
+ table.path
+ " ("
+ table.center_id
+ ":"
+ table.subcenter_id
+ ":"
+ table.table_number
+ ")");
}
table.readParameterTable();
if (table.parameters
== null) // failed - maybe theres another entry table in paramTables
continue;
// success - initialize other tables parameters with the same name
for (int j = 0; j < paramTables.length; j++) {
GribPDSParamTable tab = paramTables[j];
if (tab.path.equals(table.path)) {
tab.parameters = table.parameters;
}
}
}
return table;
}
}
}
}
// Try with the wild cards
if (number != -1) {
return readParameterTable(center, subcenter, -1, false);
} else if (subcenter != -1) {
logger.warn(
"GribPDSParamTable: Could not find table for center:"
+ center
+ " subcenter:"
+ subcenter
+ " number:"
+ wmoTable);
return readParameterTable(center, -1, -1, false);
} else if (center != -1) {
// return readParameterTable(-1, -1, -1, false);
return readParameterTable(-1, -1, wmoTable, false);
}
return null;
}
/**
* Munge a description to make it suitable as variable name
*
* @param description start with this
* @return Valid Description
*/
private static String makeValidDesc(String description) {
description = description.replaceAll("\\s+", "_");
if (valid.matcher(description).find()) return description;
// else check for special characters
if (numberFirst.matcher(description).find()) description = "N" + description;
return description.replaceAll("\\)|\\(|=|,|;|\\[|\\]", "");
}
//////////////////////////////////////////////////////////////////////////
/** Identification of center e.g. 88 for Oslo */
private int center_id;
/** Identification of center defined sub-center - not fully implemented yet. */
private int subcenter_id;
/** Identification of parameter table version number. */
private int table_number;
/** Stores the name of the file containing this table - not opened unless required for lookup. */
private String filename = null;
/** path of filename containing this table. Opened if required for lookup. */
private String path = null;
/** Map ids to GridParameter objects */
private Map<String, GridParameter> parameters = null;
private GribPDSParamTable() {}
public int getCenter_id() {
return center_id;
}
public int getSubcenter_id() {
return subcenter_id;
}
public int getTable_number() {
return table_number;
}
public String getPath() {
return path;
}
public String getFilename() {
return filename;
}
public Map<String, GridParameter> getParameters() {
if (parameters == null) readParameterTable();
return parameters;
}
/** Read parameter table. */
private void readParameterTable() {
if (path == null) {
logger.error("GribPDSParamTable: unknown path for " + this);
return;
}
try {
InputStream is = GribResourceReader.getInputStream(path);
if (is == null) {
logger.error("GribPDSParamTable: error getInputStream on " + this);
return;
}
BufferedReader br = new BufferedReader(new InputStreamReader(is));
// Read first line that has center, subcenter, and version of table
String line = br.readLine();
if (debug) System.out.println(line);
String[] tableDefArr = line.split(":");
/* LOOK - why not test values ?
center = Integer.parseInt(tableDefArr[1].trim());
subcenter = Integer.parseInt(tableDefArr[2].trim());
number = Integer.parseInt(tableDefArr[3].trim());
if ((center != center_id) && (subcenter != subcenter_id)
&& (number != table_number)) {
throw new java.io.IOException(
"parameter table header values do not "
+ " match values in GRIB file. Possible error in lookup table.");
}
*/
HashMap<String, GridParameter> tmpParameters =
new HashMap<String, GridParameter>(); // thread safe - temp hash
// rdg - added the 0 line length check to cover the case of blank lines at
// the end of the parameter table file.
while ((line = br.readLine()) != null) {
if ((line.length() == 0) || line.startsWith("#")) {
continue;
}
GridParameter parameter = new GridParameter();
tableDefArr = line.split(":");
parameter.setNumber(Integer.parseInt(tableDefArr[0].trim()));
parameter.setName(tableDefArr[1].trim());
// check to see if unit defined, if not, parameter is undefined
if (tableDefArr[2].indexOf('[') == -1) {
// Undefined unit
parameter.setDescription(tableDefArr[2].trim());
parameter.setUnit(tableDefArr[2].trim());
} else {
String[] arr2 = tableDefArr[2].split("\\[");
parameter.setDescription(makeValidDesc(arr2[0].trim()));
// System.out.println( "Desc ="+ parameter.getDescription());
// Remove "]"
parameter.setUnit(arr2[1].substring(0, arr2[1].lastIndexOf(']')).trim());
}
tmpParameters.put(Integer.toString(parameter.getNumber()), parameter);
if (debug)
System.out.println(
parameter.getNumber() + " " + parameter.getDescription() + " " + parameter.getUnit());
}
this.parameters = tmpParameters; // thread safe
} catch (IOException ioError) {
logger.warn(
"An error occurred in GribPDSParamTable while trying to open the parameter table "
+ filename
+ " : "
+ ioError);
}
}
/**
* Get the parameter with id <tt>id</tt>.
*
* @param id the parameter id
* @return the GridParameter
*/
public GridParameter getParameter(int id) {
GridParameter p = parameters.get(Integer.toString(id));
if (p != null) return p;
logger.warn(
"GribPDSParamTable: Could not find parameter "
+ id
+ " for center:"
+ center_id
+ " subcenter:"
+ subcenter_id
+ " number:"
+ table_number
+ " table "
+ filename);
String unknown = "UnknownParameter_" + Integer.toString(id) + "_table_" + filename;
return new GridParameter(id, unknown, unknown, "Unknown");
}
@Override
public String toString() {
return "GribPDSParamTable{"
+ "center_id="
+ center_id
+ ", subcenter_id="
+ subcenter_id
+ ", table_number="
+ table_number
+ ", filename='"
+ filename
+ '\''
+ ", path='"
+ path
+ '\''
+ '}';
}
public static void main(String[] args) throws IOException {
debug = true;
addParameterUserLookup(
"C:/dev/tds4.2/thredds/grib/resources/resources/grib/tables/zagreb_221_1.tab");
}
}
/**
* A Vertical Coordinate variable for a Grid variable.
*
* @author caron
* @version $Revision: 63 $ $Date: 2006-07-12 15:50:51 -0600 (Wed, 12 Jul 2006) $
*/
public class GridVertCoord implements Comparable {
/** logger */
private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GridVertCoord.class);
/** typical record for this vertical coordinate */
private GridRecord typicalRecord;
/** level name */
private String levelName;
/** lookup table */
private GridTableLookup lookup;
/** sequence # */
private int seq = 0;
/** coord values */
private double[] coordValues = null;
/** uses bounds flag */
boolean usesBounds = false;
/** don't use vertical flag */
boolean dontUseVertical = false;
/** vertical pressure factors */
double[] factors = null;
/** positive direction */
String positive = "up";
/** units */
String units;
/** levels */
private List<LevelCoord> levels = new ArrayList<LevelCoord>(); // LevelCoord
/**
* Create a new GridVertCoord with the given name
*
* @param name name
*/
GridVertCoord(String name) {
this.levelName = name;
dontUseVertical = true;
}
/**
* Create a new GridVertCoord with the appropriate params
*
* @param records list of GridRecords that make up this coord
* @param levelName the name of the level
* @param lookup the lookup table
* @param hcs Horizontal coordinate
*/
GridVertCoord(
List<GridRecord> records, String levelName, GridTableLookup lookup, GridHorizCoordSys hcs) {
this.typicalRecord = records.get(0);
this.levelName = levelName;
this.lookup = lookup;
dontUseVertical = !lookup.isVerticalCoordinate(typicalRecord);
positive = lookup.isPositiveUp(typicalRecord) ? "up" : "down";
units = lookup.getLevelUnit(typicalRecord);
usesBounds = lookup.isLayer(this.typicalRecord);
addLevels(records);
if (typicalRecord.getLevelType1() == 109 && lookup instanceof Grib1GridTableLookup)
checkForPressureLevels(records, hcs);
if (GridServiceProvider.debugVert) {
System.out.println(
"GribVertCoord: "
+ getVariableName()
+ "("
+ typicalRecord.getLevelType1()
+ ") useVertical= "
+ (!dontUseVertical)
+ " positive="
+ positive
+ " units="
+ units);
}
}
/**
* Create a new GridVertCoord for a layer
*
* @param record layer record
* @param levelName name of this level
* @param lookup lookup table
* @param level1 level 1
* @param level2 level 2
*/
GridVertCoord(
GridRecord record,
String levelName,
GridTableLookup lookup,
double[] level1,
double[] level2) {
this.typicalRecord = record;
this.levelName = levelName;
this.lookup = lookup;
// dontUseVertical = !lookup.isVerticalCoordinate(record);
positive = lookup.isPositiveUp(record) ? "up" : "down";
units = lookup.getLevelUnit(record);
usesBounds = lookup.isLayer(this.typicalRecord);
levels = new ArrayList<LevelCoord>(level1.length);
for (int i = 0; i < level1.length; i++) {
levels.add(new LevelCoord(level1[i], (level2 == null) ? 0.0 : level2[i]));
}
Collections.sort(levels);
if (positive.equals("down")) {
Collections.reverse(levels);
}
dontUseVertical = (levels.size() == 1);
}
/**
* Set the sequence number
*
* @param seq the sequence number
*/
void setSequence(int seq) {
this.seq = seq;
}
/**
* Set the level name
*
* @return the level name
*/
String getLevelName() {
return levelName;
}
/**
* Get the variable name
*
* @return the variable name
*/
String getVariableName() {
return (seq == 0) ? levelName : levelName + seq; // more than one with same levelName
}
/**
* Get the number of levels
*
* @return number of levels
*/
int getNLevels() {
return dontUseVertical ? 1 : levels.size();
}
/**
* Add levels
*
* @param records GridRecords, one for each level
*/
void addLevels(List<GridRecord> records) {
for (int i = 0; i < records.size(); i++) {
GridRecord record = records.get(i);
if (coordIndex(record) < 0) {
levels.add(new LevelCoord(record.getLevel1(), record.getLevel2()));
if (dontUseVertical && (levels.size() > 1)) {
if (GridServiceProvider.debugVert) {
logger.warn(
"GribCoordSys: unused level coordinate has > 1 levels = "
+ levelName
+ " "
+ record.getLevelType1()
+ " "
+ levels.size());
}
}
}
}
Collections.sort(levels);
if (positive.equals("down")) {
Collections.reverse(levels);
}
}
/**
* Match levels
*
* @param records records to match
* @return true if they have the same levels
*/
boolean matchLevels(List<GridRecord> records) {
// first create a new list
List<LevelCoord> levelList = new ArrayList<LevelCoord>(records.size());
for (GridRecord record : records) {
LevelCoord lc = new LevelCoord(record.getLevel1(), record.getLevel2());
if (!levelList.contains(lc)) {
levelList.add(lc);
}
}
Collections.sort(levelList);
if (positive.equals("down")) {
Collections.reverse(levelList);
}
// gotta equal existing list
return levelList.equals(levels);
}
/** check for Sigma Pressure Levels */
boolean checkForPressureLevels(List<GridRecord> records, GridHorizCoordSys hcs) {
GridDefRecord gdr = hcs.getGds();
Grib1GDSVariables g1dr = (Grib1GDSVariables) gdr.getGdsv();
if (g1dr == null || !g1dr.hasVerticalPressureLevels()) return false;
// add hybrid numbers
coordValues = new double[levels.size()];
for (int i = 0; i < levels.size(); i++) {
LevelCoord lc = levels.get(i);
coordValues[i] = lc.value1;
}
int NV = g1dr.getNV();
// add new variables
if (NV > 2 && NV < 255) { // Some data doesn't add Pressure Level values
factors = g1dr.getVerticalPressureLevels();
}
return true;
}
/**
* Add this coord as a dimension to the netCDF file
*
* @param ncfile file to add to
* @param g group in the file
*/
void addDimensionsToNetcdfFile(NetcdfFile ncfile, Group g) {
if (dontUseVertical) {
return;
}
int nlevs = levels.size();
if (coordValues != null) nlevs = coordValues.length;
ncfile.addDimension(g, new Dimension(getVariableName(), nlevs, true));
}
/**
* Add this coord as a variable in the netCDF file
*
* @param ncfile netCDF file to add to
* @param g group in file
*/
void addToNetcdfFile(NetcdfFile ncfile, Group g) {
if (dontUseVertical) {
typicalRecord = null;
return;
}
if (g == null) {
g = ncfile.getRootGroup();
}
// coordinate axis
Variable v = new Variable(ncfile, g, null, getVariableName());
v.setDataType(DataType.DOUBLE);
String desc = lookup.getLevelDescription(typicalRecord);
if (lookup instanceof Grib2GridTableLookup && usesBounds) {
desc = "Layer between " + desc;
}
v.addAttribute(new Attribute("long_name", desc));
v.addAttribute(new Attribute("units", lookup.getLevelUnit(typicalRecord)));
// positive attribute needed for CF-1 Height and Pressure
if (positive != null) {
v.addAttribute(new Attribute("positive", positive));
}
if (units != null) {
AxisType axisType;
if (SimpleUnit.isCompatible("millibar", units)) {
axisType = AxisType.Pressure;
} else if (SimpleUnit.isCompatible("m", units)) {
axisType = AxisType.Height;
} else {
axisType = AxisType.GeoZ;
}
if (lookup instanceof Grib2GridTableLookup || lookup instanceof Grib1GridTableLookup) {
v.addAttribute(
new Attribute("GRIB_level_type", Integer.toString(typicalRecord.getLevelType1())));
} else {
v.addAttribute(
new Attribute("level_type", Integer.toString(typicalRecord.getLevelType1())));
}
v.addAttribute(new Attribute(_Coordinate.AxisType, axisType.toString()));
}
if (coordValues == null) {
coordValues = new double[levels.size()];
for (int i = 0; i < levels.size(); i++) {
LevelCoord lc = (LevelCoord) levels.get(i);
coordValues[i] = lc.mid;
}
}
Array dataArray = Array.factory(DataType.DOUBLE, new int[] {coordValues.length}, coordValues);
v.setDimensions(getVariableName());
v.setCachedData(dataArray, true);
ncfile.addVariable(g, v);
if (usesBounds) {
String boundsDimName = "bounds_dim";
if (g.findDimension(boundsDimName) == null) {
ncfile.addDimension(g, new Dimension(boundsDimName, 2, true));
}
String bname = getVariableName() + "_bounds";
v.addAttribute(new Attribute("bounds", bname));
v.addAttribute(new Attribute(_Coordinate.ZisLayer, "true"));
Variable b = new Variable(ncfile, g, null, bname);
b.setDataType(DataType.DOUBLE);
b.setDimensions(getVariableName() + " " + boundsDimName);
b.addAttribute(new Attribute("long_name", "bounds for " + v.getName()));
b.addAttribute(new Attribute("units", lookup.getLevelUnit(typicalRecord)));
Array boundsArray = Array.factory(DataType.DOUBLE, new int[] {coordValues.length, 2});
ucar.ma2.Index ima = boundsArray.getIndex();
for (int i = 0; i < coordValues.length; i++) {
LevelCoord lc = (LevelCoord) levels.get(i);
boundsArray.setDouble(ima.set(i, 0), lc.value1);
boundsArray.setDouble(ima.set(i, 1), lc.value2);
}
b.setCachedData(boundsArray, true);
ncfile.addVariable(g, b);
}
if (factors != null) {
// check if already created
if (g == null) {
g = ncfile.getRootGroup();
}
if (g.findVariable("hybrida") != null) return;
v.addAttribute(new Attribute("standard_name", "atmosphere_hybrid_sigma_pressure_coordinate"));
v.addAttribute(new Attribute("formula_terms", "ap: hybrida b: hybridb ps: Pressure"));
// create hybrid factor variables
// add hybrida variable
Variable ha = new Variable(ncfile, g, null, "hybrida");
ha.setDataType(DataType.DOUBLE);
ha.addAttribute(new Attribute("long_name", "level_a_factor"));
ha.addAttribute(new Attribute("units", ""));
ha.setDimensions(getVariableName());
// add data
int middle = factors.length / 2;
double[] adata;
double[] bdata;
if (levels.size() < middle) { // only partial data wanted
adata = new double[levels.size()];
bdata = new double[levels.size()];
} else {
adata = new double[middle];
bdata = new double[middle];
}
for (int i = 0; i < middle && i < levels.size(); i++) adata[i] = factors[i];
Array haArray = Array.factory(DataType.DOUBLE, new int[] {adata.length}, adata);
ha.setCachedData(haArray, true);
ncfile.addVariable(g, ha);
// add hybridb variable
Variable hb = new Variable(ncfile, g, null, "hybridb");
hb.setDataType(DataType.DOUBLE);
hb.addAttribute(new Attribute("long_name", "level_b_factor"));
hb.addAttribute(new Attribute("units", ""));
hb.setDimensions(getVariableName());
// add data
for (int i = 0; i < middle && i < levels.size(); i++) bdata[i] = factors[i + middle];
Array hbArray = Array.factory(DataType.DOUBLE, new int[] {bdata.length}, bdata);
hb.setCachedData(hbArray, true);
ncfile.addVariable(g, hb);
/* // TODO: delete next time modifying code
double[] adata = new double[ middle ];
for( int i = 0; i < middle; i++ )
adata[ i ] = factors[ i ];
Array haArray = Array.factory(DataType.DOUBLE, new int[]{adata.length}, adata);
ha.setCachedData(haArray, true);
ncfile.addVariable(g, ha);
// add hybridb variable
Variable hb = new Variable(ncfile, g, null, "hybridb");
hb.setDataType(DataType.DOUBLE);
hb.addAttribute(new Attribute("long_name", "level_b_factor" ));
//hb.addAttribute(new Attribute("standard_name", "atmosphere_hybrid_sigma_pressure_coordinate" ));
hb.addAttribute(new Attribute("units", ""));
hb.setDimensions(getVariableName());
// add data
double[] bdata = new double[ middle ];
for( int i = 0; i < middle; i++ )
bdata[ i ] = factors[ i + middle ];
Array hbArray = Array.factory(DataType.DOUBLE, new int[]{bdata.length}, bdata);
hb.setCachedData(hbArray, true);
ncfile.addVariable(g, hb);
*/
}
}
void empty() {
// let all references to Index go, to reduce retained size
typicalRecord = null;
}
/**
* Get the index of the particular record
*
* @param record record in question
* @return the index or -1 if not found
*/
int getIndex(GridRecord record) {
if (dontUseVertical) {
return 0;
}
return coordIndex(record);
}
/**
* Compare this to another
*
* @param o the other GridVertCoord
* @return the comparison
*/
public int compareTo(Object o) {
GridVertCoord gv = (GridVertCoord) o;
return getLevelName().compareToIgnoreCase(gv.getLevelName());
}
/**
* A level coordinate
*
* @author IDV Development Team
* @version $Revision: 1.3 $
*/
private class LevelCoord implements Comparable {
/** midpoint */
double mid;
/** top/bottom values */
double value1, value2;
/**
* Create a new LevelCoord
*
* @param value1 top
* @param value2 bottom
*/
LevelCoord(double value1, double value2) {
this.value1 = value1;
this.value2 = value2;
if (usesBounds && (value1 > value2)) {
this.value1 = value2;
this.value2 = value1;
}
mid = usesBounds ? (value1 + value2) / 2 : value1;
}
/**
* Compare to another LevelCoord
*
* @param o another LevelCoord
* @return the comparison
*/
public int compareTo(Object o) {
LevelCoord other = (LevelCoord) o;
// if (closeEnough(value1, other.value1) && closeEnough(value2, other.value2)) return 0;
if (mid < other.mid) {
return -1;
}
if (mid > other.mid) {
return 1;
}
return 0;
}
/**
* Check for equality
*
* @param oo object in question
* @return true if equal
*/
public boolean equals(Object oo) {
if (this == oo) {
return true;
}
if (!(oo instanceof LevelCoord)) {
return false;
}
LevelCoord other = (LevelCoord) oo;
return (ucar.nc2.util.Misc.closeEnough(value1, other.value1)
&& ucar.nc2.util.Misc.closeEnough(value2, other.value2));
}
/**
* Generate a hashcode
*
* @return the hashcode
*/
public int hashCode() {
return (int) (value1 * 100000 + value2 * 100);
}
}
/**
* Get the coordinate index for the record
*
* @param record record in question
* @return index or -1 if not found
*/
private int coordIndex(GridRecord record) {
double val = record.getLevel1();
double val2 = record.getLevel2();
if (usesBounds && (val > val2)) {
val = record.getLevel2();
val2 = record.getLevel1();
}
for (int i = 0; i < levels.size(); i++) {
LevelCoord lc = (LevelCoord) levels.get(i);
if (usesBounds) {
if (ucar.nc2.util.Misc.closeEnough(lc.value1, val)
&& ucar.nc2.util.Misc.closeEnough(lc.value2, val2)) {
return i;
}
} else {
if (ucar.nc2.util.Misc.closeEnough(lc.value1, val)) {
return i;
}
}
}
return -1;
}
}
/**
* Read and Write Grib1 index (gbx9). Hides Grib1IndexProto
*
* <p>sample use:
*
* <pre>
* Grib1Index index = new Grib1Index();
* if (!index.readIndex(path))
* index.makeIndex(path);
*
* for (Grib1SectionGridDefinition gds : index.getGds()) {
* if (gdsSet.get(gds.calcCRC()) == null)
* gdsSet.put(gds.calcCRC(), gds);
* }
*
* for (Grib1Record gr : index.getRecords()) {
* gr.setFile(fileno);
*
* Grib1Pds pds = gr.getPDSsection().getPDS();
* int discipline = gr.getDiscipline();
*
* int id = gr.cdmVariableHash();
* Grib1ParameterBean bean = pdsSet.get(id);
* if (bean == null) {
* bean = new Grib1ParameterBean(gr);
* pdsSet.put(id, bean);
* params.add(bean);
* }
* bean.addRecord(gr);
* }
* </pre>
*
* @author John
* @since 9/5/11
*/
public class Grib1Index extends GribIndex {
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Grib1Index.class);
public static final String MAGIC_START = "Grib1Index";
private static final int version = 5;
private static final boolean debug = false;
////////////////////////////////////////////////////////////////////////////////////////////////
private List<Grib1SectionGridDefinition> gdsList;
private List<Grib1Record> records;
public List<Grib1SectionGridDefinition> getGds() {
return gdsList;
}
public List<Grib1Record> getRecords() {
return records;
}
@Override
public int getNRecords() {
return records.size();
}
public boolean readIndex(String filename, long gribLastModified) throws IOException {
return readIndex(filename, gribLastModified, CollectionManager.Force.test);
}
public boolean readIndex(String filename, long gribLastModified, CollectionManager.Force force)
throws IOException {
File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
if (!idxFile.exists()) return false;
long idxModified = idxFile.lastModified();
if ((force != CollectionManager.Force.nocheck) && (idxModified < gribLastModified))
return false;
// force new index if file was updated
FileInputStream fin =
new FileInputStream(idxFile); // LOOK need DiskCache for non-writeable directories
try {
//// check header is ok
if (!NcStream.readAndTest(fin, MAGIC_START.getBytes())) {
log.info("Bad magic number of grib index, on file" + idxFile);
return false;
}
int v = NcStream.readVInt(fin);
if (v != version) {
if ((v == 0) || (v > version))
throw new IOException(
"Grib1Index found version " + v + ", want version " + version + " on " + filename);
if (log.isDebugEnabled())
log.debug(
"Grib1Index found version " + v + ", want version " + version + " on " + filename);
return false;
}
int size = NcStream.readVInt(fin);
if (size <= 0 || size > 100 * 1000 * 1000) { // try to catch garbage
log.warn("Grib1Index bad size = {} for {} ", size, filename);
return false;
}
byte[] m = new byte[size];
NcStream.readFully(fin, m);
Grib1IndexProto.Grib1Index proto = Grib1IndexProto.Grib1Index.parseFrom(m);
String fname = proto.getFilename();
if (debug) System.out.printf("%s for %s%n", fname, filename);
gdsList = new ArrayList<Grib1SectionGridDefinition>(proto.getGdsListCount());
for (Grib1IndexProto.Grib1GdsSection pgds : proto.getGdsListList()) {
Grib1SectionGridDefinition gds = readGds(pgds);
gdsList.add(gds);
}
if (debug) System.out.printf(" read %d gds%n", gdsList.size());
records = new ArrayList<Grib1Record>(proto.getRecordsCount());
for (Grib1IndexProto.Grib1Record precord : proto.getRecordsList()) {
records.add(readRecord(precord));
}
if (debug) System.out.printf(" read %d records%n", records.size());
} catch (java.lang.NegativeArraySizeException e) {
log.error("GribIndex failed on " + filename, e);
return false;
} catch (IOException e) {
log.error("GribIndex failed on " + filename, e);
return false;
} finally {
fin.close();
}
return true;
}
// deserialize the Grib1Record object
private Grib1Record readRecord(Grib1IndexProto.Grib1Record p) {
Grib1SectionIndicator is =
new Grib1SectionIndicator(p.getGribMessageStart(), p.getGribMessageLength());
Grib1SectionProductDefinition pds = new Grib1SectionProductDefinition(p.getPds().toByteArray());
Grib1SectionGridDefinition gds =
pds.gdsExists() ? gdsList.get(p.getGdsIdx()) : new Grib1SectionGridDefinition(pds);
Grib1SectionBitMap bms = pds.bmsExists() ? new Grib1SectionBitMap(p.getBmsPos()) : null;
Grib1SectionBinaryData dataSection = new Grib1SectionBinaryData(p.getDataPos(), p.getDataLen());
return new Grib1Record(p.getHeader().toByteArray(), is, gds, pds, bms, dataSection);
}
private Grib1SectionGridDefinition readGds(Grib1IndexProto.Grib1GdsSection proto) {
ByteString bytes = proto.getGds();
return new Grib1SectionGridDefinition(bytes.toByteArray());
}
////////////////////////////////////////////////////////////////////////////////
// LOOK what about extending an index ??
public boolean makeIndex(String filename, RandomAccessFile dataRaf, Formatter f)
throws IOException {
File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
FileOutputStream fout = new FileOutputStream(idxFile);
RandomAccessFile raf = null;
try {
//// header message
fout.write(MAGIC_START.getBytes("UTF-8"));
NcStream.writeVInt(fout, version);
Map<Long, Integer> gdsMap = new HashMap<Long, Integer>();
gdsList = new ArrayList<Grib1SectionGridDefinition>();
records = new ArrayList<Grib1Record>(200);
Grib1IndexProto.Grib1Index.Builder rootBuilder = Grib1IndexProto.Grib1Index.newBuilder();
rootBuilder.setFilename(filename);
if (dataRaf == null) {
raf = new RandomAccessFile(filename, "r");
dataRaf = raf;
}
Grib1RecordScanner scan = new Grib1RecordScanner(dataRaf);
while (scan.hasNext()) {
Grib1Record r = scan.next();
if (r == null) break; // done
records.add(r);
Grib1SectionGridDefinition gds = r.getGDSsection();
Integer index = gdsMap.get(gds.calcCRC());
if (gds.getPredefinedGridDefinition()
>= 0) // skip predefined gds - they dont have raw bytes
index = 0;
else if (index == null) {
gdsList.add(gds);
index = gdsList.size() - 1;
gdsMap.put(gds.calcCRC(), index);
rootBuilder.addGdsList(makeGdsProto(gds));
}
rootBuilder.addRecords(makeRecordProto(r, index));
}
ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Index index = rootBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(fout, b.length); // message size
fout.write(b); // message - all in one gulp
f.format(" made gbx9 index for %s size=%d%n", filename, b.length);
return true;
} finally {
fout.close();
if (raf != null) raf.close();
}
}
private ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Record makeRecordProto(
Grib1Record r, int gdsIndex) throws IOException {
Grib1IndexProto.Grib1Record.Builder b = Grib1IndexProto.Grib1Record.newBuilder();
b.setHeader(ByteString.copyFrom(r.getHeader()));
b.setGribMessageStart(r.getIs().getStartPos());
b.setGribMessageLength(r.getIs().getMessageLength());
b.setGdsIdx(gdsIndex);
Grib1SectionProductDefinition pds = r.getPDSsection();
b.setPds(ByteString.copyFrom(pds.getRawBytes()));
if (pds.bmsExists()) {
Grib1SectionBitMap bms = r.getBitMapSection();
b.setBmsPos(bms.getStartingPosition());
}
Grib1SectionBinaryData ds = r.getDataSection();
b.setDataPos(ds.getStartingPosition());
b.setDataLen(ds.getLength());
return b.build();
}
private Grib1IndexProto.Grib1GdsSection makeGdsProto(Grib1SectionGridDefinition gds)
throws IOException {
Grib1IndexProto.Grib1GdsSection.Builder b = Grib1IndexProto.Grib1GdsSection.newBuilder();
b.setGds(ByteString.copyFrom(gds.getRawBytes()));
return b.build();
}
public static void main(String args[]) throws IOException {
String filename =
"G:/tigge/uv/z_tigge_c_kwbc_20110605120000_glob_prod_cf_HGHT_0000_000_10_uv.grib";
// String filename = "G:/mlode/ndfdProb/extract.Grib1";
new Grib1Index().makeIndex(filename, null, new Formatter(System.out));
}
}
/**
* A pool of precompiled XSLT stylesheets ({@link Templates}). Caching can be disabled via
* constructor parameter or via setting a system property:
*
* <pre>
* template.caching
* </pre>
*
* to <code>false</code>.
*/
public final class TemplatesPool {
private static final Logger logger = org.slf4j.LoggerFactory.getLogger(TemplatesPool.class);
/**
* Global system property disabling template caching. This property can also be set at runtime
* (after the pool is initialized).
*/
public static final String TEMPLATE_CACHING_PROPERTY = "template.caching";
/** A set of used XSLT processors. */
private static final Set<String> reportedProcessors =
Collections.synchronizedSet(new HashSet<String>());
/** A map of precompiled stylesheets ({@link Templates} objects). */
private volatile HashMap<String, Templates> stylesheets = new HashMap<String, Templates>();
/**
* If <code>true</code> the templates will not be cached until the application shuts down. This
* speeds up the application, but may be annoying, especially during development.
*/
private final boolean templateCaching;
/** {@link SAXTransformerFactory} capable of producing SAX-based transformers. */
public final SAXTransformerFactory tFactory;
/** Creates a {@link TemplatesPool} with caching enabled. */
public TemplatesPool() throws Exception {
this(true);
}
/** Check for required facilities. If not available, an exception will be thrown. */
public TemplatesPool(boolean templateCaching) throws Exception {
final TransformerFactory tFactory = TransformerFactory.newInstance();
final String processorClass = tFactory.getClass().getName();
/*
* Only report XSLT processor class once.
*/
if (!reportedProcessors.contains(processorClass)) {
logger.info("XSLT transformer factory: " + processorClass);
reportedProcessors.add(processorClass);
}
if (!tFactory.getFeature(SAXSource.FEATURE) || !tFactory.getFeature(SAXResult.FEATURE)) {
throw new Exception("Required source types not supported by the transformer factory.");
}
if (!tFactory.getFeature(SAXResult.FEATURE) || !tFactory.getFeature(StreamResult.FEATURE)) {
throw new Exception("Required result types not supported by the transformer factory.");
}
if (!(tFactory instanceof SAXTransformerFactory)) {
throw new Exception(
"TransformerFactory not an instance of SAXTransformerFactory: "
+ tFactory.getClass().getName());
}
this.tFactory = ((SAXTransformerFactory) tFactory);
this.tFactory.setErrorListener(new StylesheetErrorListener());
this.templateCaching = templateCaching;
}
/** @return returns the identity transformer handler. */
public TransformerHandler getIdentityTransformerHandler()
throws TransformerConfigurationException {
return tFactory.newTransformerHandler();
}
/** Retrieves a previously stored template, if available. */
public Templates getTemplate(String key) {
if (!isCaching()) {
return null;
}
return stylesheets.get(key);
}
/**
* Add a new template to the pool. Addition is quite costly as it replaces the internal {@link
* #stylesheets} {@link HashMap}.
*/
public void addTemplate(String key, Templates template) {
if (!isCaching()) {
return;
}
/*
* Copy-on-write.
*/
synchronized (this) {
final HashMap<String, Templates> newMap = new HashMap<String, Templates>(this.stylesheets);
newMap.put(key, template);
this.stylesheets = newMap;
}
}
/** @return <code>true</code> if template caching is enabled. */
private boolean isCaching() {
/*
* Global override takes precedence.
*/
final String global = System.getProperty(TEMPLATE_CACHING_PROPERTY);
if (global != null) {
return Boolean.parseBoolean(global);
}
return templateCaching;
}
/**
* Compile a {@link Templates} from a given system identifier. The template is not added to the
* pool, a manual call to {@link #addTemplate(String, Templates)} is required.
*/
public Templates compileTemplate(String systemId) throws SAXException {
final StreamSource source = new StreamSource(systemId);
try {
return tFactory.newTemplates(source);
} catch (Exception e) {
throw new SAXException("Could not compile stylesheet: " + systemId, e);
}
}
/**
* Compile a {@link Templates} from a given stream. The template is not added to the pool
* automatically.
*/
public Templates compileTemplate(InputStream stream) throws SAXException {
final StreamSource source = new StreamSource(stream);
try {
return tFactory.newTemplates(source);
} catch (Exception e) {
throw new SAXException("Could not compile stylesheet.", e);
}
}
/**
* Return a new {@link TransformerHandler} based on a given precompiled {@link Templates}. The
* handler {@link Transformer}'s {@link ErrorListener} is set to {@link TransformerErrorListener}
* to raise exceptions and give proper warnings.
*/
public TransformerHandler newTransformerHandler(Templates template)
throws TransformerConfigurationException {
final TransformerHandler handler = this.tFactory.newTransformerHandler(template);
/*
* We want to raise transformer exceptions on <xml:message terminate="true">, so
* we add a custom listener. Also, various XSLT processors react in different ways
* to transformation errors -- some of them report error as recoverable, some of
* them report error as unrecoverable.
*/
handler.getTransformer().setErrorListener(new TransformerErrorListener());
return handler;
}
/**
* Return a new {@link Transformer}.
*
* @see #newTransformerHandler(Templates)
*/
public Transformer newTransformer(Templates t) throws TransformerConfigurationException {
return newTransformerHandler(t).getTransformer();
}
}
/** @author pepijn */
public class WPTileProvider
implements org.pepsoft.util.swing.TileProvider, Dimension.Listener, Tile.Listener {
public WPTileProvider(
Dimension dimension,
ColourScheme colourScheme,
BiomeScheme biomeScheme,
CustomBiomeManager customBiomeManager,
Collection<Layer> hiddenLayers,
boolean contourLines,
int contourSeparation,
TileRenderer.LightOrigin lightOrigin,
boolean showBorder,
org.pepsoft.util.swing.TileProvider surroundingTileProvider,
boolean active) {
tileProvider = dimension;
this.colourScheme = colourScheme;
this.biomeScheme = biomeScheme;
this.hiddenLayers = (hiddenLayers != null) ? new HashSet<>(hiddenLayers) : null;
this.contourLines = contourLines;
this.contourSeparation = contourSeparation;
this.lightOrigin = lightOrigin;
this.active = active;
this.customBiomeManager = customBiomeManager;
this.surroundingTileProvider = surroundingTileProvider;
this.showBorder = showBorder;
tileRendererRef = createNewTileRendererRef();
}
public WPTileProvider(
TileProvider tileProvider,
ColourScheme colourScheme,
BiomeScheme biomeScheme,
CustomBiomeManager customBiomeManager,
Collection<Layer> hiddenLayers,
boolean contourLines,
int contourSeparation,
TileRenderer.LightOrigin lightOrigin,
boolean showBorder,
org.pepsoft.util.swing.TileProvider surroundingTileProvider) {
this.tileProvider = tileProvider;
this.colourScheme = colourScheme;
this.biomeScheme = biomeScheme;
this.hiddenLayers = (hiddenLayers != null) ? new HashSet<>(hiddenLayers) : null;
this.contourLines = contourLines;
this.contourSeparation = contourSeparation;
this.lightOrigin = lightOrigin;
active = false;
this.customBiomeManager = customBiomeManager;
this.surroundingTileProvider = surroundingTileProvider;
this.showBorder = showBorder;
tileRendererRef = createNewTileRendererRef();
}
public synchronized void addHiddenLayer(Layer layer) {
hiddenLayers.add(layer);
tileRendererRef = createNewTileRendererRef();
}
public synchronized void removeHiddenLayer(Layer layer) {
hiddenLayers.remove(layer);
tileRendererRef = createNewTileRendererRef();
}
@Override
public int getTileSize() {
return TILE_SIZE;
}
@Override
public boolean isTilePresent(int x, int y) {
if (zoom == 0) {
return getUnzoomedTileType(x, y) != TileType.SURROUNDS
|| ((surroundingTileProvider != null) && surroundingTileProvider.isTilePresent(x, y));
} else {
final int scale = 1 << -zoom;
for (int dx = 0; dx < scale; dx++) {
for (int dy = 0; dy < scale; dy++) {
switch (getUnzoomedTileType(x * scale + dx, y * scale + dy)) {
case WORLD:
case BORDER:
case WALL:
return true;
case SURROUNDS:
if ((surroundingTileProvider != null)
&& surroundingTileProvider.isTilePresent(x, y)) {
return true;
}
break;
}
}
}
return false;
}
}
@Override
public void paintTile(
final Image tileImage, final int x, final int y, final int imageX, final int imageY) {
try {
if (zoom == 0) {
paintUnzoomedTile(tileImage, x, y, imageX, imageY);
} else {
Graphics2D g2 = (Graphics2D) tileImage.getGraphics();
try {
BufferedImage surroundingTileImage = null;
final Color waterColour = new Color(colourScheme.getColour(BLK_WATER));
final Color lavaColour = new Color(colourScheme.getColour(BLK_LAVA));
final Color voidColour = new Color(VoidRenderer.getColour());
final Color bedrockColour = new Color(colourScheme.getColour(BLK_BEDROCK));
final int scale = 1 << -zoom;
final int subSize = TILE_SIZE / scale;
for (int dx = 0; dx < scale; dx++) {
for (int dy = 0; dy < scale; dy++) {
TileType tileType = getUnzoomedTileType(x * scale + dx, y * scale + dy);
switch (tileType) {
case WORLD:
Tile tile = tileProvider.getTile(x * scale + dx, y * scale + dy);
if (tile.hasLayer(NotPresent.INSTANCE)) {
if (surroundingTileProvider != null) {
if (surroundingTileImage == null) {
surroundingTileImage =
new BufferedImage(TILE_SIZE, TILE_SIZE, BufferedImage.TYPE_INT_ARGB);
surroundingTileProvider.paintTile(surroundingTileImage, x, y, 0, 0);
}
g2.drawImage(
surroundingTileImage,
imageX + dx * subSize,
imageY + dy * subSize,
imageX + (dx + 1) * subSize,
imageY + (dy + 1) * subSize,
imageX + dx * subSize,
imageY + dy * subSize,
imageX + (dx + 1) * subSize,
imageY + (dy + 1) * subSize,
null);
} else {
g2.setColor(voidColour);
g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, subSize);
}
}
TileRenderer tileRenderer = tileRendererRef.get();
tileRenderer.setTile(tile);
tileRenderer.renderTile(tileImage, dx * subSize, dy * subSize);
break;
case BORDER:
Color colour;
switch (((Dimension) tileProvider).getBorder()) {
case WATER:
colour = waterColour;
break;
case LAVA:
colour = lavaColour;
break;
case VOID:
colour = voidColour;
break;
default:
throw new InternalError();
}
g2.setColor(colour);
g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, subSize);
// Draw border lines
g2.setColor(Color.BLACK);
g2.setStroke(
new BasicStroke(
2,
BasicStroke.CAP_BUTT,
BasicStroke.JOIN_BEVEL,
0.0f,
new float[] {4.0f, 4.0f},
0.0f));
if (tileProvider.isTilePresent(x * scale + dx, y * scale + dy - 1)) {
g2.drawLine(
imageX + dx * subSize,
imageY + dy * subSize,
imageX + (dx + 1) * subSize - 1,
imageY + dy * subSize);
}
if (tileProvider.isTilePresent(x * scale + dx + 1, y * scale + dy)) {
g2.drawLine(
imageX + (dx + 1) * subSize - 1,
imageY + dy * subSize,
imageX + (dx + 1) * subSize - 1,
imageY + (dy + 1) * subSize - 1);
}
if (tileProvider.isTilePresent(x * scale + dx, y * scale + dy + 1)) {
g2.drawLine(
imageX + dx * subSize,
imageY + (dy + 1) * subSize - 1,
imageX + (dx + 1) * subSize - 1,
imageY + (dy + 1) * subSize - 1);
}
if (tileProvider.isTilePresent(x * scale + dx - 1, y * scale + dy)) {
g2.drawLine(
imageX + dx * subSize,
imageY + dy * subSize,
imageX + dx * subSize,
imageY + (dy + 1) * subSize - 1);
}
break;
case SURROUNDS:
case WALL:
if (surroundingTileProvider != null) {
if (surroundingTileImage == null) {
surroundingTileImage =
new BufferedImage(TILE_SIZE, TILE_SIZE, BufferedImage.TYPE_INT_ARGB);
surroundingTileProvider.paintTile(surroundingTileImage, x, y, 0, 0);
}
g2.drawImage(
surroundingTileImage,
imageX + dx * subSize,
imageY + dy * subSize,
imageX + (dx + 1) * subSize,
imageY + (dy + 1) * subSize,
imageX + dx * subSize,
imageY + dy * subSize,
imageX + (dx + 1) * subSize,
imageY + (dy + 1) * subSize,
null);
} else {
g2.setColor(voidColour);
g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, subSize);
}
if (tileType == TileType.WALL) {
g2.setColor(bedrockColour);
TileType neighbourType =
getUnzoomedTileType(x * scale + dx, y * scale + dy - 1);
int wallWidth = Math.max(subSize / 8, 1);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, wallWidth);
}
neighbourType = getUnzoomedTileType(x * scale + dx + 1, y * scale + dy);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(
imageX + (dx + 1) * subSize - wallWidth,
imageY + dy * subSize,
wallWidth,
subSize);
}
neighbourType = getUnzoomedTileType(x * scale + dx, y * scale + dy + 1);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(
imageX + dx * subSize,
imageY + (dy + 1) * subSize - wallWidth,
subSize,
wallWidth);
}
neighbourType = getUnzoomedTileType(x * scale + dx - 1, y * scale + dy);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, wallWidth, subSize);
}
}
break;
}
}
}
} finally {
g2.dispose();
}
}
} catch (Throwable e) {
// Log at debug level because this tends to happen when zooming in
// and out, probably due to some state getting out of sync. It
// doesn't so far appear to have any visible consequences.
logger.error("Exception while generating image for tile at " + x + ", " + y, e);
}
}
@Override
public int getTilePriority(int x, int y) {
if (zoom == 0) {
return (getUnzoomedTileType(x, y) == TileType.WORLD) ? 1 : 0;
} else {
final int scale = 1 << -zoom;
for (int dx = 0; dx < scale; dx++) {
for (int dy = 0; dy < scale; dy++) {
if (getUnzoomedTileType(x * scale + dx, y * scale + dy) == TileType.WORLD) {
return 1;
}
}
}
return 0;
}
}
@Override
public Rectangle getExtent() {
Rectangle sourceExtent = tileProvider.getExtent();
if (sourceExtent != null) {
if (zoom == 0) {
return sourceExtent;
} else if (zoom < 0) {
return new Rectangle(
sourceExtent.x >> -zoom,
sourceExtent.y >> -zoom,
sourceExtent.width >> -zoom,
sourceExtent.height >> -zoom);
} else {
return new Rectangle(
sourceExtent.x << zoom,
sourceExtent.y << zoom,
sourceExtent.width << zoom,
sourceExtent.height << zoom);
}
} else {
return null;
}
}
@Override
public void addTileListener(TileListener tileListener) {
if (active && listeners.isEmpty()) {
((Dimension) tileProvider).addDimensionListener(this);
for (Tile tile : ((Dimension) tileProvider).getTiles()) {
tile.addListener(this);
}
}
if (!listeners.contains(tileListener)) {
listeners.add(tileListener);
}
}
@Override
public void removeTileListener(TileListener tileListener) {
listeners.remove(tileListener);
if (active && listeners.isEmpty()) {
for (Tile tile : ((Dimension) tileProvider).getTiles()) {
tile.removeListener(this);
}
((Dimension) tileProvider).removeDimensionListener(this);
}
}
@Override
public boolean isZoomSupported() {
return true;
}
@Override
public int getZoom() {
return zoom;
}
@Override
public void setZoom(int zoom) {
if (zoom != this.zoom) {
if (zoom > 0) {
throw new UnsupportedOperationException("Zooming in not supported");
}
this.zoom = zoom;
tileRendererRef = createNewTileRendererRef();
if (surroundingTileProvider != null) {
surroundingTileProvider.setZoom(zoom);
}
}
}
// Dimension.Listener
@Override
public void tilesAdded(Dimension dimension, Set<Tile> tiles) {
for (Tile tile : tiles) {
tile.addListener(this);
}
fireTilesChangedIncludeBorder(tiles);
}
@Override
public void tilesRemoved(Dimension dimension, Set<Tile> tiles) {
for (Tile tile : tiles) {
tile.removeListener(this);
}
fireTilesChangedIncludeBorder(tiles);
}
// Tile.Listener
@Override
public void heightMapChanged(Tile tile) {
fireTileChanged(tile);
}
@Override
public void terrainChanged(Tile tile) {
fireTileChanged(tile);
}
@Override
public void waterLevelChanged(Tile tile) {
fireTileChanged(tile);
}
@Override
public void layerDataChanged(Tile tile, Set<Layer> changedLayers) {
fireTileChanged(tile);
}
@Override
public void allBitLayerDataChanged(Tile tile) {
fireTileChanged(tile);
}
@Override
public void allNonBitlayerDataChanged(Tile tile) {
fireTileChanged(tile);
}
@Override
public void seedsChanged(Tile tile) {
fireTileChanged(tile);
}
private TileType getUnzoomedTileType(int x, int y) {
if (tileProvider.isTilePresent(x, y)) {
return TileType.WORLD;
} else if (showBorder && (tileProvider instanceof Dimension)) {
Dimension dimension = (Dimension) tileProvider;
if (dimension.isBorderTile(x, y)) {
return TileType.BORDER;
} else if (dimension.isBedrockWall()
&& ((dimension.getBorder() != null)
? (dimension.isBorderTile(x - 1, y)
|| dimension.isBorderTile(x, y - 1)
|| dimension.isBorderTile(x + 1, y)
|| dimension.isBorderTile(x, y + 1))
: (tileProvider.isTilePresent(x - 1, y)
|| tileProvider.isTilePresent(x, y - 1)
|| tileProvider.isTilePresent(x + 1, y)
|| tileProvider.isTilePresent(x, y + 1)))) {
return TileType.WALL;
}
}
return TileType.SURROUNDS;
}
private void paintUnzoomedTile(
final Image tileImage, final int x, final int y, final int dx, final int dy) {
TileType tileType = getUnzoomedTileType(x, y);
switch (tileType) {
case WORLD:
Tile tile = tileProvider.getTile(x, y);
if (tile.hasLayer(NotPresent.INSTANCE) && (surroundingTileProvider != null)) {
surroundingTileProvider.paintTile(tileImage, x, y, dx, dy);
}
TileRenderer tileRenderer = tileRendererRef.get();
tileRenderer.setTile(tile);
tileRenderer.renderTile(tileImage, dx, dy);
break;
case BORDER:
int colour;
switch (((Dimension) tileProvider).getBorder()) {
case WATER:
colour = colourScheme.getColour(BLK_WATER);
break;
case LAVA:
colour = colourScheme.getColour(BLK_LAVA);
break;
case VOID:
colour = VoidRenderer.getColour();
break;
default:
throw new InternalError();
}
Graphics2D g2 = (Graphics2D) tileImage.getGraphics();
try {
g2.setColor(new Color(colour));
g2.fillRect(dx, dy, TILE_SIZE, TILE_SIZE);
// Draw border lines
g2.setColor(Color.BLACK);
g2.setStroke(
new BasicStroke(
2,
BasicStroke.CAP_BUTT,
BasicStroke.JOIN_BEVEL,
0.0f,
new float[] {4.0f, 4.0f},
0.0f));
if (tileProvider.isTilePresent(x, y - 1)) {
g2.drawLine(dx + 1, dy + 1, dx + TILE_SIZE - 1, dy + 1);
}
if (tileProvider.isTilePresent(x + 1, y)) {
g2.drawLine(dx + TILE_SIZE - 1, dy + 1, dx + TILE_SIZE - 1, dy + TILE_SIZE - 1);
}
if (tileProvider.isTilePresent(x, y + 1)) {
g2.drawLine(dx + 1, dy + TILE_SIZE - 1, dx + TILE_SIZE - 1, dy + TILE_SIZE - 1);
}
if (tileProvider.isTilePresent(x - 1, y)) {
g2.drawLine(dx + 1, dy + 1, dx + 1, dy + TILE_SIZE - 1);
}
} finally {
g2.dispose();
}
break;
case WALL:
if (surroundingTileProvider != null) {
surroundingTileProvider.paintTile(tileImage, x, y, dx, dy);
}
g2 = (Graphics2D) tileImage.getGraphics();
try {
if (surroundingTileProvider == null) {
// A surrounding tile provider would have completely
// filled the image, but since there isn't one we have
// to make sure of that ourselves
g2.setColor(new Color(VoidRenderer.getColour()));
g2.fillRect(dx, dy, TILE_SIZE, TILE_SIZE);
}
g2.setColor(new Color(colourScheme.getColour(BLK_BEDROCK)));
TileType neighbourType = getUnzoomedTileType(x, y - 1);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(dx, dy, TILE_SIZE, 16);
}
neighbourType = getUnzoomedTileType(x + 1, y);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(dx + TILE_SIZE - 16, dy, 16, TILE_SIZE);
}
neighbourType = getUnzoomedTileType(x, y + 1);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(dx, dy + TILE_SIZE - 16, TILE_SIZE, 16);
}
neighbourType = getUnzoomedTileType(x - 1, y);
if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
g2.fillRect(dx, dy, 16, TILE_SIZE);
}
} finally {
g2.dispose();
}
break;
case SURROUNDS:
if (surroundingTileProvider != null) {
surroundingTileProvider.paintTile(tileImage, x, y, dx, dy);
}
break;
default:
throw new InternalError();
}
}
private void fireTileChanged(Tile tile) {
Point coords = getTileCoordinates(tile);
for (TileListener listener : listeners) {
listener.tileChanged(this, coords.x, coords.y);
}
}
private void fireTilesChangedIncludeBorder(Set<Tile> tiles) {
if (showBorder
&& (tileProvider instanceof Dimension)
&& (((Dimension) tileProvider).getDim() == DIM_NORMAL)
&& (((Dimension) tileProvider).getBorder() != null)) {
final Set<Point> coordSet = new HashSet<>();
for (Tile tile : tiles) {
final int tileX = tile.getX(),
tileY = tile.getY(),
borderSize = ((Dimension) tileProvider).getBorderSize();
for (int dx = -borderSize; dx <= borderSize; dx++) {
for (int dy = -borderSize; dy <= borderSize; dy++) {
coordSet.add(getTileCoordinates(tileX + dx, tileY + dy));
}
}
}
for (TileListener listener : listeners) {
listener.tilesChanged(this, coordSet);
}
} else {
Set<Point> coords = tiles.stream().map(this::getTileCoordinates).collect(Collectors.toSet());
for (TileListener listener : listeners) {
listener.tilesChanged(this, coords);
}
}
}
/**
* Convert the actual tile coordinates to zoom-corrected (tile provider coordinate system)
* coordinates.
*
* @param tile The tile of which to convert the coordinates.
* @return The coordinates of the tile in the tile provider coordinate system (corrected for
* zoom).
*/
private Point getTileCoordinates(Tile tile) {
return getTileCoordinates(tile.getX(), tile.getY());
}
/**
* Convert the actual tile coordinates to zoom-corrected (tile provider coordinate system)
* coordinates.
*
* @param tileX The X tile coordinate to convert.
* @param tileY The Y tile coordinate to convert.
* @return The coordinates of the tile in the tile provider coordinate system (corrected for
* zoom).
*/
private Point getTileCoordinates(final int tileX, final int tileY) {
if (zoom == 0) {
return new Point(tileX, tileY);
} else if (zoom < 0) {
return new Point(tileX >> -zoom, tileY >> -zoom);
} else {
return new Point(tileX << zoom, tileY << zoom);
}
}
@NotNull
private ThreadLocal<TileRenderer> createNewTileRendererRef() {
return new ThreadLocal<TileRenderer>() {
@Override
protected TileRenderer initialValue() {
TileRenderer tileRenderer =
new TileRenderer(tileProvider, colourScheme, biomeScheme, customBiomeManager, zoom);
synchronized (WPTileProvider.this) {
if (hiddenLayers != null) {
tileRenderer.addHiddenLayers(hiddenLayers);
}
}
tileRenderer.setContourLines(contourLines);
tileRenderer.setContourSeparation(contourSeparation);
tileRenderer.setLightOrigin(lightOrigin);
return tileRenderer;
}
};
}
private final TileProvider tileProvider;
private final ColourScheme colourScheme;
private final BiomeScheme biomeScheme;
private final Set<Layer> hiddenLayers;
private final boolean contourLines, active, showBorder;
private final int contourSeparation;
private final TileRenderer.LightOrigin lightOrigin;
private final List<TileListener> listeners = new ArrayList<>();
private final CustomBiomeManager customBiomeManager;
private final org.pepsoft.util.swing.TileProvider surroundingTileProvider;
private int zoom = 0;
private volatile ThreadLocal<TileRenderer> tileRendererRef;
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(WPTileProvider.class);
private enum TileType {
/** The tile is part of the WorldPainter world. */
WORLD,
/** The tile is part of the WorldPainter border. */
BORDER,
/** The tile contains no WorldPainter-generated chunks. */
SURROUNDS,
/** The tile is outside the WorldPainter world and border but does contain part of a wall. */
WALL
}
}
/**
* A georeferencing "gridded" VariableEnhanced, that has a GridCoordSys. In VisAD data model, it is
* a sampled Field. The dimension are put into canonical order: (rt, e, t, z, y, x).
*
* <p>
*
* <p>Implementation note: If the Horizontal axes are 2D, the x and y dimensions are arbitrarily
* chosen to be gcs.getXHorizAxis().getDimension(1), gcs.getXHorizAxis().getDimension(0)
* respectively.
*
* <p>
*
* @author caron
*/
public class GeoGrid implements NamedObject, ucar.nc2.dt.GridDatatype {
private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GeoGrid.class);
private static final boolean debugArrayShape = false;
private final GridDataset dataset;
private final GridCoordSys gcs;
private final VariableDS vs;
private int xDimOrgIndex = -1,
yDimOrgIndex = -1,
zDimOrgIndex = -1,
tDimOrgIndex = -1,
eDimOrgIndex = -1,
rtDimOrgIndex = -1;
private int xDimNewIndex = -1,
yDimNewIndex = -1,
zDimNewIndex = -1,
tDimNewIndex = -1,
eDimNewIndex = -1,
rtDimNewIndex = -1;
private final List<Dimension> mydims;
/**
* Constructor.
*
* @param dataset belongs to this dataset
* @param dsvar wraps this Variable
* @param gcs has this grid coordinate system
*/
public GeoGrid(GridDataset dataset, VariableDS dsvar, GridCoordSys gcs) {
this.dataset = dataset;
this.vs = dsvar;
this.gcs = gcs;
CoordinateAxis xaxis = gcs.getXHorizAxis();
if (xaxis instanceof CoordinateAxis1D) {
xDimOrgIndex = findDimension(gcs.getXHorizAxis().getDimension(0));
yDimOrgIndex = findDimension(gcs.getYHorizAxis().getDimension(0));
} else { // 2D case
yDimOrgIndex = findDimension(gcs.getXHorizAxis().getDimension(0));
xDimOrgIndex = findDimension(gcs.getXHorizAxis().getDimension(1));
}
if (gcs.getVerticalAxis() != null)
zDimOrgIndex = findDimension(gcs.getVerticalAxis().getDimension(0));
if (gcs.getTimeAxis() != null) {
if (gcs.getTimeAxis1D() != null)
tDimOrgIndex = findDimension(gcs.getTimeAxis1D().getDimension(0));
else tDimOrgIndex = findDimension(gcs.getTimeAxis().getDimension(1));
}
if (gcs.getEnsembleAxis() != null)
eDimOrgIndex = findDimension(gcs.getEnsembleAxis().getDimension(0));
if (gcs.getRunTimeAxis() != null)
rtDimOrgIndex = findDimension(gcs.getRunTimeAxis().getDimension(0));
// construct canonical dimension list
int count = 0;
this.mydims = new ArrayList<Dimension>();
if ((rtDimOrgIndex >= 0) && (rtDimOrgIndex != tDimOrgIndex)) {
mydims.add(dsvar.getDimension(rtDimOrgIndex));
rtDimNewIndex = count++;
}
if (eDimOrgIndex >= 0) {
mydims.add(dsvar.getDimension(eDimOrgIndex));
eDimNewIndex = count++;
}
if (tDimOrgIndex >= 0) {
mydims.add(dsvar.getDimension(tDimOrgIndex));
tDimNewIndex = count++;
}
if (zDimOrgIndex >= 0) {
mydims.add(dsvar.getDimension(zDimOrgIndex));
zDimNewIndex = count++;
}
if (yDimOrgIndex >= 0) {
mydims.add(dsvar.getDimension(yDimOrgIndex));
yDimNewIndex = count++;
}
if (xDimOrgIndex >= 0) {
mydims.add(dsvar.getDimension(xDimOrgIndex));
xDimNewIndex = count;
}
}
private int findDimension(Dimension want) {
java.util.List dims = vs.getDimensions();
for (int i = 0; i < dims.size(); i++) {
Dimension d = (Dimension) dims.get(i);
if (d.equals(want)) return i;
}
return -1;
}
/**
* Returns an ArrayList containing the dimensions used by this geoGrid. The dimension are put into
* canonical order: (rt, e, t, z, y, x). Note that the z and t dimensions are optional. If the
* Horizontal axes are 2D, the x and y dimensions are arbitrarily chosen to be
* gcs.getXHorizAxis().getDimension(1), gcs.getXHorizAxis().getDimension(0), respectively.
*
* @return List with objects of type Dimension, in canonical order.
*/
public java.util.List<Dimension> getDimensions() {
return new ArrayList<Dimension>(mydims);
}
/**
* get the ith dimension
*
* @param i : which dimension
* @return ith Dimension
*/
public Dimension getDimension(int i) {
if ((i < 0) || (i >= mydims.size())) return null;
return mydims.get(i);
}
/** get the time Dimension, if it exists */
public Dimension getTimeDimension() {
return tDimNewIndex < 0 ? null : getDimension(tDimNewIndex);
}
/** get the z Dimension, if it exists */
public Dimension getZDimension() {
return zDimNewIndex < 0 ? null : getDimension(zDimNewIndex);
}
/** get the y Dimension, if it exists */
public Dimension getYDimension() {
return yDimNewIndex < 0 ? null : getDimension(yDimNewIndex);
}
/** get the x Dimension, if it exists */
public Dimension getXDimension() {
return xDimNewIndex < 0 ? null : getDimension(xDimNewIndex);
}
/** get the ensemble Dimension, if it exists */
public Dimension getEnsembleDimension() {
return eDimNewIndex < 0 ? null : getDimension(eDimNewIndex);
}
/** get the run time Dimension, if it exists */
public Dimension getRunTimeDimension() {
return rtDimNewIndex < 0 ? null : getDimension(rtDimNewIndex);
}
/** get the time Dimension index in the geogrid (canonical order), or -1 if none */
public int getTimeDimensionIndex() {
return tDimNewIndex;
}
/** get the z Dimension index in the geogrid (canonical order), or -1 if none */
public int getZDimensionIndex() {
return zDimNewIndex;
}
/** get the y Dimension index in the geogrid (canonical order) */
public int getYDimensionIndex() {
return yDimNewIndex;
}
/** get the x Dimension index in the geogrid (canonical order) */
public int getXDimensionIndex() {
return xDimNewIndex;
}
/** get the ensemble Dimension index in the geogrid (canonical order) */
public int getEnsembleDimensionIndex() {
return eDimNewIndex;
}
/** get the runtime Dimension index in the geogrid (canonical order) */
public int getRunTimeDimensionIndex() {
return rtDimNewIndex;
}
/**
* Convenience function; lookup Attribute by name.
*
* @param name the name of the attribute
* @return the attribute, or null if not found
*/
public Attribute findAttributeIgnoreCase(String name) {
return vs.findAttributeIgnoreCase(name);
}
/**
* Convenience function; lookup Attribute value by name. Must be String valued
*
* @param attName name of the attribute
* @param defaultValue if not found, use this as the default
* @return Attribute string value, or default if not found.
*/
public String findAttValueIgnoreCase(String attName, String defaultValue) {
return dataset.getNetcdfDataset().findAttValueIgnoreCase((Variable) vs, attName, defaultValue);
}
// implementation of GridDatatype interface
/** get the rank */
public int getRank() {
return mydims.size();
}
/** get the shape */
public int[] getShape() {
int[] shape = new int[mydims.size()];
for (int i = 0; i < mydims.size(); i++) {
Dimension d = mydims.get(i);
shape[i] = d.getLength();
}
return shape;
}
/** get the data type */
public DataType getDataType() {
return vs.getDataType();
}
public List<Attribute> getAttributes() {
return vs.getAttributes();
}
public VariableDS getVariable() {
return vs;
}
public String getFullName() {
return vs.getFullName();
}
public String getName() {
return vs.getFullName();
}
public String getShortName() {
return vs.getShortName();
}
/** get the GridCoordSys for this GeoGrid. */
public GridCoordSystem getCoordinateSystem() {
return gcs;
}
/** get the Projection. */
public ProjectionImpl getProjection() {
return gcs.getProjection();
}
/** @return ArrayList of thredds.util.NamedObject, from the GridCoordSys. */
public List<NamedObject> getLevels() {
return gcs.getLevels();
}
/** @return ArrayList of thredds.util.NamedObject, from the GridCoordSys. */
public List<NamedObject> getTimes() {
return gcs.getTimes();
}
/** get the standardized description */
public String getDescription() {
return vs.getDescription();
}
/** get the unit as a string */
public String getUnitsString() {
String units = vs.getUnitsString();
return (units == null) ? "" : units;
}
/**
* @return getUnitsString()
* @deprecated use getUnitsString()
*/
public java.lang.String getUnitString() {
return getUnitsString();
}
// public ucar.unidata.geoloc.ProjectionImpl getProjection() { return gcs.getProjection(); }
/** true if there may be missing data, see VariableDS.hasMissing() */
public boolean hasMissingData() {
return vs.hasMissing();
}
/** if val is missing data, see VariableDS.isMissingData() */
public boolean isMissingData(double val) {
return vs.isMissing(val);
}
/**
* Convert (in place) all values in the given array that are considered as "missing" to Float.NaN,
* according to isMissingData(val).
*
* @param values input array
* @return input array, with missing values converted to NaNs.
*/
public float[] setMissingToNaN(float[] values) {
if (!vs.hasMissing()) return values;
final int length = values.length;
for (int i = 0; i < length; i++) {
double value = values[i];
if (vs.isMissing(value)) values[i] = Float.NaN;
}
return values;
}
/**
* Get the minimum and the maximum data value of the previously read Array, skipping missing
* values as defined by isMissingData(double val).
*
* @param a Array to get min/max values
* @return both min and max value.
*/
public MAMath.MinMax getMinMaxSkipMissingData(Array a) {
if (!hasMissingData()) return MAMath.getMinMax(a);
IndexIterator iter = a.getIndexIterator();
double max = -Double.MAX_VALUE;
double min = Double.MAX_VALUE;
while (iter.hasNext()) {
double val = iter.getDoubleNext();
if (isMissingData(val)) continue;
if (val > max) max = val;
if (val < min) min = val;
}
return new MAMath.MinMax(min, max);
}
/**
* Reads in the data "volume" at the given time index. If its a product set, put into canonical
* order (z-y-x). If not a product set, reorder to (z,i,j), where i, j are from the original
*
* @param t time index; ignored if no time axis.
* @return data[z,y,x] or data[y,x] if no z axis.
*/
public Array readVolumeData(int t) throws java.io.IOException {
// if (gcs.isProductSet())
return readDataSlice(t, -1, -1, -1);
/* else { // 2D XY
int rank = vs.getRank();
int[] shape = vs.getShape();
int [] start = new int[rank];
CoordinateAxis taxis = gcs.getTimeAxis();
if (taxis != null) {
if ((t >= 0) && (t < taxis.getSize()))
shape[ tDim] = 1; // fix t
start[ tDim] = t;
}
if (debugArrayShape) {
System.out.println("getDataVolume shape = ");
for (int i=0; i<rank; i++)
System.out.println(" start = "+start[i]+" shape = "+ shape[i]);
}
Array dataVolume;
try {
dataVolume = vs.read( start, shape);
} catch (Exception e) {
System.out.println("Exception: GeoGridImpl.getdataSlice() on dataset "+getName());
e.printStackTrace();
throw new java.io.IOException(e.getMessage());
}
// no reordering FIX
return dataVolume.reduce();
} */
}
/**
* Reads a Y-X "horizontal slice" at the given time and vertical index. If its a product set, put
* into canonical order (y-x).
*
* @param t time index; ignored if no time axis.
* @param z vertical index; ignored if no z axis.
* @return data[y,x]
* @throws java.io.IOException on read error
*/
public Array readYXData(int t, int z) throws java.io.IOException {
return readDataSlice(t, z, -1, -1);
}
/**
* Reads a Z-Y "vertical slice" at the given time and x index. If its a product set, put into
* canonical order (z-y).
*
* @param t time index; ignored if no time axis.
* @param x x index; ignored if no x axis.
* @return data[z,y]
* @throws java.io.IOException on read error
*/
public Array readZYData(int t, int x) throws java.io.IOException {
return readDataSlice(t, -1, -1, x);
}
/**
* @throws java.io.IOException on read error
* @deprecated use readDataSlice
*/
public Array getDataSlice(int t, int z, int y, int x) throws java.io.IOException {
return readDataSlice(t, z, y, x);
}
/**
* This reads an arbitrary data slice, returning the data in canonical order (t-z-y-x). If any
* dimension does not exist, ignore it.
*
* @param t if < 0, get all of time dim; if valid index, fix slice to that value.
* @param z if < 0, get all of z dim; if valid index, fix slice to that value.
* @param y if < 0, get all of y dim; if valid index, fix slice to that value.
* @param x if < 0, get all of x dim; if valid index, fix slice to that value.
* @return data[t,z,y,x], eliminating missing or fixed dimension.
*/
public Array readDataSlice(int t, int z, int y, int x) throws java.io.IOException {
return readDataSlice(0, 0, t, z, y, x);
}
/**
* This reads an arbitrary data slice, returning the data in canonical order (rt-e-t-z-y-x). If
* any dimension does not exist, ignore it.
*
* @param rt if < 0, get all of runtime dim; if valid index, fix slice to that value.
* @param e if < 0, get all of ensemble dim; if valid index, fix slice to that value.
* @param t if < 0, get all of time dim; if valid index, fix slice to that value.
* @param z if < 0, get all of z dim; if valid index, fix slice to that value.
* @param y if < 0, get all of y dim; if valid index, fix slice to that value.
* @param x if < 0, get all of x dim; if valid index, fix slice to that value.
* @return data[rt,e,t,z,y,x], eliminating missing or fixed dimension.
*/
public Array readDataSlice(int rt, int e, int t, int z, int y, int x) throws java.io.IOException {
int rank = vs.getRank();
int[] start = new int[rank];
int[] shape = new int[rank];
for (int i = 0; i < rank; i++) {
start[i] = 0;
shape[i] = 1;
}
Dimension xdim = getXDimension();
Dimension ydim = getYDimension();
Dimension zdim = getZDimension();
Dimension tdim = getTimeDimension();
Dimension edim = getEnsembleDimension();
Dimension rtdim = getRunTimeDimension();
// construct the shape of the data volume to be read
if (rtdim != null) {
if ((rt >= 0) && (rt < rtdim.getLength())) start[rtDimOrgIndex] = rt; // fix rt
else {
shape[rtDimOrgIndex] = rtdim.getLength(); // all of rt
}
}
if (edim != null) {
if ((e >= 0) && (e < edim.getLength())) start[eDimOrgIndex] = e; // fix e
else {
shape[eDimOrgIndex] = edim.getLength(); // all of e
}
}
if (tdim != null) {
if ((t >= 0) && (t < tdim.getLength())) start[tDimOrgIndex] = t; // fix t
else {
shape[tDimOrgIndex] = tdim.getLength(); // all of t
}
}
if (zdim != null) {
if ((z >= 0) && (z < zdim.getLength())) start[zDimOrgIndex] = z; // fix z
else {
shape[zDimOrgIndex] = zdim.getLength(); // all of z
}
}
if (ydim != null) {
if ((y >= 0) && (y < ydim.getLength())) start[yDimOrgIndex] = y; // fix y
else {
shape[yDimOrgIndex] = ydim.getLength(); // all of y
}
}
if (xdim != null) {
if ((x >= 0) && (x < xdim.getLength())) // all of x
start[xDimOrgIndex] = x; // fix x
else {
shape[xDimOrgIndex] = xdim.getLength(); // all of x
}
}
if (debugArrayShape) {
System.out.println("read shape from org variable = ");
for (int i = 0; i < rank; i++)
System.out.println(
" start = "
+ start[i]
+ " shape = "
+ shape[i]
+ " name = "
+ vs.getDimension(i).getName());
}
// read it
Array dataVolume;
try {
dataVolume = vs.read(start, shape);
} catch (Exception ex) {
log.error(
"GeoGrid.getdataSlice() on dataset " + getFullName() + " " + dataset.getLocation(), ex);
throw new java.io.IOException(ex.getMessage());
}
// LOOK: the real problem is the lack of named dimensions in the Array object
// figure out correct permutation for canonical ordering for permute
List<Dimension> oldDims = new ArrayList<Dimension>(vs.getDimensions());
int[] permuteIndex = new int[dataVolume.getRank()];
int count = 0;
if (oldDims.contains(rtdim)) permuteIndex[count++] = oldDims.indexOf(rtdim);
if (oldDims.contains(edim)) permuteIndex[count++] = oldDims.indexOf(edim);
if (oldDims.contains(tdim)) permuteIndex[count++] = oldDims.indexOf(tdim);
if (oldDims.contains(zdim)) permuteIndex[count++] = oldDims.indexOf(zdim);
if (oldDims.contains(ydim)) permuteIndex[count++] = oldDims.indexOf(ydim);
if (oldDims.contains(xdim)) permuteIndex[count] = oldDims.indexOf(xdim);
if (debugArrayShape) {
System.out.println("oldDims = ");
for (Dimension oldDim : oldDims) System.out.println(" oldDim = " + oldDim.getName());
System.out.println("permute dims = ");
for (int aPermuteIndex : permuteIndex)
System.out.println(" oldDim index = " + aPermuteIndex);
}
// check to see if we need to permute
boolean needPermute = false;
for (int i = 0; i < permuteIndex.length; i++) {
if (i != permuteIndex[i]) needPermute = true;
}
// permute to the order rt,e,t,z,y,x
if (needPermute) dataVolume = dataVolume.permute(permuteIndex);
// eliminate fixed dimensions, but not all dimensions of length 1.
count = 0;
if (rtdim != null) {
if (rt >= 0) dataVolume = dataVolume.reduce(count);
else count++;
}
if (edim != null) {
if (e >= 0) dataVolume = dataVolume.reduce(count);
else count++;
}
if (tdim != null) {
if (t >= 0) dataVolume = dataVolume.reduce(count);
else count++;
}
if (zdim != null) {
if (z >= 0) dataVolume = dataVolume.reduce(count);
else count++;
}
if (ydim != null) {
if (y >= 0) dataVolume = dataVolume.reduce(count);
else count++;
}
if (xdim != null) {
if (x >= 0) dataVolume = dataVolume.reduce(count);
}
return dataVolume;
}
//////////////////////////////////
/**
* Create a new GeoGrid that is a logical subset of this GeoGrid.
*
* @param t_range subset the time dimension, or null if you want all of it
* @param z_range subset the vertical dimension, or null if you want all of it
* @param bbox a lat/lon bounding box, or null if you want all x,y
* @param z_stride use only if z_range is null, then take all z with this stride (1 means all)
* @param y_stride use this stride on the y coordinate (1 means all)
* @param x_stride use this stride on the x coordinate (1 means all)
* @return subsetted GeoGrid
* @throws InvalidRangeException if bbox does not intersect GeoGrid
*/
public GeoGrid subset(
Range t_range, Range z_range, LatLonRect bbox, int z_stride, int y_stride, int x_stride)
throws InvalidRangeException {
if ((z_range == null) && (z_stride > 1)) {
Dimension zdim = getZDimension();
if (zdim != null) z_range = new Range(0, zdim.getLength() - 1, z_stride);
}
Range y_range = null, x_range = null;
if (bbox != null) {
List yx_ranges = gcs.getRangesFromLatLonRect(bbox);
y_range = (Range) yx_ranges.get(0);
x_range = (Range) yx_ranges.get(1);
}
if (y_stride > 1) {
if (y_range == null) {
Dimension ydim = getYDimension();
y_range = new Range(0, ydim.getLength() - 1, y_stride);
} else {
y_range = new Range(y_range.first(), y_range.last(), y_stride);
}
}
if (x_stride > 1) {
if (x_range == null) {
Dimension xdim = getXDimension();
x_range = new Range(0, xdim.getLength() - 1, x_stride);
} else {
x_range = new Range(x_range.first(), x_range.last(), x_stride);
}
}
return subset(t_range, z_range, y_range, x_range);
}
public GridDatatype makeSubset(
Range t_range, Range z_range, LatLonRect bbox, int z_stride, int y_stride, int x_stride)
throws InvalidRangeException {
return subset(t_range, z_range, bbox, z_stride, y_stride, x_stride);
}
/**
* Create a new GeoGrid that is a logical subset of this GeoGrid.
*
* @param t_range subset the time dimension, or null if you want all of it
* @param z_range subset the vertical dimension, or null if you want all of it
* @param y_range subset the y dimension, or null if you want all of it
* @param x_range subset the x dimension, or null if you want all of it
* @return subsetted GeoGrid
* @throws InvalidRangeException if any of the ranges are invalid
*/
public GeoGrid subset(Range t_range, Range z_range, Range y_range, Range x_range)
throws InvalidRangeException {
return (GeoGrid) makeSubset(null, null, t_range, z_range, y_range, x_range);
}
public GridDatatype makeSubset(
Range rt_range, Range e_range, Range t_range, Range z_range, Range y_range, Range x_range)
throws InvalidRangeException {
// get the ranges list
int rank = getRank();
Range[] ranges = new Range[rank];
if (null != getXDimension()) ranges[xDimOrgIndex] = x_range;
if (null != getYDimension()) ranges[yDimOrgIndex] = y_range;
if (null != getZDimension()) ranges[zDimOrgIndex] = z_range;
if (null != getTimeDimension()) ranges[tDimOrgIndex] = t_range;
if (null != getRunTimeDimension()) ranges[rtDimOrgIndex] = rt_range;
if (null != getEnsembleDimension()) ranges[eDimOrgIndex] = e_range;
List<Range> rangesList = Arrays.asList(ranges);
// subset the variable
VariableDS v_section = (VariableDS) vs.section(new Section(rangesList));
List<Dimension> dims = v_section.getDimensions();
for (Dimension dim : dims) {
dim.setShared(true); // make them shared (section will make them unshared)
}
// subset the axes in the GridCoordSys
GridCoordSys gcs_section =
new GridCoordSys(gcs, rt_range, e_range, t_range, z_range, y_range, x_range);
// now we can make the geogrid
return new GeoGrid(dataset, v_section, gcs_section);
}
/////////////////////////////////////////////////////////////////////////////////
/** Instances which have same name and coordinate system are equal. */
public boolean equals(Object oo) {
if (this == oo) return true;
if (!(oo instanceof GeoGrid)) return false;
GeoGrid d = (GeoGrid) oo;
if (!getFullName().equals(d.getFullName())) return false;
if (!getCoordinateSystem().equals(d.getCoordinateSystem())) return false;
return true;
}
/** Override Object.hashCode() to be consistent with equals. */
public int hashCode() {
if (hashCode == 0) {
int result = 17;
// result = 37*result + dataset.getName().hashCode();
result = 37 * result + getFullName().hashCode();
result = 37 * result + getCoordinateSystem().hashCode();
hashCode = result;
}
return hashCode;
}
private int hashCode = 0; // Bloch, item 8
/** string representation */
public String toString() {
return getFullName();
}
/** nicely formatted information */
public String getInfo() {
StringBuilder buf = new StringBuilder(200);
buf.setLength(0);
buf.append(getFullName());
Format.tab(buf, 30, true);
buf.append(getUnitsString());
Format.tab(buf, 60, true);
buf.append(hasMissingData());
Format.tab(buf, 66, true);
buf.append(getDescription());
return buf.toString();
}
public int compareTo(GridDatatype g) {
return getFullName().compareTo(g.getFullName());
}
}
/**
* Jahia specific wrapper around <code>javax.jcr.Session</code> to be able to inject Jahia specific
* actions and to manage sessions to multiple repository providers in the backend.
*
* <p>Jahia services should use this wrapper rather than the original session interface to ensure
* that we manipulate wrapped nodes and not the ones from the underlying implementation.
*
* @author toto
*/
public class JCRSessionWrapper implements Session {
private static Logger logger = org.slf4j.LoggerFactory.getLogger(JCRSessionWrapper.class);
public static final String DEREF_SEPARATOR = "@/";
private JCRSessionFactory sessionFactory;
private JahiaUser user;
private Credentials credentials;
private JCRWorkspaceWrapper workspace;
private boolean isLive = true;
private Locale locale;
private List<String> tokens = new ArrayList<String>();
private Map<JCRStoreProvider, Session> sessions = new HashMap<JCRStoreProvider, Session>();
private Map<String, JCRNodeWrapper> sessionCacheByPath = new HashMap<String, JCRNodeWrapper>();
private Map<String, JCRNodeWrapper> sessionCacheByIdentifier =
new HashMap<String, JCRNodeWrapper>();
private Map<String, JCRNodeWrapper> newNodes = new HashMap<String, JCRNodeWrapper>();
private Map<String, String> nsToPrefix = new HashMap<String, String>();
private Map<String, String> prefixToNs = new HashMap<String, String>();
private Map<String, String> uuidMapping = new HashMap<String, String>();
private Map<String, String> pathMapping = new LinkedHashMap<String, String>();
private boolean isSystem;
private Date versionDate;
private Locale fallbackLocale;
private String versionLabel;
private static AtomicLong activeSessions = new AtomicLong(0L);
public JCRSessionWrapper(
JahiaUser user,
Credentials credentials,
boolean isSystem,
String workspace,
Locale locale,
JCRSessionFactory sessionFactory,
Locale fallbackLocale) {
this.user = user;
this.credentials = credentials;
this.isSystem = isSystem;
this.versionDate = null;
this.versionLabel = null;
if (workspace == null) {
this.workspace = new JCRWorkspaceWrapper("default", this, sessionFactory);
} else {
this.workspace = new JCRWorkspaceWrapper(workspace, this, sessionFactory);
}
this.locale = locale;
this.fallbackLocale = fallbackLocale;
this.sessionFactory = sessionFactory;
activeSessions.incrementAndGet();
}
public JCRNodeWrapper getRootNode() throws RepositoryException {
JCRStoreProvider provider = sessionFactory.getProvider("/");
return provider.getNodeWrapper(getProviderSession(provider).getRootNode(), "/", null, this);
}
public Repository getRepository() {
return sessionFactory;
}
public String getUserID() {
return ((SimpleCredentials) credentials).getUserID();
}
public boolean isSystem() {
return isSystem;
}
public Object getAttribute(String s) {
return null; // To change body of implemented methods use File | Settings | File Templates.
}
public String[] getAttributeNames() {
return new String
[0]; // To change body of implemented methods use File | Settings | File Templates.
}
public JCRWorkspaceWrapper getWorkspace() {
return workspace;
}
public Locale getLocale() {
return locale;
}
// public void setInterceptorsEnabled(boolean interceptorsEnabled) {
// this.interceptorsEnabled = interceptorsEnabled;
// }
public Session impersonate(Credentials credentials) throws LoginException, RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
public JCRNodeWrapper getNodeByUUID(String uuid)
throws ItemNotFoundException, RepositoryException {
return getNodeByUUID(uuid, true);
}
public JCRNodeWrapper getNodeByUUID(final String uuid, final boolean checkVersion)
throws ItemNotFoundException, RepositoryException {
if (sessionCacheByIdentifier.containsKey(uuid)) {
return sessionCacheByIdentifier.get(uuid);
}
RepositoryException originalEx = null;
for (JCRStoreProvider provider : sessionFactory.getProviderList()) {
if (!provider.isInitialized()) {
logger.debug(
"Provider "
+ provider.getKey()
+ " / "
+ provider.getClass().getName()
+ " is not yet initialized, skipping...");
continue;
}
if (provider instanceof JackrabbitStoreProvider && JCRContentUtils.isNotJcrUuid(uuid)) {
// not a valid UUID, probably a VFS node
continue;
}
try {
Session session = getProviderSession(provider);
if (session instanceof JahiaSessionImpl
&& getUser() != null
&& sessionFactory.getCurrentAliasedUser() != null
&& sessionFactory.getCurrentAliasedUser().equals(getUser())) {
((JahiaSessionImpl) session).toggleThisSessionAsAliased();
}
Node n = session.getNodeByIdentifier(uuid);
JCRNodeWrapper wrapper = provider.getNodeWrapper(n, this);
if (getUser() != null
&& sessionFactory.getCurrentAliasedUser() != null
&& !sessionFactory.getCurrentAliasedUser().equals(getUser())) {
JCRTemplate.getInstance()
.doExecuteWithUserSession(
sessionFactory.getCurrentAliasedUser().getUsername(),
session.getWorkspace().getName(),
getLocale(),
new JCRCallback<Object>() {
public Object doInJCR(JCRSessionWrapper session) throws RepositoryException {
return session.getNodeByUUID(uuid, checkVersion);
}
});
}
if (checkVersion && (versionDate != null || versionLabel != null)) {
wrapper = getFrozenVersionAsRegular(n, provider);
}
sessionCacheByIdentifier.put(uuid, wrapper);
sessionCacheByPath.put(wrapper.getPath(), wrapper);
return wrapper;
} catch (ItemNotFoundException ee) {
// All good
if (originalEx == null) {
originalEx = ee;
}
} catch (UnsupportedRepositoryOperationException uso) {
logger.debug(
"getNodeByUUID unsupported by : "
+ provider.getKey()
+ " / "
+ provider.getClass().getName());
if (originalEx == null) {
originalEx = uso;
}
} catch (RepositoryException ex) {
if (originalEx == null) {
originalEx = ex;
}
logger.warn(
"repository exception : "
+ provider.getKey()
+ " / "
+ provider.getClass().getName()
+ " : "
+ ex.getMessage());
}
}
if (originalEx != null) {
if (originalEx instanceof ItemNotFoundException) {
throw originalEx;
} else {
throw new ItemNotFoundException(uuid, originalEx);
}
}
throw new ItemNotFoundException(uuid);
}
public JCRNodeWrapper getNodeByUUID(String providerKey, String uuid)
throws ItemNotFoundException, RepositoryException {
JCRStoreProvider provider = sessionFactory.getProviders().get(providerKey);
if (provider == null) {
throw new ItemNotFoundException(uuid);
}
Session session = getProviderSession(provider);
Node n = session.getNodeByIdentifier(uuid);
return provider.getNodeWrapper(n, this);
}
public JCRItemWrapper getItem(String path) throws PathNotFoundException, RepositoryException {
return getItem(path, true);
}
public JCRItemWrapper getItem(String path, final boolean checkVersion)
throws PathNotFoundException, RepositoryException {
if (sessionCacheByPath.containsKey(path)) {
return sessionCacheByPath.get(path);
}
if (path.contains(DEREF_SEPARATOR)) {
JCRNodeWrapper parent =
(JCRNodeWrapper)
getItem(StringUtils.substringBeforeLast(path, DEREF_SEPARATOR), checkVersion);
return dereference(parent, StringUtils.substringAfterLast(path, DEREF_SEPARATOR));
}
Map<String, JCRStoreProvider> dynamicMountPoints = sessionFactory.getDynamicMountPoints();
for (Map.Entry<String, JCRStoreProvider> mp : dynamicMountPoints.entrySet()) {
if (path.startsWith(mp.getKey() + "/")) {
String localPath = path.substring(mp.getKey().length());
JCRStoreProvider provider = mp.getValue();
Item item = getProviderSession(provider).getItem(provider.getRelativeRoot() + localPath);
if (item.isNode()) {
return provider.getNodeWrapper((Node) item, localPath, null, this);
} else {
return provider.getPropertyWrapper((Property) item, this);
}
}
}
Map<String, JCRStoreProvider> mountPoints = sessionFactory.getMountPoints();
for (Map.Entry<String, JCRStoreProvider> mp : mountPoints.entrySet()) {
String key = mp.getKey();
if (key.equals("/") || path.equals(key) || path.startsWith(key + "/")) {
String localPath = path;
if (!key.equals("/")) {
localPath = localPath.substring(key.length());
}
JCRStoreProvider provider = mp.getValue();
if (localPath.equals("")) {
localPath = "/";
}
// Item item = getProviderSession(provider).getItem(localPath);
Session session = getProviderSession(provider);
if (session instanceof JahiaSessionImpl
&& getUser() != null
&& sessionFactory.getCurrentAliasedUser() != null
&& sessionFactory.getCurrentAliasedUser().equals(getUser())) {
((JahiaSessionImpl) session).toggleThisSessionAsAliased();
}
Item item = session.getItem(provider.getRelativeRoot() + localPath);
if (item.isNode()) {
final Node node = (Node) item;
JCRNodeWrapper wrapper = provider.getNodeWrapper(node, localPath, null, this);
if (getUser() != null
&& sessionFactory.getCurrentAliasedUser() != null
&& !sessionFactory.getCurrentAliasedUser().equals(getUser())) {
final JCRNodeWrapper finalWrapper = wrapper;
JCRTemplate.getInstance()
.doExecuteWithUserSession(
sessionFactory.getCurrentAliasedUser().getUsername(),
getWorkspace().getName(),
getLocale(),
new JCRCallback<Object>() {
public Object doInJCR(JCRSessionWrapper session) throws RepositoryException {
return session.getNodeByUUID(finalWrapper.getIdentifier(), checkVersion);
}
});
}
if (checkVersion
&& (versionDate != null || versionLabel != null)
&& node.isNodeType("mix:versionable")) {
wrapper = getFrozenVersionAsRegular(node, provider);
}
sessionCacheByPath.put(path, wrapper);
sessionCacheByIdentifier.put(wrapper.getIdentifier(), wrapper);
return wrapper;
} else {
return provider.getPropertyWrapper((Property) item, this);
}
}
}
throw new PathNotFoundException(path);
}
private JCRNodeWrapper dereference(JCRNodeWrapper parent, String refPath)
throws RepositoryException {
JCRStoreProvider provider = parent.getProvider();
JCRNodeWrapper wrapper;
Node referencedNode = parent.getRealNode().getProperty("j:node").getNode();
String fullPath = parent.getPath() + DEREF_SEPARATOR + refPath;
if (parent.getPath().startsWith(referencedNode.getPath())) {
throw new PathNotFoundException(fullPath);
}
String refRootName = StringUtils.substringBefore(refPath, "/");
if (!referencedNode.getName().equals(refRootName)) {
throw new PathNotFoundException(fullPath);
}
refPath = StringUtils.substringAfter(refPath, "/");
if (refPath.equals("")) {
wrapper = provider.getNodeWrapper(referencedNode, fullPath, parent, this);
} else {
Node node = referencedNode.getNode(refPath);
wrapper = provider.getNodeWrapper(node, fullPath, null, this);
}
sessionCacheByPath.put(fullPath, wrapper);
return wrapper;
}
public JCRNodeWrapper getNode(String path) throws PathNotFoundException, RepositoryException {
return getNode(path, true);
}
public JCRNodeWrapper getNode(String path, boolean checkVersion)
throws PathNotFoundException, RepositoryException {
JCRItemWrapper item = getItem(path, checkVersion);
if (item.isNode()) {
return (JCRNodeWrapper) item;
} else {
throw new PathNotFoundException();
}
}
public boolean itemExists(String path) throws RepositoryException {
try {
getItem(path);
return true;
} catch (RepositoryException e) {
return false;
}
}
public void move(String source, String dest)
throws ItemExistsException, PathNotFoundException, VersionException,
ConstraintViolationException, LockException, RepositoryException {
getWorkspace().move(source, dest, true);
if (sessionCacheByPath.containsKey(source)) {
JCRNodeWrapper n = sessionCacheByPath.get(source);
if (n instanceof JCRNodeDecorator) {
n = ((JCRNodeDecorator) n).getDecoratedNode();
}
((JCRNodeWrapperImpl) n).localPath = dest;
((JCRNodeWrapperImpl) n).localPathInProvider = dest;
}
flushCaches();
}
public void save()
throws AccessDeniedException, ItemExistsException, ConstraintViolationException,
InvalidItemStateException, VersionException, LockException, NoSuchNodeTypeException,
RepositoryException {
save(JCRObservationManager.SESSION_SAVE);
}
void registerNewNode(JCRNodeWrapper node) {
newNodes.put(node.getPath(), node);
}
void unregisterNewNode(JCRNodeWrapper node) {
if (!newNodes.isEmpty()) {
newNodes.remove(node.getPath());
try {
if (node.hasNodes()) {
NodeIterator it = node.getNodes();
while (it.hasNext()) {
unregisterNewNode((JCRNodeWrapper) it.next());
}
}
} catch (RepositoryException e) {
logger.warn("Error unregistering new nodes", e);
}
}
}
public void save(final int operationType)
throws AccessDeniedException, ItemExistsException, ConstraintViolationException,
InvalidItemStateException, VersionException, LockException, NoSuchNodeTypeException,
RepositoryException {
if (!isSystem() && getLocale() != null) {
for (JCRNodeWrapper node : newNodes.values()) {
for (String s : node.getNodeTypes()) {
ExtendedPropertyDefinition[] propDefs =
NodeTypeRegistry.getInstance().getNodeType(s).getPropertyDefinitions();
for (ExtendedPropertyDefinition propDef : propDefs) {
if (propDef.isMandatory()
&& propDef.getRequiredType() != PropertyType.WEAKREFERENCE
&& propDef.getRequiredType() != PropertyType.REFERENCE
&& !propDef.isProtected()
&& !node.hasProperty(propDef.getName())) {
throw new ConstraintViolationException("Mandatory field : " + propDef.getName());
}
}
}
}
}
newNodes.clear();
JCRObservationManager.doWorkspaceWriteCall(
this,
operationType,
new JCRCallback<Object>() {
public Object doInJCR(JCRSessionWrapper thisSession) throws RepositoryException {
for (Session session : sessions.values()) {
session.save();
}
return null;
}
});
}
public void refresh(boolean b) throws RepositoryException {
for (Session session : sessions.values()) {
session.refresh(b);
}
}
public boolean hasPendingChanges() throws RepositoryException {
for (Session session : sessions.values()) {
if (session.hasPendingChanges()) {
return true;
}
}
return false;
}
public ValueFactory getValueFactory() {
return ValueFactoryImpl.getInstance();
}
/**
* Normally determines whether this <code>Session</code> has permission to perform the specified
* actions at the specified <code>absPath</code>. This method is not supported.
*
* @param absPath an absolute path.
* @param actions a comma separated list of action strings.
* @throws UnsupportedRepositoryOperationException as long as Jahia doesn't support it
*/
public void checkPermission(String absPath, String actions)
throws AccessControlException, RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
public ContentHandler getImportContentHandler(String s, int i)
throws PathNotFoundException, ConstraintViolationException, VersionException, LockException,
RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
public void importXML(String path, InputStream inputStream, int uuidBehavior)
throws IOException, PathNotFoundException, ItemExistsException, ConstraintViolationException,
VersionException, InvalidSerializedDataException, LockException, RepositoryException {
importXML(path, inputStream, uuidBehavior, DocumentViewImportHandler.ROOT_BEHAVIOUR_REPLACE);
}
public void importXML(String path, InputStream inputStream, int uuidBehavior, int rootBehavior)
throws IOException, InvalidSerializedDataException, RepositoryException {
importXML(path, inputStream, uuidBehavior, rootBehavior, null);
}
public void importXML(
String path,
InputStream inputStream,
int uuidBehavior,
int rootBehavior,
Map<String, String> replacements)
throws IOException, InvalidSerializedDataException, RepositoryException {
JCRNodeWrapper node = getNode(path);
try {
if (!node.isCheckedOut()) {
checkout(node);
}
} catch (UnsupportedRepositoryOperationException ex) {
// versioning not supported
}
DocumentViewImportHandler documentViewImportHandler = new DocumentViewImportHandler(this, path);
documentViewImportHandler.setRootBehavior(rootBehavior);
documentViewImportHandler.setUuidBehavior(uuidBehavior);
documentViewImportHandler.setReplacements(replacements);
try {
SAXParserFactory factory;
factory = new SAXParserFactoryImpl();
factory.setNamespaceAware(true);
factory.setValidating(false);
factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);
SAXParser parser = factory.newSAXParser();
parser.parse(inputStream, documentViewImportHandler);
} catch (SAXParseException e) {
logger.error("Cannot import - File is not a valid XML", e);
} catch (Exception e) {
logger.error("Cannot import", e);
}
}
/**
* Applies the namespace prefix to the appropriate sessions, including the underlying provider
* sessions.
*
* @param prefix
* @param uri
* @throws NamespaceException
* @throws RepositoryException
*/
public void setNamespacePrefix(String prefix, String uri)
throws NamespaceException, RepositoryException {
nsToPrefix.put(uri, prefix);
prefixToNs.put(prefix, uri);
for (Session s : sessions.values()) {
s.setNamespacePrefix(prefix, uri);
try {
NamespaceRegistry nsReg = s.getWorkspace().getNamespaceRegistry();
if (nsReg != null) {
nsReg.registerNamespace(prefix, uri);
}
} catch (RepositoryException e) {
if (logger.isDebugEnabled()) {
logger.debug(
"Prefix/uri could not be registered in workspace's registry- " + prefix + "/" + uri,
e);
}
}
}
}
public String[] getNamespacePrefixes() throws RepositoryException {
Set<String> wsPrefixes =
new HashSet<String>(Arrays.asList(getWorkspace().getNamespaceRegistry().getPrefixes()));
wsPrefixes.addAll(prefixToNs.keySet());
return wsPrefixes.toArray(new String[wsPrefixes.size()]);
}
public String getNamespaceURI(String prefix) throws NamespaceException, RepositoryException {
if (prefixToNs.containsKey(prefix)) {
return prefixToNs.get(prefix);
}
return getWorkspace().getNamespaceRegistry().getURI(prefix);
}
public String getNamespacePrefix(String uri) throws NamespaceException, RepositoryException {
if (nsToPrefix.containsKey(uri)) {
return nsToPrefix.get(uri);
}
return getWorkspace().getNamespaceRegistry().getPrefix(uri);
}
public void logout() {
for (Session session : sessions.values()) {
session.logout();
}
if (credentials instanceof SimpleCredentials) {
SimpleCredentials simpleCredentials = (SimpleCredentials) credentials;
JahiaLoginModule.removeToken(
simpleCredentials.getUserID(), new String(simpleCredentials.getPassword()));
}
isLive = false;
activeSessions.decrementAndGet();
}
public boolean isLive() {
return isLive;
}
/**
* Adds the specified lock token to the wrapped sessions. Holding a lock token makes the <code>
* Session</code> the owner of the lock specified by that particular lock token.
*
* @param token a lock token (a string).
* @deprecated As of JCR 2.0, {@link LockManager#addLockToken(String)} should be used instead.
*/
public void addLockToken(String token) {
tokens.add(token);
for (Session session : sessions.values()) {
session.addLockToken(token);
}
}
public String[] getLockTokens() {
List<String> allTokens = new ArrayList<String>(tokens);
for (Session session : sessions.values()) {
String[] tokens = session.getLockTokens();
for (String token : tokens) {
if (!allTokens.contains(token)) {
allTokens.add(token);
}
}
}
return allTokens.toArray(new String[allTokens.size()]);
}
public void removeLockToken(String token) {
tokens.remove(token);
for (Session session : sessions.values()) {
session.removeLockToken(token);
}
}
/**
* Get sessions from all providers used in this wrapper.
*
* @return a <code>Collection</code> of <code>JCRSessionWrapper</code> objects
*/
public Collection<Session> getAllSessions() {
return sessions.values();
}
public Session getProviderSession(JCRStoreProvider provider) throws RepositoryException {
if (sessions.get(provider) == null) {
Session s = null;
if (credentials instanceof SimpleCredentials) {
SimpleCredentials simpleCredentials = (SimpleCredentials) credentials;
JahiaLoginModule.Token t =
JahiaLoginModule.getToken(
simpleCredentials.getUserID(), new String(simpleCredentials.getPassword()));
s = provider.getSession(credentials, workspace.getName());
credentials =
JahiaLoginModule.getCredentials(
simpleCredentials.getUserID(), t != null ? t.deniedPath : null);
} else {
s = provider.getSession(credentials, workspace.getName());
}
sessions.put(provider, s);
for (String token : tokens) {
s.addLockToken(token);
}
NamespaceRegistry namespaceRegistryWrapper = getWorkspace().getNamespaceRegistry();
NamespaceRegistry providerNamespaceRegistry = s.getWorkspace().getNamespaceRegistry();
if (providerNamespaceRegistry != null) {
for (String prefix : namespaceRegistryWrapper.getPrefixes()) {
try {
providerNamespaceRegistry.getURI(prefix);
} catch (NamespaceException ne) {
providerNamespaceRegistry.registerNamespace(
prefix, namespaceRegistryWrapper.getURI(prefix));
}
}
}
for (String prefix : prefixToNs.keySet()) {
s.setNamespacePrefix(prefix, prefixToNs.get(prefix));
}
}
return sessions.get(provider);
}
public JahiaUser getUser() {
return user;
}
public JahiaUser getAliasedUser() {
return sessionFactory.getCurrentAliasedUser();
}
public Calendar getPreviewDate() {
return sessionFactory.getCurrentPreviewDate();
}
/**
* Generates a document view export using a {@link
* org.apache.jackrabbit.commons.xml.DocumentViewExporter} instance.
*
* @param path of the node to be exported
* @param handler handler for the SAX events of the export
* @param skipBinary whether binary values should be skipped
* @param noRecurse whether to export just the identified node
* @throws PathNotFoundException if a node at the given path does not exist
* @throws SAXException if the SAX event handler failed
* @throws RepositoryException if another error occurs
*/
public void exportDocumentView(
String path, ContentHandler handler, boolean skipBinary, boolean noRecurse)
throws PathNotFoundException, SAXException, RepositoryException {
DocumentViewExporter exporter = new DocumentViewExporter(this, handler, skipBinary, noRecurse);
Item item = getItem(path);
if (item.isNode()) {
exporter.export((JCRNodeWrapper) item);
} else {
throw new PathNotFoundException("XML export is not defined for properties: " + path);
}
}
/**
* Generates a system view export using a {@link
* org.apache.jackrabbit.commons.xml.SystemViewExporter} instance.
*
* @param path of the node to be exported
* @param handler handler for the SAX events of the export
* @param skipBinary whether binary values should be skipped
* @param noRecurse whether to export just the identified node
* @throws PathNotFoundException if a node at the given path does not exist
* @throws SAXException if the SAX event handler failed
* @throws RepositoryException if another error occurs
*/
public void exportSystemView(
String path, ContentHandler handler, boolean skipBinary, boolean noRecurse)
throws PathNotFoundException, SAXException, RepositoryException {
// todo implement our own system view .. ?
SystemViewExporter exporter = new SystemViewExporter(this, handler, !noRecurse, !skipBinary);
Item item = getItem(path);
if (item.isNode()) {
exporter.export((JCRNodeWrapper) item);
} else {
throw new PathNotFoundException("XML export is not defined for properties: " + path);
}
}
/**
* Calls {@link Session#exportDocumentView(String, ContentHandler, boolean, boolean)} with the
* given arguments and a {@link ContentHandler} that serializes SAX events to the given output
* stream.
*
* @param absPath passed through
* @param out output stream to which the SAX events are serialized
* @param skipBinary passed through
* @param noRecurse passed through
* @throws IOException if the SAX serialization failed
* @throws RepositoryException if another error occurs
*/
public void exportDocumentView(
String absPath, OutputStream out, boolean skipBinary, boolean noRecurse)
throws IOException, RepositoryException {
try {
ContentHandler handler = getExportContentHandler(out);
exportDocumentView(absPath, handler, skipBinary, noRecurse);
} catch (SAXException e) {
Exception exception = e.getException();
if (exception instanceof RepositoryException) {
throw (RepositoryException) exception;
} else if (exception instanceof IOException) {
throw (IOException) exception;
} else {
throw new RepositoryException("Error serializing document view XML", e);
}
}
}
/**
* Calls {@link Session#exportSystemView(String, ContentHandler, boolean, boolean)} with the given
* arguments and a {@link ContentHandler} that serializes SAX events to the given output stream.
*
* @param absPath passed through
* @param out output stream to which the SAX events are serialized
* @param skipBinary passed through
* @param noRecurse passed through
* @throws IOException if the SAX serialization failed
* @throws RepositoryException if another error occurs
*/
public void exportSystemView(
String absPath, OutputStream out, boolean skipBinary, boolean noRecurse)
throws IOException, RepositoryException {
try {
ContentHandler handler = getExportContentHandler(out);
exportSystemView(absPath, handler, skipBinary, noRecurse);
} catch (SAXException e) {
Exception exception = e.getException();
if (exception instanceof RepositoryException) {
throw (RepositoryException) exception;
} else if (exception instanceof IOException) {
throw (IOException) exception;
} else {
throw new RepositoryException("Error serializing system view XML", e);
}
}
}
/**
* Creates a {@link ContentHandler} instance that serializes the received SAX events to the given
* output stream.
*
* @param stream output stream to which the SAX events are serialized
* @return SAX content handler
* @throws RepositoryException if an error occurs
*/
private ContentHandler getExportContentHandler(OutputStream stream) throws RepositoryException {
try {
SAXTransformerFactory stf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
TransformerHandler handler = stf.newTransformerHandler();
Transformer transformer = handler.getTransformer();
transformer.setOutputProperty(OutputKeys.METHOD, "xml");
transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
transformer.setOutputProperty(OutputKeys.INDENT, "no");
handler.setResult(new StreamResult(stream));
return handler;
} catch (TransformerFactoryConfigurationError e) {
throw new RepositoryException("SAX transformer implementation not available", e);
} catch (TransformerException e) {
throw new RepositoryException("Error creating an XML export content handler", e);
}
}
public JCRNodeWrapper getNodeByIdentifier(String id)
throws ItemNotFoundException, RepositoryException {
return getNodeByUUID(id);
}
public Property getProperty(String absPath) throws PathNotFoundException, RepositoryException {
return (Property) getItem(absPath);
}
public boolean nodeExists(String absPath) throws RepositoryException {
return itemExists(absPath);
}
public boolean propertyExists(String absPath) throws RepositoryException {
return itemExists(absPath);
}
public void removeItem(String absPath)
throws VersionException, LockException, ConstraintViolationException, AccessDeniedException,
RepositoryException {
JCRItemWrapper item = getItem(absPath);
boolean flushNeeded = false;
if (item.isNode()) {
JCRNodeWrapper node = (JCRNodeWrapper) item;
unregisterNewNode(node);
if (node.hasNodes()) {
flushNeeded = true;
}
}
item.remove();
if (flushNeeded) {
flushCaches();
} else {
removeFromCache(item);
}
}
void removeFromCache(JCRItemWrapper item) throws RepositoryException {
sessionCacheByPath.remove(item.getPath());
if (item instanceof JCRNodeWrapper) {
sessionCacheByIdentifier.remove(((JCRNodeWrapper) item).getIdentifier());
}
}
public boolean hasPermission(String absPath, String actions) throws RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
public boolean hasCapability(String s, Object o, Object[] objects) throws RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
/**
* Returns the access control manager for this <code>Session</code>.
*
* <p>Jahia throws an <code>UnsupportedRepositoryOperationException</code>.
*
* @return the access control manager for this <code>Session</code>
* @throws UnsupportedRepositoryOperationException if access control is not supported.
* @since JCR 2.0
*/
public AccessControlManager getAccessControlManager()
throws UnsupportedRepositoryOperationException, RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
public RetentionManager getRetentionManager()
throws UnsupportedRepositoryOperationException, RepositoryException {
throw new UnsupportedRepositoryOperationException();
}
public Map<String, String> getStoredPasswordsProviders() {
Map<String, String> results = new HashMap<String, String>();
results.put(null, user.getUsername());
for (JCRStoreProvider provider : sessionFactory.getProviders().values()) {
if ("storedPasswords".equals(provider.getAuthenticationType())) {
results.put(provider.getKey(), user.getProperty("storedUsername_" + provider.getKey()));
}
}
return results;
}
public void storePasswordForProvider(String providerKey, String username, String password) {
if (username == null) {
user.removeProperty("storedUsername_" + providerKey);
} else {
user.setProperty("storedUsername_" + providerKey, username);
}
if (password == null) {
user.removeProperty("storedPassword_" + providerKey);
} else {
user.setProperty("storedPassword_" + providerKey, password);
}
}
/**
* Performs check out of the specified node.
*
* @param node the node to perform the check out
* @see VersionManager#checkout(String) for details
*/
public void checkout(Node node)
throws UnsupportedRepositoryOperationException, LockException, RepositoryException {
while (!node.isCheckedOut()) {
if (!node.isNodeType("mix:versionable") && !node.isNodeType("mix:simpleVersionable")) {
node = node.getParent();
} else {
String absPath = node.getPath();
VersionManager versionManager = getWorkspace().getVersionManager();
if (!versionManager.isCheckedOut(absPath)) {
versionManager.checkout(absPath);
}
return;
}
}
}
public Map<String, String> getUuidMapping() {
return uuidMapping;
}
public Map<String, String> getPathMapping() {
return pathMapping;
}
public Locale getFallbackLocale() {
return fallbackLocale;
}
public Date getVersionDate() {
return versionDate;
}
public String getVersionLabel() {
return versionLabel;
}
public void setVersionDate(Date versionDate) {
if (this.versionDate == null) {
this.versionDate = versionDate;
} else {
throw new RuntimeException("Should not change versionDate on a session in same thread");
}
}
public void setVersionLabel(String versionLabel) {
if (this.versionLabel == null) {
if (versionLabel != null && !versionLabel.startsWith(getWorkspace().getName())) {
throw new RuntimeException(
"Cannot use label " + versionLabel + " in workspace " + getWorkspace().getName());
}
this.versionLabel = versionLabel;
} else {
throw new RuntimeException("Should not change versionLabel on a session in same thread");
}
}
/** {@inheritDoc} */
public JCRNodeWrapper getFrozenVersionAsRegular(Node objectNode, JCRStoreProvider provider)
throws RepositoryException {
try {
VersionHistory vh =
objectNode
.getSession()
.getWorkspace()
.getVersionManager()
.getVersionHistory(objectNode.getPath());
Version v = null;
if (versionLabel != null) {
v = JCRVersionService.findVersionByLabel(vh, versionLabel);
}
if (v == null && versionDate != null) {
v = JCRVersionService.findClosestVersion(vh, versionDate);
}
if (v == null) {
throw new PathNotFoundException();
}
Node frozen = v.getNode(Constants.JCR_FROZENNODE);
return provider.getNodeWrapper(frozen, this);
} catch (UnsupportedRepositoryOperationException e) {
if (getVersionDate() == null && getVersionLabel() == null) {
logger.error("Error while retrieving frozen version", e);
}
}
return null;
}
public static AtomicLong getActiveSessions() {
return activeSessions;
}
protected void flushCaches() {
sessionCacheByIdentifier.clear();
sessionCacheByPath.clear();
}
protected JCRNodeWrapper getCachedNode(String uuid) {
return sessionCacheByIdentifier.get(uuid);
}
}
/**
* Created by 4535992 on 28/12/2015.
* href:http://www.journaldev.com/2544/java-csv-parserwriter-example-using-opencsv-apache-commons-csv-and-supercsv.
*/
@SuppressWarnings("unused")
public class OpenCsvUtilities extends FileUtilities {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(OpenCsvUtilities.class);
/**
* Method to write a CSV Data List of Beans to a String.
*
* @param beans the List of Beans to convert.
* @param separator the char separator.
* @param <T> the generic variable.
* @return the String content of the List of Beans.
*/
public static <T> String writeCSVDataToStringWithBeans(List<T> beans, char separator) {
try {
Writer writer = new StringWriter();
try (CSVWriter csvWriter = new CSVWriter(writer, separator)) {
List<String[]> data = toStringArray(beans);
csvWriter.writeAll(data);
}
return writer.toString();
} catch (IOException e) {
logger.error(
"Can't write the CSV String from the Bean:"
+ beans.get(0).getClass().getName()
+ " -> "
+ e.getMessage(),
e);
return "";
}
}
/**
* Method to write a CSV Data List of Array of String to a String.
*
* @param content the List of Array of String to convert.
* @param separator the char separator.
* @return the String content of the List of Beans.
*/
public static String writeCSVDataToString(List<String[]> content, char separator) {
try {
Writer writer = new StringWriter();
CSVWriter csvWriter;
if (StringUtilities.NULL_CHAR2 == separator) {
csvWriter =
new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
} else {
csvWriter = new CSVWriter(writer, separator, CSVWriter.NO_QUOTE_CHARACTER);
}
csvWriter.writeAll(content);
csvWriter.close();
return writer.toString();
} catch (IOException e) {
logger.error("Can't write the CSV String -> " + e.getMessage(), e);
return "";
}
}
/**
* Method to write a CSV Data List of Array of String to a String.
*
* @param content the List of Array of String to convert.
* @return the String content of the List of Beans.
*/
public static String writeCSVDataToString(List<String[]> content) {
return writeCSVDataToString(content, StringUtilities.NULL_CHAR2);
}
/**
* Method to write a CSV File from List of Array of String.
*
* @param content the List of Array of String to convert.
* @param separator the char separator.
* @param fileOutputCsv the output File Csv to create.
* @return the File Csv created.
*/
public static File writeCSVDataToFile(
List<String[]> content, char separator, File fileOutputCsv) {
try {
Writer writer =
new FileWriter(fileOutputCsv, true); // the true value make append the result...
CSVWriter csvWriter;
if (StringUtilities.NULL_CHAR2 == separator) {
csvWriter =
new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
} else {
csvWriter = new CSVWriter(writer, separator, CSVWriter.NO_QUOTE_CHARACTER);
}
csvWriter.writeAll(content);
csvWriter.close();
return fileOutputCsv;
} catch (IOException e) {
logger.error("Can't write the CSV to File:" + fileOutputCsv + " -> " + e.getMessage(), e);
return null;
}
}
/**
* Method to write a CSV File from List of Array of String.
*
* @param content the List of Array of String to convert.
* @param fileOutputCsv the output File Csv to create.
* @return the File Csv created.
*/
public static File writeCSVDataToFile(List<String[]> content, File fileOutputCsv) {
return writeCSVDataToFile(content, StringUtilities.NULL_CHAR2, fileOutputCsv);
}
/**
* Method to write a CSV List of Array of String to System Console..
*
* @param content the List of Array of String to convert.
* @param separator the char separator.
*/
public static void writeCSVDataToConsole(List<String[]> content, char separator) {
try {
Writer writer = new OutputStreamWriter(System.out, StringUtilities.UTF_8);
CSVWriter csvWriter;
if (StringUtilities.NULL_CHAR2 == separator) {
csvWriter =
new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
} else {
csvWriter = new CSVWriter(writer, separator, CSVWriter.NO_QUOTE_CHARACTER);
}
csvWriter.writeAll(content, false);
csvWriter.close();
} catch (IOException e) {
logger.error("Can't write the CSV to Console -> " + e.getMessage(), e);
}
}
/**
* Method to write a CSV List of Array of String to System Console..
*
* @param content the List of Array of String to convert.
*/
public static void writeCSVDataToConsole(List<String[]> content) {
writeCSVDataToConsole(content, StringUtilities.NULL_CHAR2);
}
/**
* Method to convert a List of beans to a List of Array of Strings.
*
* @param beans the List of Beans.
* @param <T> generic value.
* @return the List of Array of String content of the csv.
*/
public static <T> List<String[]> toStringArray(List<T> beans) {
return toStringArray(beans, null);
}
/**
* Method to convert a List of beans to a List of Array of Strings.
*
* @param beans the List of Beans.
* @param addNewHeader the new String Array for the Header row.
* @param <T> generic value.
* @return the List of Array of String content of the csv.
*/
@SuppressWarnings("unchecked")
public static <T> List<String[]> toStringArray(List<T> beans, String[] addNewHeader) {
List<String[]> records = new ArrayList<>();
// add header record
// records.add(new String[]{"ID","Name","Role","Salary"});
if (addNewHeader != null) records.add(addNewHeader);
for (T bean : beans) {
// beans.stream().map((bean) -> {
List<String> record = new ArrayList<>();
// invoke getter method and convert to String
Class<T> clazz = (Class<T>) bean.getClass();
// T t = ReflectionUtilities.invokeConstructor(clazz);
List<Method> getter = (List<Method>) ReflectionUtilities.findGetters(clazz, true);
for (Method method : getter) {
record.add(String.valueOf(ReflectionUtilities.invokeGetter(bean, method)));
}
// getter.stream().forEach((method) ->
// record.add(String.valueOf(ReflectionUtilities.invokeGetter(bean,method))));
// return record;
// }).forEach((record) -> records.add(ListUtilities.toArray(record)));
records.add(ListUtilities.toArray(record));
}
return records;
}
/**
* Method use OpenCsv Library for
*
* @param columnMapping Map allow the user to pass the column Names to a Field Names of the Class.
* @param fileInputCsv the File CSV to parse.
* @param <T> the generic variable.
* @return the List of Bean parsed from the CSV file.
*/
public static <T> List<T> parseCSVFileAsList(
Map<String, String> columnMapping, File fileInputCsv) {
try {
HeaderColumnNameTranslateMappingStrategy<T> beanStrategy =
new HeaderColumnNameTranslateMappingStrategy<>();
// beanStrategy.setType(clazz); //deprecated
/*Map<String, String> columnMapping = new HashMap<>();
columnMapping.put("ID", "id");
columnMapping.put("Name", "name");
columnMapping.put("Role", "role");*/
beanStrategy.setColumnMapping(columnMapping);
CsvToBean<T> csvToBean = new CsvToBean<>();
CSVReader reader = new CSVReader(new FileReader(fileInputCsv));
return csvToBean.parse(beanStrategy, reader);
} catch (IOException e) {
logger.error(
"Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
e);
return new ArrayList<>();
}
}
/**
* Method use OpenCsv Library for
*
* @param clazz the Class of the Bean.
* @param fileInputCsv the File CSV to parse.
* @param separator the char separator.
* @param <T> the generic variable.
* @return the List of Bean parsed from the CSV file.
*/
public static <T> List<T> parseCSVFileAsList(Class<T> clazz, File fileInputCsv, char separator) {
try {
List<T> beans;
try ( // create CSVReader object
CSVReader reader = new CSVReader(new FileReader(fileInputCsv), separator)) {
beans = new ArrayList<>();
// read line by line
String[] record;
// skip header row
String[] headers = reader.readNext();
// read content
while ((record = reader.readNext()) != null) {
T t = ReflectionUtilities.invokeConstructor(clazz);
for (int i = 0; i < record.length; i++) {
String nameMethod = "set" + org.apache.commons.lang3.StringUtils.capitalize(headers[i]);
// invoke setter method
if (ReflectionUtilities.checkMethod(clazz, nameMethod)) {
ReflectionUtilities.invokeSetter(t, nameMethod, record[i]);
} else {
logger.warn(
"Not exists the Method with name:"
+ nameMethod
+ " on the Bean:"
+ t.getClass().getName());
}
}
beans.add(t);
}
}
return beans;
} catch (IOException e) {
logger.error(
"Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
e);
return new ArrayList<>();
}
}
/**
* Method use OpenCsv Library for
*
* @param fileInputCsv the File CSV to parse.
* @param separator the char separator.
* @return the List of Bean parsed from the CSV file.
*/
public static List<String[]> parseCSVFileAsList(File fileInputCsv, char separator) {
try {
List<String[]> records;
// read all lines at once
try ( // create CSVReader object
CSVReader reader = new CSVReader(new FileReader(fileInputCsv), separator)) {
// read all lines at once
records = reader.readAll();
Iterator<String[]> iterator = records.iterator();
records.clear();
// skip header row
iterator.next();
while (iterator.hasNext()) {
String[] record = iterator.next();
records.add(record);
}
}
return records;
} catch (IOException e) {
logger.error(
"Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
e);
return new ArrayList<>();
}
}
/**
* Method to get the content of a comma separated file (.csv,.input,.txt)
*
* @param CSV the File comma separated.
* @param noHeaders if true jump the first line of the content.
* @return the List of Array of the content of the File comma separated.
*/
public static List<String[]> parseCSVFileAsList(File CSV, boolean noHeaders) {
List<String[]> content;
try {
CSVReader reader1 = new CSVReader(new FileReader(CSV));
content = reader1.readAll();
/* List<String[]> myDatas = reader1.readAll();
String[] lineI = myDatas.get(i);
for (String[] line : myDatas) {
for (String value : line) {
//do stuff with value
}
}*/
if (noHeaders) content.remove(0);
if (content.get(0).length <= 1) {
logger.warn(
"Attention: You haven't parsed correctly the CSV file with OpenCSV API try with Univocity Method");
}
return content;
} catch (IOException e) {
logger.error("Can't find the CSV File:" + e.getMessage(), e);
return null;
}
}
/**
* Method to get the String array of the columns of a CSV File.
*
* @param fileCSV the File CSV.
* @param hasFirstLine if true the first line of CSV File contains the columns name.
* @return a String Array of the columns.
*/
public static String[] getHeaders(File fileCSV, boolean hasFirstLine) {
String[] columns = new String[0];
try {
CSVReader reader = new CSVReader(new FileReader(fileCSV));
columns = reader.readNext(); // assuming first read
if (!hasFirstLine) {
int columnCount = 0;
if (columns != null) columnCount = columns.length;
columns = new String[columnCount];
for (int i = 0; i < columnCount; i++) {
columns[i] = "Column#" + i;
}
}
} catch (IOException e) {
logger.error("Can't find the Headers on the CSV File", e);
}
return columns;
}
/**
* Method to get the String array of the columns of a CSV File.
*
* @param contentWithHeaders the {@link List} of {@link String} array of the content of the csv.
* @return a String Array of the columns.
*/
public static String[] getHeaders(List<String[]> contentWithHeaders) {
String[] columns = new String[0];
try {
String[] headers = contentWithHeaders.get(0);
if (headers.length <= 1) {
throw new Exception("Can't find the delimiter with openCSV try with Univicity method.");
}
} catch (Exception e) {
logger.error("Can't find the Headers on the content", e);
}
return columns;
}
/**
* Method to get the String array of the columns of a CSV File.
*
* @param fileCSV the File CSV.
* @param hasFirstLine if true the first line of CSV File contains the columns name.
* @return a String Array of the columns.
*/
public static String[] getHeadersWithUnivocity(File fileCSV, boolean hasFirstLine) {
return UnivocityUtilities.getHeaders(fileCSV, hasFirstLine);
}
// ------------------------------------------------------------------------------
/**
* Method use OpenCsv Library for
*
* @param fileInputCsv the File CSV to parse.
* @param noHeaders if true the headers are excluded from the cotent.
* @return the List of Bean parsed from the CSV file.
*/
public static List<String[]> parseCSVFileAsListWithUnivocity(
File fileInputCsv, boolean noHeaders) {
return UnivocityUtilities.parseCSVFileAsList(fileInputCsv, noHeaders);
}
/* public static void main(String[] args) throws IOException {
List<Employee> emps = parseCSVFileLineByLine();
System.out.println("**********");
parseCSVFileAsList();
System.out.println("**********");
parseCSVToBeanList();
System.out.println("**********");
writeCSVData(emps);
}*/
public static String getFieldLatitude(String[] headers) {
Pattern pattern = Pattern.compile("(L|l)(at)(itude)?", Pattern.CASE_INSENSITIVE);
return getField(headers, pattern);
}
public static String getFieldLongitude(String[] headers) {
Pattern pattern = Pattern.compile("(L|l)(on|ng)(gitude)?", Pattern.CASE_INSENSITIVE);
return getField(headers, pattern);
}
public static String getField(String[] headers, Pattern pattern) {
// String[] headers = CSVGetHeaders(headers,true);
for (String s : headers) {
if (pattern.matcher(String.valueOf(s)).matches()) {
return s;
}
}
return "NULL";
}
/**
* Parse CSV file using OpenCSV library and load in given database table. href:
* http://viralpatel.net/blogs/java-load-csv-file-to-database/. Modified by rammar:
* https://github.com/BaderLab/pharmacogenomics/blob/master/src/CSVLoader/CSVLoader.java
*
* @param connection the {@link Connection} SQL.
* @param separator the {@link Character} separator.
* @param csvFile Input CSV InputStream
* @param tableName Database table name to import data
* @param truncateBeforeLoad Truncate the table before inserting new records.
* @throws SQLException if any error is occurred with the SQL Connection.
* @throws java.io.IOException if any error is occurred with the file.
*/
public static void loadCSVToSQLTable(
Connection connection,
char separator,
InputStream csvFile,
String tableName,
boolean truncateBeforeLoad)
throws SQLException, IOException {
String SQL_INSERT = "INSERT INTO ${table}(${keys}) VALUES(${values})";
String TABLE_REGEX = "\\$\\{table\\}";
String KEYS_REGEX = "\\$\\{keys\\}";
String VALUES_REGEX = "\\$\\{values\\}";
CSVReader csvReader;
if (null == connection) {
throw new SQLException("Not a valid connection.");
}
try {
/* Modified by rammar.
*
* I was having issues with the CSVReader using the "\" to escape characters.
* A MySQL CSV file contains quote-enclosed fields and non-quote-enclosed NULL
* values written as "\N". The CSVReader was removing the "\". To detect "\N"
* I must remove the escape character, and the only character you can replace
* it with that you are pretty much guaranteed will not be used to escape
* text is '\0'.
* I read this on:
* http://stackoverflow.com/questions/6008395/opencsv-in-java-ignores-backslash-in-a-field-value
* based on:
* http://sourceforge.net/p/opencsv/support-requests/5/
*/
// PREVIOUS VERSION: csvReader = new CSVReader(new FileReader(csvFile), this.seprator);
csvReader = new CSVReader(new InputStreamReader(csvFile), separator, '"', '\0');
} catch (Exception e) {
throw new IOException("Error occured while executing file. " + e.getMessage());
}
String[] headerRow = csvReader.readNext();
if (null == headerRow) {
throw new FileNotFoundException(
"No columns defined in given CSV file." + "Please check the CSV file format.");
}
String questionmarks = StringUtils.repeat("?,", headerRow.length);
questionmarks = (String) questionmarks.subSequence(0, questionmarks.length() - 1);
/* NOTE from Ron: Header column names must match SQL table fields */
String query = SQL_INSERT.replaceFirst(TABLE_REGEX, tableName);
query = query.replaceFirst(KEYS_REGEX, StringUtils.join(headerRow, ","));
query = query.replaceFirst(VALUES_REGEX, questionmarks);
// System.out.println("Query: " + query); // Modified by rammar to suppress output
String[] nextLine;
Connection con = null;
PreparedStatement ps = null;
try {
con = connection;
con.setAutoCommit(false);
ps = con.prepareStatement(query);
if (truncateBeforeLoad) {
// delete data from table before loading csv
con.createStatement().execute("DELETE FROM " + tableName);
}
final int batchSize = 1000;
int count = 0;
Date date;
while ((nextLine = csvReader.readNext()) != null) {
int index = 1;
for (String string : nextLine) {
date = DateUtilities.convertToDate(string);
if (null != date) {
ps.setDate(index++, new java.sql.Date(date.getTime()));
} else {
/* Section modified by rammar to allow NULL values
* to be input into the DB. */
if (string.length() > 0 && !string.equals("\\N")) {
ps.setString(index++, string);
} else {
ps.setNull(index++, Types.VARCHAR);
// ps.setString(index++, null); // can use this syntax also - not sure which is better
}
}
}
ps.addBatch();
if (++count % batchSize == 0) {
ps.executeBatch();
}
}
ps.executeBatch(); // insert remaining records
logger.info(count + " records loaded into " + tableName + " DB table");
con.commit();
} catch (SQLException | IOException e) {
con.rollback();
throw new IOException(
"Error occured while loading data from file to database." + e.getMessage());
} finally {
if (null != ps) ps.close();
con.close();
csvReader.close();
}
}
}
/**
* Builds Collections of Grib1 Time Partitioned.
*
* @author caron
* @since 1/7/12
*/
public class Grib1TimePartitionBuilder extends Grib1CollectionBuilder {
public static final String MAGIC_START = "Grib1Partition0Index";
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(Grib1TimePartitionBuilder.class);
// static private final int versionTP = 4;
private static final boolean trace = false;
// called by tdm
public static boolean update(TimePartitionCollection tpc, Formatter f) throws IOException {
Grib1TimePartitionBuilder builder =
new Grib1TimePartitionBuilder(tpc.getCollectionName(), new File(tpc.getRoot()), tpc);
if (!builder.needsUpdate()) return false;
builder.readOrCreateIndex(CollectionManager.Force.always, f);
builder.gc.close();
return true;
}
// read in the index, create if it doesnt exist or is out of date
public static Grib1TimePartition factory(
TimePartitionCollection tpc, CollectionManager.Force force, Formatter f) throws IOException {
Grib1TimePartitionBuilder builder =
new Grib1TimePartitionBuilder(tpc.getCollectionName(), new File(tpc.getRoot()), tpc);
builder.readOrCreateIndex(force, f);
return builder.tp;
}
// read in the index, index raf already open
public static Grib1TimePartition createFromIndex(
String name, File directory, RandomAccessFile raf) throws IOException {
Grib1TimePartitionBuilder builder = new Grib1TimePartitionBuilder(name, directory, null);
if (builder.readIndex(raf)) {
return builder.tp;
}
throw new IOException("Reading index failed");
}
/**
* write new index if needed
*
* @param tpc use this collection
* @param force force index
* @param f put status messagess here
* @return true if index was written
* @throws IOException on error
*/
public static boolean writeIndexFile(
TimePartitionCollection tpc, CollectionManager.Force force, Formatter f) throws IOException {
Grib1TimePartitionBuilder builder = null;
try {
builder =
new Grib1TimePartitionBuilder(tpc.getCollectionName(), new File(tpc.getRoot()), tpc);
return builder.readOrCreateIndex(force, f);
} finally {
if ((builder != null) && (builder.tp != null)) builder.tp.close();
}
}
//////////////////////////////////////////////////////////////////////////////////
private final TimePartitionCollection tpc; // defines the partition
private final Grib1TimePartition tp; // build this object
private Grib1TimePartitionBuilder(String name, File directory, TimePartitionCollection tpc) {
FeatureCollectionConfig.GribConfig config =
(tpc == null)
? null
: (FeatureCollectionConfig.GribConfig)
tpc.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG);
this.tp = new Grib1TimePartition(name, directory, config);
this.gc = tp;
this.tpc = tpc;
}
private boolean readOrCreateIndex(CollectionManager.Force ff, Formatter f) throws IOException {
File idx = gc.getIndexFile();
// force new index or test for new index needed
boolean force =
((ff == CollectionManager.Force.always)
|| (ff == CollectionManager.Force.test && needsUpdate(idx.lastModified(), f)));
// otherwise, we're good as long as the index file exists and can be read
if (force || !idx.exists() || !readIndex(idx.getPath())) {
logger.info("TimePartitionBuilder createIndex {}", idx.getPath());
createPartitionedIndex(f); // write out index
readIndex(idx.getPath()); // read back in index
return true;
}
return false;
}
private boolean needsUpdate(long collectionLastModified, Formatter f) throws IOException {
CollectionManager.ChangeChecker cc = Grib1Index.getChangeChecker();
for (CollectionManager dcm :
tpc.makePartitions()) { // LOOK not really right, since we dont know if these files are the
// same as in the index
File idxFile = GribCollection.getIndexFile(dcm);
if (!idxFile.exists()) return true;
if (collectionLastModified < idxFile.lastModified()) return true;
for (MFile mfile : dcm.getFiles()) {
if (cc.hasChangedSince(mfile, idxFile.lastModified())) return true;
}
}
return false;
}
///////////////////////////////////////////////////
// create the index
private boolean createPartitionedIndex(Formatter f) throws IOException {
long start = System.currentTimeMillis();
// create partitions based on TimePartitionCollections object
for (CollectionManager dcm : tpc.makePartitions()) {
tp.addPartition(dcm);
}
List<TimePartition.Partition> bad = new ArrayList<TimePartition.Partition>();
for (TimePartition.Partition dc : tp.getPartitions()) {
try {
dc.makeGribCollection(f); // ensure collection has been read successfully
if (trace) f.format(" Open partition %s%n", dc.getDcm().getCollectionName());
} catch (Throwable t) {
logger.error(" Failed to open partition " + dc.getName(), t);
f.format(" FAIL on partition %s (remove) %n", dc.getDcm().getCollectionName());
bad.add(dc); // LOOK may be a file leak ?
}
}
// remove ones that failed
for (TimePartition.Partition p : bad) tp.removePartition(p);
// choose the "canonical" partition, aka prototype
int n = tp.getPartitions().size();
if (n == 0) {
logger.error(" Nothing in this partition = " + tp.getName());
f.format(" FAIL Partition empty collection = %s%n", tp.getName());
return false;
}
int idx = tpc.getProtoIndex(n);
TimePartition.Partition canon = tp.getPartitions().get(idx);
f.format(" Using canonical partition %s%n", canon.getDcm().getCollectionName());
// check consistency across vert and ens coords
if (!checkPartitions(canon, f)) {
logger.error(
" Partition check failed, index not written on {} message = {}",
tp.getName(),
f.toString());
f.format(" FAIL Partition check collection = %s%n", tp.getName());
return false;
}
// make the time coordinates, place results into canon
createPartitionedTimeCoordinates(canon, f);
// ready to write the index file
writeIndex(canon, f);
// close open gc's
tp.cleanup();
long took = System.currentTimeMillis() - start;
f.format(" CreatePartitionedIndex took %d msecs%n", took);
return true;
}
// consistency check on variables : compare each variable to corresponding one in proto
// also set the groupno and partno for each partition
private boolean checkPartitions(TimePartition.Partition canon, Formatter f) throws IOException {
List<TimePartition.Partition> partitions = tp.getPartitions();
int npart = partitions.size();
boolean ok = true;
// for each group in canonical Partition
GribCollection canonGc = canon.makeGribCollection(f);
for (GribCollection.GroupHcs firstGroup : canonGc.getGroups()) {
String gname = firstGroup.getId();
if (trace) f.format(" Check Group %s%n", gname);
// hash proto variables for quick lookup
Map<Integer, GribCollection.VariableIndex> check =
new HashMap<Integer, GribCollection.VariableIndex>(firstGroup.varIndex.size());
List<GribCollection.VariableIndex> varIndexP =
new ArrayList<GribCollection.VariableIndex>(firstGroup.varIndex.size());
for (GribCollection.VariableIndex vi : firstGroup.varIndex) {
TimePartition.VariableIndexPartitioned vip = tp.makeVariableIndexPartitioned(vi, npart);
varIndexP.add(vip);
check.put(vi.cdmHash, vip); // replace with its evil twin
}
firstGroup.varIndex = varIndexP; // replace with its evil twin
// for each partition
for (int partno = 0; partno < npart; partno++) {
TimePartition.Partition tpp = partitions.get(partno);
if (trace) f.format(" Check Partition %s%n", tpp.getName());
// get corresponding group
GribCollection gc = tpp.makeGribCollection(f);
int groupIdx = gc.findGroupIdxById(firstGroup.getId());
if (groupIdx < 0) {
f.format(" Cant find group %s in partition %s%n", gname, tpp.getName());
ok = false;
continue;
}
GribCollection.GroupHcs group = gc.getGroup(groupIdx);
// for each variable in partition group
for (int varIdx = 0; varIdx < group.varIndex.size(); varIdx++) {
GribCollection.VariableIndex vi2 = group.varIndex.get(varIdx);
if (trace) f.format(" Check variable %s%n", vi2);
int flag = 0;
GribCollection.VariableIndex vi1 = check.get(vi2.cdmHash); // compare with proto variable
if (vi1 == null) {
f.format(
" WARN Cant find variable %s from %s in proto - ignoring that variable%n",
vi2, tpp.getName());
continue; // we can tolerate this
}
// compare vert coordinates
VertCoord vc1 = vi1.getVertCoord();
VertCoord vc2 = vi2.getVertCoord();
if ((vc1 == null) != (vc2 == null)) {
f.format(
" ERR Vert coordinates existence on variable %s in %s doesnt match%n",
vi2, tpp.getName());
ok = false;
} else if ((vc1 != null) && !vc1.equalsData(vc2)) {
f.format(
" WARN Vert coordinates values on variable %s in %s dont match%n",
vi2, tpp.getName());
f.format(" canon vc = %s%n", vc1);
f.format(" this vc = %s%n", vc2);
flag |= TimePartition.VERT_COORDS_DIFFER;
}
// compare ens coordinates
EnsCoord ec1 = vi1.getEnsCoord();
EnsCoord ec2 = vi2.getEnsCoord();
if ((ec1 == null) != (ec2 == null)) {
f.format(
" ERR Ensemble coordinates existence on variable %s in %s doesnt match%n",
vi2, tpp.getName());
ok = false;
} else if ((ec1 != null) && !ec1.equalsData(ec2)) {
f.format(
" WARN Ensemble coordinates values on variable %s in %s dont match%n",
vi2, tpp.getName());
f.format(" canon ec = %s%n", ec1);
f.format(" this ec = %s%n", ec2);
flag |= TimePartition.ENS_COORDS_DIFFER;
}
((TimePartition.VariableIndexPartitioned) vi1)
.setPartitionIndex(partno, groupIdx, varIdx, flag);
} // loop over variable
} // loop over partition
} // loop over group
if (ok) f.format(" Partition check: vert, ens coords OK%n");
return ok;
}
private class PartGroup {
GribCollection.GroupHcs group;
TimePartition.Partition tpp;
private PartGroup(GribCollection.GroupHcs group, TimePartition.Partition tpp) {
this.group = group;
this.tpp = tpp;
}
}
private boolean createPartitionedTimeCoordinates(TimePartition.Partition canon, Formatter f)
throws IOException {
List<TimePartition.Partition> partitions = tp.getPartitions();
boolean ok = true;
// for each group in canonical Partition
for (GribCollection.GroupHcs firstGroup : canon.makeGribCollection(f).getGroups()) {
String gname = firstGroup.getId();
if (trace) f.format(" Check Group %s%n", gname);
// get list of corresponding groups from all the time partition, so we dont have to keep
// looking it up
List<PartGroup> pgList = new ArrayList<PartGroup>(partitions.size());
for (TimePartition.Partition dc : partitions) {
GribCollection.GroupHcs gg = dc.makeGribCollection(f).findGroupById(gname);
if (gg == null) logger.error(" Cant find group {} in partition {}", gname, dc.getName());
else pgList.add(new PartGroup(gg, dc));
}
// unique time coordinate unions
List<TimeCoordUnion> unionList = new ArrayList<TimeCoordUnion>();
// for each variable in canonical Partition
for (GribCollection.VariableIndex viCanon : firstGroup.varIndex) {
if (trace) f.format(" Check variable %s%n", viCanon);
TimeCoord tcCanon = viCanon.getTimeCoord();
List<TimeCoord> tcPartitions = new ArrayList<TimeCoord>(pgList.size());
// for each partition, get the time index
for (PartGroup pg : pgList) {
// get corresponding variable
GribCollection.VariableIndex vi2 = pg.group.findVariableByHash(viCanon.cdmHash);
if (vi2 == null) { // apparently not in the file
f.format(
" WARN Cant find variable %s in partition %s / %s%n",
viCanon, pg.tpp.getName(), pg.group.getId());
tcPartitions.add(null);
} else {
if (vi2.timeIdx < 0 || vi2.timeIdx >= pg.group.timeCoords.size()) {
logger.error(" timeIdx out of range var= {} on partition {}", vi2, pg.tpp.getName());
tcPartitions.add(null);
} else {
TimeCoord tc2 = vi2.getTimeCoord();
if (tc2.isInterval() != tcCanon.isInterval()) {
logger.error(
" timeIdx wrong interval type var= {} on partition {}", vi2, pg.tpp.getName());
tcPartitions.add(null);
} else {
tcPartitions.add(tc2);
}
}
}
}
// union of time coordinates
TimeCoordUnion union = new TimeCoordUnion(tcCanon.getCode(), tcPartitions, tcCanon);
// store result in the first group
viCanon.partTimeCoordIdx =
TimeCoordUnion.findUnique(unionList, union); // this merges identical TimeCoordUnion
}
/* turn TimeIndex into TimeCoord
for (int tidx = 0; tidx <unionList.size(); tidx++) {
TimeCoordUnion union = unionList.get(tidx);
f.format(" %s %d: timeIndexList=", firstGroup.hcs.getName(), tidx);
for (int idx : union.) f.format("%d,",idx);
f.format("%n");
} */
// store results in first group
firstGroup.timeCoordPartitions = unionList;
}
return ok;
}
//////////////////////////////////////////////////////////
@Override
public String getMagicStart() {
return MAGIC_START;
}
// writing ncx
/*
MAGIC_START
version
sizeRecords
VariableRecords (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
*/
private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
File file = tp.getIndexFile();
if (file.exists()) {
if (!file.delete()) logger.error("Cant delete " + file.getPath());
}
RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
raf.writeLong(0); // no record section
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(tp.getName());
GribCollection canonGc = canon.makeGribCollection(f);
for (GribCollection.GroupHcs g : canonGc.getGroups())
indexBuilder.addGroups(writeGroupProto(g));
indexBuilder.setCenter(canonGc.getCenter());
indexBuilder.setSubcenter(canonGc.getSubcenter());
indexBuilder.setMaster(canonGc.getMaster());
indexBuilder.setLocal(canonGc.getLocal());
for (TimePartition.Partition p : tp.getPartitions()) {
indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
}
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);
} finally {
f.format("file size = %d bytes%n", raf.length());
raf.close();
}
return true;
}
private GribCollectionProto.Group writeGroupProto(GribCollection.GroupHcs g) throws IOException {
GribCollectionProto.Group.Builder b = GribCollectionProto.Group.newBuilder();
b.setGds(ByteString.copyFrom(g.rawGds));
b.setGdsHash(g.gdsHash);
for (GribCollection.VariableIndex vb : g.varIndex)
b.addVariables(writeVariableProto((TimePartition.VariableIndexPartitioned) vb));
for (int i = 0; i < g.timeCoordPartitions.size(); i++)
b.addTimeCoordUnions(writeTimeCoordUnionProto(g.timeCoordPartitions.get(i), i));
List<VertCoord> vertCoords = g.vertCoords;
for (int i = 0; i < vertCoords.size(); i++)
b.addVertCoords(writeCoordProto(vertCoords.get(i), i));
List<EnsCoord> ensCoords = g.ensCoords;
for (int i = 0; i < ensCoords.size(); i++) b.addEnsCoords(writeCoordProto(ensCoords.get(i), i));
return b.build();
}
private GribCollectionProto.Variable writeVariableProto(TimePartition.VariableIndexPartitioned v)
throws IOException {
GribCollectionProto.Variable.Builder b = GribCollectionProto.Variable.newBuilder();
b.setDiscipline(v.discipline);
b.setCategory(v.category);
b.setParameter(v.parameter);
b.setLevelType(v.levelType);
b.setIsLayer(v.isLayer);
b.setIntervalType(v.intvType);
if (v.intvName != null) b.setIntvName(v.intvName);
b.setCdmHash(v.cdmHash);
b.setRecordsPos(0);
b.setRecordsLen(0);
b.setTimeIdx(v.partTimeCoordIdx); // note
if (v.vertIdx >= 0) b.setVertIdx(v.vertIdx);
if (v.ensIdx >= 0) b.setEnsIdx(v.ensIdx);
if (v.ensDerivedType >= 0) b.setEnsDerivedType(v.ensDerivedType); // derived type (table 4.7)
if (v.probabilityName != null) b.setProbabilityName(v.probabilityName);
if (v.probType >= 0) b.setProbabilityType(v.probType);
for (int idx : v.groupno) b.addGroupno(idx);
for (int idx : v.varno) b.addVarno(idx);
for (int idx : v.flag) b.addFlag(idx);
return b.build();
}
protected GribCollectionProto.TimeCoordUnion writeTimeCoordUnionProto(
TimeCoordUnion tcu, int index) throws IOException {
GribCollectionProto.TimeCoordUnion.Builder b = GribCollectionProto.TimeCoordUnion.newBuilder();
b.setCode(index);
b.setUnit(tcu.getUnits());
if (tcu.isInterval()) {
for (TimeCoord.Tinv tinv : tcu.getIntervals()) {
b.addValues((float) tinv.getBounds1());
b.addBound((float) tinv.getBounds2());
}
} else {
for (int value : tcu.getCoords()) b.addValues((float) value);
}
for (TimeCoordUnion.Val val : tcu.getValues()) {
b.addPartition(val.getPartition());
b.addIndex(val.getIndex());
}
return b.build();
}
private GribCollectionProto.Partition writePartitionProto(String name, TimePartition.Partition p)
throws IOException {
GribCollectionProto.Partition.Builder b = GribCollectionProto.Partition.newBuilder();
b.setFilename(p.getIndexFilename());
b.setName(name);
return b.build();
}
///////////////////////////////////////////////////////////////////////////
// reading ncx
@Override
protected boolean readPartitions(GribCollectionProto.GribCollectionIndex proto) {
for (int i = 0; i < proto.getPartitionsCount(); i++) {
GribCollectionProto.Partition pp = proto.getPartitions(i);
tp.addPartition(pp.getName(), pp.getFilename());
}
return proto.getPartitionsCount() > 0;
}
@Override
protected void readTimePartitions(
GribCollection.GroupHcs group, GribCollectionProto.Group proto) {
List<TimeCoord> list = new ArrayList<TimeCoord>(proto.getTimeCoordUnionsCount());
for (int i = 0; i < proto.getTimeCoordUnionsCount(); i++) {
GribCollectionProto.TimeCoordUnion tpu = proto.getTimeCoordUnions(i);
list.add(readTimePartition(tpu, i));
}
group.timeCoords = list;
}
/*
protected TimeCoord readTimePartition(GribCollectionProto.TimeCoordUnion pc, int timeIndex) {
int[] partition = new int[pc.getPartitionCount()];
int[] index = new int[pc.getPartitionCount()]; // better be the same
for (int i = 0; i < pc.getPartitionCount(); i++) {
partition[i] = pc.getPartition(i);
index[i] = pc.getIndex(i);
}
if (pc.getBoundCount() > 0) { // its an interval
List<TimeCoord.Tinv> coords = new ArrayList<TimeCoord.Tinv>(pc.getValuesCount());
for (int i = 0; i < pc.getValuesCount(); i++)
coords.add(new TimeCoord.Tinv((int) pc.getValues(i), (int) pc.getBound(i)));
TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
return tc.setIndex( timeIndex);
} else {
List<Integer> coords = new ArrayList<Integer>(pc.getValuesCount());
for (float value : pc.getValuesList())
coords.add((int) value);
TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
return tc.setIndex( timeIndex);
}
}
*/
protected TimeCoord readTimePartition(GribCollectionProto.TimeCoordUnion pc, int timeIndex) {
int[] partition = new int[pc.getPartitionCount()];
int[] index = new int[pc.getPartitionCount()]; // better be the same
for (int i = 0; i < pc.getPartitionCount(); i++) {
partition[i] = pc.getPartition(i);
index[i] = pc.getIndex(i);
}
if (pc.getBoundCount() > 0) { // its an interval
List<TimeCoord.Tinv> coords = new ArrayList<TimeCoord.Tinv>(pc.getValuesCount());
for (int i = 0; i < pc.getValuesCount(); i++)
coords.add(new TimeCoord.Tinv((int) pc.getValues(i), (int) pc.getBound(i)));
TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
return tc.setIndex(timeIndex);
} else {
List<Integer> coords = new ArrayList<Integer>(pc.getValuesCount());
for (float value : pc.getValuesList()) coords.add((int) value);
TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
return tc.setIndex(timeIndex);
}
}
@Override
protected GribCollection.VariableIndex readVariable(
GribCollectionProto.Variable pv, GribCollection.GroupHcs group) {
int discipline = pv.getDiscipline();
int category = pv.getCategory();
int param = pv.getParameter();
int levelType = pv.getLevelType();
int intvType = pv.getIntervalType();
String intvName = pv.getIntvName();
boolean isLayer = pv.getIsLayer();
int ensDerivedType = pv.getEnsDerivedType();
int probType = pv.getProbabilityType();
String probabilityName = pv.getProbabilityName();
int cdmHash = pv.getCdmHash();
long recordsPos = pv.getRecordsPos();
int recordsLen = pv.getRecordsLen();
int timeIdx = pv.getTimeIdx();
int vertIdx = pv.getVertIdx();
int ensIdx = pv.getEnsIdx();
int tableVersion = pv.getTableVersion();
List<Integer> groupnoList = pv.getGroupnoList();
List<Integer> varnoList = pv.getVarnoList();
List<Integer> flagList = pv.getFlagList();
return tp.makeVariableIndex(
group,
tableVersion,
discipline,
category,
param,
levelType,
isLayer,
intvType,
intvName,
ensDerivedType,
probType,
probabilityName,
-1,
cdmHash,
timeIdx,
vertIdx,
ensIdx,
recordsPos,
recordsLen,
groupnoList,
varnoList,
flagList);
}
public static void main(String[] args) throws IOException {
Formatter f = new Formatter();
String indexName = (args.length > 0) ? args[0] : "F:/nomads/NOMADS-cfsrr-timeseries.ncx";
RandomAccessFile raf = new RandomAccessFile(indexName, "r");
Grib1TimePartition gtc = Grib1TimePartitionBuilder.createFromIndex("test", null, raf);
gtc.showIndex(f);
System.out.printf("%s%n", f);
}
}
/**
* Forecast Model Run Collection, manages dynamic collections of GridDatasets. Fmrc represents a
* virtual dataset. To instantiate, you obtain an FmrcInv "snapshot" from which you can call
* getDatatset().
*
* <p>Assumes that we dont have multiple runtimes in the same file. Can handle different time steps
* in different files. Can handle different grids in different files. However this creates problems
* for the "typical dataset". Cannot handle different ensembles in different files. (LOOK fix)
* Cannot handle different levels in different files. ok
*
* @author caron
* @since Jan 11, 2010
*/
@ThreadSafe
public class Fmrc {
private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fmrc.class);
/**
* Factory method
*
* @param collection describes the collection. May be one of:
* <ol>
* <li>collection specification string
* <li>catalog:catalogURL
* <li>filename.ncml
* <li>
* </ol>
* collectionSpec date extraction is used to get rundates
* @param errlog place error messages here
* @return Fmrc or null on error
* @throws IOException on read error
* @see
* "http://www.unidata.ucar.edu/software/netcdf-java/reference/collections/CollectionSpecification.html"
*/
public static Fmrc open(String collection, Formatter errlog) throws IOException {
if (collection.startsWith(MFileCollectionManager.CATALOG)) {
CatalogCollectionManager manager = new CatalogCollectionManager(collection);
return new Fmrc(manager, new FeatureCollectionConfig());
} else if (collection.endsWith(".ncml")) {
NcmlCollectionReader ncmlCollection = NcmlCollectionReader.open(collection, errlog);
if (ncmlCollection == null) return null;
Fmrc fmrc = new Fmrc(ncmlCollection.getCollectionManager(), new FeatureCollectionConfig());
fmrc.setNcml(ncmlCollection.getNcmlOuter(), ncmlCollection.getNcmlInner());
return fmrc;
}
return new Fmrc(collection, errlog);
}
public static Fmrc open(FeatureCollectionConfig config, Formatter errlog) throws IOException {
if (config.spec.startsWith(MFileCollectionManager.CATALOG)) {
CatalogCollectionManager manager = new CatalogCollectionManager(config.spec);
return new Fmrc(manager, config);
}
return new Fmrc(config, errlog);
}
////////////////////////////////////////////////////////////////////////
private final CollectionManager manager;
private final FeatureCollectionConfig config;
// should be final
// private Element ncmlOuter, ncmlInner;
// the current state - changing must be thread safe
private final Object lock = new Object();
private FmrcDataset fmrcDataset;
private volatile boolean forceProto = false;
private volatile long lastInvChanged;
private volatile long lastProtoChanged;
private Fmrc(String collectionSpec, Formatter errlog) throws IOException {
this.manager = MFileCollectionManager.open(collectionSpec, null, errlog);
this.config = new FeatureCollectionConfig();
this.config.spec = collectionSpec;
}
private Fmrc(FeatureCollectionConfig config, Formatter errlog) {
this.manager = new MFileCollectionManager(config, errlog);
this.config = config;
}
// from AggregationFmrc
public Fmrc(CollectionManager manager, FeatureCollectionConfig config) {
this.manager = manager;
this.config = config;
}
public void setNcml(Element outerNcml, Element innerNcml) {
config.protoConfig.outerNcml = outerNcml;
config.innerNcml = innerNcml;
}
public void close() {
manager.close();
}
// exposed for debugging
public CollectionManager getManager() {
return manager;
}
public FmrcInv getFmrcInv(Formatter debug) throws IOException {
return makeFmrcInv(debug);
}
/////////////////////////////////////////////////////////////////////////////////////////
public CalendarDateRange getDateRangeForRun(CalendarDate run) {
return fmrcDataset.getDateRangeForRun(run);
}
public CalendarDateRange getDateRangeForOffset(double offset) {
return fmrcDataset.getDateRangeForOffset(offset);
}
public List<CalendarDate> getRunDates() throws IOException {
checkNeeded(false); // ??
return fmrcDataset.getRunDates();
}
public List<CalendarDate> getForecastDates() throws IOException {
checkNeeded(false); // ??
return fmrcDataset.getForecastDates();
}
// for making offset datasets
public double[] getForecastOffsets() throws IOException {
checkNeeded(false); // ??
return fmrcDataset.getForecastOffsets();
}
// LOOK : all of these guys could use ehcache
public GridDataset getDataset2D(NetcdfDataset result) throws IOException {
checkNeeded(false);
GridDataset gds = fmrcDataset.getNetcdfDataset2D(result);
return gds;
}
public GridDataset getDatasetBest() throws IOException {
checkNeeded(false);
GridDataset gds = fmrcDataset.getBest();
return gds;
}
public GridDataset getDatasetBest(FeatureCollectionConfig.BestDataset bd) throws IOException {
checkNeeded(false);
GridDataset gds = fmrcDataset.getBest(bd);
return gds;
}
public GridDataset getRunTimeDataset(CalendarDate run) throws IOException {
checkNeeded(false);
GridDataset gds = fmrcDataset.getRunTimeDataset(run);
return gds;
}
public GridDataset getConstantForecastDataset(CalendarDate time) throws IOException {
checkNeeded(false);
GridDataset gds = fmrcDataset.getConstantForecastDataset(time);
return gds;
}
public GridDataset getConstantOffsetDataset(double hour) throws IOException {
checkNeeded(false);
GridDataset gds = fmrcDataset.getConstantOffsetDataset(hour);
return gds;
}
/////////////////////////////////////////
public void updateProto() {
forceProto = true;
}
public void update() {
synchronized (lock) {
boolean forceProtoLocal = forceProto;
if (fmrcDataset == null) {
try {
fmrcDataset = new FmrcDataset(config);
} catch (Throwable t) {
logger.error(config.spec + ": initial fmrcDataset creation failed", t);
// throw new RuntimeException(t);
}
}
try {
FmrcInv fmrcInv = makeFmrcInv(null);
fmrcDataset.setInventory(fmrcInv, forceProtoLocal);
logger.debug(config.spec + ": make new Dataset, new proto = {}", forceProtoLocal);
if (forceProtoLocal) forceProto = false;
this.lastInvChanged = System.currentTimeMillis();
if (forceProtoLocal) this.lastProtoChanged = this.lastInvChanged;
} catch (Throwable t) {
logger.error(config.spec + ": makeFmrcInv failed", t);
// throw new RuntimeException(t);
}
}
}
// true if things have changed since given time
public boolean checkInvState(long lastInvChange) throws IOException {
return this.lastInvChanged > lastInvChange;
}
// true if things have changed since given time
public boolean checkProtoState(long lastProtoChanged) throws IOException {
return this.lastProtoChanged > lastProtoChanged;
}
public void checkNeeded(boolean force) {
synchronized (lock) {
if (fmrcDataset == null) {
try {
manager.scan(true);
update();
return;
} catch (Throwable t) {
logger.error(config.spec + ": rescan failed");
throw new RuntimeException(t);
}
}
if (!force && !manager.isScanNeeded()) return;
try {
if (!manager.scan(true)) return;
update();
} catch (Throwable t) {
logger.error(config.spec + ": rescan failed");
throw new RuntimeException(t);
}
}
}
// scan has been done, create FmrcInv
private FmrcInv makeFmrcInv(Formatter debug) throws IOException {
try {
Map<CalendarDate, FmrInv> fmrMap =
new HashMap<CalendarDate, FmrInv>(); // all files are grouped by run date in an FmrInv
List<FmrInv> fmrList = new ArrayList<FmrInv>(); // an fmrc is a collection of fmr
// get the inventory, sorted by path
for (MFile f : manager.getFiles()) {
if (logger.isDebugEnabled()) logger.debug("Fmrc: " + config.spec + ": file=" + f.getPath());
GridDatasetInv inv = null;
try {
inv =
GridDatasetInv.open(
manager, f, config.innerNcml); // inventory is discovered for each GDS
} catch (IOException ioe) {
logger.warn("Error opening " + f.getPath() + "(skipped)", ioe);
continue; // skip
}
CalendarDate runDate = inv.getRunDate();
if (debug != null)
debug.format(" opened %s rundate = %s%n", f.getPath(), inv.getRunDateString());
// add to fmr for that rundate
FmrInv fmr = fmrMap.get(runDate);
if (fmr == null) {
fmr = new FmrInv(runDate);
fmrMap.put(runDate, fmr);
fmrList.add(fmr);
}
fmr.addDataset(inv, debug);
}
if (debug != null) debug.format("%n");
// finish the FmrInv
Collections.sort(fmrList);
for (FmrInv fmr : fmrList) {
fmr.finish();
if (logger.isDebugEnabled())
logger.debug(
"Fmrc: spec="
+ config.spec
+ ": fmr rundate="
+ fmr.getRunDate()
+ " nfiles= "
+ fmr.getFiles().size());
}
return new FmrcInv(
"fmrc:" + manager.getCollectionName(), fmrList, config.fmrcConfig.regularize);
} catch (Throwable t) {
logger.error("makeFmrcInv", t);
throw new RuntimeException(t);
}
}
public void showDetails(Formatter out) throws IOException {
checkNeeded(false);
fmrcDataset.showDetails(out);
}
public static void main(String[] args) throws IOException {
Formatter errlog = new Formatter();
String spec1 = "/data/testdata/ncml/nc/nam_c20s/NAM_CONUS_20km_surface_#yyyyMMdd_HHmm#.grib1";
String spec2 = "/data/testdata/grid/grib/grib1/data/agg/.*grb";
String spec3 = "/data/testdata/ncml/nc/ruc_conus40/RUC_CONUS_40km_#yyyyMMdd_HHmm#.grib1";
String spec4 = "/data/testdata/cdmUnitTest/rtmodels/.*_nmm\\.GrbF[0-9]{5}$";
String cat1 =
"catalog:http://motherlode.ucar.edu:8080/thredds/catalog/fmrc/NCEP/RUC2/CONUS_40km/files/catalog.xml";
String cat2 =
"catalog:http://motherlode.ucar.edu:8080/thredds/catalog/fmrc/NCEP/NDFD/CONUS_5km/files/catalog.xml";
String specH = "C:/data/datasets/nogaps/US058GMET-GR1mdl.*air_temp";
String specH2 = "C:/data/ft/grid/cg/.*nc$";
String specH3 = "C:/data/ft/grid/namExtract/#yyyyMMdd_HHmm#.*nc$";
Fmrc fmrc = new Fmrc(specH3, errlog);
System.out.printf("errlog = %s%n", errlog);
}
}
