private void setVariables(Field.BeanTableField beanTableField) { List<ThreddsMetadata.Variables> variableLists = dataset.getLocalMetadata().getVariables(); if ((variableLists != null) && (variableLists.size() > 0)) { ThreddsMetadata.Variables vars = variableLists.get(0); beanTableField.setValue(vars.getVariableList()); setMode(beanTableField, 0); return; } variableLists = dataset.getLocalMetadataInheritable().getVariables(); if ((variableLists != null) && (variableLists.size() > 0)) { ThreddsMetadata.Variables vars = variableLists.get(0); beanTableField.setValue(vars.getVariableList()); setMode(beanTableField, 1); return; } variableLists = dataset.getVariables(); if ((variableLists != null) && (variableLists.size() > 0)) { ThreddsMetadata.Variables vars = variableLists.get(0); beanTableField.setValue(vars.getVariableList()); setMode(beanTableField, vars.getVariableList().size() == 0 ? 1 : 2); return; } // clear out the table beanTableField.setValue(new ArrayList()); }
@Test public void testDatasetPathMatchSuccess() { String matchPattern = "prod/sref.([0-9]{4})([0-9]{2})([0-9]{2})/([0-9]{2})/pgrb_biasc/sref_([^.]*)\\.t\\4z\\.pgrb([0-9]{3})\\.(.*)\\.grib2$"; String substitutionPattern = "$1-$2-$3T$4"; // String substitutionPattern = "$1-$2-$3T$4 Grid $6 member $5-$7"; String duration = "96 hours"; String dsName = "/data/nccf/com/sref/prod/sref.20090603/03/pgrb_biasc/sref_eta.t03z.pgrb212.n2.grib2"; RegExpAndDurationTimeCoverageEnhancer timeCoverageEnhancer = RegExpAndDurationTimeCoverageEnhancer.getInstanceToMatchOnDatasetPath( matchPattern, substitutionPattern, duration); assertTrue(timeCoverageEnhancer != null); InvDatasetImpl ds = new InvDatasetImpl(null, dsName); CrawlableDataset crDs = new MockCrawlableDataset(dsName, false); assertTrue("Failed to add metadata.", timeCoverageEnhancer.addMetadata(ds, crDs)); // This dataset hasn't been finalized so ds.getTimeCoverage() doesn't work. String startDateString = ds.getLocalMetadata().getTimeCoverage().getStart().getText(); String expectedStartDateString = "2009-06-03T03"; assertTrue( "TimeCoverage start date [" + startDateString + "] not as expected [" + expectedStartDateString + "].", startDateString.equals(expectedStartDateString)); }
private NetcdfDataset openDataset( InvDataset invDataset, boolean acquire, ucar.nc2.util.CancelTask task, Result result) throws IOException { IOException saveException = null; List<InvAccess> accessList = new ArrayList<InvAccess>(invDataset.getAccess()); // a list of all the accesses while (accessList.size() > 0) { InvAccess access = chooseDatasetAccess(accessList); // no valid access if (access == null) { result.errLog.format("No access that could be used in dataset %s %n", invDataset); if (saveException != null) throw saveException; return null; } String datasetLocation = access.getStandardUrlName(); ServiceType serviceType = access.getService().getServiceType(); if (debugOpen) System.out.println("ThreddsDataset.openDataset try " + datasetLocation + " " + serviceType); // deal with RESOLVER type if (serviceType == ServiceType.RESOLVER) { InvDatasetImpl rds = openResolver(datasetLocation, task, result); if (rds == null) return null; accessList = new ArrayList<InvAccess>(rds.getAccess()); continue; } // ready to open it through netcdf API NetcdfDataset ds; // try to open try { ds = openDataset(access, acquire, task, result); } catch (IOException e) { result.errLog.format("Cant open %s %n err=%s%n", datasetLocation, e.getMessage()); if (debugOpen) { System.out.println("Cant open= " + datasetLocation + " " + serviceType); e.printStackTrace(); } accessList.remove(access); saveException = e; continue; } result.accessUsed = access; return ds; } // loop over accesses if (saveException != null) throw saveException; return null; }
private InvDataset createDataset( InvCatalog catalog, String datasetName, final String serviceName) { final InvDatasetImpl dapDataset = new InvDatasetImpl( null, datasetName, FeatureType.NONE, serviceName, "http://wherever.you.want.bc"); dapDataset.setCatalog(catalog); final InvService dapService = new InvService(serviceName, serviceName, "irrelevant", "irrelevant", "irrelevant"); dapDataset.addAccess(new InvAccessImpl(dapDataset, "http://y.z", dapService)); dapDataset.finish(); return dapDataset; }
private void level2level3catalog( RadarType radarType, String pathInfo, PrintWriter pw, HttpServletRequest req, HttpServletResponse res) throws IOException { try { String type; if (pathInfo.contains("level2")) type = radarType.toString() + "/level2"; else type = radarType.toString() + "/level3"; ByteArrayOutputStream os = new ByteArrayOutputStream(10000); InvCatalogFactory factory = InvCatalogFactory.getDefaultFactory(false); factory.writeXML(cat, os, true); InvCatalogImpl tCat = factory.readXML(new ByteArrayInputStream(os.toByteArray()), catURI); Iterator parents = tCat.getDatasets().iterator(); while (parents.hasNext()) { ArrayList<InvDatasetImpl> delete = new ArrayList<InvDatasetImpl>(); InvDatasetImpl top = (InvDatasetImpl) parents.next(); Iterator tDatasets = top.getDatasets().iterator(); while (tDatasets.hasNext()) { InvDatasetImpl ds = (InvDatasetImpl) tDatasets.next(); if (ds instanceof InvDatasetScan) { InvDatasetScan ids = (InvDatasetScan) ds; if (ids.getPath() == null) continue; if (ids.getPath().contains(type)) { ids.setXlinkHref(ids.getPath() + "/dataset.xml"); } else { delete.add(ds); } } } // remove datasets for (InvDatasetImpl idi : delete) { top.removeDataset(idi); } } if (pathInfo.endsWith("xml")) { String catAsString = factory.writeXML(tCat); pw.println(catAsString); pw.flush(); } else { HtmlWriter.getInstance().writeCatalog(req, res, tCat, true); // show catalog as HTML } } catch (Throwable e) { log.error("RadarServer.level2level3catalog failed", e); if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } return; }
private void extractVariables() { if (leafDataset == null) leafDataset = findLeafDataset(dataset); if (leafDataset == null) return; ThreddsMetadata.Variables vars; try { vars = MetadataExtractor.extractVariables(leafDataset); } catch (IOException e) { return; } if (vars != null) { ThreddsMetadata tm = dataset.getLocalMetadataInheritable(); List<ThreddsMetadata.Variables> varsList = tm.getVariables(); boolean replaced = false; for (int i = 0; i < varsList.size(); i++) { ThreddsMetadata.Variables vs = varsList.get(i); if (vs.getVocabulary().equals(vars.getVocabulary())) { varsList.set(i, vars); // replace replaced = true; break; } } if (!replaced) tm.addVariables(vars); variablesFld.setValue(vars.getVariableList()); setMode(variablesFld, 1); } }
public static ThreddsMetadata.Variables extractVariables( InvDatasetImpl threddsDataset, GridDataset gridDataset) { thredds.catalog.DataFormatType fileFormat = threddsDataset.getDataFormatType(); if ((fileFormat != null) && (fileFormat.equals(DataFormatType.GRIB1) || fileFormat.equals(DataFormatType.GRIB2))) { boolean isGrib1 = fileFormat.equals(DataFormatType.GRIB1); ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables(fileFormat.toString()); for (GridDatatype grid : gridDataset.getGrids()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); v.setName(grid.getName()); v.setDescription(grid.getDescription()); v.setUnits(grid.getUnitsString()); // ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("GRIB_param_number"); // String paramNumber = (att != null) ? att.getNumericValue().toString() : null; if (isGrib1) { v.setVocabularyName(grid.findAttValueIgnoreCase("GRIB_param_name", "ERROR")); v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id")); } else { String paramDisc = grid.findAttValueIgnoreCase("GRIB_param_discipline", ""); String paramCategory = grid.findAttValueIgnoreCase("GRIB_param_category", ""); String paramName = grid.findAttValueIgnoreCase("GRIB_param_name", ""); v.setVocabularyName(paramDisc + " / " + paramCategory + " / " + paramName); v.setVocabularyId(grid.findAttributeIgnoreCase("GRIB_param_id")); } vars.addVariable(v); } vars.sort(); return vars; } else { // GRID but not GRIB ThreddsMetadata.Variables vars = new ThreddsMetadata.Variables("CF-1.0"); for (GridDatatype grid : gridDataset.getGrids()) { ThreddsMetadata.Variable v = new ThreddsMetadata.Variable(); vars.addVariable(v); v.setName(grid.getName()); v.setDescription(grid.getDescription()); v.setUnits(grid.getUnitsString()); ucar.nc2.Attribute att = grid.findAttributeIgnoreCase("standard_name"); v.setVocabularyName((att != null) ? att.getStringValue() : "N/A"); } vars.sort(); return vars; } }
@Override public String getViewerLinkHtml(InvDatasetImpl ds, HttpServletRequest req) { InvAccess access = ds.getAccess(ServiceType.WMS); URI dataURI = access.getStandardUri(); try { URI base = new URI(req.getRequestURL().toString()); dataURI = base.resolve(dataURI); } catch (URISyntaxException e) { return "Error generating viewer link"; } // ToDo Switch to use TdsContext.getContextPath() return "<a href='" + ServletUtil.getContextPath() + "/godiva2/godiva2.html?server=" + dataURI.toString() + "'>Godiva2 (browser-based)</a>"; }
/** * Open an ADDE Station Dataset from an InvAccess, which must be type ADDE and Station. * * @param access open Invdataset from this access. * @throws IOException */ public AddeStationObsDataset(InvAccess access, ucar.nc2.util.CancelTask cancelTask) throws IOException { super(); InvDataset invDs = access.getDataset(); this.location = (invDs.getID() != null) ? "thredds:" + access.getDataset().getCatalogUrl() : access.getStandardUrlName(); addeURL = access.getStandardUrlName(); // see if we have a stationDB file InvDataset invds = access.getDataset(); String pv = invds.findProperty("_StationDBlocation"); if (pv != null) { stationDBlocation = InvDatasetImpl.resolve(invds, pv); } init(); // Get the bounding box if possible ThreddsMetadata.GeospatialCoverage geoCoverage = invds.getGeospatialCoverage(); if (null != geoCoverage) boundingBox = geoCoverage.getBoundingBox(); else // otherwise, stationHelper constructs from the station locations boundingBox = stationHelper.getBoundingBox(); // get the date range if possible DateRange timeCoverage = invds.getTimeCoverage(); if (timeCoverage != null) { startDate = timeCoverage.getStart().getDate(); endDate = timeCoverage.getEnd().getDate(); } else { startDate = new Date(0); // fake endDate = new Date(); } /* // LOOK maybe its already annotated ?? LOOK set title, description ThreddsDataFactory.annotate( access.getDataset(), this); finish(); */ }
private InvDatasetImpl getLeafDataset(InvDatasetImpl ds) { if (ds == null) return null; System.out.println("getLeafDataset " + ds.getFullName()); List nestedList = ds.getDatasets(); for (int i = 0; i < nestedList.size(); i++) { InvDatasetImpl nestedDS = (InvDatasetImpl) nestedList.get(i); if (nestedDS.hasAccess() && !nestedDS.hasNestedDatasets()) { InvDatasetImpl result = chooseLeafDataset(nestedList); // this assumes that they are uniform !! return (result.hasAccess()) ? result : nestedDS; } // depth first InvDatasetImpl leaf = getLeafDataset(nestedDS); if (leaf != null) return leaf; } return null; }
private boolean nameDatasetRegExp(InvDatasetImpl dataset) { boolean isMatch; Matcher matcher; // Test for a match on the urlPath if (dataset.getUrlPath() != null) { logger.debug("nameDatasetRegExp(): try naming on urlPath <{}>", dataset.getUrlPath()); matcher = this.regExpPattern.matcher(dataset.getUrlPath()); isMatch = matcher.find(); } else { matcher = this.regExpPattern.matcher(dataset.getName()); isMatch = matcher.find(); } if (isMatch) { // Test for substitution. StringBuffer resultingName = new StringBuffer(); matcher.appendReplacement(resultingName, this.substitutePattern); resultingName.delete(0, matcher.start()); if (resultingName.length() != 0) { logger.debug("nameDatasetRegExp(): Setting name to \"" + resultingName + "\"."); dataset.setName(resultingName.toString()); return true; } else { logger.debug("nameDatasetRegExp(): No name for regEx substitution."); return false; } } if (logger.isDebugEnabled()) logger.debug( "nameDatasetRegExp(): Neither URL <" + dataset.getUrlPath() + "> or name <" + dataset.getName() + "> matched pattern <" + this.matchPattern + "> ."); return false; }
public InvCatalog getDirCatalog( CrawlableDataset catalogCrDs, String filterPattern, boolean sortInIncreasingOrder, String addIdBase, boolean addDatasetSize, String dsNameMatchPattern, String startTimeSubstitutionPattern, String duration) { // Setup the filter CrawlableDatasetFilter filter = null; if (filterPattern != null) { // Include atomic datasets that match the given filter string. MultiSelectorFilter.Selector selector = new MultiSelectorFilter.Selector( new RegExpMatchOnNameFilter(filterPattern), true, true, false); filter = new MultiSelectorFilter(selector); } else { filter = new RegExpMatchOnNameFilter(".*"); } List enhancerList = null; if (dsNameMatchPattern != null && startTimeSubstitutionPattern != null && duration != null) { enhancerList = new ArrayList(); enhancerList.add( RegExpAndDurationTimeCoverageEnhancer.getInstanceToMatchOnDatasetName( dsNameMatchPattern, startTimeSubstitutionPattern, duration)); } CatalogBuilder catBuilder = new StandardCatalogBuilder( prefixPath, null, collectionCrDs, filter, service, addIdBase, null, null, addDatasetSize, new LexigraphicByNameSorter(sortInIncreasingOrder), null, enhancerList, null, new BooleanCatalogRefExpander(!this.createCatalogRefs)); InvCatalog catalog; try { catalog = catBuilder.generateCatalog(catalogCrDs); } catch (IOException e) { throw new IllegalArgumentException("Could not generate catalog: " + e.getMessage()); } // log.debug( "getDirCatalog(): directory=" + directory + "; filterPattern=" + filterPattern // + "; sortInIncreasingOrder=" + sortInIncreasingOrder + "; addIdBase="+addIdBase+"; // dsNameMatchPattern=" + dsNameMatchPattern + "; startTimeSubstitutionPattern=" + // startTimeSubstitutionPattern + "; duration=" + duration ); // if ( !directory.isDirectory() ) // { // String tmpMsg = "Given directory is not a directory <" + directory.getAbsolutePath() + // ">."; // log.warn( tmpMsg); // throw new IllegalArgumentException( tmpMsg ); // } // DatasetSource dsSource = DatasetSource.newDatasetSource( directory.getName(), // DatasetSourceType.LOCAL, // DatasetSourceStructure.DIRECTORY_TREE, // directory.getAbsolutePath(), // this.service ); // dsSource.setPrefixUrlPath( this.prefixPath ); // dsSource.setCreateCatalogRefs( this.createCatalogRefs ); // dsSource.setAddDatasetSize( addDatasetSize ); // if ( filterPattern != null ) // { // DatasetFilter datasetFilter = new DatasetFilter( dsSource, "Filter files on \"" + // filterPattern + "\"", // DatasetFilter.Type.REGULAR_EXPRESSION, // filterPattern ); // datasetFilter.setMatchPatternTarget( "name"); // dsSource.addDatasetFilter( datasetFilter ); // datasetFilter = new DatasetFilter( dsSource, "Allow all dirs", // DatasetFilter.Type.REGULAR_EXPRESSION, // "", true, false, false ); // datasetFilter.setMatchPatternTarget( "name" ); // dsSource.addDatasetFilter( datasetFilter ); // } // dsSource.setDatasetSorter( new DatasetSorter( sortInIncreasingOrder)); // if ( dsNameMatchPattern != null // && startTimeSubstitutionPattern != null // && duration != null) // { // dsSource.addDatasetEnhancer( DatasetEnhancer1.createAddTimeCoverageEnhancer( // dsNameMatchPattern, startTimeSubstitutionPattern, duration)); // } // // if ( addIdBase != null) // { // dsSource.addDatasetEnhancer( DatasetEnhancer1.createAddIdEnhancer( addIdBase)); // } // // InvCatalog cat = null; // try // { // cat = dsSource.fullExpand(); // } // catch ( IOException e ) // { // throw new IllegalArgumentException( "Given directory is not a collection dataset <" + // directory.getAbsolutePath() + ">: " + e.getMessage() ); // } InvDataset topDs = catalog.getDataset(); if (collectionCrDs.getPath().equals(catalogCrDs.getPath()) && this.serviceTitle != null) // if ( topDs.getName().equals( "" ) && this.serviceTitle != null ) { logger.warn( "getDirCatalog(): top dataset name is null, setting to serviceTitle <" + this.serviceTitle + ">"); ((InvDatasetImpl) topDs).setName(this.serviceTitle); } return (catalog); }
/** Returns true if this is a gridded dataset that is accessible via WMS. */ @Override public boolean isViewable(InvDatasetImpl ds) { InvAccess access = ds.getAccess(ServiceType.WMS); return (access != null) && (ds.getDataType() == FeatureType.GRID); }
public void store2Dataset() { PersistentBean persBean = new PersistentBean(dataset); // reset the BeanMaps setStoreValue(NAME, persBean, false); setStoreValue(ID, persBean, false); setStoreValue(AUTHORITY, persBean, true); setStoreValue(SERVICE_NAME, persBean, true); setStoreValue(FORMAT_TYPE, persBean, true); setStoreValue(DATA_TYPE, persBean, true); setStoreValue(COLLECTION_TYPE, persBean, false); setStoreValue(HARVEST, persBean, false); String gcType = (String) metadataPP.getFieldValue(GC_TYPE); if (!gcType.equals(INHERITED)) { ThreddsMetadata.GeospatialCoverage gc = new ThreddsMetadata.GeospatialCoverage(); storeGC(new PersistentBean(gc)); if (gcType.equals(LOCAL)) { dataset.getLocalMetadata().setGeospatialCoverage(gc); dataset.getLocalMetadataInheritable().setGeospatialCoverage(null); } else { dataset.getLocalMetadata().setGeospatialCoverage(null); dataset.getLocalMetadataInheritable().setGeospatialCoverage(gc); } } String tcType = (String) metadataPP.getFieldValue(TC_TYPE); if (!tcType.equals(INHERITED)) { DateRange dateRange = dateRangeSelector.getDateRange(); if (tcType.equals(LOCAL)) { dataset.getLocalMetadata().setTimeCoverage(dateRange); dataset.getLocalMetadataInheritable().setTimeCoverage((DateRange) null); } else { dataset.getLocalMetadata().setTimeCoverage((DateRange) null); dataset.getLocalMetadataInheritable().setTimeCoverage(dateRange); } } setStoreValue(SUMMARY, persBean, true); setStoreValue(RIGHTS, persBean, true); setStoreValue(HISTORY, persBean, true); setStoreValue(PROCESSING, persBean, true); // storeBeanList( variablesFld, VARIABLES, persBean); storeBeanList(creatorsFld, CREATORS, persBean); storeBeanList(publishersFld, PUBLISHERS, persBean); storeBeanList(projectsFld, PROJECTS, persBean); storeBeanList(keywordsFld, KEYWORDS, persBean); storeBeanList(datesFld, DATES, persBean); storeBeanList(contributorsFld, CONTRIBUTORS, persBean); storeBeanList(docsFld, DOCUMENTATION, persBean); if (dataset instanceof InvDatasetScan) { setStoreValue(dscanPP, DSCAN_PATH, persBean, false); setStoreValue(dscanPP, DSCAN_DIR, persBean, false); // setStoreValue( dscanPP, DSCAN_FILTER, persBean, false); setStoreValue(dscanPP, DSCAN_ADDSIZE, persBean, false); // setStoreValue( dscanPP, DSCAN_ADDLATEST, persBean, false); // setStoreValue( dscanPP, DSCAN_TC_MATCH, persBean, false); // setStoreValue( dscanPP, DSCAN_TC_SUBS, persBean, false); // setStoreValue( dscanPP, DSCAN_TC_DURATOPN, persBean, false); } dataset.finish(); }
public boolean setDataset(InvDatasetImpl ds) { if (!accept()) return false; this.dataset = ds; this.leafDataset = null; exampleButton.setText("Example Dataset"); ////////// PersistentBean persBean = new PersistentBean(ds); setEditValue(NAME, persBean, 0); setEditValue(ID, persBean, 0); setEditValueWithInheritence(AUTHORITY, persBean); setEditValueWithInheritence(SERVICE_NAME, persBean); setEditValueWithInheritence(FORMAT_TYPE, persBean); setEditValueWithInheritence(DATA_TYPE, persBean); setEditValue(COLLECTION_TYPE, persBean, 0); setEditValue(HARVEST, persBean, 0); // gotta find which GeospatialCoverage to use. int mode = 0; ThreddsMetadata.GeospatialCoverage gc = ds.getLocalMetadata().getGeospatialCoverage(); if ((gc == null) || gc.isEmpty()) { gc = ds.getLocalMetadataInheritable().getGeospatialCoverage(); mode = 1; } if ((gc == null) || gc.isEmpty()) { gc = ds.getGeospatialCoverage(); mode = 2; // inherited } metadataPP.setFieldValue(GC_TYPE, inherit_types.get(mode)); setGC(gc, mode); // gotta find which TimeCoverage to use. mode = 0; CalendarDateRange tc = ds.getLocalMetadata().getCalendarDateCoverage(); if (tc == null) { tc = ds.getLocalMetadataInheritable().getCalendarDateCoverage(); mode = 1; } if (tc == null) { tc = ds.getCalendarDateCoverage(); mode = 2; // inherited } metadataPP.setFieldValue(TC_TYPE, inherit_types.get(mode)); if (tc != null) dateRangeSelector.setDateRange(tc.toDateRange()); setTCmode(mode); setEditValueWithInheritence(SUMMARY, persBean); setEditValueWithInheritence(RIGHTS, persBean); setEditValueWithInheritence(HISTORY, persBean); setEditValueWithInheritence(PROCESSING, persBean); setVariables(variablesFld); setBeanList(creatorsFld, CREATORS, persBean); setBeanList(publishersFld, PUBLISHERS, persBean); setBeanList(projectsFld, PROJECTS, persBean); setBeanList(keywordsFld, KEYWORDS, persBean); setBeanList(datesFld, DATES, persBean); setBeanList(contributorsFld, CONTRIBUTORS, persBean); setBeanList(docsFld, DOCUMENTATION, persBean); if (ds instanceof InvDatasetScan) { dscanPP.setEnabled(true); setEditValue(dscanPP, DSCAN_PATH, persBean, 0); setEditValue(dscanPP, DSCAN_DIR, persBean, 0); // setEditValue( dscanPP, DSCAN_FILTER, persBean, 0); setEditValue(dscanPP, DSCAN_ADDSIZE, persBean, 0); // setEditValue( dscanPP, DSCAN_ADDLATEST, persBean, 0); // setEditValue( dscanPP, DSCAN_TC_MATCH, persBean, 0); // setEditValue( dscanPP, DSCAN_TC_SUBS, persBean, 0); // setEditValue( dscanPP, DSCAN_TC_DURATOPN, persBean, 0); } else dscanPP.setEnabled(false); return true; }