Exemplo n.º 1
0
  public void init() throws javax.servlet.ServletException {
    super.init();

    org.slf4j.Logger logServerStartup = org.slf4j.LoggerFactory.getLogger("serverStartup");
    logServerStartup.info(
        getClass().getName() + " initialization start - " + UsageLog.setupNonRequestContext());

    this.ascLimit = ThreddsConfig.getInt("Opendap.ascLimit", ascLimit);
    this.binLimit = ThreddsConfig.getInt("Opendap.binLimit", binLimit);

    this.odapVersionString = ThreddsConfig.get("Opendap.serverVersion", odapVersionString);
    logServerStartup.info(
        getClass().getName()
            + " version= "
            + odapVersionString
            + " ascLimit = "
            + ascLimit
            + " binLimit = "
            + binLimit);

    // debugging actions
    makeDebugActions();

    logServerStartup.info(
        getClass().getName()
            + " initialization done - "
            + UsageLog.closingMessageNonRequestContext());
  }
Exemplo n.º 2
0
/** by Mr Skip on 09.02.2016. */
@Service
public class BusInfoService implements BusInfoLogic {
  private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BusInfoService.class);

  @Autowired private BusInfoRepo busInfoRepo;

  @Override
  public BusInfo getByDriverName(String name) {
    BusInfo busInfo;
    if ((busInfo = busInfoRepo.getBusByDriverName(name)) == null) {
      LOGGER.warn(
          "Not find any Bus for driver name:" + name + ". In class: " + getClass().getName());
      busInfo = new BusInfo();
    }
    return busInfo;
  }

  @Override
  public BusInfo getByDriverPhone(String phone) {
    BusInfo busInfo;
    if ((busInfo = busInfoRepo.getBusByDriverPhone(phone)) == null) {
      LOGGER.warn(
          "Not find any Bus for driver phone:" + phone + ". In class: " + getClass().getName());
      busInfo = new BusInfo();
    }
    return busInfo;
  }

  @Override
  public List<BusInfo> getAll() {
    List<BusInfo> list;
    if ((list = busInfoRepo.findAll()).isEmpty()) {
      LOGGER.warn("Cant find any Bus in class: " + getClass().getName());
    }
    return list;
  }

  @Override
  public Map<Integer, String> getAllBusNumber() {
    Map<Integer, String> list = new HashMap<>();
    for (String s : busInfoRepo.getAllBusNumber()) {
      list.put(list.size() + 1, s);
    }
    return list;
  }
}
Exemplo n.º 3
0
/** @author pepijn */
public class WPPluginManager {
  private WPPluginManager(UUID uuid, String logPrefix, ClassLoader classLoader) {
    allPlugins = org.pepsoft.util.PluginManager.findPlugins(Plugin.class, FILENAME, classLoader);
    Set<String> namesEncountered = new HashSet<>();
    for (Iterator<Plugin> i = allPlugins.iterator(); i.hasNext(); ) {
      Plugin plugin = i.next();
      if ((plugin.getUUIDs() != null) && (uuid != null) && (!plugin.getUUIDs().contains(uuid))) {
        logger.error(
            logPrefix
                + plugin.getName()
                + " plugin is not authorised for this installation; not loading it");
        i.remove();
        continue;
      }
      String name = plugin.getName();
      if (namesEncountered.contains(name)) {
        throw new RuntimeException("Multiple plugins with the same name (" + name + ") detected!");
      } else {
        namesEncountered.add(name);
      }
      logger.info(logPrefix + "Loaded plugin: " + name + " (version " + plugin.getVersion() + ")");
    }
  }

  public List<Plugin> getAllPlugins() {
    return Collections.unmodifiableList(allPlugins);
  }

  @SuppressWarnings("unchecked") // Guaranteed by Java
  public <T extends Plugin> List<T> getPlugins(Class<T> type) {
    List<T> plugins = new ArrayList<>(allPlugins.size());
    plugins.addAll(
        allPlugins
            .stream()
            .filter(plugin -> type.isAssignableFrom(plugin.getClass()))
            .map(plugin -> (T) plugin)
            .collect(Collectors.toList()));
    return plugins;
  }

  public static synchronized void initialise(UUID uuid) {
    initialise(uuid, "", ClassLoader.getSystemClassLoader());
  }

  public static synchronized void initialise(UUID uuid, String logPrefix, ClassLoader classLoader) {
    if (instance != null) {
      throw new IllegalStateException("Already initialised");
    }
    instance = new WPPluginManager(uuid, logPrefix, classLoader);
  }

  public static synchronized WPPluginManager getInstance() {
    return instance;
  }

  private final List<Plugin> allPlugins;

  private static WPPluginManager instance;
  private static final String FILENAME = "org.pepsoft.worldpainter.plugins";
  private static final org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(WPPluginManager.class);
}
Exemplo n.º 4
0
public class HTMLDriver extends PanelDriver implements Exportable {
  private static transient org.slf4j.Logger log =
      org.slf4j.LoggerFactory.getLogger(HTMLDriver.class.getName());

  public static final String PARAMETER_HTML = "html_code";
  public static final String PARAMETER_EDITING_LANG = "edit_lang";

  private static final String PAGE_SHOW = "show";
  private static final String PAGE_EDIT = "edit";

  private static final String PAGE_CHOOSE_IMAGE = "imageSelect";
  private static final String PAGE_CHOOSE_LINK = "linkSelect";

  public static final String PARAMETER_USE_DEFAULTS = "useDefaultLanguage";

  private static final String ATTR_TEXT = "text";
  private static final String ATTR_EDITING_LANGUAGE = "lang";

  /** The locale manager. */
  protected LocaleManager localeManager;

  public HTMLDriver() {
    localeManager = LocaleManager.lookup();
  }

  public void init(PanelProvider provider) throws Exception {
    super.init(provider);
    addParameter(new BooleanParameter(provider, PARAMETER_USE_DEFAULTS, false, true));
    String[] methodsForEditMode = new String[] {"actionChangeEditingLanguage", "actionSaveChanges"};
    for (int i = 0; i < methodsForEditMode.length; i++) {
      String method = methodsForEditMode[i];
      addMethodPermission(method, PanelPermission.class, PanelPermission.ACTION_EDIT);
    }
  }

  public void initPanelSession(PanelSession status, HttpSession session) {
    status.setCurrentPageId(PAGE_SHOW);
  }

  protected void beforePanelInstanceRemove(final PanelInstance instance) throws Exception {
    super.beforePanelInstanceRemove(instance);

    new HibernateTxFragment() {
      protected void txFragment(Session session) throws Exception {
        HTMLText htmlText = load(instance);
        if (htmlText != null) htmlText.delete();
      }
    }.execute();
  }

  /** Returns if this driver defines support to activate edit mode. */
  public boolean supportsEditMode(Panel panel) {
    return true;
  }

  public int getEditWidth(Panel panel, CommandRequest request) {
    return 1000;
  }

  public int getEditHeight(Panel panel, CommandRequest request) {
    return 695;
  }

  /** Defines the action to be taken when activating edit mode */
  public void activateEditMode(Panel panel, CommandRequest request) throws Exception {
    super.activateEditMode(panel, request);
    HTMLText text = load(panel.getInstance());
    SessionManager.getPanelSession(panel).setAttribute(ATTR_TEXT, toEditableObject(text));
  }

  protected Map toEditableObject(HTMLText text) {
    Map m = new HashMap();
    for (Iterator it = text.getText().keySet().iterator(); it.hasNext(); ) {
      String lang = (String) it.next();
      String val = text.getText(lang);
      m.put(lang, val);
    }
    return m;
  }

  /** Defines the action to be taken when activating edit mode */
  public void activateNormalMode(Panel panel, CommandRequest request) throws Exception {
    super.activateNormalMode(panel, request);
    SessionManager.getPanelSession(panel).removeAttribute(ATTR_TEXT);
    SessionManager.getPanelSession(panel).removeAttribute(ATTR_EDITING_LANGUAGE);
  }

  /**
   * Returns if this driver is using default language
   *
   * @param panel
   * @return if this driver is using default language
   */
  public boolean isUsingDefaultLanguage(Panel panel) {
    return Boolean.valueOf(panel.getParameterValue(PARAMETER_USE_DEFAULTS)).booleanValue();
  }

  /**
   * Returns if this driver is using default language
   *
   * @param panel
   * @return if this driver is using default language
   */
  public boolean isUsingDefaultLanguage(PanelInstance panel) {
    return Boolean.valueOf(panel.getParameterValue(PARAMETER_USE_DEFAULTS)).booleanValue();
  }

  /**
   * Determine the text being shown for given panel.
   *
   * @param panel
   * @return The text shown, i18n.
   */
  public Map getHtmlCode(Panel panel) {
    PanelSession pSession = SessionManager.getPanelSession(panel);
    Map m = (Map) pSession.getAttribute(ATTR_TEXT);
    if (m != null) return m;
    HTMLText text = load(panel.getInstance());
    if (text != null) return text.getText();
    try {
      HTMLText textToCreate = new HTMLText();
      textToCreate.setPanelInstance(panel.getInstance());
      Locale[] locales = LocaleManager.lookup().getPlatformAvailableLocales();
      for (int i = 0; i < locales.length; i++) {
        Locale locale = locales[i];
        ResourceBundle i18n =
            localeManager.getBundle("org.jboss.dashboard.ui.panel.advancedHTML.messages", locale);
        textToCreate.setText(locale.getLanguage(), i18n.getString("defaultContent"));
      }
      textToCreate.save();
    } catch (Exception e) {
      log.error("Error creating empty text for panel: ", e);
    }
    text = load(panel.getInstance());
    if (text != null) return text.getText();
    log.error("Current HTML code is null for panel " + panel);
    return null;
  }

  /**
   * Determine the editing language.
   *
   * @return The text shown, i18n.
   */
  public String getEditingLanguage(Panel panel) {
    String lang =
        (String) SessionManager.getPanelSession(panel).getAttribute(ATTR_EDITING_LANGUAGE);
    return lang == null ? LocaleManager.lookup().getDefaultLang() : lang;
  }

  public CommandResponse actionChangeEditingLanguage(Panel panel, CommandRequest request)
      throws Exception {
    String currentText = request.getRequestObject().getParameter(PARAMETER_HTML);
    Map text = (Map) SessionManager.getPanelSession(panel).getAttribute(ATTR_TEXT);
    if (text == null) {
      text = toEditableObject(load(panel.getInstance()));
      SessionManager.getPanelSession(panel).setAttribute(ATTR_TEXT, text);
    }
    text.put(getEditingLanguage(panel), currentText);
    SessionManager.getPanelSession(panel).setAttribute(ATTR_TEXT, text);
    String paramLang = request.getRequestObject().getParameter(PARAMETER_EDITING_LANG);
    SessionManager.getPanelSession(panel).setAttribute(ATTR_EDITING_LANGUAGE, paramLang);
    return new ShowPanelPage();
  }

  public CommandResponse actionSaveChanges(Panel panel, CommandRequest request) throws Exception {
    String currentText = request.getRequestObject().getParameter(PARAMETER_HTML);
    Map m = (Map) SessionManager.getPanelSession(panel).getAttribute(ATTR_TEXT);
    HTMLText text = load(panel.getInstance());
    for (Iterator it = m.keySet().iterator(); it.hasNext(); ) {
      String lang = (String) it.next();
      String val = (String) m.get(lang);
      text.setText(lang, val);
    }
    text.setText(getEditingLanguage(panel), currentText);
    text.save();
    activateNormalMode(panel, request);
    return new ShowPanelPage();
  }

  public CommandResponse actionSelectImage(Panel panel, CommandRequest request) throws Exception {
    return new ShowPopupPanelPage(panel, PAGE_CHOOSE_IMAGE);
  }

  public CommandResponse actionSelectLink(Panel panel, CommandRequest request) throws Exception {
    return new ShowPopupPanelPage(panel, PAGE_CHOOSE_LINK);
  }

  public HTMLText load(final PanelInstance instance) {
    final List results = new ArrayList();
    try {
      new HibernateTxFragment() {
        protected void txFragment(Session session) throws Exception {
          FlushMode oldFlushMode = session.getFlushMode();
          session.setFlushMode(FlushMode.NEVER);
          Query query =
              session.createQuery(
                  " from "
                      + HTMLText.class.getName()
                      + " as text where text.panelInstance = :instance");
          query.setParameter("instance", instance);
          query.setCacheable(true);
          results.addAll(query.list());
          session.setFlushMode(oldFlushMode);
        }
      }.execute();
      HTMLText text = null;
      if (results.size() > 0) text = (HTMLText) results.get(0);
      else log.debug("Does not exist a html_text for HTML panel");
      return text;
    } catch (Exception e) {
      log.error("Can't retrive a data for HTML panel ", e);
      return null;
    }
  }

  /**
   * Replicates panel data.
   *
   * @param src Source PanelInstance
   * @param dest Destinaton PanelInstance
   */
  public void replicateData(final PanelInstance src, PanelInstance dest) throws Exception {
    super.replicateData(src, dest);
    log.debug(
        "HTMLDriver replicating Data from PanelInstance "
            + src.getDbid()
            + " to "
            + dest.getDbid()
            + ").");
    if (src.equals(dest)) {
      log.debug("Ignoring replication, panel instance is the same.");
      return;
    }

    final HTMLText[] textArray = new HTMLText[1];
    try {
      new HibernateTxFragment() {
        protected void txFragment(Session session) throws Exception {
          log.debug("Getting text to duplicate for instance " + src.getDbid());
          FlushMode oldMode = session.getFlushMode();
          session.setFlushMode(
              FlushMode
                  .COMMIT); // Avoids flushing, as we know the text was not modified in this
                            // transaction.
          textArray[0] = load(src);
          session.setFlushMode(oldMode);
          log.debug("Got text to duplicate for instance " + src.getDbid());
        }
      }.execute();
    } catch (Exception e) {
      log.error("Error loading text for instance. ", e);
    }
    HTMLText text = textArray[0];
    if (text == null) {
      log.debug(
          "Nothing to replicate from PanelInstance " + src.getDbid() + " to " + dest.getDbid());
      return;
    }
    Map htmlSrc = text.getText();

    log.debug("htmlCode to replicate = " + htmlSrc);
    HTMLText htmlDest = new HTMLText();
    htmlDest.setPanelInstance(dest.getInstance());
    for (Iterator it = htmlSrc.keySet().iterator(); it.hasNext(); ) {
      String key = (String) it.next();
      String val = (String) htmlSrc.get(key);
      htmlDest.setText(key, val);
    }
    try {
      log.debug("Updating HTMLText: IDText " + htmlDest.getDbid() + " Text " + htmlDest.getText());
      htmlDest.save();
    } catch (Exception e) {
      log.error("Replicating panel data", e);
    }
  }

  /** Write instance content to given OutputStream, which must not be closed. */
  public void exportContent(PanelInstance instance, OutputStream os) throws Exception {
    HTMLText text = load(instance);
    if (text == null) {
      try {
        text = new HTMLText();
        text.setPanelInstance(instance);
        text.save();
      } catch (Exception e) {
        log.error("Error creating empty HTMLText object", e);
      }
    }
    ObjectOutputStream oos = new ObjectOutputStream(os);
    if (log.isDebugEnabled()) log.debug("Exporting content: " + text.getText());
    HashMap h = new HashMap(); // Avoids serializing a hibernate map
    h.putAll(text.getText());
    oos.writeObject(h);
  }

  /** Read instance content from given InputStream, which must not be closed. */
  public void importContent(PanelInstance instance, InputStream is) throws Exception {
    HTMLText currentText = new HTMLText();
    currentText.setPanelInstance(instance);
    ObjectInputStream ois = new ObjectInputStream(is);
    Map text = (Map) ois.readObject();
    if (log.isDebugEnabled()) log.debug("Importing content: " + text);
    for (Iterator it = text.keySet().iterator(); it.hasNext(); ) {
      String lang = (String) it.next();
      String value = (String) text.get(lang);
      currentText.setText(lang, value);
    }
    currentText.save();
  }

  protected String getPanelHTMLContent(PanelInstance instance, String lang) {
    HTMLText text = load(instance);
    if (text != null) {
      String val = text.getText(lang);
      if (StringUtils.isEmpty(val) && isUsingDefaultLanguage(instance)) {
        LocaleManager localeManager = LocaleManager.lookup();
        val = text.getText(localeManager.getDefaultLang());
      }
      return val;
    }
    return null;
  }
}
/**
 * Choice list initializer that looks up direct sub types from the specified type If no param is
 * specified, get the list of nodeTypes
 */
public class SubNodeTypesChoiceListInitializerImpl implements ChoiceListInitializer {

  private static transient Logger logger =
      org.slf4j.LoggerFactory.getLogger(SubNodeTypesChoiceListInitializerImpl.class);

  @SuppressWarnings("unchecked")
  public List<ChoiceListValue> getChoiceListValues(
      ExtendedPropertyDefinition epd,
      String param,
      List<ChoiceListValue> values,
      Locale locale,
      Map<String, Object> context) {
    final SortedSet<ChoiceListValue> listValues = new TreeSet<ChoiceListValue>();
    if (StringUtils.isEmpty(param)) {
      param = "jmix:editorialContent";
    }
    try {
      String includedTypes = StringUtils.substringBefore(param, ";");

      Set<String> excludedTypes = new HashSet<String>();
      String exclusion = StringUtils.substringAfter(param, ";");
      if (StringUtils.isNotBlank(exclusion)) {
        excludedTypes.addAll(
            CollectionUtils.collect(
                Arrays.asList(StringUtils.substringAfter(param, ";").split(",")),
                new Transformer() {
                  public Object transform(Object input) {
                    return ((String) input).trim();
                  }
                }));
      }

      for (String nodeTypeName : includedTypes.split(",")) {
        nodeTypeName = nodeTypeName.trim();
        ExtendedNodeType nodeType = NodeTypeRegistry.getInstance().getNodeType(nodeTypeName);
        if (!isExcludedType(nodeType, excludedTypes)) {
          listValues.add(new ChoiceListValue(nodeType.getLabel(locale), nodeType.getName()));
        }
        NodeTypeIterator nti = nodeType.getSubtypes();
        while (nti.hasNext()) {
          ExtendedNodeType type = (ExtendedNodeType) nti.next();
          if (!isExcludedType(type, excludedTypes)) {
            listValues.add(new ChoiceListValue(type.getLabel(locale), type.getName()));
          }
        }
      }
    } catch (NoSuchNodeTypeException e) {
      logger.error("Cannot get type", e);
    }

    return new LinkedList<ChoiceListValue>(listValues);
  }

  private boolean isExcludedType(ExtendedNodeType nodeType, Set<String> excludedTypes) {
    if (excludedTypes.contains(nodeType.getName())) {
      return true;
    }
    boolean isExcluded = false;
    for (String excludedType : excludedTypes) {
      if (nodeType.isNodeType(excludedType)) {
        isExcluded = true;
        break;
      }
    }
    return isExcluded;
  }
}
Exemplo n.º 6
0
/**
 * _more_
 *
 * @author edavis
 * @since 4.0
 */
public class WcsCoverage {
  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(WcsCoverage.class);

  private GridDataset.Gridset coverage;
  private WcsDataset dataset;

  private String name;
  private String label;
  private String description;

  private GridCoordSystem coordSys;
  private String nativeCRS;

  private String defaultRequestCrs;

  private List<WcsRequest.Format> supportedCoverageFormatList;

  private HashMap<String, WcsRangeField> range;

  public WcsCoverage(GridDataset.Gridset coverage, WcsDataset dataset) {
    this.dataset = dataset;
    if (this.dataset == null) {
      log.error("WcsCoverage(): non-null dataset required.");
      throw new IllegalArgumentException("Non-null dataset required.");
    }

    this.coverage = coverage;
    if (this.coverage == null) {
      log.error("WcsCoverage(): non-null coverage required.");
      throw new IllegalArgumentException("Non-null coverage required.");
    }
    this.coordSys = coverage.getGeoCoordSystem();
    if (this.coordSys == null) {
      log.error("WcsCoverage(): Coverage must have non-null coordinate system.");
      throw new IllegalArgumentException("Non-null coordinate system required.");
    }

    this.name = this.coordSys.getName();
    this.label = this.coordSys.getName();

    this.range = new HashMap<String, WcsRangeField>();
    StringBuilder descripSB =
        new StringBuilder("All parameters on the \"")
            .append(this.name)
            .append("\" coordinate system: ");
    for (GridDatatype curField : this.coverage.getGrids()) {
      String stdName = curField.findAttValueIgnoreCase("standard_name", "");
      descripSB.append(stdName.length() == 0 ? curField.getFullName() : stdName).append(",");

      WcsRangeField field = new WcsRangeField(curField);
      range.put(field.getName(), field);
    }
    descripSB.setCharAt(descripSB.length() - 1, '.');
    this.description = descripSB.toString();

    this.nativeCRS = EPSG_OGC_CF_Helper.getWcs1_0CrsId(this.coordSys.getProjection());

    this.defaultRequestCrs = "OGC:CRS84";

    this.supportedCoverageFormatList = new ArrayList<WcsRequest.Format>();
    // this.supportedCoverageFormatList = "application/x-netcdf";
    this.supportedCoverageFormatList.add(WcsRequest.Format.GeoTIFF);
    this.supportedCoverageFormatList.add(WcsRequest.Format.GeoTIFF_Float);
    this.supportedCoverageFormatList.add(WcsRequest.Format.NetCDF3);
  }

  GridDataset.Gridset getGridset() {
    return coverage;
  }

  public String getName() {
    return this.name;
  }

  public String getLabel() {
    return this.label;
  }

  public String getDescription() {
    return this.description;
  }

  public GridCoordSystem getCoordinateSystem() {
    return coordSys;
  }

  public String getDefaultRequestCrs() {
    return defaultRequestCrs;
  }

  public String getNativeCrs() {
    return nativeCRS;
  }

  public List<WcsRequest.Format> getSupportedCoverageFormatList() {
    return Collections.unmodifiableList(supportedCoverageFormatList);
  }

  public boolean isSupportedCoverageFormat(WcsRequest.Format covFormat) {
    return this.supportedCoverageFormatList.contains(covFormat);
  }

  public boolean isRangeFieldName(String fieldName) {
    return range.containsKey(fieldName);
  }

  public Set<String> getRangeFieldNames() {
    return range.keySet();
  }

  public Collection<WcsRangeField> getRange() {
    return range.values();
  }

  private static DiskCache2 diskCache = null;

  public static void setDiskCache(DiskCache2 _diskCache) {
    diskCache = _diskCache;
  }

  private static DiskCache2 getDiskCache() {
    if (diskCache == null) {
      log.error("getDiskCache(): Disk cache has not been set.");
      throw new IllegalStateException(
          "Disk cache must be set before calling GetCoverage.getDiskCache().");
    }
    return diskCache;
  }

  public File writeCoverageDataToFile(
      WcsRequest.Format format,
      LatLonRect bboxLatLonRect,
      AxisSubset vertSubset,
      List<String> rangeSubset,
      CalendarDateRange timeRange)
      throws WcsException {
    boolean zRangeDone = false;
    boolean tRangeDone = false;

    try {
      // Get the height range.
      Range zRange = vertSubset != null ? vertSubset.getRange() : null;
      zRangeDone = true;

      // Get the time range.
      Range tRange = null;
      if (timeRange != null) {
        CoordinateAxis1DTime timeAxis = this.coordSys.getTimeAxis1D();
        int startIndex = timeAxis.findTimeIndexFromCalendarDate(timeRange.getStart());
        int endIndex = timeAxis.findTimeIndexFromCalendarDate(timeRange.getEnd());
        tRange = new Range(startIndex, endIndex);
        tRangeDone = true;
      }

      if (format == WcsRequest.Format.GeoTIFF || format == WcsRequest.Format.GeoTIFF_Float) {
        if (rangeSubset.size() != 1) {
          String msg =
              "GeoTIFF response encoding only available for single range field selection ["
                  + rangeSubset
                  + "].";
          log.error("writeCoverageDataToFile(): " + msg);
          throw new WcsException(WcsException.Code.InvalidParameterValue, "RangeSubset", msg);
        }
        String reqRangeFieldName = rangeSubset.get(0);

        File dir = new File(getDiskCache().getRootDirectory());
        File tifFile = File.createTempFile("WCS", ".tif", dir);
        if (log.isDebugEnabled())
          log.debug("writeCoverageDataToFile(): tifFile=" + tifFile.getPath());

        WcsRangeField rangeField = this.range.get(reqRangeFieldName);
        GridDatatype subset =
            rangeField.getGridDatatype().makeSubset(tRange, zRange, bboxLatLonRect, 1, 1, 1);
        Array data = subset.readDataSlice(0, 0, -1, -1);

        GeotiffWriter writer = new GeotiffWriter(tifFile.getPath());
        writer.writeGrid(
            this.dataset.getDataset(), subset, data, format == WcsRequest.Format.GeoTIFF);

        writer.close();

        return tifFile;
      } else if (format == WcsRequest.Format.NetCDF3) {
        File dir = new File(getDiskCache().getRootDirectory());
        File outFile = File.createTempFile("WCS", ".nc", dir);
        if (log.isDebugEnabled())
          log.debug("writeCoverageDataToFile(): ncFile=" + outFile.getPath());
        // WTF ?? this.coordSys.getVerticalAxis().isNumeric();

        NetcdfFileWriter writer =
            NetcdfFileWriter.createNew(NetcdfFileWriter.Version.netcdf3, outFile.getAbsolutePath());
        CFGridWriter2.writeFile(
            this.dataset.getDataset(),
            rangeSubset,
            bboxLatLonRect,
            null,
            1,
            zRange,
            timeRange,
            1,
            true,
            writer);
        return outFile;
      } else {
        log.error(
            "writeCoverageDataToFile(): Unsupported response encoding format [" + format + "].");
        throw new WcsException(
            WcsException.Code.InvalidFormat,
            "Format",
            "Unsupported response encoding format [" + format + "].");
      }
    } catch (InvalidRangeException e) {
      String msg = "Failed to subset coverage [" + this.getName();
      if (!zRangeDone) msg += "] along vertical axis [" + vertSubset + "]. ";
      else if (!tRangeDone) msg += "] along time axis [" + timeRange + "]. ";
      else msg += "] in horizontal plane [" + bboxLatLonRect + "]. ";
      log.error("writeCoverageDataToFile(): " + msg + e.getMessage());
      throw new WcsException(WcsException.Code.CoverageNotDefined, "", msg);
    } catch (IOException e) {
      log.error(
          "writeCoverageDataToFile(): Failed to write file for requested coverage <"
              + this.getName()
              + ">: "
              + e.getMessage());
      throw new WcsException(
          WcsException.Code.UNKNOWN, "", "Problem creating coverage [" + this.getName() + "].");
    }
  }
}
Exemplo n.º 7
0
/**
 * THREDDS opendap server.
 *
 * @author jcaron
 * @author Nathan David Potter
 * @since Apr 27, 2009 (branched)
 */
public class OpendapServlet extends AbstractServlet {
  static final String DEFAULTCONTEXTPATH = "/thredds";
  static final String GDATASET = "guarded_dataset";
  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(OpendapServlet.class);

  private boolean allowSessions = false;
  private boolean allowDeflate = false; // handled by Tomcat

  private String odapVersionString = "opendap/3.7";

  private URI baseURI = null;

  private int ascLimit = 50;
  private int binLimit = 500;

  private boolean debugSession = false;

  public String getDefaultContextPath() {
    return DEFAULTCONTEXTPATH;
  }

  public void init() throws javax.servlet.ServletException {
    super.init();

    org.slf4j.Logger logServerStartup = org.slf4j.LoggerFactory.getLogger("serverStartup");
    logServerStartup.info(
        getClass().getName() + " initialization start - " + UsageLog.setupNonRequestContext());

    this.ascLimit = ThreddsConfig.getInt("Opendap.ascLimit", ascLimit);
    this.binLimit = ThreddsConfig.getInt("Opendap.binLimit", binLimit);

    this.odapVersionString = ThreddsConfig.get("Opendap.serverVersion", odapVersionString);
    logServerStartup.info(
        getClass().getName()
            + " version= "
            + odapVersionString
            + " ascLimit = "
            + ascLimit
            + " binLimit = "
            + binLimit);

    // debugging actions
    makeDebugActions();

    logServerStartup.info(
        getClass().getName()
            + " initialization done - "
            + UsageLog.closingMessageNonRequestContext());
  }

  public String getServerVersion() {
    return this.odapVersionString;
  }

  // Servlets that support HTTP GET requests and can quickly determine their last modification time
  // should
  // override this method. This makes browser and proxy caches work more effectively, reducing the
  // load on
  // server and network resources.
  protected long getLastModified(HttpServletRequest req) {
    String query = req.getQueryString();
    if (query != null) return -1;

    String path = req.getPathInfo();
    if (path == null) return -1;

    if (path.endsWith(".asc")) path = path.substring(0, path.length() - 4);
    else if (path.endsWith(".ascii")) path = path.substring(0, path.length() - 6);
    else if (path.endsWith(".das")) path = path.substring(0, path.length() - 4);
    else if (path.endsWith(".dds")) path = path.substring(0, path.length() - 4);
    else if (path.endsWith(".ddx")) path = path.substring(0, path.length() - 4);
    else if (path.endsWith(".dods")) path = path.substring(0, path.length() - 5);
    else if (path.endsWith(".html")) path = path.substring(0, path.length() - 5);
    else if (path.endsWith(".info")) path = path.substring(0, path.length() - 5);
    else if (path.endsWith(".opendap")) path = path.substring(0, path.length() - 5);
    else return -1;

    // if (null != DatasetHandler.findResourceControl( path)) return -1; // LOOK weird Firefox
    // beahviour?

    File file = DataRootHandler.getInstance().getCrawlableDatasetAsFile(path);
    if ((file != null) && file.exists()) return file.lastModified();

    return -1;
  }

  /////////////////////////////////////////////////////////////////////////////

  public void doGet(HttpServletRequest request, HttpServletResponse response) {
    log.info("doGet(): " + UsageLog.setupRequestContext(request));
    // System.out.printf("opendap doGet: req=%s%n%s%n", ServletUtil.getRequest(request),
    // ServletUtil.showRequestDetail(this, request));

    String path = null;

    ReqState rs = getRequestState(request, response);

    try {
      path = request.getPathInfo();
      log.debug("doGet path={}", path);

      if (thredds.servlet.Debug.isSet("showRequestDetail"))
        log.debug(ServletUtil.showRequestDetail(this, request));

      if (path == null) {
        log.info(
            "doGet(): "
                + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
        response.sendError(HttpServletResponse.SC_NOT_FOUND);
        return;
      }

      if (baseURI == null) { // first time, set baseURI
        URI reqURI = ServletUtil.getRequestURI(request);
        // Build base URI from request (rather than hard-coding "/thredds/dodsC/").
        String baseUriString = request.getContextPath() + request.getServletPath() + "/";
        baseURI = reqURI.resolve(baseUriString);
        log.debug("doGet(): baseURI was set = {}", baseURI);
      }

      if (path.endsWith("latest.xml")) {
        DataRootHandler.getInstance().processReqForLatestDataset(this, request, response);
        return;
      }

      // Redirect all catalog requests at the root level.
      if (path.equals("/") || path.equals("/catalog.html") || path.equals("/catalog.xml")) {
        ServletUtil.sendPermanentRedirect(ServletUtil.getContextPath() + path, request, response);
        return;
      }

      // Make sure catalog requests match a dataRoot before trying to handle.
      if (path.endsWith("/") || path.endsWith("/catalog.html") || path.endsWith("/catalog.xml")) {
        if (!DataRootHandler.getInstance().hasDataRootMatch(path)) {
          log.info(
              "doGet(): "
                  + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
          response.sendError(HttpServletResponse.SC_NOT_FOUND);
          return;
        }

        if (!DataRootHandler.getInstance().processReqForCatalog(request, response))
          log.error(
              "doGet(): "
                  + UsageLog.closingMessageForRequestContext(
                      ServletUtil.STATUS_FORWARD_FAILURE, -1));

        return;
      }

      if (rs != null) {
        String dataSet = rs.getDataSet();
        String requestSuffix = rs.getRequestSuffix();

        if ((dataSet == null) || dataSet.equals("/") || dataSet.equals("")) {
          doGetDIR(rs);
        } else if (requestSuffix.equalsIgnoreCase("blob")) {
          doGetBLOB(rs);
        } else if (requestSuffix.equalsIgnoreCase("close")) {
          doClose(rs);
        } else if (requestSuffix.equalsIgnoreCase("dds")) {
          doGetDDS(rs);
        } else if (requestSuffix.equalsIgnoreCase("das")) {
          doGetDAS(rs);
        } else if (requestSuffix.equalsIgnoreCase("ddx")) {
          doGetDDX(rs);
        } else if (requestSuffix.equalsIgnoreCase("dods")) {
          doGetDAP2Data(rs);
        } else if (requestSuffix.equalsIgnoreCase("asc")
            || requestSuffix.equalsIgnoreCase("ascii")) {
          doGetASC(rs);
        } else if (requestSuffix.equalsIgnoreCase("info")) {
          doGetINFO(rs);
        } else if (requestSuffix.equalsIgnoreCase("html")
            || requestSuffix.equalsIgnoreCase("htm")) {
          doGetHTML(rs);
        } else if (requestSuffix.equalsIgnoreCase("ver")
            || requestSuffix.equalsIgnoreCase("version")
            || dataSet.equalsIgnoreCase("/version")
            || dataSet.equalsIgnoreCase("/version/")) {
          doGetVER(rs);
        } else if (dataSet.equalsIgnoreCase("/help")
            || dataSet.equalsIgnoreCase("/help/")
            || dataSet.equalsIgnoreCase("/" + requestSuffix)
            || requestSuffix.equalsIgnoreCase("help")) {
          doGetHELP(rs);
        } else {
          sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
          return;
        }

      } else {
        sendErrorResponse(response, HttpServletResponse.SC_BAD_REQUEST, "Unrecognized request");
        return;
      }

      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, -1));

      // plain ol' 404
    } catch (FileNotFoundException e) {
      sendErrorResponse(response, HttpServletResponse.SC_NOT_FOUND, e.getMessage());

      // DAP2Exception bad url
    } catch (BadURLException e) {
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      dap2ExceptionHandler(e, rs);

      // all other DAP2Exception
    } catch (DAP2Exception de) {
      int status =
          (de.getErrorCode() == DAP2Exception.NO_SUCH_FILE)
              ? HttpServletResponse.SC_NOT_FOUND
              : HttpServletResponse.SC_BAD_REQUEST;
      if ((de.getErrorCode() != DAP2Exception.NO_SUCH_FILE) && (de.getErrorMessage() != null))
        log.debug(de.getErrorMessage());
      log.info(UsageLog.closingMessageForRequestContext(status, -1));
      response.setStatus(status);
      dap2ExceptionHandler(de, rs);

      // parsing, usually the CE
    } catch (ParseException pe) {
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
      response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
      parseExceptionHandler(pe, response);

      // 403 - request too big
    } catch (UnsupportedOperationException e) {
      sendErrorResponse(response, HttpServletResponse.SC_FORBIDDEN, e.getMessage());

    } catch (java.net.SocketException e) {
      log.info("SocketException: " + e.getMessage(), e);
      log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));

    } catch (IOException e) {
      String eName =
          e.getClass().getName(); // dont want compile time dependency on ClientAbortException
      if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
        log.debug("ClientAbortException: " + e.getMessage());
        log.info(UsageLog.closingMessageForRequestContext(ServletUtil.STATUS_CLIENT_ABORT, -1));
        return;
      }

      log.error("path= " + path, e);
      sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.getMessage());

      // everything else
    } catch (Throwable t) {
      log.error("path= " + path, t);
      t.printStackTrace();
      sendErrorResponse(response, HttpServletResponse.SC_INTERNAL_SERVER_ERROR, t.getMessage());
    }
  }

  public void doGetASC(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (ds == null) return;

      response.setHeader("XDODS-Server", getServerVersion());
      response.setContentType("text/plain");
      response.setHeader("Content-Description", "dods-ascii");

      log.debug(
          "Sending OPeNDAP ASCII Data For: " + rs + "  CE: '" + rs.getConstraintExpression() + "'");

      ServerDDS dds = ds.getDDS();
      CEEvaluator ce = new CEEvaluator(dds);
      ce.parseConstraint(rs);
      checkSize(dds, true);

      PrintWriter pw = new PrintWriter(response.getOutputStream());
      dds.printConstrained(pw);
      pw.println("---------------------------------------------");

      AsciiWriter writer = new AsciiWriter(); // could be static
      writer.toASCII(pw, dds, ds);

      // the way that getDAP2Data works
      // DataOutputStream sink = new DataOutputStream(bOut);
      // ce.send(myDDS.getName(), sink, ds);

      pw.flush();

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  public void doGetDAS(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (ds == null) return;

      response.setContentType("text/plain");
      response.setHeader("XDODS-Server", getServerVersion());
      response.setHeader("Content-Description", "dods-das");

      OutputStream Out = new BufferedOutputStream(response.getOutputStream());

      DAS myDAS = ds.getDAS();
      myDAS.print(Out);

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  public void doGetDDS(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (null == ds) return;

      response.setContentType("text/plain");
      response.setHeader("XDODS-Server", getServerVersion());
      response.setHeader("Content-Description", "dods-dds");

      OutputStream out = new BufferedOutputStream(response.getOutputStream());
      ServerDDS myDDS = ds.getDDS();

      if (rs.getConstraintExpression().equals("")) { // No Constraint Expression?
        // Send the whole DDS
        myDDS.print(out);
        out.flush();

      } else { // Otherwise, send the constrained DDS
        // Instantiate the CEEvaluator and parse the constraint expression
        CEEvaluator ce = new CEEvaluator(myDDS);
        ce.parseConstraint(rs);

        // Send the constrained DDS back to the client
        PrintWriter pw = new PrintWriter(new OutputStreamWriter(out));
        myDDS.printConstrained(pw);
        pw.flush();
      }

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  public void doGetDDX(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (null == ds) return;

      response.setContentType("text/plain");
      response.setHeader("XDODS-Server", getServerVersion());
      response.setHeader("Content-Description", "dods-ddx");

      OutputStream out = new BufferedOutputStream(response.getOutputStream());

      ServerDDS myDDS = ds.getDDS();
      myDDS.ingestDAS(ds.getDAS());

      if (rs.getConstraintExpression().equals("")) { // No Constraint Expression?
        // Send the whole DDS
        myDDS.printXML(out);
        out.flush();
      } else { // Otherwise, send the constrained DDS

        // Instantiate the CEEvaluator and parse the constraint expression
        CEEvaluator ce = new CEEvaluator(myDDS);
        ce.parseConstraint(rs);

        // Send the constrained DDS back to the client
        PrintWriter pw = new PrintWriter(new OutputStreamWriter(out));
        myDDS.printConstrainedXML(pw);
        pw.flush();
      }

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  public void doGetBLOB(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (null == ds) return;

      response.setContentType("application/octet-stream");
      response.setHeader("XDODS-Server", getServerVersion());
      response.setHeader("Content-Description", "dods-blob");

      ServletOutputStream sOut = response.getOutputStream();
      OutputStream bOut;
      DeflaterOutputStream dOut = null;
      if (rs.getAcceptsCompressed() && allowDeflate) {
        response.setHeader("Content-Encoding", "deflate");
        dOut = new DeflaterOutputStream(sOut);
        bOut = new BufferedOutputStream(dOut);
      } else {
        bOut = new BufferedOutputStream(sOut);
      }

      ServerDDS myDDS = ds.getDDS();
      CEEvaluator ce = new CEEvaluator(myDDS);
      ce.parseConstraint(rs);
      checkSize(myDDS, false);

      // Send the binary data back to the client
      DataOutputStream sink = new DataOutputStream(bOut);
      ce.send(myDDS.getEncodedName(), sink, ds);
      sink.flush();

      // Finish up sending the compressed stuff, but don't
      // close the stream (who knows what the Servlet may expect!)
      if (null != dOut) dOut.finish();
      bOut.flush();

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  private void doClose(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();
    HttpServletRequest request = rs.getRequest();
    String reqPath = rs.getDataSet();
    HttpSession session = request.getSession();
    session.removeAttribute(reqPath); // work done in the listener

    response.setHeader("XDODS-Server", getServerVersion()); // needed by client

    /* if (path.endsWith(".close")) {
      closeSession(request, response);
      response.setContentLength(0);
      log.info("doGet(): " + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, 0));
      return;

    }

    // so we need to worry about deleting sessions?
    session.invalidate();  */
  }

  public void doGetDAP2Data(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (null == ds) return;

      response.setContentType("application/octet-stream");
      response.setHeader("XDODS-Server", getServerVersion());
      response.setHeader("Content-Description", "dods-data");

      ServletOutputStream sOut = response.getOutputStream();
      OutputStream bOut;
      DeflaterOutputStream dOut = null;
      if (rs.getAcceptsCompressed() && allowDeflate) {
        response.setHeader("Content-Encoding", "deflate");
        dOut = new DeflaterOutputStream(sOut);
        bOut = new BufferedOutputStream(dOut);

      } else {
        bOut = new BufferedOutputStream(sOut);
      }

      ServerDDS myDDS = ds.getDDS();
      CEEvaluator ce = new CEEvaluator(myDDS);
      ce.parseConstraint(rs);
      checkSize(myDDS, false);

      // Send the constrained DDS back to the client
      PrintWriter pw = new PrintWriter(new OutputStreamWriter(bOut));
      myDDS.printConstrained(pw);

      // Send the Data delimiter back to the client
      pw.flush();
      bOut.write("\nData:\n".getBytes());
      bOut.flush();

      // Send the binary data back to the client
      DataOutputStream sink = new DataOutputStream(bOut);
      ce.send(myDDS.getEncodedName(), sink, ds);
      sink.flush();

      // Finish up sending the compressed stuff, but don't
      // close the stream (who knows what the Servlet may expect!)
      if (null != dOut) dOut.finish();
      bOut.flush();

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  public void doGetVER(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    response.setContentType("text/plain");
    response.setHeader("XDODS-Server", getServerVersion());
    response.setHeader("Content-Description", "dods-version");

    PrintWriter pw = new PrintWriter(new OutputStreamWriter(response.getOutputStream()));

    pw.println("Server Version: " + getServerVersion());
    pw.flush();
  }

  public void doGetHELP(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();

    response.setContentType("text/html");
    response.setHeader("XDODS-Server", getServerVersion());
    response.setHeader("Content-Description", "dods-help");

    PrintWriter pw = new PrintWriter(new OutputStreamWriter(response.getOutputStream()));
    printHelpPage(pw);
    pw.flush();
  }

  public void doGetDIR(ReqState rs) throws Exception {
    // rather dangerous here, since you can go into an infinite loop
    // so we're going to insist that there's  no suffix
    HttpServletResponse response = rs.getResponse();
    HttpServletRequest request = rs.getRequest();
    if ((rs.getRequestSuffix() == null) || (rs.getRequestSuffix().length() == 0)) {
      ServletUtil.forwardToCatalogServices(request, response);
      return;
    }

    sendErrorResponse(response, 0, "Unrecognized request");
  }

  public void doGetINFO(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();
    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (null == ds) return;

      PrintStream pw = new PrintStream(response.getOutputStream());
      response.setHeader("XDODS-Server", getServerVersion());
      response.setContentType("text/html");
      response.setHeader("Content-Description", "dods-description");

      GetInfoHandler di = new GetInfoHandler();
      di.sendINFO(pw, ds, rs);

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  public void doGetHTML(ReqState rs) throws Exception {
    HttpServletResponse response = rs.getResponse();
    HttpServletRequest request = rs.getRequest();
    GuardedDataset ds = null;
    try {
      ds = getDataset(rs);
      if (ds == null) return;

      response.setHeader("XDODS-Server", getServerVersion());
      response.setContentType("text/html");
      response.setHeader("Content-Description", "dods-form");

      // Utilize the getDDS() method to get	a parsed and populated DDS
      // for this server.
      ServerDDS myDDS = ds.getDDS();
      DAS das = ds.getDAS();
      GetHTMLInterfaceHandler2 di = new GetHTMLInterfaceHandler2();
      di.sendDataRequestForm(request, response, rs.getDataSet(), myDDS, das);

    } finally { // release lock if needed
      if (ds != null) ds.release();
    }
  }

  ///////////////////////////////////////////////////////////////////////////////////////////////
  // debugging
  private void makeDebugActions() {
    DebugHandler debugHandler = DebugHandler.get("ncdodsServer");
    DebugHandler.Action act;

    act =
        new DebugHandler.Action("help", "Show help page") {
          public void doAction(DebugHandler.Event e) {
            try {
              doGetHELP(getRequestState(e.req, e.res));
            } catch (Exception ioe) {
              log.error("ShowHelp", ioe);
            }
          }
        };
    debugHandler.addAction(act);

    act =
        new DebugHandler.Action("version", "Show server version") {
          public void doAction(DebugHandler.Event e) {
            e.pw.println("  version= " + getServerVersion());
          }
        };
    debugHandler.addAction(act);
  }

  public String getServerName() {
    return this.getClass().getName();
  }

  /*protected ReqState getRequestState(HttpServletRequest request, HttpServletResponse response) {

    ReqState rs = null;
    // The url and query strings will come to us in encoded form
    // (see HTTPmethod.newMethod())
    String baseurl = request.getRequestURL().toString();
    baseurl = EscapeStrings.escapeURL(baseurl);
    log.debug("doGet baseurl={}", baseurl);

    String query = request.getQueryString();
    query = EscapeStrings.unescapeURLQuery(query);
    log.debug("doGet query={}", query);

    try {
      rs = new ReqState(request, response, getServletConfig(), getServerName(), baseurl, query);
    } catch (BadURLException bue) {
      rs = null;
    }

    return rs;
  }*/

  /**
   * ************************************************************************ Prints the OPeNDAP
   * Server help page to the passed PrintWriter
   *
   * @param pw PrintWriter stream to which to dump the help page.
   */
  private void printHelpPage(PrintWriter pw) {

    pw.println("<h3>OPeNDAP Server Help</h3>");
    pw.println("To access most of the features of this OPeNDAP server, append");
    pw.println(
        "one of the following a eight suffixes to a URL: .das, .dds, .dods, .ddx, .blob, .info,");
    pw.println(".ver or .help. Using these suffixes, you can ask this server for:");
    pw.println("<dl>");
    pw.println("<dt> das  </dt> <dd> Dataset Attribute Structure (DAS)</dd>");
    pw.println("<dt> dds  </dt> <dd> Dataset Descriptor Structure (DDS)</dd>");
    pw.println("<dt> dods </dt> <dd> DataDDS object (A constrained DDS populated with data)</dd>");
    pw.println("<dt> ddx  </dt> <dd> XML version of the DDS/DAS</dd>");
    pw.println(
        "<dt> blob </dt> <dd> Serialized binary data content for requested data set, "
            + "with the constraint expression applied.</dd>");
    pw.println("<dt> info </dt> <dd> info object (attributes, types and other information)</dd>");
    pw.println("<dt> html </dt> <dd> html form for this dataset</dd>");
    pw.println("<dt> ver  </dt> <dd> return the version number of the server</dd>");
    pw.println("<dt> help </dt> <dd> help information (this text)</dd>");
    pw.println("</dl>");
    pw.println("For example, to request the DAS object from the FNOC1 dataset at URI/GSO (a");
    pw.println("test dataset) you would appand `.das' to the URL:");
    pw.println("http://opendap.gso.url.edu/cgi-bin/nph-nc/data/fnoc1.nc.das.");

    pw.println("<p><b>Note</b>: Many OPeNDAP clients supply these extensions for you so you don't");
    pw.println("need to append them (for example when using interfaces supplied by us or");
    pw.println("software re-linked with a OPeNDAP client-library). Generally, you only need to");
    pw.println("add these if you are typing a URL directly into a WWW browser.");
    pw.println("<p><b>Note</b>: If you would like version information for this server but");
    pw.println("don't know a specific data file or data set name, use `/version' for the");
    pw.println("filename. For example: http://opendap.gso.url.edu/cgi-bin/nph-nc/version will");
    pw.println("return the version number for the netCDF server used in the first example. ");

    pw.println("<p><b>Suggestion</b>: If you're typing this URL into a WWW browser and");
    pw.println("would like information about the dataset, use the `.info' extension.");

    pw.println("<p>If you'd like to see a data values, use the `.html' extension and submit a");
    pw.println("query using the customized form.");
  }
  // **************************************************************************

  /**
   * ************************************************************************ Prints the Bad URL
   * Page page to the passed PrintWriter
   *
   * @param pw PrintWriter stream to which to dump the bad URL page.
   */
  private void printBadURLPage(PrintWriter pw) {

    String serverContactName = ThreddsConfig.get("serverInformation.contact.name", "UNKNOWN");
    String serverContactEmail = ThreddsConfig.get("serverInformation.contact.email", "UNKNOWN");
    pw.println("<h3>Error in URL</h3>");
    pw.println("The URL extension did not match any that are known by this");
    pw.println("server. Below is a list of the five extensions that are be recognized by");
    pw.println("all OPeNDAP servers. If you think that the server is broken (that the URL you");
    pw.println("submitted should have worked), then please contact the");
    pw.println("administrator of this server [" + serverContactName + "] at: ");
    pw.println("<a href='mailto:" + serverContactEmail + "'>" + serverContactEmail + "</a><p>");
  }

  ///////////////////////////////////////////////////////
  // utils
  /**
   * @param request
   * @param response
   * @return the request state
   */
  protected ReqState getRequestState(HttpServletRequest request, HttpServletResponse response) {
    ReqState rs = null;

    // Assume url was encoded
    String baseurl = request.getRequestURL().toString();
    baseurl = EscapeStrings.unescapeURL(baseurl);

    // Assume query  was encoded
    String query = request.getQueryString();
    query = EscapeStrings.unescapeURLQuery(query);

    log.info(String.format("OpendapServlet: nominal url: %s?%s", baseurl, query));

    try {
      rs = new ReqState(request, response, this, getServerName(), baseurl, query);
    } catch (Exception bue) {
      rs = null;
    }

    return rs;
  }

  private void checkSize(ServerDDS dds, boolean isAscii) throws Exception {
    long size = computeSize(dds, isAscii);
    // System.err.printf("total (constrained) size=%s\n", size);
    log.debug("total (constrained) size={}", size);
    double dsize = size / (1000 * 1000);
    double maxSize = isAscii ? ascLimit : binLimit; // Mbytes
    if (dsize > maxSize) {
      log.info("Reject request size = {} Mbytes", dsize);
      throw new UnsupportedOperationException(
          "Request too big=" + dsize + " Mbytes, max=" + maxSize);
    }
  }

  // Recursively compute size of the dds to be returned
  private long computeSize(DConstructor ctor, boolean isAscii) throws Exception {
    long projectsize = 0; // accumulate size of projected variables
    long othersize = 0; // accumulate size of non-projected variables
    long fieldsize = 0;
    int projectedcount = 0;
    int fieldcount = 0;
    Enumeration vars = ctor.getVariables();
    while (vars.hasMoreElements()) {
      fieldcount++;
      BaseType field = (BaseType) vars.nextElement();
      fieldsize = computeFieldSize(field, isAscii);
      // accumulate the field sizes
      if (field.isProject()) {
        projectsize += fieldsize;
        projectedcount++;
      } else {
        othersize += fieldsize;
      }
    }
    // Cases to consider:
    // 1. If all of the fields of this ctor are projected,
    //    then return projectsize
    // 2. If none of the fields of this ctor are projected,
    //    then return othersize
    // 3. otherwise, at least one field, but not all, is projected,
    //    => return projectsize;
    if (projectedcount == fieldcount) return projectsize;
    else if (projectedcount == 0) return othersize;
    else {
      assert (projectedcount > 0 && projectedcount < fieldcount);
      return projectsize;
    }
  }

  long computeFieldSize(BaseType bt, boolean isAscii) throws Exception {
    long fieldsize = 0;
    // Figure out what this field is (e.g. primitive or not)
    // Somewhat convoluted.
    if (bt instanceof DConstructor) {
      // simple struct, seq, or grid => recurse
      fieldsize = computeSize((DConstructor) bt, isAscii);
    } else if (bt instanceof DArray) {
      SDArray da = (SDArray) bt;
      // Separate structure arrays from primitive arrays
      if (da.getContainerVar() instanceof DPrimitive) {
        fieldsize = computeArraySize(da);
      } else if (da.getContainerVar() instanceof DStructure) {
        fieldsize = computeSize((DStructure) da.getContainerVar(), isAscii); // recurse
      } else { // Some kind of problem
        throw new NoSuchTypeException("Computesize: unexpected type for " + bt.getLongName());
      }
    } else if (bt instanceof DPrimitive) {
      DPrimitive dp = (DPrimitive) bt;
      if (dp instanceof DString) {
        String v = ((DString) dp).getValue();
        fieldsize = (v == null ? 0 : v.length());
      } else {
        DataType dtype = DODSNetcdfFile.convertToNCType(bt);
        fieldsize = dtype.getSize();
      }
    } else { // Some kind of problem
      throw new NoSuchTypeException("Computesize: unknown type for " + bt.getLongName());
    }
    return fieldsize;
  }

  long computeArraySize(SDArray da) throws Exception {
    assert (da.getContainerVar() instanceof DPrimitive);
    BaseType base = da.getPrimitiveVector().getTemplate();
    DataType dtype = DODSNetcdfFile.convertToNCType(base);
    int elemSize = dtype.getSize();
    int n = da.numDimensions();
    List<Range> ranges = new ArrayList<Range>(n);
    long size = 0;
    for (int i = 0; i < n; i++) {
      ranges.add(new Range(da.getStart(i), da.getStop(i), da.getStride(i)));
      Section s = new Section(ranges);
      size += s.computeSize() * elemSize;
    }
    return size;
  }

  /*
   * *********************** dataset caching ***********************************************
   */

  // any time the server needs access to the dataset, it gets a "GuardedDataset" which allows us to
  // add caching
  // optionally, a session may be established, which allows us to reserve the dataset for that
  // session.
  protected GuardedDataset getDataset(ReqState preq) throws Exception {
    HttpServletRequest req = preq.getRequest();
    String reqPath = preq.getDataSet();

    // see if the client wants sessions
    boolean acceptSession = false;
    String s = req.getHeader("X-Accept-Session");
    if (s != null && s.equalsIgnoreCase("true") && allowSessions) acceptSession = true;

    HttpSession session = null;
    if (acceptSession) {
      // see if theres already a session established, create one if not
      session = req.getSession();
      if (!session.isNew()) {
        GuardedDataset gdataset = (GuardedDataset) session.getAttribute(reqPath);
        if (null != gdataset) {
          if (debugSession)
            System.out.printf(" found gdataset %s in session %s %n", reqPath, session.getId());
          if (log.isDebugEnabled())
            log.debug(" found gdataset " + gdataset + " in session " + session.getId());
          return gdataset;
        }
      }
    }

    NetcdfFile ncd = DatasetHandler.getNetcdfFile(req, preq.getResponse(), reqPath);
    if (null == ncd) return null;

    GuardedDataset gdataset = new GuardedDatasetCacheAndClone(reqPath, ncd, acceptSession);
    // GuardedDataset gdataset = new GuardedDatasetImpl(reqPath, ncd, acceptSession);

    if (acceptSession) {
      String cookiePath = req.getRequestURI();
      String suffix = "." + preq.getRequestSuffix();
      if (cookiePath.endsWith(suffix)) // snip off the suffix
      cookiePath = cookiePath.substring(0, cookiePath.length() - suffix.length());
      session.setAttribute(reqPath, gdataset);
      session.setAttribute(CookieFilter.SESSION_PATH, cookiePath);
      // session.setAttribute("dataset", ncd.getLocation());  // for UsageValve
      // session.setMaxInactiveInterval(30); // 30 second timeout !!
      if (debugSession)
        System.out.printf(
            " added gdataset %s in session %s cookiePath %s %n",
            reqPath, session.getId(), cookiePath);
      if (log.isDebugEnabled())
        log.debug(" added gdataset " + gdataset + " in session " + session.getId());
    } /* else {
        session = req.getSession();
        session.setAttribute("dataset", ncd.getLocation()); // for UsageValve
      } */

    return gdataset;
  }

  //////////////////////////////////////////////////////////////////////////////

  public void parseExceptionHandler(ParseException pe, HttpServletResponse response) {
    try {
      BufferedOutputStream eOut = new BufferedOutputStream(response.getOutputStream());
      response.setHeader("Content-Description", "dods-error");
      response.setContentType("text/plain");

      String msg = pe.getMessage().replace('\"', '\'');

      DAP2Exception de2 = new DAP2Exception(opendap.dap.DAP2Exception.CANNOT_READ_FILE, msg);
      de2.print(eOut);
    } catch (Exception e) {
      System.err.println("parseExceptionHandler: " + e);
    }
  }

  public void dap2ExceptionHandler(DAP2Exception de, ReqState rs) {
    rs.getResponse().setHeader("Content-Description", "dods-error");
    rs.getResponse().setContentType("text/plain");
    try {
      de.print(rs.getResponse().getOutputStream());
    } catch (Exception e) {
      System.err.println("dap2ExceptionHandler: " + e);
    }
  }

  private void sendErrorResponse(HttpServletResponse response, int errorCode, String errorMessage) {
    try {
      log.info(UsageLog.closingMessageForRequestContext(errorCode, -1));
      response.setStatus(errorCode);
      response.setHeader("Content-Description", "dods-error");
      response.setContentType("text/plain");

      PrintWriter pw = new PrintWriter(response.getOutputStream());
      pw.println("Error {");
      pw.println("    code = " + errorCode + ";");
      pw.println("    message = \"" + errorMessage + "\";");

      pw.println("};");
      pw.flush();
    } catch (Exception e) {
      System.err.println("sendErrorResponse: " + e);
    }
  }
}
/** Created by Martin on 26.6.2015. */
public class ProjectsClient {
  private static final Logger logger = org.slf4j.LoggerFactory.getLogger(ProjectsClient.class);

  private GerritApi api;
  private IProjectsRepository repo;
  private ProjectsCache cache;
  private Boolean downloadParents;
  private WaitCaller caller;

  public ProjectsClient(
      GerritApi api, WaitCaller caller, IProjectsRepository repository, Boolean downloadParents) {
    this.api = api;
    this.repo = repository;
    this.caller = caller;

    this.cache = new ProjectsCache();
    this.downloadParents = downloadParents;
  }

  public void prepare() {
    cache.restore(repo.getAllProjects());
  }

  public ProjectDto getProject(String id) {
    if (cache.isCached(id)) {
      return cache.tryGetCached(id);
    }

    ProjectInfo info = null;

    try {
      info = caller.waitOrCall(() -> api.projects().name(id).get());
    } catch (Exception e) {

      logger.error(Logging.prepare("getProject", id), e);
    }

    if (info != null) {
      ProjectDto project = new ProjectDto(info.id, info.name);

      if (info.parent != null && downloadParents) {
        ProjectDto parent = getProject(info.parent);
        project.parentId = Optional.of(parent.id);
      }

      project = repo.add(project);

      // getProjectBranches(project);
      // getApprovals(project);

      cache.cache(project);

      return project;
    }

    return null;
  }

  //
  //    private void getApprovals(ProjectDto project) {
  //        int start = 0;
  //        int limit = 1;
  //
  //        boolean getCommands = true;
  //
  //        List<ApprovalTypeDto> approvals = new ArrayList<>();
  //        while (approvals.size() <= 0) {
  //            List<ChangeInfo> changes = null;
  //
  //            try {
  //                Changes.QueryRequest request = api.changes().query()
  //                        .withStart(start)
  //                        .withLimit(limit)
  //                        .withOptions(ListChangesOption.DETAILED_LABELS);
  //                if (getCommands) {
  //                    request.withOption(ListChangesOption.DOWNLOAD_COMMANDS);
  //                }
  //
  //                changes = caller.waitOrCall(() -> request.get());
  //            } catch (Exception e) {
  //                e.printStackTrace();
  //                getCommands = false;
  //            }
  //
  //            if (changes != null && changes.size() > 0) {
  //                ChangeInfo change = changes.get(0);
  //
  //                if (change.labels != null && change.labels.size() > 0) {
  //                    approvals.addAll(createApprovalTypes(project, change.labels));
  //                }
  //
  //            }
  //
  //            start += limit;
  //        }
  //
  //        approvals.forEach(repo::addApprovalType);
  //
  //        approvals.forEach(a -> project.approvals.put(a.name, a));
  //    }

  private List<ApprovalTypeDto> createApprovalTypes(
      ProjectDto project, Map<String, LabelInfo> labels) {
    return labels
        .entrySet()
        .stream()
        .filter(e -> project.approvals.containsKey(e.getKey()) == false)
        .map(e -> createApprovalType(project, e.getKey(), e.getValue()))
        .collect(Collectors.toList());
  }

  private ApprovalTypeDto createApprovalType(ProjectDto project, String type, LabelInfo label) {
    short defaultValue = 0;
    if (label != null && label.defaultValue != null) {
      defaultValue = label.defaultValue;
    }

    return new ApprovalTypeDto(
        project,
        type,
        defaultValue,
        label
            .values
            .entrySet()
            .stream()
            .map(ev -> new ApprovalValueDto(ev.getKey(), ev.getValue()))
            .toArray(ApprovalValueDto[]::new));
  }

  public void addApprovals(ProjectDto project, Map<String, LabelInfo> labels) {
    if (labels.size() > 0) {
      labels.entrySet().forEach(e -> addApproval(project, e.getKey(), e.getValue()));
    }
  }

  public void addApproval(ProjectDto project, String key, LabelInfo label) {
    if (project.approvals.containsKey(key) == false) {
      ApprovalTypeDto approval = createApprovalType(project, key, label);
      repo.addApprovalType(approval);
      project.approvals.put(approval.name, approval);
    }
  }

  private void getProjectBranches(ProjectDto project) {
    List<BranchInfo> branches = null;
    try {
      ProjectApi.ListBranchesRequest request = api.projects().name(project.projectId).branches();
      branches = caller.waitOrCall(() -> request.get());
    } catch (Exception e) {
      logger.error(Logging.prepare("getProjectBranches", project.projectId), e);
    }

    if (branches != null) {
      List<BranchDto> projectBranches =
          branches
              .stream()
              .filter(b -> project.hasBranch(b.ref) == false)
              .map(b -> repo.addBranch(project, b.ref, b.revision))
              .filter(b -> b != null)
              .collect(Collectors.toList());

      projectBranches.forEach(project.branches::add);
    }
  }

  public BranchDto getBranch(ProjectDto project, String name) {
    if (project.hasBranch(name) == false) {
      repo.loadProjectBranches(project);
    }

    if (project.hasBranch(name) == false) {
      getProjectBranches(project);
    }
    return project.getBranch(name);
  }
}
Exemplo n.º 9
0
/**
 * @author : rincevent
 * @since JAHIA 6.5 Created : 14 sept. 2010
 */
public class JBPMMailProducer extends MailProducerImpl {
  private static final long serialVersionUID = -5084848266010688683L;
  private static transient Logger logger =
      org.slf4j.LoggerFactory.getLogger(JBPMMailProducer.class);
  ScriptEngine scriptEngine;
  private Bindings bindings;

  private String templateKey;

  public String getTemplateKey() {
    return templateKey;
  }

  public void setTemplateKey(String templateKey) {
    this.templateKey = templateKey;
  }

  public Collection<Message> produce(final Execution execution) {
    final Map<String, Object> vars = ((ExecutionImpl) execution).getVariables();
    Locale locale = (Locale) vars.get("locale");

    if (templateKey != null) {
      MailTemplate template = null;
      MailTemplateRegistry templateRegistry =
          ((ProcessEngine) SpringContextSingleton.getBean("processEngine"))
              .get(MailTemplateRegistry.class);
      if (locale != null) {
        template = (templateRegistry.getTemplate(templateKey + "." + locale.toString()));
        if (template == null) {
          template = (templateRegistry.getTemplate(templateKey + "." + locale.getLanguage()));
        }
      }
      if (template == null) {
        template = templateRegistry.getTemplate(templateKey);
      }
      setTemplate(template);
    }

    if (ServicesRegistry.getInstance().getMailService().isEnabled() && getTemplate() != null) {
      try {
        return JCRTemplate.getInstance()
            .doExecuteWithSystemSession(
                null,
                "default",
                locale,
                new JCRCallback<Collection<Message>>() {
                  public Collection<Message> doInJCR(JCRSessionWrapper session)
                      throws RepositoryException {
                    try {
                      scriptEngine =
                          ScriptEngineUtils.getInstance()
                              .getEngineByName(getTemplate().getLanguage());
                      bindings = null;
                      Message email = instantiateEmail();
                      fillFrom(email, execution, session);
                      fillRecipients(email, execution, session);
                      fillSubject(email, execution, session);
                      fillContent(email, execution, session);
                      Address[] addresses = email.getRecipients(Message.RecipientType.TO);
                      if (addresses != null && addresses.length > 0) {
                        return Collections.singleton(email);
                      } else {
                        return Collections.emptyList();
                      }
                    } catch (MessagingException e) {
                      logger.error(e.getMessage(), e);
                    } catch (ScriptException e) {
                      logger.error(e.getMessage(), e);
                    }
                    return Collections.emptyList();
                  }
                });
      } catch (RepositoryException e) {
        logger.error(e.getMessage(), e);
      }
    }
    return Collections.emptyList();
  }

  /**
   * Fills the <code>from</code> attribute of the given email. The sender addresses are an optional
   * element in the mail template. If absent, each mail server supplies the current user's email
   * address.
   *
   * @see {@link InternetAddress#getLocalAddress(Session)}
   */
  protected void fillFrom(Message email, Execution execution, JCRSessionWrapper session)
      throws MessagingException {
    try {
      AddressTemplate fromTemplate = getTemplate().getFrom();
      // "from" attribute is optional
      if (fromTemplate == null) return;

      // resolve and parse addresses
      String addresses = fromTemplate.getAddresses();
      if (addresses != null) {
        addresses = evaluateExpression(execution, addresses, session);
        // non-strict parsing applies to a list of mail addresses entered by a human
        email.addFrom(InternetAddress.parse(addresses, false));
      }

      EnvironmentImpl environment = EnvironmentImpl.getCurrent();
      IdentitySession identitySession = environment.get(IdentitySession.class);
      AddressResolver addressResolver = environment.get(AddressResolver.class);

      // resolve and tokenize users
      String userList = fromTemplate.getUsers();
      if (userList != null) {
        String[] userIds = tokenizeActors(userList, execution, session);
        List<User> users = identitySession.findUsersById(userIds);
        email.addFrom(resolveAddresses(users, addressResolver));
      }

      // resolve and tokenize groups
      String groupList = fromTemplate.getGroups();
      if (groupList != null) {
        for (String groupId : tokenizeActors(groupList, execution, session)) {
          Group group = identitySession.findGroupById(groupId);
          email.addFrom(addressResolver.resolveAddresses(group));
        }
      }
    } catch (ScriptException e) {
      logger.error(e.getMessage(), e);
    } catch (RepositoryException e) {
      logger.error(e.getMessage(), e);
    }
  }

  protected void fillRecipients(Message email, Execution execution, JCRSessionWrapper session)
      throws MessagingException {
    try {
      // to
      AddressTemplate to = getTemplate().getTo();
      if (to != null) {
        fillRecipients(to, email, Message.RecipientType.TO, execution, session);
      }

      // cc
      AddressTemplate cc = getTemplate().getCc();
      if (cc != null) {
        fillRecipients(cc, email, Message.RecipientType.CC, execution, session);
      }

      // bcc
      AddressTemplate bcc = getTemplate().getBcc();
      if (bcc != null) {
        fillRecipients(bcc, email, Message.RecipientType.BCC, execution, session);
      }
    } catch (ScriptException e) {
      logger.error(e.getMessage(), e);
    } catch (RepositoryException e) {
      logger.error(e.getMessage(), e);
    }
  }

  private void fillRecipients(
      AddressTemplate addressTemplate,
      Message email,
      Message.RecipientType recipientType,
      Execution execution,
      JCRSessionWrapper session)
      throws MessagingException, RepositoryException, ScriptException {
    // resolve and parse addresses
    String addresses = addressTemplate.getAddresses();
    if (addresses != null) {
      addresses = evaluateExpression(execution, addresses, session);
      // non-strict parsing applies to a list of mail addresses entered by a human
      email.addRecipients(recipientType, InternetAddress.parse(addresses, false));
    }

    EnvironmentImpl environment = EnvironmentImpl.getCurrent();
    IdentitySession identitySession = environment.get(IdentitySession.class);
    AddressResolver addressResolver = environment.get(AddressResolver.class);

    // resolve and tokenize users
    String userList = addressTemplate.getUsers();
    if (userList != null) {
      String[] userIds = tokenizeActors(userList, execution, session);
      List<User> users = identitySession.findUsersById(userIds);
      email.addRecipients(recipientType, resolveAddresses(users, addressResolver));
    }

    // resolve and tokenize groups
    String groupList = addressTemplate.getGroups();
    if (groupList != null) {
      for (String groupId : tokenizeActors(groupList, execution, session)) {
        Group group = identitySession.findGroupById(groupId);
        email.addRecipients(recipientType, addressResolver.resolveAddresses(group));
      }
    }
  }

  private String[] tokenizeActors(String recipients, Execution execution, JCRSessionWrapper session)
      throws RepositoryException, ScriptException {
    String[] actors = evaluateExpression(execution, recipients, session).split("[,;\\s]+");
    if (actors.length == 0) throw new JbpmException("recipient list is empty: " + recipients);
    return actors;
  }

  /** construct recipient addresses from user entities */
  private Address[] resolveAddresses(List<User> users, AddressResolver addressResolver) {
    int userCount = users.size();
    List<Address> addresses = new ArrayList<Address>();
    for (int i = 0; i < userCount; i++) {
      if (!StringUtils.isEmpty(users.get(i).getBusinessEmail())) {
        addresses.add(addressResolver.resolveAddress(users.get(i)));
      }
    }
    return addresses.toArray(new Address[addresses.size()]);
  }

  protected void fillSubject(Message email, Execution execution, JCRSessionWrapper session)
      throws MessagingException {
    String subject = getTemplate().getSubject();
    if (subject != null) {
      try {
        String evaluatedSubject =
            evaluateExpression(execution, subject, session).replaceAll("[\r\n]", "");
        email.setSubject(WordUtils.abbreviate(evaluatedSubject, 60, 74, "..."));
      } catch (RepositoryException e) {
        logger.error(e.getMessage(), e);
      } catch (ScriptException e) {
        logger.error(e.getMessage(), e);
      }
    }
  }

  protected void fillContent(Message email, Execution execution, JCRSessionWrapper session)
      throws MessagingException {
    String text = getTemplate().getText();
    String html = getTemplate().getHtml();
    List<AttachmentTemplate> attachmentTemplates = getTemplate().getAttachmentTemplates();

    try {
      if (html != null || !attachmentTemplates.isEmpty()) {
        // multipart
        MimeMultipart multipart = new MimeMultipart("related");

        BodyPart p = new MimeBodyPart();
        Multipart alternatives = new MimeMultipart("alternative");
        p.setContent(alternatives, "multipart/alternative");
        multipart.addBodyPart(p);

        // html
        if (html != null) {
          BodyPart htmlPart = new MimeBodyPart();
          html = evaluateExpression(execution, html, session);
          htmlPart.setContent(html, "text/html; charset=UTF-8");
          alternatives.addBodyPart(htmlPart);
        }

        // text
        if (text != null) {
          BodyPart textPart = new MimeBodyPart();
          text = evaluateExpression(execution, text, session);
          textPart.setContent(text, "text/plain; charset=UTF-8");
          alternatives.addBodyPart(textPart);
        }

        // attachments
        if (!attachmentTemplates.isEmpty()) {
          addAttachments(execution, multipart);
        }

        email.setContent(multipart);
      } else if (text != null) {
        // unipart
        text = evaluateExpression(execution, text, session);
        email.setText(text);
      }
    } catch (RepositoryException e) {
      logger.error(e.getMessage(), e);
    } catch (ScriptException e) {
      logger.error(e.getMessage(), e);
    }
  }

  private String evaluateExpression(
      Execution execution, String scriptToExecute, JCRSessionWrapper session)
      throws RepositoryException, ScriptException {
    ScriptContext scriptContext = scriptEngine.getContext();
    if (bindings == null) {
      bindings = getBindings(execution, session);
    }
    scriptContext.setWriter(new StringWriter());
    scriptContext.setErrorWriter(new StringWriter());
    scriptEngine.eval(scriptToExecute, bindings);
    String error = scriptContext.getErrorWriter().toString();
    if (error.length() > 0) {
      logger.error("Scripting error : " + error);
    }
    return scriptContext.getWriter().toString().trim();
  }

  private Bindings getBindings(Execution execution, JCRSessionWrapper session)
      throws RepositoryException {
    EnvironmentImpl environment = EnvironmentImpl.getCurrent();
    final Map<String, Object> vars = ((ExecutionImpl) execution).getVariables();
    Locale locale = (Locale) vars.get("locale");
    final Bindings bindings = new MyBindings(environment);
    ResourceBundle resourceBundle =
        JahiaResourceBundle.lookupBundle(
            "org.jahia.services.workflow."
                + ((ExecutionImpl) execution).getProcessDefinition().getKey(),
            locale);
    bindings.put("bundle", resourceBundle);
    JahiaUser jahiaUser =
        ServicesRegistry.getInstance()
            .getJahiaUserManagerService()
            .lookupUserByKey((String) vars.get("user"));
    bindings.put("user", jahiaUser);
    bindings.put("date", new DateTool());
    bindings.put("submissionDate", Calendar.getInstance());
    bindings.put("locale", locale);
    bindings.put("workspace", vars.get("workspace"));

    List<JCRNodeWrapper> nodes = new LinkedList<JCRNodeWrapper>();
    @SuppressWarnings("unchecked")
    List<String> stringList = (List<String>) vars.get("nodeIds");
    for (String s : stringList) {
      JCRNodeWrapper nodeByUUID = session.getNodeByUUID(s);
      if (!nodeByUUID.isNodeType("jnt:translation")) {
        nodes.add(nodeByUUID);
      }
    }
    bindings.put("nodes", nodes);
    return bindings;
  }

  public class MyBindings extends SimpleBindings {
    private final Environment environment;

    public MyBindings(Environment environment) {
      super();
      this.environment = environment;
    }

    @Override
    public boolean containsKey(Object key) {
      return super.containsKey(key) || environment.get((String) key) != null;
    }

    @Override
    public Object get(Object key) {
      return super.containsKey(key) ? super.get(key) : environment.get((String) key);
    }
  }
}
Exemplo n.º 10
0
public class RC {
  static boolean showlog = false; /* do not do any logging */
  public static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(RC.class);

  //////////////////////////////////////////////////
  // Predefined flags
  // To add a new flag:
  // 1. choose a name for the flag
  // 2. Define the protected static field with default value
  // 3. Define a get function
  // 4. Add an arm to the set function
  // 5. Add any usefull utilities like booleanize()

  public static final String USEGROUPSKEY = "ucar.nc2.cdm.usegroups";
  public static final String VERIFYSERVERKEY = "ucar.nc2.net.verifyserver";
  public static final String ALLOWSELFSIGNEDKEY = "ucar.nc2.net.allowselfsigned";

  protected static boolean useGroups = true;
  protected static boolean verifyServer = false;
  protected static boolean allowSelfSigned = true;

  public static boolean getUseGroups() {
    if (!initialized) RC.initialize();
    return useGroups;
  }

  public static boolean getVerifyServer() {
    if (!initialized) RC.initialize();
    return verifyServer;
  }

  public static boolean getAllowSelfSigned() {
    if (!initialized) RC.initialize();
    return allowSelfSigned;
  }

  public static void set(String key, String value) {
    // TODO: think about the rc properties naming hierarchy
    assert (key != null);
    if (USEGROUPSKEY.equals(key)) {
      useGroups = booleanize(value);
    } else if (VERIFYSERVERKEY.equals(key)) {
      verifyServer = booleanize(value);
    } else if (ALLOWSELFSIGNEDKEY.equals(key)) {
      allowSelfSigned = booleanize(value);
    }
  }

  static boolean booleanize(String value) {
    // canonical boolean values
    if (value == null || "0".equals(value) || "false".equalsIgnoreCase(value)) return false;
    if (value.length() == 0 || "1".equals(value) || "true".equalsIgnoreCase(value)) return true;
    return value != null; // any non-null value?
  }

  //////////////////////////////////////////////////

  static final String DFALTRCFILE = ".threddsrc";

  static final char LTAG = '[';
  static final char RTAG = ']';

  static final String[] rcfilelist = new String[] {".dodsrc", ".tdsrc"};

  static int urlCompare(URL u1, URL u2) {
    int relation;
    if (u1 == null && u2 == null) return 0;
    if (u1 == null) return -1;
    if (u2 == null) return +1;
    // 1. host test
    String host1 = (new StringBuilder(u1.getHost())).reverse().toString();
    String host2 = (new StringBuilder(u2.getHost())).reverse().toString();
    // Use lexical order on the reversed host names
    relation = host1.compareTo(host2);
    if (relation != 0) return relation;
    // 2. path test
    relation = (u1.getPath().compareTo(u2.getPath()));
    if (relation != 0) return relation;
    // 3. port number
    relation = (u1.getPort() - u2.getPort());
    if (relation != 0) return relation;
    // 4. note: all other fields are ignored
    return 0;
  }

  // Match has different semantics than urlCompare
  static boolean urlMatch(URL pattern, URL url) {
    int relation;

    if (pattern == null) return (url == null);

    if (!(url.getHost().endsWith(pattern.getHost())))
      return false; // e.g. pattern=x.y.org url=y.org

    if (!(url.getPath().startsWith(pattern.getPath())))
      return false; // e.g. pattern=y.org/a/b url=y.org/a

    if (pattern.getPort() > 0 && pattern.getPort() != url.getPort()) return false;

    // note: all other fields are ignored

    return true;
  }

  public static class Triple implements Comparable {
    public String key; // also sort key
    public String value;
    public URL url;

    public Triple(String key, String value, String url) {
      URL u = null;
      if (url != null && url.length() > 0)
        try {
          u = new URL(url);
        } catch (MalformedURLException e) {
          u = null;
        }
      set(key, value, u);
    }

    public Triple(String key, String value, URL url) {
      set(key, value, url);
    }

    void set(String key, String value, URL url) {
      this.key = key.trim().toLowerCase();
      this.url = url;
      this.value = value;
      if (this.value == null) this.value = "";
    }

    public boolean equals(Object o) {
      if (o == null || !(o instanceof Triple)) return false;
      return (compareTo((Triple) o) == 0);
    }

    public int compareTo(Object o) {
      if (o == null) throw new NullPointerException();
      Triple t = (Triple) o;
      int relation = key.compareTo(t.key);
      if (relation != 0) return relation;
      relation = urlCompare(this.url, t.url);
      return relation;
    }

    // toString produces an rc line
    public String toString() {
      StringBuilder line = new StringBuilder();
      if (url != null) {
        line.append("[");
        line.append(url.toString());
        line.append("]");
      }
      line.append(key);
      line.append("=");
      line.append(value);
      return line.toString();
    }
  }

  // Define a singlton RC instance for general global use
  static RC dfaltRC = null;

  private static boolean initialized = false;

  static {
    RC.initialize();
  }

  public static synchronized void initialize() {
    if (!initialized) {
      initialized = true;
      RC.loadDefaults();
      RC.setWellKnown();
      RC.loadFromJava();
    }
  }

  /**
   * Allow users to add to the default rc
   *
   * @param key
   * @param value
   * @param url null => not url specific
   */
  public static synchronized void add(String key, String value, String url) {
    if (key == null) return;
    if (!initialized) RC.initialize();
    Triple t = new Triple(key, value, url);
    dfaltRC.insert(t);
    // recompute well-knowns
    setWellKnown();
  }

  /**
   * Allow users to search the default rc
   *
   * @param key
   * @param url null => not url specific
   * @return value corresponding to key+url, or null if does not exist
   */
  public static synchronized String find(String key, String url) {
    if (key == null) return null;
    if (!initialized) RC.initialize();
    Triple t = dfaltRC.lookup(key, url);
    return (t == null ? null : t.value);
  }

  /** Record some well known parameters */
  static void setWellKnown() {
    if (dfaltRC.triplestore.size() == 0) return;
    // Walk the set of triples looking for those that have no url
    for (String key : dfaltRC.keySet()) {
      Triple triple = dfaltRC.lookup(key);
      if (triple.url == null) {
        RC.set(key, triple.value); // let set sort it out
      }
    }
  }

  static void loadDefaults() {
    RC rc0 = new RC();
    String[] locations =
        new String[] {
          System.getProperty("user.home"), System.getProperty("user.dir"),
        };

    boolean found1 = false;
    for (String loc : locations) {
      if (loc == null) continue;
      String dir = loc.replace('\\', '/');
      if (dir.endsWith("/")) dir = dir.substring(0, dir.length() - 1);
      for (String rcpath : rcfilelist) {
        String filepath = loc + "/" + rcpath;
        if (rc0.load(filepath)) found1 = true;
      }
    }
    if (!found1) if (showlog) log.debug("No .rc file found");
    dfaltRC = rc0;
  }

  static void loadFromJava() {
    String[] flags = new String[] {USEGROUPSKEY, VERIFYSERVERKEY, ALLOWSELFSIGNEDKEY};
    for (String flag : flags) {
      String value = System.getProperty(flag);
      if (value != null) {
        set(flag, value);
      }
    }
  }

  static RC getDefault() {
    return dfaltRC;
  }

  //////////////////////////////////////////////////
  // Instance Data

  Map<String, List<Triple>> triplestore;

  //////////////////////////////////////////////////
  // constructors

  public RC() {
    triplestore = new HashMap<String, List<Triple>>();
  }

  //////////////////////////////////////////////////
  // Loaders

  // Load this triple store from an rc file
  // overwrite existing entries

  public boolean load(String abspath) {
    abspath = abspath.replace('\\', '/');
    File rcFile = new File(abspath);
    if (!rcFile.exists() || !rcFile.canRead()) {
      return false;
    }
    if (showlog) log.debug("Loading rc file: " + abspath);
    try (BufferedReader rdr =
        new BufferedReader(new InputStreamReader(new FileInputStream(rcFile), CDM.UTF8))) {
      for (int lineno = 1; ; lineno++) {
        URL url = null;
        String line = rdr.readLine();
        if (line == null) break;
        // trim leading blanks
        line = line.trim();
        if (line.length() == 0) continue; // empty line
        if (line.charAt(0) == '#') continue; // check for comment
        // parse the line
        if (line.charAt(0) == LTAG) {
          int rindex = line.indexOf(RTAG);
          if (rindex < 0) return false;
          if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
          String surl = line.substring(1, rindex);
          try {
            url = new URL(surl);
          } catch (MalformedURLException mue) {
            if (showlog) log.error("Malformed [url] at " + abspath + "." + lineno);
          }
          line = line.substring(rindex + 1);
          // trim again
          line = line.trim();
        }
        // Get the key,value part
        String[] pieces = line.split("\\s*=\\s*");
        assert (pieces.length == 1 || pieces.length == 2);
        // Create the triple
        String value = "1";
        if (pieces.length == 2) value = pieces[1].trim();
        Triple triple = new Triple(pieces[0].trim(), value, url);
        List<Triple> list = triplestore.get(triple.key);
        if (list == null) list = new ArrayList<Triple>();
        Triple prev = addtriple(list, triple);
        triplestore.put(triple.key, list);
      }

    } catch (FileNotFoundException fe) {
      if (showlog) log.debug("Loading rc file: " + abspath);
      return false;

    } catch (IOException ioe) {
      if (showlog) log.error("File " + abspath + ": IO exception: " + ioe.getMessage());
      return false;
    }
    return true;
  }

  public Set<String> keySet() {
    return triplestore.keySet();
  }

  public List<Triple> getTriples(String key) {
    List<Triple> list = triplestore.get(key);
    if (list == null) list = new ArrayList<Triple>();
    return list;
  }

  public Triple lookup(String key) {
    return lookup(key, (URL) null);
  }

  public Triple lookup(String key, String url) {
    if (url == null || url.length() == 0) return lookup(key);
    try {
      URL u = new URL(url);
      return lookup(key, u);
    } catch (MalformedURLException m) {
    }
    return null;
  }

  public Triple lookup(String key, URL url) {
    List<Triple> list = triplestore.get(key);
    if (list == null) return null;
    if (url == null) {
      if (list.size() == 0) return null;
      return list.get(0);
    } else
      for (Triple t : list) {
        if (urlMatch(t.url, url)) return t;
      }
    return null;
  }

  Triple addtriple(List<Triple> list, Triple triple) {
    Triple prev = null;
    assert (list != null);
    // Look for duplicates
    int i = list.indexOf(triple);
    if (i >= 0) {
      prev = list.remove(i);
    }
    list.add(triple);
    Collections.sort(list);
    return prev;
  }

  // Allow for external loading
  public Triple insert(Triple t) {
    if (t.key == null) return null;
    List<Triple> list = triplestore.get(t.key);
    if (list == null) list = new ArrayList<Triple>();
    Triple prev = addtriple(list, t);
    triplestore.put(t.key, list);
    return prev;
  }

  // Output in .rc form
  public String toString() {
    StringBuilder rc = new StringBuilder();
    for (String key : keySet()) {
      List<Triple> list = getTriples(key);
      for (Triple triple : list) {
        String line = triple.toString();
        rc.append(line);
        rc.append("\n");
      }
    }
    return rc.toString();
  }
} // class RC
Exemplo n.º 11
0
/**
 * An IOServiceProvider for CINRAD level II files.
 *
 * @author caron
 */
public class Cinrad2IOServiceProvider extends AbstractIOServiceProvider {
  private static org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(Cinrad2IOServiceProvider.class);
  private static final int MISSING_INT = -9999;
  private static final float MISSING_FLOAT = Float.NaN;

  public boolean isValidFileOld(RandomAccessFile raf) {
    try {
      String loc = raf.getLocation();
      int posFirst = loc.lastIndexOf('/') + 1;
      if (posFirst < 0) posFirst = 0;
      String stationId = loc.substring(posFirst, posFirst + 4);
      NexradStationDB.init();
      NexradStationDB.Station station = NexradStationDB.get("K" + stationId);
      if (station != null) return true;
      else return false;
    } catch (IOException ioe) {
      return false;
    }
  }

  public boolean isValidFile(RandomAccessFile raf) {
    int data_msecs = 0;
    short data_julian_date = 0;

    try {
      raf.order(RandomAccessFile.LITTLE_ENDIAN);
      raf.seek(0);
      raf.skipBytes(14);
      short message_type = raf.readShort();
      if (message_type != 1) return false;

      raf.skipBytes(12);
      // data header
      byte[] b4 = raf.readBytes(4);
      data_msecs = bytesToInt(b4, true);
      byte[] b2 = raf.readBytes(2);
      data_julian_date = (short) bytesToShort(b2, true);
      java.util.Date dd = Cinrad2Record.getDate(data_julian_date, data_msecs);

      Calendar cal = new GregorianCalendar(new SimpleTimeZone(0, "GMT"));
      cal.clear();
      cal.setTime(dd);
      int year = cal.get(Calendar.YEAR);
      cal.setTime(new Date());
      int cyear = cal.get(Calendar.YEAR);
      if (year < 1990 || year > cyear) return false;
      return true;
    } catch (IOException ioe) {
      return false;
    }
  }

  public static int bytesToInt(byte[] bytes, boolean swapBytes) {
    byte a = bytes[0];
    byte b = bytes[1];
    byte c = bytes[2];
    byte d = bytes[3];
    if (swapBytes) {
      return ((a & 0xff)) + ((b & 0xff) << 8) + ((c & 0xff) << 16) + ((d & 0xff) << 24);
    } else {
      return ((a & 0xff) << 24) + ((b & 0xff) << 16) + ((c & 0xff) << 8) + ((d & 0xff));
    }
  }

  public static int bytesToShort(byte[] bytes, boolean swapBytes) {
    byte a = bytes[0];
    byte b = bytes[1];

    if (swapBytes) {
      return ((a & 0xff)) + ((b & 0xff) << 8);

    } else {
      return ((a & 0xff) << 24) + ((b & 0xff) << 16);
    }
  }

  public String getFileTypeId() {
    return "CINRAD";
  }

  public String getFileTypeDescription() {
    return "Chinese Level-II Base Data";
  }

  private Cinrad2VolumeScan volScan;
  private Dimension radialDim;
  private double radarRadius;
  private DateFormatter formatter = new DateFormatter();

  public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask)
      throws IOException {
    NexradStationDB.init();

    volScan = new Cinrad2VolumeScan(raf, cancelTask);
    if (volScan.hasDifferentDopplarResolutions())
      throw new IllegalStateException("volScan.hasDifferentDopplarResolutions");

    radialDim = new Dimension("radial", volScan.getMaxRadials());
    ncfile.addDimension(null, radialDim);

    makeVariable(
        ncfile,
        Cinrad2Record.REFLECTIVITY,
        "Reflectivity",
        "Reflectivity",
        "R",
        volScan.getReflectivityGroups());
    int velocity_type =
        (volScan.getDopplarResolution() == Cinrad2Record.DOPPLER_RESOLUTION_HIGH_CODE)
            ? Cinrad2Record.VELOCITY_HI
            : Cinrad2Record.VELOCITY_LOW;
    Variable v =
        makeVariable(
            ncfile,
            velocity_type,
            "RadialVelocity",
            "Radial Velocity",
            "V",
            volScan.getVelocityGroups());
    makeVariableNoCoords(
        ncfile, Cinrad2Record.SPECTRUM_WIDTH, "SpectrumWidth", "Spectrum Width", v);

    if (volScan.getStationId() != null) {
      ncfile.addAttribute(null, new Attribute("Station", volScan.getStationId()));
      ncfile.addAttribute(null, new Attribute("StationName", volScan.getStationName()));
      ncfile.addAttribute(
          null, new Attribute("StationLatitude", new Double(volScan.getStationLatitude())));
      ncfile.addAttribute(
          null, new Attribute("StationLongitude", new Double(volScan.getStationLongitude())));
      ncfile.addAttribute(
          null,
          new Attribute("StationElevationInMeters", new Double(volScan.getStationElevation())));

      double latRadiusDegrees = Math.toDegrees(radarRadius / ucar.unidata.geoloc.Earth.getRadius());
      ncfile.addAttribute(
          null,
          new Attribute(
              "geospatial_lat_min", new Double(volScan.getStationLatitude() - latRadiusDegrees)));
      ncfile.addAttribute(
          null,
          new Attribute(
              "geospatial_lat_max", new Double(volScan.getStationLatitude() + latRadiusDegrees)));
      double cosLat = Math.cos(Math.toRadians(volScan.getStationLatitude()));
      double lonRadiusDegrees =
          Math.toDegrees(radarRadius / cosLat / ucar.unidata.geoloc.Earth.getRadius());
      ncfile.addAttribute(
          null,
          new Attribute(
              "geospatial_lon_min", new Double(volScan.getStationLongitude() - lonRadiusDegrees)));
      ncfile.addAttribute(
          null,
          new Attribute(
              "geospatial_lon_max", new Double(volScan.getStationLongitude() + lonRadiusDegrees)));

      // add a radial coordinate transform (experimental)
      Variable ct = new Variable(ncfile, null, null, "radialCoordinateTransform");
      ct.setDataType(DataType.CHAR);
      ct.setDimensions(""); // scalar
      ct.addAttribute(new Attribute("transform_name", "Radial"));
      ct.addAttribute(new Attribute("center_latitude", new Double(volScan.getStationLatitude())));
      ct.addAttribute(new Attribute("center_longitude", new Double(volScan.getStationLongitude())));
      ct.addAttribute(new Attribute("center_elevation", new Double(volScan.getStationElevation())));
      ct.addAttribute(new Attribute(_Coordinate.TransformType, "Radial"));
      ct.addAttribute(
          new Attribute(_Coordinate.AxisTypes, "RadialElevation RadialAzimuth RadialDistance"));

      Array data =
          Array.factory(DataType.CHAR.getPrimitiveClassType(), new int[0], new char[] {' '});
      ct.setCachedData(data, true);
      ncfile.addVariable(null, ct);
    }

    DateFormatter formatter = new DateFormatter();

    ncfile.addAttribute(null, new Attribute(CDM.CONVENTIONS, _Coordinate.Convention));
    ncfile.addAttribute(null, new Attribute("format", volScan.getDataFormat()));
    ncfile.addAttribute(null, new Attribute(CF.FEATURE_TYPE, FeatureType.RADIAL.toString()));
    // Date d = Cinrad2Record.getDate(volScan.getTitleJulianDays(), volScan.getTitleMsecs());
    // ncfile.addAttribute(null, new Attribute("base_date", formatter.toDateOnlyString(d)));

    ncfile.addAttribute(
        null,
        new Attribute(
            "time_coverage_start", formatter.toDateTimeStringISO(volScan.getStartDate())));
    ; // .toDateTimeStringISO(d)));
    ncfile.addAttribute(
        null,
        new Attribute("time_coverage_end", formatter.toDateTimeStringISO(volScan.getEndDate())));

    ncfile.addAttribute(
        null,
        new Attribute(CDM.HISTORY, "Direct read of Nexrad Level 2 file into NetCDF-Java 2.2 API"));
    ncfile.addAttribute(null, new Attribute("DataType", "Radial"));

    ncfile.addAttribute(
        null,
        new Attribute(
            "Title",
            "Nexrad Level 2 Station "
                + volScan.getStationId()
                + " from "
                + formatter.toDateTimeStringISO(volScan.getStartDate())
                + " to "
                + formatter.toDateTimeStringISO(volScan.getEndDate())));

    ncfile.addAttribute(
        null,
        new Attribute(
            "Summary",
            "Weather Surveillance Radar-1988 Doppler (WSR-88D) "
                + "Level II data are the three meteorological base data quantities: reflectivity, mean radial velocity, and "
                + "spectrum width."));

    ncfile.addAttribute(
        null,
        new Attribute(
            "keywords",
            "WSR-88D; NEXRAD; Radar Level II; reflectivity; mean radial velocity; spectrum width"));

    ncfile.addAttribute(
        null,
        new Attribute(
            "VolumeCoveragePatternName",
            Cinrad2Record.getVolumeCoveragePatternName(volScan.getVCP())));
    ncfile.addAttribute(
        null, new Attribute("VolumeCoveragePattern", new Integer(volScan.getVCP())));
    ncfile.addAttribute(
        null,
        new Attribute(
            "HorizonatalBeamWidthInDegrees", new Double(Cinrad2Record.HORIZONTAL_BEAM_WIDTH)));

    ncfile.finish();
  }

  public Variable makeVariable(
      NetcdfFile ncfile,
      int datatype,
      String shortName,
      String longName,
      String abbrev,
      List groups)
      throws IOException {
    int nscans = groups.size();

    if (nscans == 0) {
      throw new IllegalStateException("No data for " + shortName);
    }

    // get representative record
    List firstGroup = (List) groups.get(0);
    Cinrad2Record firstRecord = (Cinrad2Record) firstGroup.get(0);
    int ngates = firstRecord.getGateCount(datatype);

    String scanDimName = "scan" + abbrev;
    String gateDimName = "gate" + abbrev;
    Dimension scanDim = new Dimension(scanDimName, nscans);
    Dimension gateDim = new Dimension(gateDimName, ngates);
    ncfile.addDimension(null, scanDim);
    ncfile.addDimension(null, gateDim);

    ArrayList dims = new ArrayList();
    dims.add(scanDim);
    dims.add(radialDim);
    dims.add(gateDim);

    Variable v = new Variable(ncfile, null, null, shortName);
    v.setDataType(DataType.BYTE);
    v.setDimensions(dims);
    ncfile.addVariable(null, v);

    v.addAttribute(new Attribute(CDM.UNITS, Cinrad2Record.getDatatypeUnits(datatype)));
    v.addAttribute(new Attribute(CDM.LONG_NAME, longName));

    byte[] b = new byte[2];
    b[0] = Cinrad2Record.MISSING_DATA;
    b[1] = Cinrad2Record.BELOW_THRESHOLD;
    Array missingArray = Array.factory(DataType.BYTE.getPrimitiveClassType(), new int[] {2}, b);

    v.addAttribute(new Attribute(CDM.MISSING_VALUE, missingArray));
    v.addAttribute(
        new Attribute("signal_below_threshold", new Byte(Cinrad2Record.BELOW_THRESHOLD)));
    v.addAttribute(
        new Attribute(CDM.SCALE_FACTOR, new Float(Cinrad2Record.getDatatypeScaleFactor(datatype))));
    v.addAttribute(
        new Attribute(CDM.ADD_OFFSET, new Float(Cinrad2Record.getDatatypeAddOffset(datatype))));
    v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));

    ArrayList dim2 = new ArrayList();
    dim2.add(scanDim);
    dim2.add(radialDim);

    // add time coordinate variable
    String timeCoordName = "time" + abbrev;
    Variable timeVar = new Variable(ncfile, null, null, timeCoordName);
    timeVar.setDataType(DataType.INT);
    timeVar.setDimensions(dim2);
    ncfile.addVariable(null, timeVar);

    // int julianDays = volScan.getTitleJulianDays();
    // Date d = Cinrad2Record.getDate( julianDays, 0);
    // Date d = Cinrad2Record.getDate(volScan.getTitleJulianDays(), volScan.getTitleMsecs());
    Date d = volScan.getStartDate();
    String units = "msecs since " + formatter.toDateTimeStringISO(d);

    timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "time since base date"));
    timeVar.addAttribute(new Attribute(CDM.UNITS, units));
    timeVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Integer(MISSING_INT)));
    timeVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Time.toString()));

    // add elevation coordinate variable
    String elevCoordName = "elevation" + abbrev;
    Variable elevVar = new Variable(ncfile, null, null, elevCoordName);
    elevVar.setDataType(DataType.FLOAT);
    elevVar.setDimensions(dim2);
    ncfile.addVariable(null, elevVar);

    elevVar.addAttribute(new Attribute(CDM.UNITS, "degrees"));
    elevVar.addAttribute(
        new Attribute(
            CDM.LONG_NAME,
            "elevation angle in degres: 0 = parallel to pedestal base, 90 = perpendicular"));
    elevVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(MISSING_FLOAT)));
    elevVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialElevation.toString()));

    // add azimuth coordinate variable
    String aziCoordName = "azimuth" + abbrev;
    Variable aziVar = new Variable(ncfile, null, null, aziCoordName);
    aziVar.setDataType(DataType.FLOAT);
    aziVar.setDimensions(dim2);
    ncfile.addVariable(null, aziVar);

    aziVar.addAttribute(new Attribute(CDM.UNITS, "degrees"));
    aziVar.addAttribute(
        new Attribute(CDM.LONG_NAME, "azimuth angle in degrees: 0 = true north, 90 = east"));
    aziVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(MISSING_FLOAT)));
    aziVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialAzimuth.toString()));

    // add gate coordinate variable
    String gateCoordName = "distance" + abbrev;
    Variable gateVar = new Variable(ncfile, null, null, gateCoordName);
    gateVar.setDataType(DataType.FLOAT);
    gateVar.setDimensions(gateDimName);
    Array data =
        Array.makeArray(
            DataType.FLOAT,
            ngates,
            (double) firstRecord.getGateStart(datatype),
            (double) firstRecord.getGateSize(datatype));
    gateVar.setCachedData(data, false);
    ncfile.addVariable(null, gateVar);
    radarRadius = firstRecord.getGateStart(datatype) + ngates * firstRecord.getGateSize(datatype);

    gateVar.addAttribute(new Attribute(CDM.UNITS, "m"));
    gateVar.addAttribute(new Attribute(CDM.LONG_NAME, "radial distance to start of gate"));
    gateVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialDistance.toString()));

    // add number of radials variable
    String nradialsName = "numRadials" + abbrev;
    Variable nradialsVar = new Variable(ncfile, null, null, nradialsName);
    nradialsVar.setDataType(DataType.INT);
    nradialsVar.setDimensions(scanDim.getName());
    nradialsVar.addAttribute(new Attribute(CDM.LONG_NAME, "number of valid radials in this scan"));
    ncfile.addVariable(null, nradialsVar);

    // add number of gates variable
    String ngateName = "numGates" + abbrev;
    Variable ngateVar = new Variable(ncfile, null, null, ngateName);
    ngateVar.setDataType(DataType.INT);
    ngateVar.setDimensions(scanDim.getName());
    ngateVar.addAttribute(new Attribute(CDM.LONG_NAME, "number of valid gates in this scan"));
    ncfile.addVariable(null, ngateVar);

    makeCoordinateDataWithMissing(
        datatype, timeVar, elevVar, aziVar, nradialsVar, ngateVar, groups);

    // back to the data variable
    String coordinates =
        timeCoordName + " " + elevCoordName + " " + aziCoordName + " " + gateCoordName;
    v.addAttribute(new Attribute(_Coordinate.Axes, coordinates));

    // make the record map
    int nradials = radialDim.getLength();
    Cinrad2Record[][] map = new Cinrad2Record[nscans][nradials];
    for (int i = 0; i < groups.size(); i++) {
      Cinrad2Record[] mapScan = map[i];
      List group = (List) groups.get(i);
      for (int j = 0; j < group.size(); j++) {
        Cinrad2Record r = (Cinrad2Record) group.get(j);
        int radial = r.radial_num - 1;
        mapScan[radial] = r;
      }
    }

    Vgroup vg = new Vgroup(datatype, map);
    v.setSPobject(vg);

    return v;
  }

  private void makeVariableNoCoords(
      NetcdfFile ncfile, int datatype, String shortName, String longName, Variable from) {

    Variable v = new Variable(ncfile, null, null, shortName);
    v.setDataType(DataType.BYTE);
    v.setDimensions(from.getDimensions());
    ncfile.addVariable(null, v);

    v.addAttribute(new Attribute(CDM.UNITS, Cinrad2Record.getDatatypeUnits(datatype)));
    v.addAttribute(new Attribute(CDM.LONG_NAME, longName));

    byte[] b = new byte[2];
    b[0] = Cinrad2Record.MISSING_DATA;
    b[1] = Cinrad2Record.BELOW_THRESHOLD;
    Array missingArray = Array.factory(DataType.BYTE.getPrimitiveClassType(), new int[] {2}, b);

    v.addAttribute(new Attribute(CDM.MISSING_VALUE, missingArray));
    v.addAttribute(
        new Attribute("signal_below_threshold", new Byte(Cinrad2Record.BELOW_THRESHOLD)));
    v.addAttribute(
        new Attribute(CDM.SCALE_FACTOR, new Float(Cinrad2Record.getDatatypeScaleFactor(datatype))));
    v.addAttribute(
        new Attribute(CDM.ADD_OFFSET, new Float(Cinrad2Record.getDatatypeAddOffset(datatype))));
    v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));

    Attribute fromAtt = from.findAttribute(_Coordinate.Axes);
    v.addAttribute(new Attribute(_Coordinate.Axes, fromAtt));

    Vgroup vgFrom = (Vgroup) from.getSPobject();
    Vgroup vg = new Vgroup(datatype, vgFrom.map);
    v.setSPobject(vg);
  }

  private void makeCoordinateData(
      int datatype,
      Variable time,
      Variable elev,
      Variable azi,
      Variable nradialsVar,
      Variable ngatesVar,
      List groups) {

    Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape());
    IndexIterator timeDataIter = timeData.getIndexIterator();

    Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape());
    IndexIterator elevDataIter = elevData.getIndexIterator();

    Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape());
    IndexIterator aziDataIter = aziData.getIndexIterator();

    Array nradialsData =
        Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape());
    IndexIterator nradialsIter = nradialsData.getIndexIterator();

    Array ngatesData =
        Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape());
    IndexIterator ngatesIter = ngatesData.getIndexIterator();

    int last_msecs = Integer.MIN_VALUE;
    int nscans = groups.size();
    int maxRadials = volScan.getMaxRadials();
    for (int i = 0; i < nscans; i++) {
      List scanGroup = (List) groups.get(i);
      int nradials = scanGroup.size();

      Cinrad2Record first = null;
      for (int j = 0; j < nradials; j++) {
        Cinrad2Record r = (Cinrad2Record) scanGroup.get(j);
        if (first == null) first = r;

        timeDataIter.setIntNext(r.data_msecs);
        elevDataIter.setFloatNext(r.getElevation());
        aziDataIter.setFloatNext(r.getAzimuth());

        if (r.data_msecs < last_msecs)
          logger.warn("makeCoordinateData time out of order " + r.data_msecs);
        last_msecs = r.data_msecs;
      }

      for (int j = nradials; j < maxRadials; j++) {
        timeDataIter.setIntNext(MISSING_INT);
        elevDataIter.setFloatNext(MISSING_FLOAT);
        aziDataIter.setFloatNext(MISSING_FLOAT);
      }

      nradialsIter.setIntNext(nradials);
      ngatesIter.setIntNext(first.getGateCount(datatype));
    }

    time.setCachedData(timeData, false);
    elev.setCachedData(elevData, false);
    azi.setCachedData(aziData, false);
    nradialsVar.setCachedData(nradialsData, false);
    ngatesVar.setCachedData(ngatesData, false);
  }

  private void makeCoordinateDataWithMissing(
      int datatype,
      Variable time,
      Variable elev,
      Variable azi,
      Variable nradialsVar,
      Variable ngatesVar,
      List groups) {

    Array timeData = Array.factory(time.getDataType().getPrimitiveClassType(), time.getShape());
    Index timeIndex = timeData.getIndex();

    Array elevData = Array.factory(elev.getDataType().getPrimitiveClassType(), elev.getShape());
    Index elevIndex = elevData.getIndex();

    Array aziData = Array.factory(azi.getDataType().getPrimitiveClassType(), azi.getShape());
    Index aziIndex = aziData.getIndex();

    Array nradialsData =
        Array.factory(nradialsVar.getDataType().getPrimitiveClassType(), nradialsVar.getShape());
    IndexIterator nradialsIter = nradialsData.getIndexIterator();

    Array ngatesData =
        Array.factory(ngatesVar.getDataType().getPrimitiveClassType(), ngatesVar.getShape());
    IndexIterator ngatesIter = ngatesData.getIndexIterator();

    // first fill with missing data
    IndexIterator ii = timeData.getIndexIterator();
    while (ii.hasNext()) ii.setIntNext(MISSING_INT);

    ii = elevData.getIndexIterator();
    while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT);

    ii = aziData.getIndexIterator();
    while (ii.hasNext()) ii.setFloatNext(MISSING_FLOAT);

    // now set the  coordinate variables from the Cinrad2Record radial
    int last_msecs = Integer.MIN_VALUE;
    int nscans = groups.size();
    try {
      for (int scan = 0; scan < nscans; scan++) {
        List scanGroup = (List) groups.get(scan);
        int nradials = scanGroup.size();

        Cinrad2Record first = null;
        for (int j = 0; j < nradials; j++) {
          Cinrad2Record r = (Cinrad2Record) scanGroup.get(j);
          if (first == null) first = r;

          int radial = r.radial_num - 1;
          timeData.setInt(timeIndex.set(scan, radial), r.data_msecs);
          elevData.setFloat(elevIndex.set(scan, radial), r.getElevation());
          aziData.setFloat(aziIndex.set(scan, radial), r.getAzimuth());

          if (r.data_msecs < last_msecs)
            logger.warn("makeCoordinateData time out of order " + r.data_msecs);
          last_msecs = r.data_msecs;
        }

        nradialsIter.setIntNext(nradials);
        ngatesIter.setIntNext(first.getGateCount(datatype));
      }
    } catch (java.lang.ArrayIndexOutOfBoundsException ae) {
      logger.debug("Cinrad2IOSP.uncompress ", ae);
    }
    time.setCachedData(timeData, false);
    elev.setCachedData(elevData, false);
    azi.setCachedData(aziData, false);
    nradialsVar.setCachedData(nradialsData, false);
    ngatesVar.setCachedData(ngatesData, false);
  }

  public Array readData(Variable v2, Section section) throws IOException, InvalidRangeException {
    Vgroup vgroup = (Vgroup) v2.getSPobject();

    Range scanRange = section.getRange(0);
    Range radialRange = section.getRange(1);
    Range gateRange = section.getRange(2);

    Array data = Array.factory(v2.getDataType().getPrimitiveClassType(), section.getShape());
    IndexIterator ii = data.getIndexIterator();

    for (int i = scanRange.first(); i <= scanRange.last(); i += scanRange.stride()) {
      Cinrad2Record[] mapScan = vgroup.map[i];
      readOneScan(mapScan, radialRange, gateRange, vgroup.datatype, ii);
    }

    return data;
  }

  private void readOneScan(
      Cinrad2Record[] mapScan, Range radialRange, Range gateRange, int datatype, IndexIterator ii)
      throws IOException {
    for (int i = radialRange.first(); i <= radialRange.last(); i += radialRange.stride()) {
      Cinrad2Record r = mapScan[i];
      readOneRadial(r, datatype, gateRange, ii);
    }
  }

  private void readOneRadial(Cinrad2Record r, int datatype, Range gateRange, IndexIterator ii)
      throws IOException {
    if (r == null) {
      for (int i = gateRange.first(); i <= gateRange.last(); i += gateRange.stride())
        ii.setByteNext(Cinrad2Record.MISSING_DATA);
      return;
    }
    r.readData(volScan.raf, datatype, gateRange, ii);
  }

  private class Vgroup {
    Cinrad2Record[][] map;
    int datatype;

    Vgroup(int datatype, Cinrad2Record[][] map) {
      this.datatype = datatype;
      this.map = map;
    }
  }

  /////////////////////////////////////////////////////////////////////

  public void close() throws IOException {
    volScan.raf.close();
  }
}
Exemplo n.º 12
0
/**
 * A session is encapsulated in an instance of the class HTTPSession. The encapsulation is with
 * respect to a specific HttpHost "realm", where the important part is is host+port. This means that
 * once a session is specified, it is tied permanently to that realm.
 *
 * <p>A Session encapsulate a number of other objects:
 *
 * <ul>
 *   <li>An instance of an Apache HttpClient.
 *   <li>A http session id
 *   <li>A RequestContext object; this also includes authentication: specifically a credential and a
 *       credentials provider.
 * </ul>
 *
 * <p>Currently, it is assumed that only one set of credentials is needed, whether directly for
 * server X or for server Y. This may change in the future.
 *
 * <p>As a rule, if the client gives an HTTPSession object to the "create method" procedures of
 * HTTPFactory (e.g. HTTPFactory.Get or HTTPFactory.Post) then that creation call must specify a url
 * that is "compatible" with the scope of the session. The method url is <it>compatible</i> if its
 * host+port is the same as the session's host+port (=scope) and its scheme is compatible, where
 * e.g. http is compatible with https (see HTTPAuthUtil.httphostCompatible)
 *
 * <p>If the HTTPFactory method creation call does not specify a session object, then one is created
 * (and destroyed) behind the scenes along with the method.
 *
 * <p>Note that the term legalurl in the following code means that the url has reserved characters
 * within identifieers in escaped form. This is particularly and issue for queries. Especially:
 * ?x[0:5] is legal and the square brackets need not be encoded.
 *
 * <p>As of the move to Apache Httpclient 4.4 and later, the underlying HttpClient objects are
 * generally immutable. This means that at least this class (HTTPSession) and the HTTPMethod class
 * must store the relevant info and create the HttpClient and HttpMethod objects dynamically. This
 * also means that when a parameter is changed (Agent, for example), any existing cached HttpClient
 * must be thrown away and reconstructed using the change. As a rule, the HttpClient object will be
 * created at the last minute so that multiple parameter changes can be effected without have to
 * re-create the HttpClient for each parameter change. Also note that the immutable objects will be
 * cached and reused if no parameters are changed.
 *
 * <p><em>Authorization</em> We assume that the session supports two CredentialsProvider instances:
 * one global to all HTTPSession objects and one specific to each HTTPSession object.
 *
 * <p>As an aside, authentication is a bit tricky because some authorization schemes use
 * redirection. That is, the initial request is made to server X, but X says: goto to server Y" to
 * get, say, and authorization token. Then Y says: return to X with this token and proceed.
 *
 * <p><em>SSL</em> TBD.
 */
public class HTTPSession implements Closeable {
  //////////////////////////////////////////////////
  // Constants

  // Define all the legal properties
  // Previously taken from class AllClientPNames, but that is now
  // deprecated, so just use an enum

  static /*package*/ enum Prop {
    ALLOW_CIRCULAR_REDIRECTS,
    HANDLE_REDIRECTS,
    HANDLE_AUTHENTICATION,
    MAX_REDIRECTS,
    MAX_THREADS,
    SO_TIMEOUT,
    CONN_TIMEOUT,
    CONN_REQ_TIMEOUT,
    USER_AGENT,
    COOKIE_STORE,
    RETRIES,
    UNAVAILRETRIES,
    COMPRESSION,
    CREDENTIALS,
    USESESSIONS,
  }

  // Header names
  // from: http://en.wikipedia.org/wiki/List_of_HTTP_header_fields
  public static final String HEADER_USERAGENT = "User-Agent";
  public static final String ACCEPT_ENCODING = "Accept-Encoding";

  static final int DFALTTHREADCOUNT = 50;
  static final int DFALTREDIRECTS = 25;
  static final int DFALTCONNTIMEOUT = 1 * 60 * 1000; // 1 minutes (60000 milliseconds)
  static final int DFALTCONNREQTIMEOUT = DFALTCONNTIMEOUT;
  static final int DFALTSOTIMEOUT = 5 * 60 * 1000; // 5 minutes (300000 milliseconds)

  static final int DFALTRETRIES = 3;
  static final int DFALTUNAVAILRETRIES = 3;
  static final int DFALTUNAVAILINTERVAL = 3000; // 3 seconds
  static final String DFALTUSERAGENT = "/NetcdfJava/HttpClient4.4";

  static final String[] KNOWNCOMPRESSORS = {"gzip", "deflate"};

  //////////////////////////////////////////////////////////////////////////

  static final boolean IGNORECERTS = false;

  //////////////////////////////////////////////////////////////////////////
  // Type Declaration(s)

  // Support loose certificate acceptance
  static class LooseTrustStrategy extends TrustSelfSignedStrategy {
    @Override
    public boolean isTrusted(final X509Certificate[] chain, String authType)
        throws CertificateException {
      try {
        if (super.isTrusted(chain, authType)) return true;
        // check expiration dates
        for (X509Certificate x5 : chain) {
          try {
            x5.checkValidity();
          } catch (CertificateExpiredException | CertificateNotYetValidException ce) {
            return true;
          }
        }
      } catch (CertificateException e) {
        return true; // temporary
      }
      return false;
    }
  }

  /** Sub-class HashTable<String,Object> for mnemonic convenience and for synchronized access. */
  static class Settings extends Hashtable<Prop, Object> {
    public Settings() {}

    public Set<Prop> getKeys() {
      return keySet();
    }

    public Object getParameter(Prop param) {
      return super.get(param);
    }

    public long getIntParameter(Prop param) {
      return (Long) super.get(param);
    }

    public Settings setParameter(Prop param, Object value) {
      super.put(param, value);
      return this;
    }

    public Object removeParameter(Prop param) {
      return super.remove(param);
    }
  }

  // For communication between HTTPSession.execute and HTTPMethod.execute.
  static /*package*/ class ExecState {
    public HttpRequestBase request = null;
    public CloseableHttpResponse response = null;
  }

  static /*package*/ enum Methods {
    Get("get"),
    Head("head"),
    Put("put"),
    Post("post"),
    Options("options");
    private final String name;

    Methods(String name) {
      this.name = name;
    }

    public String getName() {
      return name;
    }
  }

  static class GZIPResponseInterceptor implements HttpResponseInterceptor {
    public void process(final HttpResponse response, final HttpContext context)
        throws HttpException, IOException {
      HttpEntity entity = response.getEntity();
      if (entity != null) {
        Header ceheader = entity.getContentEncoding();
        if (ceheader != null) {
          HeaderElement[] codecs = ceheader.getElements();
          for (HeaderElement h : codecs) {
            if (h.getName().equalsIgnoreCase("gzip")) {
              response.setEntity(new GzipDecompressingEntity(response.getEntity()));
              return;
            }
          }
        }
      }
    }
  }

  static class DeflateResponseInterceptor implements HttpResponseInterceptor {
    public void process(final HttpResponse response, final HttpContext context)
        throws HttpException, IOException {
      HttpEntity entity = response.getEntity();
      if (entity != null) {
        Header ceheader = entity.getContentEncoding();
        if (ceheader != null) {
          HeaderElement[] codecs = ceheader.getElements();
          for (HeaderElement h : codecs) {
            if (h.getName().equalsIgnoreCase("deflate")) {
              response.setEntity(new DeflateDecompressingEntity(response.getEntity()));
              return;
            }
          }
        }
      }
    }
  }

  static class ZipStreamFactory implements InputStreamFactory {
    // InputStreamFactory methods
    @Override
    public InputStream create(InputStream instream) throws IOException {
      return new ZipInputStream(instream, HTTPUtil.UTF8);
    }
  }

  static class GZIPStreamFactory implements InputStreamFactory {
    // InputStreamFactory methods
    @Override
    public InputStream create(InputStream instream) throws IOException {
      return new GZIPInputStream(instream);
    }
  }

  ////////////////////////////////////////////////////////////////////////
  // Static variables

  public static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(HTTPSession.class);

  protected static PoolingHttpClientConnectionManager connmgr;

  // Define a settings object to hold all the
  // settable values; there will be one
  // instance for global and one for local.

  static Settings globalsettings;

  // Define interceptor instances
  static List<HttpRequestInterceptor> reqintercepts = new ArrayList<HttpRequestInterceptor>();
  static List<HttpResponseInterceptor> rspintercepts = new ArrayList<HttpResponseInterceptor>();
  // This is a hack to suppress content-encoding headers from request
  protected static HttpResponseInterceptor CEKILL;
  // Debug Header interceptors
  protected static List<HttpRequestInterceptor> dbgreq = new ArrayList<>();
  protected static List<HttpResponseInterceptor> dbgrsp = new ArrayList<>();

  protected static Map<String, InputStreamFactory> contentDecoderMap;

  // public final HttpClientBuilder setContentDecoderRegistry(Map<String,InputStreamFactory>
  // contentDecoderMap)

  // Since can't access CredentialsProvider map, mimic
  protected static Map<AuthScope, CredentialsProvider> globalcreds = new HashMap<>();

  // As taken from the command line, usually
  protected static KeyStore keystore = null;
  protected static KeyStore truststore = null;
  protected static String keypath = null;
  protected static String keypassword = null;
  protected static String trustpath = null;
  protected static String trustpassword = null;
  protected static SSLConnectionSocketFactory globalsslfactory = null;
  protected static Registry<ConnectionSocketFactory> sslregistry = null;

  protected static HttpHost httpproxy = null;
  protected static HttpHost httpsproxy = null;
  protected static String proxyuser = null;
  protected static String proxypwd = null;

  // For debugging
  protected static Boolean globaldebugheaders = null;

  static { // watch out: order is important for these initializers
    CEKILL = new HTTPUtil.ContentEncodingInterceptor();
    contentDecoderMap = new HashMap<String, InputStreamFactory>();
    contentDecoderMap.put("zip", new ZipStreamFactory());
    contentDecoderMap.put("gzip", new GZIPStreamFactory());
    globalsettings = new Settings();
    setDefaults(globalsettings);
    processDFlags(); // Process all -D flags
    connmgr = new PoolingHttpClientConnectionManager(sslregistry);
    setGlobalUserAgent(DFALTUSERAGENT);
    // does not work setGlobalThreadCount(DFALTTHREADCOUNT);
    setGlobalConnectionTimeout(DFALTCONNTIMEOUT);
    setGlobalSoTimeout(DFALTSOTIMEOUT);
  }

  //////////////////////////////////////////////////////////////////////////
  // Static Initialization

  // Provide defaults for a settings map
  protected static void setDefaults(Settings props) {
    if (false) { // turn off for now
      props.setParameter(Prop.HANDLE_AUTHENTICATION, Boolean.TRUE);
    }
    props.setParameter(Prop.HANDLE_REDIRECTS, Boolean.TRUE);
    props.setParameter(Prop.ALLOW_CIRCULAR_REDIRECTS, Boolean.TRUE);
    props.setParameter(Prop.MAX_REDIRECTS, (Integer) DFALTREDIRECTS);
    props.setParameter(Prop.SO_TIMEOUT, (Integer) DFALTSOTIMEOUT);
    props.setParameter(Prop.CONN_TIMEOUT, (Integer) DFALTCONNTIMEOUT);
    props.setParameter(Prop.CONN_REQ_TIMEOUT, (Integer) DFALTCONNREQTIMEOUT);
    props.setParameter(Prop.USER_AGENT, DFALTUSERAGENT);
  }

  static synchronized void processDFlags() {
    // SSL flags
    keypath = cleanproperty("keystore");
    keypassword = cleanproperty("keystorepassword");
    trustpath = cleanproperty("truststore");
    trustpassword = cleanproperty("truststorepassword");
    setGlobalSSLAuth(keypath, keypassword, trustpath, trustpassword);

    // Proxy flags
    String proxyurl = cleanproperty("proxyurl");
    if (proxyurl != null) setGlobalProxy(proxyurl);
    else { // Check the java.net flags
      String proxyhost = cleanproperty("https.proxyHost");
      if (proxyhost != null) {
        StringBuilder buf = new StringBuilder();
        buf.append("https://");
        buf.append(proxyhost);
        String proxyport = cleanproperty("https.proxyPort");
        if (proxyport != null) {
          buf.append(":");
          buf.append(proxyport);
        }
        setGlobalProxy(buf.toString());
      }
    }
  }

  //////////////////////////////////////////////////////////////////////////
  // Static Methods (Mostly global accessors)

  public static synchronized void setGlobalUserAgent(String userAgent) {
    globalsettings.setParameter(Prop.USER_AGENT, userAgent);
  }

  public static synchronized String getGlobalUserAgent() {
    return (String) globalsettings.getParameter(Prop.USER_AGENT);
  }

  public static synchronized void setGlobalThreadCount(int nthreads) {
    // globalsettings.setParameter(Prop.MAX_THREADS,nthreads);
    throw new UnsupportedOperationException(
        "HTTPSession.setGlobalThreadCount is currently not working");
  }

  // Alias
  public static void setGlobalMaxConnections(int nthreads) {
    setGlobalThreadCount(nthreads);
  }

  public static synchronized int getGlobalThreadCount() {
    return connmgr.getMaxTotal();
  }

  // Timeouts

  public static synchronized void setGlobalConnectionTimeout(int timeout) {
    if (timeout >= 0) {
      globalsettings.setParameter(Prop.CONN_TIMEOUT, (Integer) timeout);
      globalsettings.setParameter(Prop.CONN_REQ_TIMEOUT, (Integer) timeout);
    }
  }

  public static synchronized void setGlobalSoTimeout(int timeout) {
    if (timeout >= 0) globalsettings.setParameter(Prop.SO_TIMEOUT, (Integer) timeout);
  }

  /** Enable/disable redirection following Default is yes. */
  public static synchronized void setGlobalFollowRedirects(boolean tf) {
    globalsettings.setParameter(Prop.HANDLE_REDIRECTS, (Boolean) tf);
  }

  /**
   * Set the max number of redirects to follow
   *
   * @param n
   */
  public static synchronized void setGlobalMaxRedirects(int n) {
    if (n < 0) // validate
    throw new IllegalArgumentException("setMaxRedirects");
    globalsettings.setParameter(Prop.MAX_REDIRECTS, n);
  }

  public static synchronized Object getGlobalSetting(String key) {
    return globalsettings.get(key);
  }

  //////////////////////////////////////////////////
  // Compression

  public static synchronized void setGlobalCompression(String compressors) {
    if (globalsettings.getParameter(Prop.COMPRESSION) != null) removeGlobalCompression();
    String compresslist = checkCompressors(compressors);
    if (HTTPUtil.nullify(compresslist) == null)
      throw new IllegalArgumentException("Bad compressors: " + compressors);
    globalsettings.setParameter(Prop.COMPRESSION, compresslist);
    HttpResponseInterceptor hrsi;
    if (compresslist.contains("gzip")) {
      hrsi = new GZIPResponseInterceptor();
      rspintercepts.add(hrsi);
    }
    if (compresslist.contains("deflate")) {
      hrsi = new DeflateResponseInterceptor();
      rspintercepts.add(hrsi);
    }
  }

  public static void removeGlobalCompression() {
    if (globalsettings.removeParameter(Prop.COMPRESSION) != null) {
      for (int i = rspintercepts.size() - 1; i >= 0; i--) { // walk backwards
        HttpResponseInterceptor hrsi = rspintercepts.get(i);
        if (hrsi instanceof GZIPResponseInterceptor || hrsi instanceof DeflateResponseInterceptor)
          rspintercepts.remove(i);
      }
    }
  }

  protected static synchronized String checkCompressors(String compressors) {
    // Syntactic check of compressors
    Set<String> cset = new HashSet<>();
    compressors = compressors.replace(',', ' ');
    compressors = compressors.replace('\t', ' ');
    String[] pieces = compressors.split("[ ]+");
    for (String p : pieces) {
      for (String c : KNOWNCOMPRESSORS) {
        if (p.equalsIgnoreCase(c)) {
          cset.add(c);
          break;
        }
      }
    }
    StringBuilder buf = new StringBuilder();
    for (String s : cset) {
      if (buf.length() > 0) buf.append(",");
      buf.append(s);
    }
    return buf.toString();
  }
  //////////////////////////////////////////////////
  // Authorization

  /**
   * @param provider
   * @throws HTTPException
   */
  public static void setGlobalCredentialsProvider(CredentialsProvider provider)
      throws HTTPException {
    setGlobalCredentialsProvider(provider, (AuthScope) null);
  }

  /**
   * This is the most general case
   *
   * @param provider the credentials provider
   * @param scope where to use it (i.e. on what host)
   * @throws HTTPException
   */
  public static void setGlobalCredentialsProvider(CredentialsProvider provider, AuthScope scope)
      throws HTTPException {
    mapcreds(provider, scope, globalcreds);
  }

  /**
   * It is convenient to be able to directly set the Credentials (not the provider) when those
   * credentials are fixed. Scope defaults to ANY
   *
   * @param creds
   * @throws HTTPException
   */
  public static void setGlobalCredentials(Credentials creds) throws HTTPException {
    setGlobalCredentials(creds, null);
  }

  /**
   * It is convenient to be able to directly set the Credentials (not the provider) when those
   * credentials are fixed.
   *
   * @param creds
   * @param scope where to use it (i.e. on what host)
   * @throws HTTPException
   */
  public static void setGlobalCredentials(Credentials creds, AuthScope scope) throws HTTPException {
    assert (creds != null);
    if (scope == null) scope = AuthScope.ANY;
    CredentialsProvider provider = new BasicCredentialsProvider();
    provider.setCredentials(scope, creds);
    setGlobalCredentialsProvider(provider, scope);
  }

  /* Make this externally accessible primarily for testing */

  public static synchronized void setGlobalSSLAuth(
      String keypath, String keypassword, String trustpath, String trustpassword) {
    // load the stores if defined
    try {
      if (trustpath != null && trustpassword != null) {
        truststore = KeyStore.getInstance(KeyStore.getDefaultType());
        try (FileInputStream instream = new FileInputStream(new File(trustpath))) {
          truststore.load(instream, trustpassword.toCharArray());
        }
      } else truststore = null;
      if (keypath != null && keypassword != null) {
        keystore = KeyStore.getInstance(KeyStore.getDefaultType());
        try (FileInputStream instream = new FileInputStream(new File(keypath))) {
          keystore.load(instream, keypassword.toCharArray());
        }
      } else keystore = null;
    } catch (IOException | NoSuchAlgorithmException | CertificateException | KeyStoreException ex) {
      log.error("Illegal -D keystore parameters: " + ex.getMessage());
      truststore = null;
      keystore = null;
    }
    try {
      // set up the context
      SSLContext scxt = null;
      if (IGNORECERTS) {
        scxt = SSLContext.getInstance("TLS");
        TrustManager[] trust_mgr =
            new TrustManager[] {
              new X509TrustManager() {
                public X509Certificate[] getAcceptedIssuers() {
                  return null;
                }

                public void checkClientTrusted(X509Certificate[] certs, String t) {}

                public void checkServerTrusted(X509Certificate[] certs, String t) {}
              }
            };
        scxt.init(
            null, // key manager
            trust_mgr, // trust manager
            new SecureRandom()); // random number generator
      } else {
        SSLContextBuilder sslbuilder = SSLContexts.custom();
        TrustStrategy strat = new LooseTrustStrategy();
        if (truststore != null) sslbuilder.loadTrustMaterial(truststore, strat);
        else sslbuilder.loadTrustMaterial(strat);
        sslbuilder.loadTrustMaterial(truststore, new LooseTrustStrategy());
        if (keystore != null) sslbuilder.loadKeyMaterial(keystore, keypassword.toCharArray());
        scxt = sslbuilder.build();
      }
      globalsslfactory = new SSLConnectionSocketFactory(scxt, new NoopHostnameVerifier());

      RegistryBuilder rb = RegistryBuilder.<ConnectionSocketFactory>create();
      rb.register("https", globalsslfactory);
      sslregistry = rb.build();
    } catch (KeyStoreException
        | NoSuchAlgorithmException
        | KeyManagementException
        | UnrecoverableEntryException e) {
      log.error("Failed to set key/trust store(s): " + e.getMessage());
      sslregistry = null;
      globalsslfactory = null;
    }
  }

  public static synchronized void setGlobalProxy(String proxyurl) {
    if (proxyurl == null) throw new IllegalArgumentException("Bad proxy URL: " + proxyurl);
    URI uri;
    try {
      uri = HTTPUtil.parseToURI(proxyurl);
    } catch (URISyntaxException e) {
      throw new IllegalArgumentException("Bad proxy URL: " + proxyurl);
    }
    if (uri.getScheme().equals("http"))
      httpproxy = new HttpHost(uri.getHost(), uri.getPort(), "http");
    else if (uri.getScheme().equals("https"))
      httpsproxy = new HttpHost(uri.getHost(), uri.getPort(), "https");
    String upw = uri.getUserInfo();
    if (upw != null) {
      String[] pieces = upw.split("[:]");
      if (pieces.length != 2
          || HTTPUtil.nullify(pieces[0]) == null
          || HTTPUtil.nullify(pieces[1]) == null)
        throw new IllegalArgumentException("Bad userinfo: " + proxyurl);
      proxyuser = pieces[0];
      proxypwd = pieces[1];
    }
  }

  //////////////////////////////////////////////////
  // Instance variables

  // Currently, the granularity of authorization is host+port.
  protected String sessionURI = null; // This is either a real url
  // or one constructed from an AuthScope
  protected URI scopeURI = null; // constructed
  protected AuthScope scope = null;
  protected boolean closed = false;

  // Since can't access CredentialsProvider map, mimic
  protected Map<AuthScope, CredentialsProvider> localcreds = new HashMap<>();

  protected List<ucar.httpservices.HTTPMethod> methodList = new Vector<HTTPMethod>();
  protected String identifier = "Session";
  protected Settings localsettings = new Settings();

  // We currently only allow the use of global interceptors
  //    protected List<Object> intercepts = new ArrayList<Object>(); // current set of interceptors;

  // This context is re-used over all method executions so that we maintain
  // cookies, credentials, etc.
  // But we do need away to clear so that e.g. we can clear credentials cache
  protected HttpClientContext sessioncontext = HttpClientContext.create();

  // cached and recreated as needed
  protected boolean cachevalid = false; // Are cached items up-to-date?
  protected CloseableHttpClient cachedclient = null;
  protected RequestConfig cachedconfig = null;
  protected URI requestURI = null; // full uri from the HTTPMethod call
  protected ExecState execution = new ExecState();

  //////////////////////////////////////////////////
  // Constructor(s)
  // All are package level so that only HTTPFactory can be used externally

  protected HTTPSession() throws HTTPException {}

  HTTPSession(String host, int port) throws HTTPException {
    init(new AuthScope(host, port, null, null), null);
  }

  HTTPSession(String uri) throws HTTPException {
    init(HTTPAuthUtil.uriToAuthScope(uri), uri);
  }

  HTTPSession(HttpHost httphost) throws HTTPException {
    init(HTTPAuthUtil.hostToAuthScope(httphost), null);
  }

  protected void init(AuthScope scope, String actualurl) throws HTTPException {
    assert (scope != null);
    if (actualurl != null) this.sessionURI = actualurl;
    else this.sessionURI = HTTPAuthUtil.authscopeToURI(scope).toString();
    this.scope = scope;
    this.scopeURI = HTTPAuthUtil.authscopeToURI(scope);
    this.cachevalid = false; // Force build on first use
    this.sessioncontext.setCookieStore(new BasicCookieStore());
    this.sessioncontext.setAttribute(HttpClientContext.AUTH_CACHE, new BasicAuthCache());
  }

  //////////////////////////////////////////////////
  // Interceptors: Only supported at global level

  protected static void setInterceptors(HttpClientBuilder cb) {
    for (HttpRequestInterceptor hrq : reqintercepts) {
      cb.addInterceptorLast(hrq);
    }
    for (HttpResponseInterceptor hrs : rspintercepts) {
      cb.addInterceptorLast(hrs);
    }
    // Add debug interceptors
    for (HttpRequestInterceptor hrq : dbgreq) {
      cb.addInterceptorFirst(hrq);
    }
    for (HttpResponseInterceptor hrs : dbgrsp) {
      cb.addInterceptorFirst(hrs);
    }
    // Hack: add Content-Encoding suppressor
    cb.addInterceptorFirst(CEKILL);
  }

  //////////////////////////////////////////////////
  // Accessor(s)

  public AuthScope getAuthScope() {
    return this.scope;
  }

  public String getSessionURI() {
    return this.sessionURI;
  }

  /**
   * Extract the sessionid cookie value
   *
   * @return sessionid string
   */
  public String getSessionID() {
    String sid = null;
    String jsid = null;
    List<Cookie> cookies = this.sessioncontext.getCookieStore().getCookies();
    for (Cookie cookie : cookies) {
      if (cookie.getName().equalsIgnoreCase("sessionid")) sid = cookie.getValue();
      if (cookie.getName().equalsIgnoreCase("jsessionid")) jsid = cookie.getValue();
    }
    return (sid == null ? jsid : sid);
  }

  public HTTPSession setUserAgent(String agent) {
    if (agent == null || agent.length() == 0) throw new IllegalArgumentException("null argument");
    localsettings.setParameter(Prop.USER_AGENT, agent);
    this.cachevalid = false;
    return this;
  }

  public HTTPSession setSoTimeout(int timeout) {
    if (timeout <= 0) throw new IllegalArgumentException("setSoTimeout");
    localsettings.setParameter(Prop.SO_TIMEOUT, timeout);
    this.cachevalid = false;
    return this;
  }

  public HTTPSession setConnectionTimeout(int timeout) {
    if (timeout <= 0) throw new IllegalArgumentException("setConnectionTImeout");
    localsettings.setParameter(Prop.CONN_TIMEOUT, timeout);
    localsettings.setParameter(Prop.CONN_REQ_TIMEOUT, timeout);
    this.cachevalid = false;
    return this;
  }

  /**
   * Set the max number of redirects to follow
   *
   * @param n
   */
  public HTTPSession setMaxRedirects(int n) {
    if (n < 0) // validate
    throw new IllegalArgumentException("setMaxRedirects");
    localsettings.setParameter(Prop.MAX_REDIRECTS, n);
    this.cachevalid = false;
    return this;
  }

  /** Enable/disable redirection following Default is yes. */
  public HTTPSession setFollowRedirects(boolean tf) {
    localsettings.setParameter(Prop.HANDLE_REDIRECTS, (Boolean) tf);
    this.cachevalid = false;
    return this;
  }

  /**
   * Should we use sessionid's?
   *
   * @param tf
   */
  public HTTPSession setUseSessions(boolean tf) {
    localsettings.setParameter(Prop.USESESSIONS, (Boolean) tf);
    this.cachevalid = false;
    return this;
  }

  public List<Cookie> getCookies() {
    if (this.sessioncontext == null) return null;
    List<Cookie> cookies = this.sessioncontext.getCookieStore().getCookies();
    return cookies;
  }

  public HTTPSession clearCookies() {
    BasicCookieStore cookies = (BasicCookieStore) this.sessioncontext.getCookieStore();
    if (cookies != null) cookies.clear();
    return this;
  }

  public HTTPSession clearCredentialsCache() {
    BasicAuthCache ac =
        (BasicAuthCache) this.sessioncontext.getAttribute(HttpClientContext.AUTH_CACHE);
    if (ac != null) ac.clear();
    return this;
  }

  // make package specific

  HttpClient getClient() {
    return this.cachedclient;
  }

  HttpClientContext getExecutionContext() {
    return this.sessioncontext;
  }

  public Object getSetting(String key) {
    return localsettings.get(key);
  }

  //////////////////////////////////////////////////

  /** Close the session. This implies closing any open methods. */
  public synchronized void close() {
    if (this.closed) return; // multiple calls ok
    while (methodList.size() > 0) {
      HTTPMethod m = methodList.get(0);
      m.close(); // forcibly close; will invoke removemethod().
    }
    closed = true;
  }

  synchronized HTTPSession addMethod(HTTPMethod m) {
    if (!methodList.contains(m)) methodList.add(m);
    return this;
  }

  synchronized HTTPSession removeMethod(HTTPMethod m) {
    methodList.remove(m);
    return this;
  }

  //////////////////////////////////////////////////
  // Authorization
  // per-session versions of the global accessors

  /**
   * @param provider
   * @throws HTTPException
   */
  public HTTPSession setCredentialsProvider(CredentialsProvider provider) throws HTTPException {
    setCredentialsProvider(provider, null);
    return this;
  }

  /**
   * This is the most general case
   *
   * @param provider the credentials provider
   * @param scope where to use it (i.e. on what host+port)
   * @throws HTTPException
   */
  public HTTPSession setCredentialsProvider(CredentialsProvider provider, AuthScope scope)
      throws HTTPException {
    mapcreds(provider, scope, localcreds);
    return this;
  }

  /**
   * It is convenient to be able to directly set the Credentials (not the provider) when those
   * credentials are fixed. Scope defaults to ANY
   *
   * @param creds
   * @throws HTTPException
   */
  public HTTPSession setCredentials(Credentials creds) throws HTTPException {
    setCredentials(creds, null);
    return this;
  }

  /**
   * It is convenient to be able to directly set the Credentials (not the provider) when those
   * credentials are fixed.
   *
   * @param creds
   * @param scope where to use it (i.e. on what host)
   * @throws HTTPException
   */
  public HTTPSession setCredentials(Credentials creds, AuthScope scope) throws HTTPException {
    assert (creds != null);
    if (scope == null) scope = AuthScope.ANY;
    CredentialsProvider provider = new BasicCredentialsProvider();
    provider.setCredentials(scope, creds);
    setCredentialsProvider(provider, scope);
    return this;
  }

  //////////////////////////////////////////////////
  // Execution (do an actual execution)
  // Package visible

  /**
   * Called primarily from HTTPMethod to do the bulk of the execution. Assumes HTTPMethod has
   * inserted its headers into request.
   *
   * @param method
   * @param methoduri
   * @param rb
   * @return Request+Response pair
   * @throws HTTPException
   */
  ExecState execute(HTTPMethod method, URI methoduri, RequestBuilder rb) throws HTTPException {
    this.execution = new ExecState();
    this.requestURI = methoduri;
    AuthScope methodscope = HTTPAuthUtil.uriToAuthScope(methoduri);
    AuthScope target = HTTPAuthUtil.authscopeUpgrade(this.scope, methodscope);
    synchronized (this) { // keep coverity happy
      // Merge Settings;
      Settings merged = HTTPUtil.merge(globalsettings, localsettings);
      if (!this.cachevalid) {
        RequestConfig.Builder rcb = RequestConfig.custom();
        this.cachedconfig = configureRequest(rcb, merged);
        HttpClientBuilder cb = HttpClients.custom();
        configClient(cb, merged);
        setAuthenticationAndProxy(cb);
        this.cachedclient = cb.build();
        rb.setConfig(this.cachedconfig);
        this.cachevalid = true;
      }
    }
    this.execution.request = (HttpRequestBase) rb.build();
    try {
      HttpHost targethost = HTTPAuthUtil.authscopeToHost(target);
      this.execution.response =
          cachedclient.execute(targethost, this.execution.request, this.sessioncontext);
    } catch (IOException ioe) {
      throw new HTTPException(ioe);
    }
    return this.execution;
  }

  protected RequestConfig configureRequest(RequestConfig.Builder rcb, Settings settings)
      throws HTTPException {
    // Configure the RequestConfig
    for (Prop key : settings.getKeys()) {
      Object value = settings.getParameter(key);
      boolean tf = (value instanceof Boolean ? (Boolean) value : false);
      if (key == Prop.ALLOW_CIRCULAR_REDIRECTS) {
        rcb.setCircularRedirectsAllowed(tf);
      } else if (key == Prop.HANDLE_REDIRECTS) {
        rcb.setRedirectsEnabled(tf);
        rcb.setRelativeRedirectsAllowed(tf);
      } else if (key == Prop.MAX_REDIRECTS) {
        rcb.setMaxRedirects((Integer) value);
      } else if (key == Prop.SO_TIMEOUT) {
        rcb.setSocketTimeout((Integer) value);
      } else if (key == Prop.CONN_TIMEOUT) {
        rcb.setConnectTimeout((Integer) value);
      } else if (key == Prop.CONN_REQ_TIMEOUT) {
        rcb.setConnectionRequestTimeout((Integer) value);
      } else if (key == Prop.MAX_THREADS) {
        connmgr.setMaxTotal((Integer) value);
        connmgr.setDefaultMaxPerRoute((Integer) value);
      } /* else ignore */
    }
    RequestConfig cfg = rcb.build();
    return cfg;
  }

  protected void configClient(HttpClientBuilder cb, Settings settings) throws HTTPException {
    cb.useSystemProperties();
    String agent = (String) settings.get(Prop.USER_AGENT);
    if (agent != null) cb.setUserAgent(agent);
    setInterceptors(cb);
    cb.setContentDecoderRegistry(contentDecoderMap);
  }

  /**
   * Handle authentication and Proxy'ing
   *
   * @param cb
   * @throws HTTPException
   */
  protected synchronized void setAuthenticationAndProxy(HttpClientBuilder cb) throws HTTPException {
    // First, setup the ssl factory
    cb.setSSLSocketFactory(globalsslfactory);

    // Second, Construct a CredentialsProvider that is
    // the union of the Proxy credentials plus
    // either the global local credentials; local overrides global
    // Unfortunately, we cannot either clone or extract the contents
    // of the client supplied provider, so we are forced (for now)
    // to modify the client supplied provider.

    // Look in the local authcreds for best scope match
    AuthScope bestMatch = HTTPAuthUtil.bestmatch(scope, localcreds.keySet());
    CredentialsProvider cp = null;
    if (bestMatch != null) {
      cp = localcreds.get(bestMatch);
    } else {
      bestMatch = HTTPAuthUtil.bestmatch(scope, globalcreds.keySet());
      if (bestMatch != null) cp = globalcreds.get(bestMatch);
    }
    // Build the proxy credentials and AuthScope
    Credentials proxycreds = null;
    AuthScope proxyscope = null;
    if (proxyuser != null && (httpproxy != null || httpsproxy != null)) {
      if (httpproxy != null) proxyscope = HTTPAuthUtil.hostToAuthScope(httpproxy);
      else // httpsproxy != null
      proxyscope = HTTPAuthUtil.hostToAuthScope(httpsproxy);
      proxycreds = new UsernamePasswordCredentials(proxyuser, proxypwd);
    }
    if (cp == null && proxycreds != null && proxyscope != null) {
      // If client provider is null and proxycreds are not,
      // then use proxycreds alone
      cp = new BasicCredentialsProvider();
      cp.setCredentials(proxyscope, proxycreds);
    } else if (cp != null && proxycreds != null && proxyscope != null) {
      // If client provider is not null and proxycreds are not,
      // then add proxycreds to the client provider
      cp.setCredentials(proxyscope, proxycreds);
    }
    if (cp != null) this.sessioncontext.setCredentialsProvider(cp);
  }

  //////////////////////////////////////////////////
  // Utilities Static and Per-Instance

  static String getCanonicalURL(String legalurl) {
    if (legalurl == null) return null;
    int index = legalurl.indexOf('?');
    if (index >= 0) legalurl = legalurl.substring(0, index);
    // remove any trailing extension
    // index = legalurl.lastIndexOf('.');
    // if(index >= 0) legalurl = legalurl.substring(0,index);
    return HTTPUtil.canonicalpath(legalurl);
  }

  static String getUrlAsString(String url) throws HTTPException {
    try (HTTPMethod m = HTTPFactory.Get(url); ) {
      int status = m.execute();
      String content = null;
      if (status == 200) {
        content = m.getResponseAsString();
      }
      return content;
    }
  }

  static int putUrlAsString(String content, String url) throws HTTPException {
    int status = 0;
    try {
      try (HTTPMethod m = HTTPFactory.Put(url)) {
        m.setRequestContent(
            new StringEntity(content, ContentType.create("application/text", "UTF-8")));
        status = m.execute();
      }
    } catch (UnsupportedCharsetException uce) {
      throw new HTTPException(uce);
    }
    return status;
  }

  static String getstorepath(String prefix) {
    String path = System.getProperty(prefix + "store");
    if (path != null) {
      path = path.trim();
      if (path.length() == 0) path = null;
    }
    return path;
  }

  static String getpassword(String prefix) {
    String password = System.getProperty(prefix + "storepassword");
    if (password != null) {
      password = password.trim();
      if (password.length() == 0) password = null;
    }
    return password;
  }

  static String cleanproperty(String property) {
    String value = System.getProperty(property);
    if (value != null) {
      value = value.trim();
      if (value.length() == 0) value = null;
    }
    return value;
  }

  static void mapcreds(
      CredentialsProvider provider,
      AuthScope scope,
      Map<AuthScope, CredentialsProvider> authcreds) {
    assert (provider != null);
    if (scope == null) scope = AuthScope.ANY;
    authcreds.put(scope, provider);
  }

  //////////////////////////////////////////////////
  // Testing support

  // Expose the state for testing purposes
  public synchronized boolean isClosed() {
    return this.closed;
  }

  public synchronized int getMethodcount() {
    return methodList.size();
  }

  public RequestConfig getDebugConfig() {
    return (this.cachevalid ? this.cachedconfig : null);
  }

  public Header[] getRequestHeaders() {
    if (!this.cachevalid) return null;
    Header[] hdrs = null;
    if (this.execution.request != null) hdrs = this.execution.request.getAllHeaders();
    return hdrs;
  }

  //////////////////////////////////////////////////
  // Debug interface

  // Provide a way to kill everything at the end of a Test

  // When testing, we need to be able to clean up
  // all existing sessions because JUnit can run all
  // test within a single jvm.
  static List<HTTPSession> sessionList = null; // List of all HTTPSession instances

  // only used when testing flag is set
  public static boolean TESTING = false; // set to true during testing, should be false otherwise

  protected static synchronized void kill() {
    if (sessionList != null) {
      for (HTTPSession session : sessionList) {
        session.close();
      }
      sessionList.clear();
      // Rebuild the connection manager
      connmgr.shutdown();
      connmgr = new PoolingHttpClientConnectionManager(sslregistry);
      setGlobalThreadCount(DFALTTHREADCOUNT);
    }
  }

  // If we are testing, then track the sessions for kill
  protected static synchronized void track(HTTPSession session) {
    if (sessionList == null) sessionList = new ArrayList<HTTPSession>();
    sessionList.add(session);
  }

  public static synchronized void debugHeaders(boolean print) {
    HTTPUtil.InterceptRequest rq = new HTTPUtil.InterceptRequest();
    HTTPUtil.InterceptResponse rs = new HTTPUtil.InterceptResponse();
    rq.setPrint(print);
    rs.setPrint(print);
    /* remove any previous */
    for (int i = reqintercepts.size() - 1; i >= 0; i--) {
      HttpRequestInterceptor hr = reqintercepts.get(i);
      if (hr instanceof HTTPUtil.InterceptCommon) reqintercepts.remove(i);
    }
    for (int i = rspintercepts.size() - 1; i >= 0; i--) {
      HttpResponseInterceptor hr = rspintercepts.get(i);
      if (hr instanceof HTTPUtil.InterceptCommon) rspintercepts.remove(i);
    }
    reqintercepts.add(rq);
    rspintercepts.add(rs);
  }

  public static void debugReset() {
    for (HttpRequestInterceptor hri : reqintercepts) {
      if (hri instanceof HTTPUtil.InterceptCommon) ((HTTPUtil.InterceptCommon) hri).clear();
    }
  }

  public static HTTPUtil.InterceptRequest debugRequestInterceptor() {
    for (HttpRequestInterceptor hri : reqintercepts) {
      if (hri instanceof HTTPUtil.InterceptRequest) return ((HTTPUtil.InterceptRequest) hri);
    }
    return null;
  }

  public static HTTPUtil.InterceptResponse debugResponseInterceptor() {
    for (HttpResponseInterceptor hri : rspintercepts) {
      if (hri instanceof HTTPUtil.InterceptResponse) return ((HTTPUtil.InterceptResponse) hri);
    }
    return null;
  }

  //////////////////////////////////////////////////
  // Deprecated, but here for back compatibility

  @Deprecated
  public static void setGlobalCredentialsProvider(AuthScope scope, CredentialsProvider provider)
      throws HTTPException {
    setGlobalCredentialsProvider(provider, scope);
  }

  @Deprecated
  public static void setGlobalCredentialsProvider(String url, CredentialsProvider provider)
      throws HTTPException {
    assert (url != null && provider != null);
    AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
    setGlobalCredentialsProvider(provider, scope);
  }

  @Deprecated
  public static void setGlobalCredentials(String url, Credentials creds) throws HTTPException {
    assert (url != null && creds != null);
    AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
    CredentialsProvider provider = new BasicCredentialsProvider();
    provider.setCredentials(scope, creds);
    setGlobalCredentialsProvider(provider, scope);
  }

  @Deprecated
  public void setCredentials(String url, Credentials creds) throws HTTPException {
    assert (creds != null);
    AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
    setCredentials(creds, scope);
  }

  @Deprecated
  public void setCredentialsProvider(String url, CredentialsProvider provider)
      throws HTTPException {
    assert (url != null && provider != null);
    AuthScope scope = HTTPAuthUtil.uriToAuthScope(url);
    setCredentialsProvider(provider, scope);
  }

  @Deprecated
  public void setCredentialsProvider(AuthScope scope, CredentialsProvider provider)
      throws HTTPException {
    setCredentialsProvider(provider, scope);
  }

  @Deprecated
  public static int getRetryCount() {
    throw new UnsupportedOperationException();
  }

  @Deprecated
  public static void setGlobalCompression() {
    setGlobalCompression("gzip,deflate");
  }

  @Deprecated
  public static void setGlobalProxy(String host, int port) {
    try {
      URL u = new URL("http", host, port, null);
      setGlobalProxy(u.toString());
    } catch (MalformedURLException e) {
      throw new IllegalArgumentException(e);
    }
  }

  @Deprecated
  public void setProxy(String host, int port) {
    setGlobalProxy(host, port);
  }

  @Deprecated
  public static void setGlobalCredentialsProvider(CredentialsProvider provider, String scheme)
      throws HTTPException {
    setGlobalCredentialsProvider(provider);
  }

  @Deprecated
  public static void setRetryCount(int count) {
    throw new UnsupportedOperationException();
  }

  @Deprecated
  public void clearState() {
    // no-op
  }

  @Deprecated
  public String getSessionURL() {
    return getSessionURI();
  }
}
/**
 * This class reads a NEXRAD level II data file. It can handle NCDC archives (ARCHIVE2), as well as
 * CRAFT/IDD compressed files (AR2V0001).
 *
 * <p>Adapted with permission from the Java Iras software developed by David Priegnitz at NSSL.
 *
 * <p>
 *
 * <p>Documentation on Archive Level II data format can be found at: <a
 * href="http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html">
 * http://www.ncdc.noaa.gov/oa/radar/leveliidoc.html</a>
 *
 * @author caron
 * @author David Priegnitz
 */
public class Level2VolumeScan {

  // data formats
  public static final String ARCHIVE2 = "ARCHIVE2";
  public static final String AR2V0001 = "AR2V0001";
  public static final String AR2V0002 = "AR2V0002";
  public static final String AR2V0003 = "AR2V0003";
  public static final String AR2V0004 = "AR2V0004";
  public static final String AR2V0006 = "AR2V0006";

  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Level2VolumeScan.class);
  ////////////////////////////////////////////////////////////////////////////////////

  // Data file
  RandomAccessFile raf;

  private String dataFormat = null; // ARCHIVE2 or AR2V0001
  private int title_julianDay; // days since 1/1/70
  private int title_msecs; // milliseconds since midnight
  private String stationId; // 4 letter station assigned by ICAO
  private NexradStationDB.Station station; // from lookup table, may be null
  private Level2Record first, last;

  private int vcp = 0; // Volume coverage pattern
  private int max_radials = 0;
  private int min_radials = Integer.MAX_VALUE;
  private int max_radials_hr = 0;
  private int min_radials_hr = Integer.MAX_VALUE;
  private int dopplarResolution;
  private boolean hasDifferentDopplarResolutions;
  private boolean hasHighResolutionData;

  private boolean hasHighResolutionREF;
  private boolean hasHighResolutionVEL;
  private boolean hasHighResolutionSW;
  private boolean hasHighResolutionZDR;
  private boolean hasHighResolutionPHI;
  private boolean hasHighResolutionRHO;
  // List of List of Level2Record
  private List<List<Level2Record>> reflectivityGroups, dopplerGroups;

  // private ArrayList reflectivityGroups, dopplerGroups;

  private List<List<Level2Record>> reflectivityHighResGroups;
  private List<List<Level2Record>> velocityHighResGroups;
  private List<List<Level2Record>> spectrumHighResGroups;
  private ArrayList diffReflectHighResGroups;
  private ArrayList diffPhaseHighResGroups;
  private ArrayList coefficientHighResGroups;

  private boolean showMessages = false,
      showData = false,
      debugScans = false,
      debugGroups2 = false,
      debugRadials = false,
      debugStats = false;
  private boolean runCheck = false;

  Level2VolumeScan(RandomAccessFile orgRaf, CancelTask cancelTask) throws IOException {
    this.raf = orgRaf;

    if (log.isDebugEnabled()) log.debug("Level2VolumeScan on " + raf.getLocation());

    raf.seek(0);
    raf.order(RandomAccessFile.BIG_ENDIAN);

    // volume scan header
    dataFormat = raf.readString(8);
    raf.skipBytes(1);
    String volumeNo = raf.readString(3);
    title_julianDay = raf.readInt(); // since 1/1/70
    title_msecs = raf.readInt();
    stationId = raf.readString(4).trim(); // only in AR2V0001
    if (log.isDebugEnabled()) log.debug(" dataFormat= " + dataFormat + " stationId= " + stationId);

    if (stationId.length() == 0) {
      // try to get it from the filename LOOK

      stationId = null;
    }

    // try to find the station
    if (stationId != null) {
      if (!stationId.startsWith("K") && stationId.length() == 4) {
        String _stationId = "K" + stationId;
        station = NexradStationDB.get(_stationId);
      } else station = NexradStationDB.get(stationId);
    }

    // see if we have to uncompress
    if (dataFormat.equals(AR2V0001)
        || dataFormat.equals(AR2V0003)
        || dataFormat.equals(AR2V0004)
        || dataFormat.equals(AR2V0006)) {
      raf.skipBytes(4);
      String BZ = raf.readString(2);
      if (BZ.equals("BZ")) {
        RandomAccessFile uraf;
        File uncompressedFile = DiskCache.getFileStandardPolicy(raf.getLocation() + ".uncompress");

        if (uncompressedFile.exists() && uncompressedFile.length() > 0) {
          // see if its locked - another thread is writing it
          FileInputStream fstream = null;
          FileLock lock = null;
          try {
            fstream = new FileInputStream(uncompressedFile);
            // lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked

            while (true) { // loop waiting for the lock
              try {
                lock = fstream.getChannel().lock(0, 1, true); // wait till its unlocked
                break;

              } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
                try {
                  Thread.sleep(100); // msecs
                } catch (InterruptedException e1) {
                  break;
                }
              }
            }

          } finally {
            if (lock != null) lock.release();
            if (fstream != null) fstream.close();
          }
          uraf = new ucar.unidata.io.RandomAccessFile(uncompressedFile.getPath(), "r");

        } else {
          // nope, gotta uncompress it
          uraf = uncompress(raf, uncompressedFile.getPath());
          if (log.isDebugEnabled())
            log.debug("made uncompressed file= " + uncompressedFile.getPath());
        }

        // switch to uncompressed file
        raf.close();
        raf = uraf;
        raf.order(RandomAccessFile.BIG_ENDIAN);
      }

      raf.seek(Level2Record.FILE_HEADER_SIZE);
    }

    List<Level2Record> reflectivity = new ArrayList<Level2Record>();
    List<Level2Record> doppler = new ArrayList<Level2Record>();
    List<Level2Record> highReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highVelocity = new ArrayList<Level2Record>();
    List<Level2Record> highSpectrum = new ArrayList<Level2Record>();
    List<Level2Record> highDiffReflectivity = new ArrayList<Level2Record>();
    List<Level2Record> highDiffPhase = new ArrayList<Level2Record>();
    List<Level2Record> highCorreCoefficient = new ArrayList<Level2Record>();

    long message_offset31 = 0;
    int recno = 0;
    while (true) {

      Level2Record r = Level2Record.factory(raf, recno++, message_offset31);
      if (r == null) break;
      if (showData) r.dump2(System.out);
      // skip non-data messages
      if (r.message_type == 31) {
        message_offset31 = message_offset31 + (r.message_size * 2 + 12 - 2432);
      }

      if (r.message_type != 1 && r.message_type != 31) {
        if (showMessages) r.dumpMessage(System.out);
        continue;
      }

      //  if (showData) r.dump2(System.out);

      /* skip bad
      if (!r.checkOk()) {
        r.dump(System.out);
        continue;
      }   */

      // some global params
      if (vcp == 0) vcp = r.vcp;
      if (first == null) first = r;
      last = r;

      if (runCheck && !r.checkOk()) {
        continue;
      }

      if (r.hasReflectData) reflectivity.add(r);
      if (r.hasDopplerData) doppler.add(r);

      if (r.message_type == 31) {
        if (r.hasHighResREFData) highReflectivity.add(r);
        if (r.hasHighResVELData) highVelocity.add(r);
        if (r.hasHighResSWData) highSpectrum.add(r);
        if (r.hasHighResZDRData) highDiffReflectivity.add(r);
        if (r.hasHighResPHIData) highDiffPhase.add(r);
        if (r.hasHighResRHOData) highCorreCoefficient.add(r);
      }

      if ((cancelTask != null) && cancelTask.isCancel()) return;
    }
    if (debugRadials)
      System.out.println(" reflect ok= " + reflectivity.size() + " doppler ok= " + doppler.size());
    if (highReflectivity.size() == 0) {
      reflectivityGroups = sortScans("reflect", reflectivity, 600);
      dopplerGroups = sortScans("doppler", doppler, 600);
    }
    if (highReflectivity.size() > 0)
      reflectivityHighResGroups = sortScans("reflect_HR", highReflectivity, 720);
    if (highVelocity.size() > 0)
      velocityHighResGroups = sortScans("velocity_HR", highVelocity, 720);
    if (highSpectrum.size() > 0)
      spectrumHighResGroups = sortScans("spectrum_HR", highSpectrum, 720);
    if (highDiffReflectivity.size() > 0)
      diffReflectHighResGroups = sortScans("diffReflect_HR", highDiffReflectivity, 720);
    if (highDiffPhase.size() > 0)
      diffPhaseHighResGroups = sortScans("diffPhase_HR", highDiffPhase, 720);
    if (highCorreCoefficient.size() > 0)
      coefficientHighResGroups = sortScans("coefficient_HR", highCorreCoefficient, 720);
  }

  private ArrayList sortScans(String name, List<Level2Record> scans, int siz) {

    // now group by elevation_num
    Map<Short, List<Level2Record>> groupHash = new HashMap<Short, List<Level2Record>>(siz);
    for (Level2Record record : scans) {
      List<Level2Record> group = groupHash.get(record.elevation_num);
      if (null == group) {
        group = new ArrayList<Level2Record>();
        groupHash.put(record.elevation_num, group);
      }
      group.add(record);
    }

    // sort the groups by elevation_num
    ArrayList groups = new ArrayList(groupHash.values());
    Collections.sort(groups, new GroupComparator());

    // use the maximum radials
    for (int i = 0; i < groups.size(); i++) {
      ArrayList group = (ArrayList) groups.get(i);
      Level2Record r = (Level2Record) group.get(0);
      if (runCheck) testScan(name, group);
      if (r.getGateCount(REFLECTIVITY_HIGH) > 500 || r.getGateCount(VELOCITY_HIGH) > 1000) {
        max_radials_hr = Math.max(max_radials_hr, group.size());
        min_radials_hr = Math.min(min_radials_hr, group.size());
      } else {
        max_radials = Math.max(max_radials, group.size());
        min_radials = Math.min(min_radials, group.size());
      }
    }

    if (debugRadials) {
      System.out.println(name + " min_radials= " + min_radials + " max_radials= " + max_radials);
      for (int i = 0; i < groups.size(); i++) {
        ArrayList group = (ArrayList) groups.get(i);
        Level2Record lastr = (Level2Record) group.get(0);
        for (int j = 1; j < group.size(); j++) {
          Level2Record r = (Level2Record) group.get(j);
          if (r.data_msecs < lastr.data_msecs) System.out.println(" out of order " + j);
          lastr = r;
        }
      }
    }

    testVariable(name, groups);
    if (debugScans) System.out.println("-----------------------------");

    return groups;
  }

  public int getMaxRadials(int r) {
    if (r == 0) return max_radials;
    else if (r == 1) return max_radials_hr;
    else return 0;
  }

  public int getMinRadials(int r) {
    if (r == 0) return min_radials;
    else if (r == 1) return min_radials_hr;
    else return 0;
  }

  public int getDopplarResolution() {
    return dopplarResolution;
  }

  public boolean hasDifferentDopplarResolutions() {
    return hasDifferentDopplarResolutions;
  }

  public boolean hasHighResolutions(int dt) {
    if (dt == 0) return hasHighResolutionData;
    else if (dt == 1) return hasHighResolutionREF;
    else if (dt == 2) return hasHighResolutionVEL;
    else if (dt == 3) return hasHighResolutionSW;
    else if (dt == 4) return hasHighResolutionZDR;
    else if (dt == 5) return hasHighResolutionPHI;
    else if (dt == 6) return hasHighResolutionRHO;
    else return false;
  }

  // do we have same characteristics for all records in a scan?
  private int MAX_RADIAL = 721;
  private int[] radial = new int[MAX_RADIAL];

  private boolean testScan(String name, ArrayList group) {
    int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
    Level2Record first = (Level2Record) group.get(0);

    int n = group.size();
    if (debugScans) {
      boolean hasBoth = first.hasDopplerData && first.hasReflectData;
      System.out.println(
          name
              + " "
              + first
              + " has "
              + n
              + " radials resolution= "
              + first.resolution
              + " has both = "
              + hasBoth);
    }

    boolean ok = true;
    double sum = 0.0;
    double sum2 = 0.0;

    for (int i = 0; i < MAX_RADIAL; i++) radial[i] = 0;

    for (int i = 0; i < group.size(); i++) {
      Level2Record r = (Level2Record) group.get(i);

      /* this appears to be common - seems to be ok, we put missing values in
      if (r.getGateCount(datatype) != first.getGateCount(datatype)) {
        log.error(raf.getLocation()+" different number of gates ("+r.getGateCount(datatype)+
                "!="+first.getGateCount(datatype)+") in record "+name+ " "+r);
        ok = false;
      } */

      if (r.getGateSize(datatype) != first.getGateSize(datatype)) {
        log.warn(
            raf.getLocation()
                + " different gate size ("
                + r.getGateSize(datatype)
                + ") in record "
                + name
                + " "
                + r);
        ok = false;
      }
      if (r.getGateStart(datatype) != first.getGateStart(datatype)) {
        log.warn(
            raf.getLocation()
                + " different gate start ("
                + r.getGateStart(datatype)
                + ") in record "
                + name
                + " "
                + r);
        ok = false;
      }
      if (r.resolution != first.resolution) {
        log.warn(
            raf.getLocation()
                + " different resolution ("
                + r.resolution
                + ") in record "
                + name
                + " "
                + r);
        ok = false;
      }

      if ((r.radial_num < 0) || (r.radial_num >= MAX_RADIAL)) {
        log.info(
            raf.getLocation()
                + " radial out of range= "
                + r.radial_num
                + " in record "
                + name
                + " "
                + r);
        continue;
      }
      if (radial[r.radial_num] > 0) {
        log.warn(
            raf.getLocation()
                + " duplicate radial = "
                + r.radial_num
                + " in record "
                + name
                + " "
                + r);
        ok = false;
      }
      radial[r.radial_num] = r.recno + 1;

      sum += r.getElevation();
      sum2 += r.getElevation() * r.getElevation();
      // System.out.println("  elev="+r.getElevation()+" azi="+r.getAzimuth());
    }

    for (int i = 1; i < radial.length; i++) {
      if (0 == radial[i]) {
        if (n != (i - 1)) {
          log.warn(" missing radial(s)");
          ok = false;
        }
        break;
      }
    }

    double avg = sum / n;
    double sd = Math.sqrt((n * sum2 - sum * sum) / (n * (n - 1)));
    // System.out.println(" avg elev="+avg+" std.dev="+sd);

    return ok;
  }

  // do we have same characteristics for all groups in a variable?
  private boolean testVariable(String name, List scans) {
    int datatype = name.equals("reflect") ? Level2Record.REFLECTIVITY : Level2Record.VELOCITY_HI;
    if (scans.size() == 0) {
      log.warn(" No data for = " + name);
      return false;
    }

    boolean ok = true;
    List firstScan = (List) scans.get(0);
    Level2Record firstRecord = (Level2Record) firstScan.get(0);
    dopplarResolution = firstRecord.resolution;

    if (debugGroups2)
      System.out.println(
          "Group "
              + Level2Record.getDatatypeName(datatype)
              + " ngates = "
              + firstRecord.getGateCount(datatype)
              + " start = "
              + firstRecord.getGateStart(datatype)
              + " size = "
              + firstRecord.getGateSize(datatype));

    for (int i = 1; i < scans.size(); i++) {
      List scan = (List) scans.get(i);
      Level2Record record = (Level2Record) scan.get(0);

      if ((datatype == Level2Record.VELOCITY_HI)
          && (record.resolution
              != firstRecord.resolution)) { // do all velocity resolutions match ??
        log.warn(
            name
                + " scan "
                + i
                + " diff resolutions = "
                + record.resolution
                + ", "
                + firstRecord.resolution
                + " elev= "
                + record.elevation_num
                + " "
                + record.getElevation());
        ok = false;
        hasDifferentDopplarResolutions = true;
      }

      if (record.getGateSize(datatype) != firstRecord.getGateSize(datatype)) {
        log.warn(
            name
                + " scan "
                + i
                + " diff gates size = "
                + record.getGateSize(datatype)
                + " "
                + firstRecord.getGateSize(datatype)
                + " elev= "
                + record.elevation_num
                + " "
                + record.getElevation());
        ok = false;

      } else if (debugGroups2)
        System.out.println(
            " ok gates size elev= " + record.elevation_num + " " + record.getElevation());

      if (record.getGateStart(datatype) != firstRecord.getGateStart(datatype)) {
        log.warn(
            name
                + " scan "
                + i
                + " diff gates start = "
                + record.getGateStart(datatype)
                + " "
                + firstRecord.getGateStart(datatype)
                + " elev= "
                + record.elevation_num
                + " "
                + record.getElevation());
        ok = false;

      } else if (debugGroups2)
        System.out.println(
            " ok gates start elev= " + record.elevation_num + " " + record.getElevation());

      if (record.message_type == 31) {
        hasHighResolutionData = true;
        // each data type
        if (record.hasHighResREFData) hasHighResolutionREF = true;
        if (record.hasHighResVELData) hasHighResolutionVEL = true;
        if (record.hasHighResSWData) hasHighResolutionSW = true;
        if (record.hasHighResZDRData) hasHighResolutionZDR = true;
        if (record.hasHighResPHIData) hasHighResolutionPHI = true;
        if (record.hasHighResRHOData) hasHighResolutionRHO = true;
      }
    }

    return ok;
  }

  /**
   * Get Reflectivity Groups Groups are all the records for a variable and elevation_num;
   *
   * @return List of type List of type Level2Record
   */
  public List getReflectivityGroups() {
    return reflectivityGroups;
  }

  /**
   * Get Velocity Groups Groups are all the records for a variable and elevation_num;
   *
   * @return List of type List of type Level2Record
   */
  public List getVelocityGroups() {
    return dopplerGroups;
  }

  public List getHighResVelocityGroups() {
    return velocityHighResGroups;
  }

  public List getHighResReflectivityGroups() {
    return reflectivityHighResGroups;
  }

  public List getHighResSpectrumGroups() {
    return spectrumHighResGroups;
  }

  public List getHighResDiffReflectGroups() {
    return diffReflectHighResGroups;
  }

  public List getHighResDiffPhaseGroups() {
    return diffPhaseHighResGroups;
  }

  public List getHighResCoeffocientGroups() {
    return coefficientHighResGroups;
  }

  private class GroupComparator implements Comparator<List<Level2Record>> {

    public int compare(List<Level2Record> group1, List<Level2Record> group2) {
      Level2Record record1 = group1.get(0);
      Level2Record record2 = group2.get(0);

      // if (record1.elevation_num != record2.elevation_num)
      return record1.elevation_num - record2.elevation_num;
      // return record1.cut - record2.cut;
    }
  }

  /**
   * Get data format (ARCHIVE2, AR2V0001) for this file.
   *
   * @return data format (ARCHIVE2, AR2V0001) for this file.
   */
  public String getDataFormat() {
    return dataFormat;
  }

  /**
   * Get the starting Julian day for this volume
   *
   * @return days since 1/1/70.
   */
  public int getTitleJulianDays() {
    return title_julianDay;
  }

  /**
   * Get the starting time in seconds since midnight.
   *
   * @return Generation time of data in milliseconds of day past midnight (UTC).
   */
  public int getTitleMsecs() {
    return title_msecs;
  }

  /**
   * Get the Volume Coverage Pattern number for this data.
   *
   * @return VCP
   * @see Level2Record#getVolumeCoveragePatternName
   */
  public int getVCP() {
    return vcp;
  }

  /**
   * Get the 4-char station ID for this data
   *
   * @return station ID (may be null)
   */
  public String getStationId() {
    return stationId;
  }

  public String getStationName() {
    return station == null ? "unknown" : station.name;
  }

  public double getStationLatitude() {
    return station == null ? 0.0 : station.lat;
  }

  public double getStationLongitude() {
    return station == null ? 0.0 : station.lon;
  }

  public double getStationElevation() {
    return station == null ? 0.0 : station.elev;
  }

  public Date getStartDate() {
    return first.getDate();
  }

  public Date getEndDate() {
    return last.getDate();
  }

  /**
   * Write equivilent uncompressed version of the file.
   *
   * @param inputRaf file to uncompress
   * @param ufilename write to this file
   * @return raf of uncompressed file
   * @throws IOException on read error
   */
  private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
      throws IOException {
    RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
    FileLock lock = null;

    while (true) { // loop waiting for the lock
      try {
        lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
        break;

      } catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
        try {
          Thread.sleep(100); // msecs
        } catch (InterruptedException e1) {
        }
      }
    }

    try {
      inputRaf.seek(0);
      byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
      inputRaf.read(header);
      outputRaf.write(header);

      boolean eof = false;
      int numCompBytes;
      byte[] ubuff = new byte[40000];
      byte[] obuff = new byte[40000];
      try {
        CBZip2InputStream cbzip2 = new CBZip2InputStream();
        while (!eof) {
          try {
            numCompBytes = inputRaf.readInt();
            if (numCompBytes == -1) {
              if (log.isDebugEnabled()) log.debug("  done: numCompBytes=-1 ");
              break;
            }
          } catch (EOFException ee) {
            log.warn("  got EOFException ");
            break; // assume this is ok
          }

          if (log.isDebugEnabled()) {
            log.debug(
                "reading compressed bytes "
                    + numCompBytes
                    + " input starts at "
                    + inputRaf.getFilePointer()
                    + "; output starts at "
                    + outputRaf.getFilePointer());
          }
          /*
           * For some stupid reason, the last block seems to
           * have the number of bytes negated.  So, we just
           * assume that any negative number (other than -1)
           * is the last block and go on our merry little way.
           */
          if (numCompBytes < 0) {
            if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
            numCompBytes = -numCompBytes;
            eof = true;
          }
          byte[] buf = new byte[numCompBytes];
          inputRaf.readFully(buf);
          ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);

          // CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
          cbzip2.setStream(bis);
          int total = 0;
          int nread;
          /*
          while ((nread = cbzip2.read(ubuff)) != -1) {
            dout2.write(ubuff, 0, nread);
            total += nread;
          }
          */
          try {
            while ((nread = cbzip2.read(ubuff)) != -1) {
              if (total + nread > obuff.length) {
                byte[] temp = obuff;
                obuff = new byte[temp.length * 2];
                System.arraycopy(temp, 0, obuff, 0, temp.length);
              }
              System.arraycopy(ubuff, 0, obuff, total, nread);
              total += nread;
            }
            if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
          } catch (BZip2ReadException ioe) {
            log.warn("Nexrad2IOSP.uncompress ", ioe);
          }
          float nrecords = (float) (total / 2432.0);
          if (log.isDebugEnabled())
            log.debug(
                "  unpacked "
                    + total
                    + " num bytes "
                    + nrecords
                    + " records; ouput ends at "
                    + outputRaf.getFilePointer());
        }

      } catch (Exception e) {
        if (outputRaf != null) outputRaf.close();
        outputRaf = null;

        // dont leave bad files around
        File ufile = new File(ufilename);
        if (ufile.exists()) {
          if (!ufile.delete())
            log.warn("failed to delete uncompressed file (IOException)" + ufilename);
        }

        if (e instanceof IOException) throw (IOException) e;
        else throw new RuntimeException(e);
      }

    } finally {
      if (null != outputRaf) outputRaf.flush();
      if (lock != null) lock.release();
    }

    return outputRaf;
  }

  // debugging

  static void bdiff(String filename) throws IOException {

    InputStream in1 = new FileInputStream(filename + ".tmp");
    InputStream in2 = new FileInputStream(filename + ".tmp2");

    int count = 0;
    int bad = 0;
    while (true) {
      int b1 = in1.read();
      int b2 = in2.read();
      if (b1 < 0) break;
      if (b2 < 0) break;

      if (b1 != b2) {
        System.out.println(count + " in1=" + b1 + " in2= " + b2);
        bad++;
        if (bad > 130) break;
      }
      count++;
    }
    System.out.println("total read = " + count);
  }

  // check if compressed file seems ok
  public static long testValid(String ufilename) throws IOException {
    boolean lookForHeader = false;

    // gotta make it
    RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    raf.seek(0);
    byte[] b = new byte[8];
    raf.read(b);
    String test = new String(b);
    if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
      System.out.println("--Good header= " + test);
      raf.seek(24);
    } else {
      System.out.println("--No header ");
      lookForHeader = true;
      raf.seek(0);
    }

    boolean eof = false;
    int numCompBytes;
    try {

      while (!eof) {

        if (lookForHeader) {
          raf.read(b);
          test = new String(b);
          if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
            System.out.println("  found header= " + test);
            raf.skipBytes(16);
            lookForHeader = false;
          } else {
            raf.skipBytes(-8);
          }
        }

        try {
          numCompBytes = raf.readInt();
          if (numCompBytes == -1) {
            System.out.println("\n--done: numCompBytes=-1 ");
            break;
          }
        } catch (EOFException ee) {
          System.out.println("\n--got EOFException ");
          break; // assume this is ok
        }

        System.out.print(" " + numCompBytes + ",");
        if (numCompBytes < 0) {
          System.out.println("\n--last block " + numCompBytes);
          numCompBytes = -numCompBytes;
          if (!lookForHeader) eof = true;
        }

        raf.skipBytes(numCompBytes);
      }
    } catch (EOFException e) {
      e.printStackTrace();
    }

    return raf.getFilePointer();
  }

  /** test */
  public static void main2(String[] args) throws IOException {
    File testDir = new File("C:/data/bad/radar2/");

    File[] files = testDir.listFiles();
    for (int i = 0; i < files.length; i++) {
      File file = files[i];
      if (!file.getPath().endsWith(".ar2v")) continue;
      System.out.println(file.getPath() + " " + file.length());
      long pos = testValid(file.getPath());
      if (pos == file.length()) {
        System.out.println("OK");
        try {
          NetcdfFile.open(file.getPath());
        } catch (Throwable t) {
          System.out.println("ERROR=  " + t);
        }
      } else System.out.println("NOT pos=" + pos);

      System.out.println();
    }
  }

  public static void main(String args[]) throws IOException {
    NexradStationDB.init();

    RandomAccessFile raf =
        new RandomAccessFile(
            "/upc/share/testdata/radar/nexrad/level2/Level2_KFTG_20060818_1814.ar2v.uncompress.missingradials",
            "r");
    // RandomAccessFile raf = new
    // RandomAccessFile("R:/testdata2/radar/nexrad/level2/problem/KCCX_20060627_1701", "r");
    new Level2VolumeScan(raf, null);
  }
}
Exemplo n.º 14
0
/** Handles the Ensemble coordinate dimension. Assumes GribGridRecord */
public class GridEnsembleCoord {
  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GridEnsembleCoord.class);

  private List<EnsCoord> ensCoords;
  private int seq = 0;

  /**
   * Create a new GridEnsembleCoord with the list of records
   *
   * @param records records to use
   */
  GridEnsembleCoord(List<GridRecord> records) {
    Map<Integer, EnsCoord> map = new HashMap<Integer, EnsCoord>();

    for (GridRecord record : records) {
      GribGridRecord ggr = (GribGridRecord) record;
      int ensNumber = ggr.getPds().getPerturbationNumber();
      int ensType = ggr.getPds().getPerturbationType();
      int h = 1000 * ensNumber + ensType; // unique perturbation number and type
      map.put(h, new EnsCoord(ensNumber, ensType));
    }

    ensCoords = new ArrayList<EnsCoord>(map.values());
    Collections.sort(ensCoords);
  }

  private class EnsCoord implements Comparable<EnsCoord> {
    int number, type;

    private EnsCoord(int number, int type) {
      this.number = number;
      this.type = type;
    }

    @Override
    public int compareTo(EnsCoord o) {
      int r = number - o.number;
      return (r == 0) ? type - o.type : r;
    }

    @Override
    public boolean equals(Object o) {
      if (this == o) return true;
      if (o == null || getClass() != o.getClass()) return false;

      EnsCoord ensCoord = (EnsCoord) o;

      if (number != ensCoord.number) return false;
      if (type != ensCoord.type) return false;

      return true;
    }

    @Override
    public int hashCode() {
      return 1000 * number + type;
    }

    @Override
    public String toString() {
      return "number=" + number + ", type=" + type;
    }
  }

  @Override
  public boolean equals(Object o) {
    if (this == o) return true;
    if (o == null || getClass() != o.getClass()) return false;

    GridEnsembleCoord that = (GridEnsembleCoord) o;
    return ensCoords.equals(that.ensCoords);
  }

  @Override
  public int hashCode() {
    return ensCoords.hashCode();
  }

  /**
   * Set the sequence number
   *
   * @param seq the sequence number
   */
  void setSequence(int seq) {
    this.seq = seq;
  }

  /**
   * Get the name
   *
   * @return the name
   */
  String getName() {
    return (seq == 0) ? "ens" : "ens" + seq;
  }

  /**
   * Add this as a dimension to a netCDF file
   *
   * @param ncfile the netCDF file
   * @param g the group in the file
   */
  void addDimensionsToNetcdfFile(NetcdfFile ncfile, Group g) {
    ncfile.addDimension(g, new Dimension(getName(), getNEnsembles(), true));
  }

  /**
   * Add this as a variable to the netCDF file
   *
   * @param ncfile the netCDF file
   * @param g the group in the file
   */
  void addToNetcdfFile(NetcdfFile ncfile, Group g) {
    Variable v = new Variable(ncfile, g, null, getName());
    v.setDimensions(v.getShortName());
    v.setDataType(DataType.STRING);
    v.addAttribute(new Attribute("long_name", "ensemble"));
    v.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Ensemble.toString()));

    String[] data = new String[getNEnsembles()];

    for (int i = 0; i < getNEnsembles(); i++) {
      EnsCoord ec = ensCoords.get(i);
      data[i] = Grib2Tables.codeTable4_6(ec.type) + " " + ec.number;
    }
    Array dataArray = Array.factory(DataType.STRING, new int[] {getNEnsembles()}, data);
    v.setCachedData(dataArray, false);

    ncfile.addVariable(g, v);
  }

  /**
   * Get the index of a GridRecord
   *
   * @param ggr the grib record
   * @return the index or -1 if not found
   */
  int getIndex(GribGridRecord ggr) {
    int ensNumber = ggr.getPds().getPerturbationNumber();
    int ensType = ggr.getPds().getPerturbationType();

    int count = 0;
    for (EnsCoord coord : ensCoords) {
      if ((coord.number == ensNumber) && (coord.type == ensType)) return count;
      count++;
    }
    return -1;
  }

  /**
   * Get the number of Ensembles
   *
   * @return the number of Ensembles
   */
  int getNEnsembles() {
    return ensCoords.size();
  }
}
public class DashboardFilterHandler extends UIComponentHandlerFactoryElement {
  private static transient org.slf4j.Logger log =
      org.slf4j.LoggerFactory.getLogger(DashboardFilterHandler.class.getName());

  // Component JSPs
  protected String componentIncludeJSP;
  protected String componentIncludeJSPshow;
  protected String componentIncludeJSPedit;
  protected String componentIncludeJSPproperties;

  protected String componentIncludeJSPshort_show;
  protected String componentIncludeJSPshort_edit;
  protected String componentIncludeJSPshort_properties;

  public static final String I18N_PREFFIX = "dashboardFilter.";
  public static final String PARAM_VALUE = "value";
  public static final String PARAM_VALUE_MIN = "minValue";
  public static final String PARAM_VALUE_MAX = "maxValue";
  public static final String PARAM_CUSTOM_VALUE = "customValue";
  public static final String PARAM_LAST_HOUR = "lastHour";
  public static final String PARAM_LAST_12HOURS = "last12Hours";
  public static final String PARAM_TODAY = "today";
  public static final String PARAM_YESTERDAY = "yesterday";
  public static final String PARAM_LAST_7DAYS = "last7Days";
  public static final String PARAM_THIS_MONTH = "thisMonth";
  public static final String PARAM_LAST_MONTH = "lastMonth";
  public static final String PARAM_THIS_QUARTER = "thisQuarter";
  public static final String PARAM_LAST_QUARTER = "lastQuarter";
  public static final String PARAM_LAST_6MONTHS = "last12Months";
  public static final String PARAM_THIS_YEAR = "thisYear";
  public static final String PARAM_LAST_YEAR = "lastYear";
  public static final String PARAM_NULL_VALUE = "---";
  public static final String PARAM_VISIBLE = "visible";
  public static final String PARAM_DRILLDOWN_DISABLED = "drillDownDisabled";
  public static final String PARAM_SECTION = "section";
  public static final String PARAM_SHOW_REFRESH_BUTTON = "showRefreshButton";
  public static final String PARAM_SHOW_APPLY_BUTTON = "showApplyButton";
  public static final String PARAM_SHOW_CLEAR_BUTTON = "showClearButton";
  public static final String PARAM_SHOW_PROPERTY_NAMES = "showPropertyNames";
  public static final String PARAM_SHOW_SUBMIT_ON_CHANGE = "showSubmitOnChange";
  public static final String PARAM_SHOW_AUTO_REFRESH = "showAutoRefresh";
  public static final String PARAM_SHORT_MODE = "shortMode";
  public static final String PARAM_SHOW_LEGEND = "showLegend";

  protected DashboardHandler dashboardHandler;
  protected DashboardFilterRequestProcessor requestProcessor;

  protected String serializedProperties;

  // Component options.
  protected boolean isShowMode;
  protected boolean isEditMode;
  protected boolean isShortMode;
  protected boolean showPropertyNames;
  protected boolean showRefreshButton;
  protected boolean showApplyButton;
  protected boolean showClearButton;
  protected boolean showSubmitOnChange;
  protected boolean showLegend;
  protected boolean showAutoRefresh;

  // Handle component properties.
  protected List
      properties; // Properties selected in edit mode. Filter can be executed with these properties.
  protected List
      notAllowedProperties; // Properties that cannot be displayed because there is another property
                            // visible with same property identifier.
  protected Set filterPropertyErrors; // Errors when data input to filter cannot be parsed.
  protected boolean
      panelDuplicated; // Flag which indicate if a panel instance is duplicated in different
                       // sections.

  // Auto-Refresh control
  protected int autoRefreshTimeout;
  protected boolean refreshEnabled;

  public DashboardFilterHandler() {
    serializedProperties = null;
    isShowMode = true;
    isEditMode = false;
    isShortMode = true;
    properties = new ArrayList();
    filterPropertyErrors = new HashSet();
    notAllowedProperties = new ArrayList();
    showPropertyNames = true;
    showRefreshButton = true;
    showApplyButton = true;
    showClearButton = true;
    showSubmitOnChange = true;
    showLegend = true;
    showAutoRefresh = false;
    autoRefreshTimeout = 15;
    panelDuplicated = false;
  }

  // -------------- START GETTERS AND SETTERS --------------------- //

  public boolean isPanelDuplicated() {
    return panelDuplicated;
  }

  public void setPanelDuplicated(boolean panelDuplicated) {
    this.panelDuplicated = panelDuplicated;
  }

  public boolean isRefreshEnabled() {
    return refreshEnabled;
  }

  public void setRefreshEnabled(boolean refreshEnabled) {
    this.refreshEnabled = refreshEnabled;
  }

  public int getAutoRefreshTimeout() {
    return autoRefreshTimeout;
  }

  public void setAutoRefreshTimeout(int autoRefreshTimeout) {
    this.autoRefreshTimeout = autoRefreshTimeout;
  }

  public boolean isShowAutoRefresh() {
    return showAutoRefresh;
  }

  public void setShowAutoRefresh(boolean showAutoRefresh) {
    this.showAutoRefresh = showAutoRefresh;
  }

  public List getNotAllowedProperties() {
    return notAllowedProperties;
  }

  public void setNotAllowedProperties(List notAllowedProperties) {
    this.notAllowedProperties = notAllowedProperties;
  }

  public boolean isShowSubmitOnChange() {
    return showSubmitOnChange;
  }

  public void setShowSubmitOnChange(boolean showSubmitOnChange) {
    this.showSubmitOnChange = showSubmitOnChange;
  }

  public boolean isShowLegend() {
    return showLegend;
  }

  public void setShowLegend(boolean showLegend) {
    this.showLegend = showLegend;
  }

  public DashboardFilterRequestProcessor getRequestProcessor() {
    return requestProcessor;
  }

  public void setRequestProcessor(DashboardFilterRequestProcessor requestProcessor) {
    this.requestProcessor = requestProcessor;
  }

  public boolean isShowApplyButton() {
    return showApplyButton;
  }

  public void setShowApplyButton(boolean showApplyButton) {
    this.showApplyButton = showApplyButton;
  }

  public boolean isShowClearButton() {
    return showClearButton;
  }

  public void setShowClearButton(boolean showClearButton) {
    this.showClearButton = showClearButton;
  }

  public boolean isShowRefreshButton() {
    return showRefreshButton;
  }

  public void setShowRefreshButton(boolean showRefreshButton) {
    this.showRefreshButton = showRefreshButton;
  }

  public boolean isShowPropertyNames() {
    return showPropertyNames;
  }

  public void setShowPropertyNames(boolean showPropertyNames) {
    this.showPropertyNames = showPropertyNames;
  }

  public List getProperties() {
    return properties;
  }

  public void setProperties(List properties) {
    this.properties = properties;
  }

  public DashboardHandler getDashboardHandler() {
    return dashboardHandler;
  }

  public void setDashboardHandler(DashboardHandler dashboardHandler) {
    this.dashboardHandler = dashboardHandler;
  }

  public boolean isShortMode() {
    return isShortMode;
  }

  public void setShortMode(boolean shortMode) {
    isShortMode = shortMode;
  }

  public boolean isEditMode() {
    return isEditMode;
  }

  public boolean isShowMode() {
    return isShowMode;
  }

  public String getComponentIncludeJSPshort_edit() {
    return componentIncludeJSPshort_edit;
  }

  public void setComponentIncludeJSPshort_edit(String componentIncludeJSPshort_edit) {
    this.componentIncludeJSPshort_edit = componentIncludeJSPshort_edit;
  }

  public String getComponentIncludeJSPshort_properties() {
    return componentIncludeJSPshort_properties;
  }

  public void setComponentIncludeJSPshort_properties(String componentIncludeJSPshort_properties) {
    this.componentIncludeJSPshort_properties = componentIncludeJSPshort_properties;
  }

  public String getComponentIncludeJSPshort_show() {
    return componentIncludeJSPshort_show;
  }

  public void setComponentIncludeJSPshort_show(String componentIncludeJSPshort_show) {
    this.componentIncludeJSPshort_show = componentIncludeJSPshort_show;
  }

  public String getComponentIncludeJSPproperties() {
    return componentIncludeJSPproperties;
  }

  public String getJSPForShowMode() {
    if (isShortMode) return componentIncludeJSPshort_show;
    else return componentIncludeJSPshow;
  }

  public String getJSPForEditMode() {
    return componentIncludeJSPedit;
  }

  public String getJSPForProperties() {
    if (!isShortMode) return componentIncludeJSPproperties;
    else return componentIncludeJSPshort_properties;
  }

  public String getJSP() {
    if (isEditMode) return getJSPForEditMode();
    return getJSPForShowMode();
  }

  public void setComponentIncludeJSPproperties(String componentIncludeJSPproperties) {
    this.componentIncludeJSPproperties = componentIncludeJSPproperties;
  }

  public String getComponentIncludeJSPedit() {
    return componentIncludeJSPedit;
  }

  public void setComponentIncludeJSPedit(String componentIncludeJSPedit) {
    this.componentIncludeJSPedit = componentIncludeJSPedit;
  }

  public String getComponentIncludeJSPshow() {
    return componentIncludeJSPshow;
  }

  public void setComponentIncludeJSPshow(String componentIncludeJSPshow) {
    this.componentIncludeJSPshow = componentIncludeJSPshow;
  }

  public String getComponentIncludeJSP() {
    return getJSP();
  }

  public void setComponentIncludeJSP(String componentIncludeJSP) {
    this.componentIncludeJSP = componentIncludeJSP;
  }

  // -------------- END GETTERS AND SETTERS --------------------- //

  // -------------- START HANDLING METHODS ---------------------- //

  public static DashboardFilterHandler lookup(String code) {
    DashboardFilterHandler handler = null;

    // Get handler for code;
    if (!StringUtils.isBlank(code))
      handler =
          (DashboardFilterHandler)
              Factory.lookup("org.jboss.dashboard.ui.components.DashboardFilterHandler_" + code);
    if (handler == null)
      handler =
          (DashboardFilterHandler)
              Factory.lookup("org.jboss.dashboard.ui.components.DashboardFilterHandler");
    return handler;
  }

  public String getComponentPath() {
    return getComponentName();
  }

  public Dashboard getDashboard() {
    return dashboardHandler.getCurrentDashboard();
  }

  public DashboardFilter getFilter() {
    return getDashboard().getDashboardFilter();
  }

  public void enableEditMode() {
    isEditMode = true;
    isShowMode = false;
  }

  public void enableShowMode() {
    isShowMode = true;
    isEditMode = false;

    // Enable or disable autorefresh on show mode.
    if (showAutoRefresh) setRefreshEnabled(true);
    else setRefreshEnabled(false);
  }

  public String getSerializedProperties() {
    return serializedProperties;
  }

  public void setSerializedProperties(String serializedProperties) {
    this.serializedProperties = serializedProperties;
  }

  public DashboardFilterProperty getDashboardFilterPropertyForCurrentFilter(
      String dataProviderCode, String propertyId) {
    Iterator it = properties.iterator();
    while (it.hasNext()) {
      DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
      if (dataProviderCode.equals(dashboardFilterProperty.getDataProviderCode())
          && propertyId.equals(dashboardFilterProperty.getPropertyId()))
        return dashboardFilterProperty;
    }
    return null;
  }

  public DashboardFilterProperty getDashboardFilterProperty(String propertyId) {
    Iterator it = properties.iterator();
    while (it.hasNext()) {
      DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
      if (propertyId.equals(dashboardFilterProperty.getPropertyId()))
        return dashboardFilterProperty;
    }
    return null;
  }

  // Calls dashboard filter to get static  properties but keep properties instance configuration. Is
  // propety is found on this instance properties List thsi instance is returned.
  public DashboardFilterProperty[] getStaticPropertiesForCurrentFilter() {
    DashboardFilterProperty[] staticProperties = getFilter().getStaticProperties();
    if (staticProperties == null) return null;
    DashboardFilterProperty[] results = new DashboardFilterProperty[staticProperties.length];
    for (int i = 0; i < staticProperties.length; i++) {
      DashboardFilterProperty staticProperty = staticProperties[i];
      DashboardFilterProperty property =
          getDashboardFilterPropertyForCurrentFilter(
              staticProperty.getDataProviderCode(), staticProperty.getPropertyId());
      if (property != null) results[i] = property;
      else results[i] = staticProperty;
    }
    return results;
  }

  public DashboardFilterProperty[] getAllPropertiesForCurrentFilter() {
    List results = new ArrayList();
    try {
      // Static properties.
      DashboardFilterProperty[] staticProps = getStaticPropertiesForCurrentFilter();
      if (staticProps != null) results.addAll(Arrays.asList(staticProps));

      // Dynamic properties.
      Iterator it = getDashboard().getDataProviders().iterator();
      while (it.hasNext()) {
        DataProvider dataProvider = (DataProvider) it.next();
        DataProperty[] allProperties = dataProvider.getDataSet().getProperties();
        for (int i = 0; i < allProperties.length; i++) {
          DataProperty property = allProperties[i];
          DashboardFilterProperty prop =
              getDashboardFilterPropertyForCurrentFilter(
                  dataProvider.getCode(), property.getPropertyId());
          if (prop == null)
            prop =
                new DashboardFilterProperty(
                    dataProvider.getCode(), property.getPropertyId(), getFilter(), null, false);
          results.add(prop);
        }
      }
    } catch (Exception e) {
      log.error("Cannot get data provider results.", e);
    }
    return (DashboardFilterProperty[]) results.toArray(new DashboardFilterProperty[results.size()]);
  }

  public DashboardFilterProperty[] getBeingFilteredProperties() {
    List results = new ArrayList();
    Iterator it = properties.iterator();
    while (it.hasNext()) {
      DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
      if (dashboardFilterProperty.isBeingFiltered()) results.add(dashboardFilterProperty);
    }

    return (DashboardFilterProperty[]) results.toArray(new DashboardFilterProperty[results.size()]);
  }

  public List<DashboardFilterProperty> getVisibleProperties() {
    List<DashboardFilterProperty> results = new ArrayList<DashboardFilterProperty>();
    Iterator it = properties.iterator();
    while (it.hasNext()) {
      DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
      if (dashboardFilterProperty.isVisible()) results.add(dashboardFilterProperty);
    }
    return results;
  }

  public boolean hasError(String propertyId) {
    return filterPropertyErrors.contains(propertyId);
  }

  // --------------- END HANDLING METHODS ------------------------ //

  // --------------- START ACTIONS ------------------------------- //

  public void actionStore(CommandRequest request) {
    Map parameters = request.getRequestObject().getParameterMap();

    // Initialize parameters and properties to default.
    showPropertyNames = false;
    showRefreshButton = false;
    showApplyButton = false;
    showClearButton = false;
    showSubmitOnChange = false;
    isShortMode = false;
    showLegend = false;
    showAutoRefresh = false;
    properties.clear();
    notAllowedProperties.clear();

    // Component options.
    if (parameters.containsKey(PARAM_SHOW_REFRESH_BUTTON)) showRefreshButton = true;
    if (parameters.containsKey(PARAM_SHOW_PROPERTY_NAMES)) showPropertyNames = true;
    if (parameters.containsKey(PARAM_SHOW_CLEAR_BUTTON)) showClearButton = true;
    if (parameters.containsKey(PARAM_SHOW_APPLY_BUTTON)) showApplyButton = true;
    if (parameters.containsKey(PARAM_SHOW_SUBMIT_ON_CHANGE)) showSubmitOnChange = true;
    if (parameters.containsKey(PARAM_SHORT_MODE)) isShortMode = true;
    if (parameters.containsKey(PARAM_SHOW_LEGEND)) showLegend = true;
    if (parameters.containsKey(PARAM_SHOW_AUTO_REFRESH)) showAutoRefresh = true;

    // Component properties.
    DashboardFilterProperty[] allProperties = getAllPropertiesForCurrentFilter();
    for (int i = 0; i < allProperties.length; i++) {
      DashboardFilterProperty property = allProperties[i];
      String dataProviderCode = property.getDataProviderCode();
      String propertyId = property.getPropertyId();

      String visibleParamKey =
          new StringBuffer()
              .append(PARAM_VISIBLE)
              .append("/")
              .append(dataProviderCode)
              .append("/")
              .append(propertyId)
              .toString();
      String drillDownParamKey =
          new StringBuffer()
              .append(PARAM_SECTION)
              .append("/")
              .append(dataProviderCode)
              .append("/")
              .append(propertyId)
              .toString();
      boolean isVisible = parameters.containsKey(visibleParamKey);
      Long sectionId = null;
      if (parameters.containsKey(drillDownParamKey)) {
        String sectionIdStr = ((String[]) parameters.get(drillDownParamKey))[0];
        if (!PARAM_DRILLDOWN_DISABLED.equals(sectionIdStr)) sectionId = Long.decode(sectionIdStr);
      }

      if (!isVisible && sectionId == null) continue;

      // Property must be added?
      DashboardFilterProperty prop =
          getDashboardFilterPropertyForCurrentFilter(dataProviderCode, propertyId);
      if (prop == null)
        prop = new DashboardFilterProperty(dataProviderCode, propertyId, getFilter(), null, false);

      // Check if another property with same identifier.
      if (getDashboardFilterProperty(propertyId) != null) {
        // Another property with same id is already set to the filter.
        // Filter cannot use two properties with same property id., so show warning.
        notAllowedProperties.add(prop);
        continue;
      }

      // Add property to this component.
      properties.add(prop);

      // Set property parameters
      prop.setBeignFiltered(false);
      prop.setVisible(isVisible);
      prop.setSectionId(sectionId);
    }
  }

  public CommandResponse actionFilter(CommandRequest request) {
    try {
      // Init attributes for start applying the filter.
      filterPropertyErrors.clear();

      // Parse parameters and set the filter.
      Iterator visiblePropertiesIt = properties.iterator();
      while (visiblePropertiesIt.hasNext()) {
        DashboardFilterProperty dashboardFilterProperty =
            (DashboardFilterProperty) visiblePropertiesIt.next();

        // Is property already in the dashboard filter?. Then is not possible to filter by this
        // property, it's already added to dashboard filter.
        if (dashboardFilterProperty.isBeingFiltered()) continue;
        if (!dashboardFilterProperty.isPropertyAlive()) {
          log.warn(
              "Trying to filter by "
                  + dashboardFilterProperty.getPropertyId()
                  + ". This property is not in any dataset.");
          continue;
        }
        if (!dashboardFilterProperty.isVisible()) continue;

        Object[] result;
        try {
          result =
              requestProcessor.parseDashboardProperty(
                  request.getRequestObject(), dashboardFilterProperty);
        } catch (Exception e) {
          log.error("Error parsing property " + dashboardFilterProperty.getPropertyId() + ".", e);
          continue;
        }

        if (result.length != 3) {
          log.error(
              "Error parsing property: '"
                  + dashboardFilterProperty.getPropertyId()
                  + "' for dataProvider: '"
                  + dashboardFilterProperty.getDataProviderCode()
                  + "'");
          continue;
        }

        Collection allowedValues = (Collection) result[0];
        Object minValue = result[1];
        Object maxValue = result[2];
        if (allowedValues == null && minValue == null && maxValue == null) continue;

        // Set the filter with this property.
        Dashboard currentDashboard = DashboardHandler.lookup().getCurrentDashboard();
        if (currentDashboard.filter(
            dashboardFilterProperty.getPropertyId(),
            minValue,
            true,
            maxValue,
            true,
            allowedValues,
            FilterByCriteria.ALLOW_ANY)) {
          return new ShowCurrentScreenResponse();
        }
      }
    } catch (Exception e) {
      log.error("Error trying to filter properties for dashboard", e);
    }
    return null;
  }

  public void actionRefresh(CommandRequest request) {
    try {
      String timeOutValue = request.getRequestObject().getParameter("refreshTimeOut");
      if (!StringUtils.isBlank(timeOutValue)) {
        try {
          autoRefreshTimeout = Integer.decode(timeOutValue).intValue();
        } catch (NumberFormatException e) {
          log.warn("Cannot parse auto refresh value as a number.");
        }
      }
      getDashboard().refresh();
    } catch (Exception e) {
      log.error("Cannot refresh dashboard.", e);
    }
  }

  public CommandResponse actionClear(CommandRequest request) {
    try {
      Dashboard dashboard = getDashboard();
      if (dashboard.unfilter()) {
        return new ShowCurrentScreenResponse();
      }
    } catch (Exception e) {
      log.error("Cannot refresh dashboard.", e);
    }
    return null;
  }

  public CommandResponse actionDeleteFilteredProperty(CommandRequest request) {
    String propertyToDelete = request.getRequestObject().getParameter("filteredPropertyToDelete");
    if (propertyToDelete == null || propertyToDelete.trim().length() == 0) return null;

    try {
      Dashboard dashboard = getDashboard();
      if (dashboard.unfilter(propertyToDelete)) {
        return new ShowCurrentScreenResponse();
      }
    } catch (Exception e) {
      log.error("Cannot remove filter property.", e);
    }
    return null;
  }

  public void actionPlay(CommandRequest request) {
    setRefreshEnabled(true);
  }

  public void actionStop(CommandRequest request) {
    setRefreshEnabled(false);
  }

  // --------- END ACTIONS ---------------------------------- //

  // --------- SERIALIZATION / DESERIALIZATION -------------- //

  public String serializeComponentData() throws Exception {
    // Serialize visible properties and options.
    StringWriter sw = new StringWriter();
    PrintWriter out = new PrintWriter(sw);
    int indent = 0;
    printIndent(out, indent);
    out.println("<dashboard_filter>");
    Iterator it = properties.iterator();
    while (it.hasNext()) {
      DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) it.next();
      printIndent(out, indent + 1);
      out.println(
          "<property id=\""
              + StringEscapeUtils.escapeXml(dashboardFilterProperty.getPropertyId())
              + "\" providerCode =\""
              + StringEscapeUtils.escapeXml(dashboardFilterProperty.getDataProviderCode())
              + "\">");
      printIndent(out, indent + 2);
      out.println("<visible>" + dashboardFilterProperty.isVisible() + "</visible>");
      if (dashboardFilterProperty.getSectionId() != null) {
        printIndent(out, indent + 2);
        out.println("<section>" + dashboardFilterProperty.getSectionId() + "</section>");
      }
      printIndent(out, indent + 1);
      out.println("</property>");
    }

    // Serialize options.
    printIndent(out, indent + 1);
    out.println("<options>");
    printIndent(out, indent + 2);
    out.println("<shortViewMode>" + isShortMode + "</shortViewMode>");
    printIndent(out, indent + 2);
    out.println("<showLegend>" + showLegend + "</showLegend>");
    printIndent(out, indent + 2);
    out.println("<showRefreshButton>" + showRefreshButton + "</showRefreshButton>");
    printIndent(out, indent + 2);
    out.println("<showApplyhButton>" + showApplyButton + "</showApplyhButton>");
    printIndent(out, indent + 2);
    out.println("<showClearButton>" + showClearButton + "</showClearButton>");
    printIndent(out, indent + 2);
    out.println("<showPropertyNames>" + showPropertyNames + "</showPropertyNames>");
    printIndent(out, indent + 2);
    out.println("<showSubmitOnChange>" + showSubmitOnChange + "</showSubmitOnChange>");
    printIndent(out, indent + 1);
    out.println("<showAutoRefresh>" + showAutoRefresh + "</showAutoRefresh>");
    printIndent(out, indent + 1);
    out.println("</options>");
    printIndent(out, indent);

    out.println("</dashboard_filter>");
    serializedProperties = sw.toString();
    return sw.toString();
  }

  protected void printIndent(PrintWriter out, int indent) {
    for (int i = 0; i < indent; i++) {
      out.print("  ");
    }
  }

  // Return if is needed to serialize and save properties after this call because properties that
  // does not exist on current filter or data providers must be deleted from persistence.
  // return: must clear serialized trash properties after deserialize process saving this data.
  public boolean deserializeComponentData(String serializedData) throws Exception {
    // Load options and visible properties
    if (serializedData == null || serializedData.trim().length() == 0) {
      log.info("No data to deserialize.");
      return false;
    }

    DOMParser parser = new DOMParser();
    parser.parse(new InputSource(new StringReader(serializedData)));
    Document doc = parser.getDocument();
    NodeList nodes = doc.getElementsByTagName("dashboard_filter");
    if (nodes.getLength() > 1) {
      log.error("Each dashboard filter component just can parse one <dashboard_filter>");
      return false;
    }
    if (nodes.getLength() == 0) {
      log.info("No data to deserialize.");
      return false;
    }

    boolean needsToSerializeAfter = false;
    serializedProperties = serializedData;
    Node rootNode = nodes.item(0);
    nodes = rootNode.getChildNodes();
    for (int x = 0; x < nodes.getLength(); x++) {
      Node node = nodes.item(x);
      if (node.getNodeName().equals("property")) {
        // Parse visible properties.
        String dataProviderCode = node.getAttributes().getNamedItem("providerCode").getNodeValue();
        String propertyId = node.getAttributes().getNamedItem("id").getNodeValue();
        String sectionId = null;
        boolean visible = false;
        NodeList subnodes = node.getChildNodes();
        for (int i = 0; i < subnodes.getLength(); i++) {
          Node subnode = subnodes.item(i);
          if (subnode.getNodeName().equals("section")) {
            sectionId = subnode.getFirstChild().getNodeValue();
          }
          if (subnode.getNodeName().equals("visible")) {
            visible = Boolean.valueOf(subnode.getFirstChild().getNodeValue()).booleanValue();
          }
        }
        Long lSectionId = sectionId != null ? Long.decode(sectionId) : null;
        DashboardFilterProperty filterProp =
            new DashboardFilterProperty(
                dataProviderCode, propertyId, getFilter(), lSectionId, true);
        filterProp.setVisible(visible);
        if (filterProp.isPropertyAlive()) properties.add(filterProp);
        else needsToSerializeAfter = true;
      } else if (node.getNodeName().equals("options")) {
        // Parse component options.
        NodeList options = node.getChildNodes();
        String showRefreshButton = null;
        String showPropertyNames = null;
        String showClearButton = null;
        String showApplyButton = null;
        String showSubmitOnChange = null;
        String showShortViewMode = null;
        String showLegend = null;
        String showAutoRefresh = null;
        for (int i = 0; i < options.getLength(); i++) {
          Node option = options.item(i);
          if (option.getNodeName().equals("showRefreshButton"))
            showRefreshButton = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("showPropertyNames"))
            showPropertyNames = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("showClearButton"))
            showClearButton = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("showApplyhButton"))
            showApplyButton = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("showSubmitOnChange"))
            showSubmitOnChange = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("shortViewMode"))
            showShortViewMode = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("showLegend"))
            showLegend = option.getFirstChild().getNodeValue();
          if (option.getNodeName().equals("showAutoRefresh"))
            showAutoRefresh = option.getFirstChild().getNodeValue();
        }
        this.showPropertyNames = Boolean.valueOf(showPropertyNames).booleanValue();
        this.showRefreshButton = Boolean.valueOf(showRefreshButton).booleanValue();
        this.showApplyButton = Boolean.valueOf(showApplyButton).booleanValue();
        this.showClearButton = Boolean.valueOf(showClearButton).booleanValue();
        this.showSubmitOnChange = Boolean.valueOf(showSubmitOnChange).booleanValue();
        this.isShortMode = Boolean.valueOf(showShortViewMode).booleanValue();
        this.showLegend = Boolean.valueOf(showLegend).booleanValue();
        this.showAutoRefresh = Boolean.valueOf(showAutoRefresh).booleanValue();

        // Enable auto-refresh if necessary on start.
        if (this.showAutoRefresh) setRefreshEnabled(true);
      }
    }
    return needsToSerializeAfter;
  }

  public void beforeRenderComponent() {
    super.beforeRenderComponent();

    // Initialize the dashboard (loads all its kpi panels)
    DashboardHandler.lookup().getCurrentDashboard();

    // Get the filter.
    DashboardFilter filter = getFilter();

    // Check all visible properties exist.
    Iterator props = properties.iterator();
    while (props.hasNext()) {
      DashboardFilterProperty dashboardFilterProperty = (DashboardFilterProperty) props.next();
      if (!dashboardFilterProperty.isPropertyAlive()) props.remove();
    }

    // Check if filtered properties for this filter component are already in dashboard filter.
    DashboardFilterProperty[] beingFilteredProps = getBeingFilteredProperties();
    for (int i = 0; i < beingFilteredProps.length; i++) {
      List dfProperties = Arrays.asList(filter.getPropertyIds());
      DashboardFilterProperty beignFilteredProp = beingFilteredProps[i];
      if (!dfProperties.contains(beignFilteredProp.getPropertyId()))
        beignFilteredProp.setBeignFiltered(false);
    }

    // Check filtered properties and hide from available filter properties (set property not
    // visible)
    String[] propIds = filter.getPropertyIds();
    for (int i = 0; i < propIds.length; i++) {
      String propId = propIds[i];
      DashboardFilterProperty prop = getDashboardFilterProperty(propId);
      if (prop == null) {
        DashboardFilterProperty parentProperty = filter.getPropertyInParentDashboards(propId);
        if (parentProperty != null) {
          prop =
              new DashboardFilterProperty(
                  parentProperty.getDataProviderCode(), propId, getFilter(), null, true);
          properties.add(prop);
        }
      } else {
        prop.setBeignFiltered(true);
      }
    }
  }
}
Exemplo n.º 16
0
/**
 * Helper class for using the netcdf-3 record dimension.
 *
 * @author caron
 * @since Feb 29, 2008
 */
public class RecordDatasetHelper {
  private static org.slf4j.Logger log =
      org.slf4j.LoggerFactory.getLogger(RecordDatasetHelper.class);

  protected NetcdfDataset ncfile;
  protected String obsTimeVName, nomTimeVName;
  protected String latVName, lonVName, zcoordVName, zcoordUnits;
  protected String stnIdVName, stnIndexVName, stnDescVName;
  protected StationHelper stationHelper;
  protected DataType stationIdType;

  protected StructureDS recordVar;
  protected Dimension obsDim;

  protected LatLonRect boundingBox;
  protected double minDate, maxDate;
  protected DateUnit timeUnit;

  protected double altScaleFactor = 1.0;

  protected Formatter errs = null;
  protected boolean showErrors = true;

  /**
   * Constructor.
   *
   * @param ncfile the netccdf file
   * @param typedDataVariables list of data variables; all record variables will be added to this
   *     list, except . You can remove extra
   * @param obsTimeVName observation time variable name (required)
   * @param nomTimeVName nominal time variable name (may be null)
   * @throws IllegalArgumentException if ncfile has no unlimited dimension and recDimName is null.
   */
  public RecordDatasetHelper(
      NetcdfDataset ncfile,
      String obsTimeVName,
      String nomTimeVName,
      List<VariableSimpleIF> typedDataVariables,
      String recDimName,
      Formatter errBuffer) {
    this.ncfile = ncfile;
    this.obsTimeVName = obsTimeVName;
    this.nomTimeVName = nomTimeVName;
    this.errs = errBuffer;

    // check if we already have a structure vs if we have to add it.

    if (this.ncfile.hasUnlimitedDimension()) {
      this.ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
      this.recordVar = (StructureDS) this.ncfile.getRootGroup().findVariable("record");
      this.obsDim = ncfile.getUnlimitedDimension();

    } else {
      if (recDimName == null)
        throw new IllegalArgumentException(
            "File <"
                + this.ncfile.getLocation()
                + "> has no unlimited dimension, specify psuedo record dimension with observationDimension global attribute.");
      this.obsDim = this.ncfile.getRootGroup().findDimension(recDimName);
      this.recordVar = new StructurePseudoDS(this.ncfile, null, "record", null, obsDim);
    }

    // create member variables
    List<Variable> recordMembers = ncfile.getVariables();
    for (Variable v : recordMembers) {
      if (v == recordVar) continue;
      if (v.isScalar()) continue;
      if (v.getDimension(0) == this.obsDim) typedDataVariables.add(v);
    }

    // need the time units
    Variable timeVar = ncfile.findVariable(obsTimeVName);
    String timeUnitString =
        ncfile.findAttValueIgnoreCase(timeVar, CDM.UNITS, "seconds since 1970-01-01");
    try {
      timeUnit = new DateUnit(timeUnitString);
    } catch (Exception e) {
      if (null != errs) errs.format("Error on string = %s == %s%n", timeUnitString, e.getMessage());
      try {
        timeUnit = new DateUnit("seconds since 1970-01-01");
      } catch (Exception e1) {
        // cant happen
      }
    }
  }

  /**
   * Set extra information used by station obs datasets. Use stnIdVName or stnIndexVName.
   *
   * @param stnIdVName the obs variable that is used to find the station in the stnHash; may be type
   *     int or a String (char).
   * @param stnDescVName optional station var containing station description
   */
  public void setStationInfo(
      String stnIdVName, String stnDescVName, String stnIndexVName, StationHelper stationHelper) {
    this.stnIdVName = stnIdVName;
    this.stnDescVName = stnDescVName;
    this.stnIndexVName = stnIndexVName;
    this.stationHelper = stationHelper;

    if (stnIdVName != null) {
      Variable stationVar = ncfile.findVariable(stnIdVName);
      stationIdType = stationVar.getDataType();
    }
  }

  public void setLocationInfo(String latVName, String lonVName, String zcoordVName) {
    this.latVName = latVName;
    this.lonVName = lonVName;
    this.zcoordVName = zcoordVName;

    // check for meter conversion
    if (zcoordVName != null) {
      Variable v = ncfile.findVariable(zcoordVName);
      zcoordUnits = ncfile.findAttValueIgnoreCase(v, CDM.UNITS, null);
      if (zcoordUnits != null)
        try {
          altScaleFactor = getMetersConversionFactor(zcoordUnits);
        } catch (Exception e) {
          if (errs != null) errs.format("%s", e.getMessage());
        }
    }
  }

  // make structure variable names to shortNames so StructureData sdata can
  // access it members
  public void setShortNames(
      String latVName, String lonVName, String altVName, String obsTimeVName, String nomTimeVName) {
    this.latVName = latVName;
    this.lonVName = lonVName;
    this.zcoordVName = altVName;
    this.obsTimeVName = obsTimeVName;
    this.nomTimeVName = nomTimeVName;
  }

  protected static double getMetersConversionFactor(String unitsString) throws Exception {
    SimpleUnit unit = SimpleUnit.factoryWithExceptions(unitsString);
    return unit.convertTo(1.0, SimpleUnit.meterUnit);
  }

  public Structure getRecordVar() {
    return (this.recordVar);
  }

  public int getRecordCount() {
    Dimension unlimitedDim = ncfile.getUnlimitedDimension();
    return unlimitedDim.getLength();
  }

  public void setTimeUnit(DateUnit timeUnit) {
    this.timeUnit = timeUnit;
  }

  public DateUnit getTimeUnit() {
    return this.timeUnit;
  }

  public LatLonPoint getLocation(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    double lat = sdata.convertScalarDouble(members.findMember(latVName));
    double lon = sdata.convertScalarDouble(members.findMember(lonVName));
    return new LatLonPointImpl(lat, lon);
  }

  public double getLatitude(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    return sdata.convertScalarDouble(members.findMember(latVName));
  }

  public double getLongitude(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    return sdata.convertScalarDouble(members.findMember(lonVName));
  }

  public double getZcoordinate(StructureData sdata) {
    StructureMembers members = sdata.getStructureMembers();
    return (zcoordVName == null)
        ? Double.NaN
        : sdata.convertScalarDouble(members.findMember(zcoordVName));
  }

  public String getZcoordUnits() {
    return zcoordUnits;
  }

  public Date getObservationTimeAsDate(StructureData sdata) {
    return timeUnit.makeDate(getObservationTime(sdata));
  }

  public double getObservationTime(StructureData sdata) {
    return getTime(sdata.findMember(obsTimeVName), sdata);
  }

  private double getTime(StructureMembers.Member timeVar, StructureData sdata) {
    if (timeVar == null) return 0.0;

    if ((timeVar.getDataType() == DataType.CHAR) || (timeVar.getDataType() == DataType.STRING)) {
      String time = sdata.getScalarString(timeVar);
      CalendarDate date = CalendarDateFormatter.isoStringToCalendarDate(null, time);
      if (date == null) {
        log.error("Cant parse date - not ISO formatted, = " + time);
        return 0.0;
      }
      return date.getMillis() / 1000.0;

    } else {
      return sdata.convertScalarDouble(timeVar);
    }
  }

  /*
   * This reads through all the records in the dataset, and constructs a list of
   * RecordPointObs or RecordStationObs. It does not cache the data.
   * <p>If stnIdVName is not null, its a StationDataset, then construct a Station HashMap of StationImpl
   * objects. Add the RecordStationObs into the list of obs for that station.
   *
   * @param cancel allow user to cancel
   * @return List of RecordPointObs or RecordStationObs
   * @throws IOException on read error
   *
  public List<RecordPointObs> readAllCreateObs(CancelTask cancel) throws IOException {

    // see if its a station or point dataset
    boolean hasStations = stnIdVName != null;
    if (hasStations)
      stnHash = new HashMap<Object, Station>();

    // get min and max date and lat,lon
    double minDate = Double.MAX_VALUE;
    double maxDate = -Double.MAX_VALUE;

    double minLat = Double.MAX_VALUE;
    double maxLat = -Double.MAX_VALUE;

    double minLon = Double.MAX_VALUE;
    double maxLon = -Double.MAX_VALUE;

    // read all the data, create a RecordObs
    StructureMembers members = null;
    List<RecordPointObs> records = new ArrayList<RecordPointObs>();
    int recno = 0;
    Structure.Iterator ii = recordVar.getStructureIterator();
    while (ii.hasNext()) {
      StructureData sdata = ii.next();
      if (members == null)
        members = sdata.getStructureMembers();

      Object stationId = null;
      if (hasStations) {
        if (stationIdType == DataType.INT) {
          stationId = sdata.getScalarInt(stnIdVName);
        } else
          stationId = sdata.getScalarString(stnIdVName).trim();
      }

      String desc = (stnDescVName == null) ? null : sdata.getScalarString(stnDescVName);
      double lat = sdata.getScalarDouble(latVName);
      double lon = sdata.getScalarDouble(lonVName);
      double alt = (altVName == null) ? 0.0 : altScaleFactor * sdata.getScalarDouble(altVName);
      double obsTime = sdata.convertScalarDouble(members.findMember(obsTimeVName));
      double nomTime = (nomTimeVName == null) ? obsTime : sdata.convertScalarDouble(members.findMember(nomTimeVName));

      //double obsTime = sdata.convertScalarDouble( members.findMember( obsTimeVName) );
      //double nomTime = (nomTimeVName == null) ? obsTime : sdata.convertScalarDouble( members.findMember( nomTimeVName));

      if (hasStations) {
        Station stn = stnHash.get(stationId);
        if (stn == null) {
          stn = new Station(stationId.toString(), desc, lat, lon, alt);
          stnHash.put(stationId, stn);
        }
        RecordStationObs stnObs = new RecordStationObs(stn, obsTime, nomTime, timeUnit, recno);
        records.add(stnObs);
        //stn.addObs( stnObs);

      } else {
        records.add(new RecordPointObs(new EarthLocation(lat, lon, alt), obsTime, nomTime, timeUnit, recno));
      }

      // track date range and bounding box
      minDate = Math.min(minDate, obsTime);
      maxDate = Math.max(maxDate, obsTime);

      minLat = Math.min(minLat, lat);
      maxLat = Math.max(maxLat, lat);
      minLon = Math.min(minLon, lon);
      maxLon = Math.max(maxLon, lon);

      recno++;
      if ((cancel != null) && cancel.isCancel()) return null;
    }
    boundingBox = new LatLonRect(new LatLonPointImpl(minLat, minLon), new LatLonPointImpl(maxLat, maxLon));

    return records;
  }

  /* private boolean debugBB = false;
  public List getData(ArrayList records, LatLonRect boundingBox, CancelTask cancel) throws IOException {
    if (debugBB) System.out.println("Want bb= "+boundingBox);
    ArrayList result = new ArrayList();
    for (int i = 0; i < records.size(); i++) {
      RecordDatasetHelper.RecordPointObs r =  (RecordDatasetHelper.RecordPointObs) records.get(i);
      if (boundingBox.contains(r.getLatLon())) {
        if (debugBB) System.out.println(" ok latlon= "+r.getLatLon());
        result.add( r);
      }
      if ((cancel != null) && cancel.isCancel()) return null;
    }
    return result;
  }

  // return List<PointObsDatatype>
  public List getData(ArrayList records, LatLonRect boundingBox, double startTime, double endTime, CancelTask cancel) throws IOException {
    if (debugBB) System.out.println("Want bb= "+boundingBox);
    ArrayList result = new ArrayList();
    for (int i = 0; i < records.size(); i++) {
      RecordDatasetHelper.RecordPointObs r =  (RecordDatasetHelper.RecordPointObs) records.get(i);
      if (boundingBox.contains(r.getLatLon())) {
        if (debugBB) System.out.println(" ok latlon= "+r.getLatLon());
        double timeValue = r.getObservationTime();
        if ((timeValue >= startTime) && (timeValue <= endTime))
          result.add( r);
      }
      if ((cancel != null) && cancel.isCancel()) return null;
    }
    return result;
  }  */

  //////////////////////////////////////////////////////////////////////////////////////
  public PointFeature factory(StationImpl s, StructureData sdata, int recno) {
    if (s == null) return new RecordPointObs(sdata, recno);
    else return new RecordStationObs(s, sdata, recno);
  }

  class RecordPointObs extends PointFeatureImpl {
    protected int recno;
    protected StructureData sdata;

    RecordPointObs(int recno) {
      super(RecordDatasetHelper.this.timeUnit);
      this.recno = recno;
    }

    // Constructor for the case where you keep track of the location, time of each record, but not
    // the data.
    protected RecordPointObs(
        EarthLocation location, double obsTime, double nomTime, DateUnit timeUnit, int recno) {
      super(location, obsTime, nomTime, timeUnit);
      this.recno = recno;
    }

    // Constructor for when you already have the StructureData and want to wrap it in a
    // StationObsDatatype
    protected RecordPointObs(StructureData sdata, int recno) {
      super(RecordDatasetHelper.this.timeUnit);
      this.sdata = sdata;
      this.recno = recno;

      StructureMembers members = sdata.getStructureMembers();
      obsTime = getTime(members.findMember(obsTimeVName), sdata);
      nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);

      // this assumes the lat/lon/alt is stored in the obs record
      double lat = sdata.convertScalarDouble(members.findMember(latVName));
      double lon = sdata.convertScalarDouble(members.findMember(lonVName));
      double alt =
          (zcoordVName == null)
              ? 0.0
              : altScaleFactor * sdata.convertScalarDouble(members.findMember(zcoordVName));
      location = new EarthLocationImpl(lat, lon, alt);
    }

    public String getId() {
      return Integer.toString(recno);
    }

    public LatLonPoint getLatLon() {
      return new LatLonPointImpl(location.getLatitude(), location.getLongitude());
    }

    public StructureData getFeatureData() throws IOException {
      if (null == sdata) {
        try {
          // deal with files that are updating // LOOK kludge?
          if (recno > getRecordCount()) {
            int n = getRecordCount();
            ncfile.syncExtend();
            log.info(
                "RecordPointObs.getData recno="
                    + recno
                    + " > "
                    + n
                    + "; after sync= "
                    + getRecordCount());
          }

          sdata = recordVar.readStructure(recno);
        } catch (ucar.ma2.InvalidRangeException e) {
          e.printStackTrace();
          throw new IOException(e.getMessage());
        }
      }
      return sdata;
    }

    public ucar.ma2.StructureData getDataAll() throws java.io.IOException {
      return getFeatureData();
    }
  }

  //////////////////////////////////////////////////////////////////////////////////////

  // a PointObs with the location info stored as a Station
  class RecordStationObs extends RecordPointObs {
    private Station station;

    /**
     * Constructor for the case where you keep track of the station, time of each record, but the
     * data reading is deferred.
     *
     * @param station data is for this Station
     * @param obsTime observation time
     * @param nomTime nominal time (may be NaN)
     * @param recno data is at this record number
     */
    protected RecordStationObs(
        Station station, double obsTime, double nomTime, DateUnit timeUnit, int recno) {
      super(station, obsTime, nomTime, timeUnit, recno);
      this.station = station;
    }

    // Constructor for when you have everything
    protected RecordStationObs(
        Station station, double obsTime, double nomTime, StructureData sdata, int recno) {
      super(recno);
      this.station = station;
      this.location = station;
      this.obsTime = obsTime;
      this.nomTime = nomTime;
      this.sdata = sdata;
    }

    // Constructor for when you already have the StructureData and Station, and calculate times
    protected RecordStationObs(Station station, StructureData sdata, int recno) {
      super(recno);
      this.station = station;
      this.location = station;
      this.sdata = sdata;

      StructureMembers members = sdata.getStructureMembers();
      obsTime = getTime(members.findMember(obsTimeVName), sdata);
      nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);
    }

    // Constructor for when you already have the StructureData, and need to find Station and times
    protected RecordStationObs(StructureData sdata, int recno, boolean useId) {
      super(recno);
      this.recno = recno;
      this.sdata = sdata;
      this.timeUnit = RecordDatasetHelper.this.timeUnit;

      StructureMembers members = sdata.getStructureMembers();
      obsTime = getTime(members.findMember(obsTimeVName), sdata);
      nomTime = (nomTimeVName == null) ? obsTime : getTime(members.findMember(nomTimeVName), sdata);

      if (useId) {
        // this assumes the station id/name is stored in the obs record
        String stationId;
        if (stationIdType == DataType.INT) {
          stationId = Integer.toString(sdata.getScalarInt(stnIdVName));
        } else stationId = sdata.getScalarString(stnIdVName).trim();
        station = stationHelper.getStation(stationId);
        if (null != errs)
          errs.format(" cant find station id = <%s> when reading record %d%n", stationId, recno);
        log.error(" cant find station id = <" + stationId + "> when reading record " + recno);

      } else {
        // use a station index
        List<Station> stations = stationHelper.getStations();
        int stationIndex = sdata.getScalarInt(stnIndexVName);
        if (stationIndex < 0 || stationIndex >= stations.size()) {
          if (null != errs)
            errs.format(
                " cant find station at index =%d when reading record %d%n", stationIndex, recno);
          log.error(
              "cant find station at index = " + stationIndex + " when reading record " + recno);
        } else station = stations.get(stationIndex);
      }

      location = station;
    }
  }
}
Exemplo n.º 17
0
/**
 * Build a GribCollection object. Rectilyse and manage grib collection index. Covers
 * GribCollectionProto.
 *
 * @author caron
 * @since 4/6/11
 */
public class Grib2CollectionBuilder {
  private static final org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(GribCollection.class);

  public static final String MAGIC_START = "Grib2CollectionIndex";
  protected static final int version = 5;
  private static final boolean debug = false;

  // from a single file, read in the index, create if it doesnt exist
  public static GribCollection createFromSingleFile(
      File file, CollectionManager.Force force, Formatter f) throws IOException {
    Grib2CollectionBuilder builder = new Grib2CollectionBuilder(file, f);
    builder.readOrCreateIndex(force, f);
    return builder.gc;
  }

  // from a collection, read in the index, create if it doesnt exist or is out of date
  // assume that the CollectionManager is up to date, eg doesnt need to be scanned
  public static GribCollection factory(
      CollectionManager dcm, CollectionManager.Force force, Formatter f) throws IOException {
    Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm);
    builder.readOrCreateIndex(force, f);
    return builder.gc;
  }

  // read in the index, index raf already open
  public static GribCollection createFromIndex(String name, File directory, RandomAccessFile raf)
      throws IOException {
    Grib2CollectionBuilder builder = new Grib2CollectionBuilder(name, directory);
    if (builder.readIndex(raf)) return builder.gc;
    throw new IOException("Reading index failed");
  }

  // this writes the index always
  public static boolean writeIndexFile(File indexFile, CollectionManager dcm, Formatter f)
      throws IOException {
    Grib2CollectionBuilder builder = new Grib2CollectionBuilder(dcm);
    return builder.createIndex(indexFile, CollectionManager.Force.always, f);
  }

  ////////////////////////////////////////////////////////////////

  private final List<CollectionManager> collections = new ArrayList<CollectionManager>();
  protected GribCollection gc;

  // single file
  private Grib2CollectionBuilder(File file, Formatter f) throws IOException {
    try {
      // String spec = StringUtil2.substitute(file.getPath(), "\\", "/");
      CollectionManager dcm = new DatasetCollectionSingleFile(file);
      this.collections.add(dcm);
      this.gc = new Grib2Collection(file.getName(), new File(dcm.getRoot()));

    } catch (Exception e) {
      ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
      e.printStackTrace(new PrintStream(bos));
      f.format("%s", bos.toString());
      throw new IOException(e);
    }
  }

  private Grib2CollectionBuilder(CollectionManager dcm) {
    this.collections.add(dcm);
    this.gc = new Grib2Collection(dcm.getCollectionName(), new File(dcm.getRoot()));
  }

  private Grib2CollectionBuilder(String name, File directory) {
    this.gc = new Grib2Collection(name, directory);
  }

  protected Grib2CollectionBuilder() {
    this.gc = null;
  }

  protected int getVersion() {
    return version;
  }

  // read or create index
  private void readOrCreateIndex(CollectionManager.Force ff, Formatter f) throws IOException {

    // force new index or test for new index needed
    boolean force =
        ((ff == CollectionManager.Force.always)
            || (ff == CollectionManager.Force.test && needsUpdate()));

    // otherwise, we're good as long as the index file exists
    File idx = gc.getIndexFile();
    if (force || !idx.exists() || !readIndex(idx.getPath())) {
      logger.info("GribCollection {}: createIndex {}", gc.getName(), idx.getPath());
      createIndex(idx, ff, f); // write out index
      gc.rafLocation = idx.getPath();
      gc.setRaf(new RandomAccessFile(idx.getPath(), "r"));
      readIndex(gc.getRaf()); // read back in index
    }
  }

  public boolean needsUpdate() {
    File idx = gc.getIndexFile();
    return !idx.exists() || needsUpdate(idx.lastModified());
  }

  private boolean needsUpdate(long idxLastModified) {
    CollectionManager.ChangeChecker cc = Grib2Index.getChangeChecker();
    for (CollectionManager dcm : collections) {
      for (MFile mfile : dcm.getFiles()) {
        if (cc.hasChangedSince(mfile, idxLastModified)) return true;
      }
    }
    return false;
  }

  ////////////////////////////////////////////////////////////////////////////////////////////////////
  // reading

  public boolean readIndex(String filename) throws IOException {
    return readIndex(new RandomAccessFile(filename, "r"));
  }

  public boolean readIndex(RandomAccessFile raf) {
    gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
    try {
      raf.order(RandomAccessFile.BIG_ENDIAN);
      raf.seek(0);

      //// header message
      if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
        logger.error("GribCollection {}: invalid index", gc.getName());
        return false;
      }

      int v = raf.readInt();
      if (v != getVersion()) {
        logger.warn(
            "GribCollection {}: index found version={}, want version= {} on file {}",
            new Object[] {gc.getName(), v, version, raf.getLocation()});
        return false;
      }

      long skip = raf.readLong();
      raf.skipBytes(skip);

      int size = NcStream.readVInt(raf);
      if ((size < 0) || (size > 100 * 1000 * 1000)) {
        logger.warn("GribCollection {}: invalid index ", gc.getName());
        return false;
      }

      byte[] m = new byte[size];
      raf.readFully(m);

      GribCollectionProto.GribCollectionIndex proto =
          GribCollectionProto.GribCollectionIndex.parseFrom(m);

      gc.center = proto.getCenter();
      gc.subcenter = proto.getSubcenter();
      gc.master = proto.getMaster();
      gc.local = proto.getLocal();
      gc.genProcessType = proto.getGenProcessType();
      gc.genProcessId = proto.getGenProcessId();
      gc.backProcessId = proto.getBackProcessId();
      gc.local = proto.getLocal();
      // gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);

      gc.filenames = new ArrayList<String>(proto.getFilesCount());
      for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));

      // error condition on a GribCollection Index
      if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
        logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
        return false;
      }

      gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
      for (int i = 0; i < proto.getGroupsCount(); i++)
        gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
      Collections.sort(gc.groups);

      gc.params = new ArrayList<Parameter>(proto.getParamsCount());
      for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));

      if (!readPartitions(proto)) {
        logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
        return false;
      }

      return true;

    } catch (Throwable t) {
      logger.error("Error reading index " + raf.getLocation(), t);
      return false;
    }
  }

  protected boolean readPartitions(GribCollectionProto.GribCollectionIndex proto) {
    return true;
  }

  protected void readTimePartitions(
      GribCollection.GroupHcs group, GribCollectionProto.Group proto) {
    // NOOP
  }

  GribCollection.GroupHcs readGroup(GribCollectionProto.Group p, GribCollection.GroupHcs group)
      throws IOException {

    Grib2SectionGridDefinition gdss = new Grib2SectionGridDefinition(p.getGds().toByteArray());
    Grib2Gds gds = gdss.getGDS();
    group.setHorizCoordSystem(gds.makeHorizCoordSys());

    group.varIndex = new ArrayList<GribCollection.VariableIndex>();
    for (int i = 0; i < p.getVariablesCount(); i++)
      group.varIndex.add(readVariable(p.getVariables(i), group));
    Collections.sort(group.varIndex);

    group.timeCoords = new ArrayList<TimeCoord>(p.getTimeCoordsCount());
    for (int i = 0; i < p.getTimeCoordsCount(); i++)
      group.timeCoords.add(readTimeCoord(p.getTimeCoords(i)));

    group.vertCoords = new ArrayList<VertCoord>(p.getVertCoordsCount());
    for (int i = 0; i < p.getVertCoordsCount(); i++)
      group.vertCoords.add(readVertCoord(p.getVertCoords(i)));

    group.ensCoords = new ArrayList<EnsCoord>(p.getEnsCoordsCount());
    for (int i = 0; i < p.getEnsCoordsCount(); i++)
      group.ensCoords.add(readEnsCoord(p.getEnsCoords(i)));

    group.filenose = new int[p.getFilenoCount()];
    for (int i = 0; i < p.getFilenoCount(); i++) group.filenose[i] = p.getFileno(i);

    readTimePartitions(group, p);

    // finish
    for (GribCollection.VariableIndex vi : group.varIndex) {
      TimeCoord tc = group.timeCoords.get(vi.timeIdx);
      vi.ntimes = tc.getSize();
      VertCoord vc = (vi.vertIdx < 0) ? null : group.vertCoords.get(vi.vertIdx);
      vi.nverts = (vc == null) ? 0 : vc.getSize();
      EnsCoord ec = (vi.ensIdx < 0) ? null : group.ensCoords.get(vi.ensIdx);
      vi.nens = (ec == null) ? 0 : ec.getSize();
    }

    // group.assignVertNames();

    return group;
  }

  private Parameter readParam(GribCollectionProto.Parameter pp) throws IOException {
    if (pp.hasSdata()) return new Parameter(pp.getName(), pp.getSdata());

    int count = 0;
    double[] vals = new double[pp.getDataCount()];
    for (double val : pp.getDataList()) vals[count++] = val;

    return new Parameter(pp.getName(), vals);
  }

  private TimeCoord readTimeCoord(GribCollectionProto.Coord pc) throws IOException {
    if (pc.getBoundCount() > 0) { // its an interval
      List<TimeCoord.Tinv> coords = new ArrayList<TimeCoord.Tinv>(pc.getValuesCount());
      for (int i = 0; i < pc.getValuesCount(); i++)
        coords.add(new TimeCoord.Tinv((int) pc.getValues(i), (int) pc.getBound(i)));
      return new TimeCoord(pc.getCode(), pc.getUnit(), coords);
    } else {
      List<Integer> coords = new ArrayList<Integer>(pc.getValuesCount());
      for (float value : pc.getValuesList()) coords.add((int) value);
      return new TimeCoord(pc.getCode(), pc.getUnit(), coords);
    }
  }

  private VertCoord readVertCoord(GribCollectionProto.Coord pc) throws IOException {
    boolean isLayer = (pc.getBoundCount() > 0);
    List<VertCoord.Level> coords = new ArrayList<VertCoord.Level>(pc.getValuesCount());
    for (int i = 0; i < pc.getValuesCount(); i++)
      coords.add(new VertCoord.Level(pc.getValues(i), isLayer ? pc.getBound(i) : 0));
    return new VertCoord(pc.getCode(), coords, isLayer);
  }

  private EnsCoord readEnsCoord(GribCollectionProto.Coord pc) throws IOException {
    List<EnsCoord.Coord> coords = new ArrayList<EnsCoord.Coord>(pc.getValuesCount());
    for (int i = 0; i < pc.getValuesCount(); i += 2)
      coords.add(new EnsCoord.Coord((int) pc.getValues(i), (int) pc.getValues(i + 1)));
    return new EnsCoord(coords);
  }

  protected GribCollection.VariableIndex readVariable(
      GribCollectionProto.Variable pv, GribCollection.GroupHcs group) {
    int discipline = pv.getDiscipline();
    int category = pv.getCategory();
    int param = pv.getParameter();
    int levelType = pv.getLevelType();
    int intvType = pv.getIntervalType();
    boolean isLayer = pv.getIsLayer();
    int ensDerivedType = pv.getEnsDerivedType();
    int probType = pv.getProbabilityType();
    String probabilityName = pv.getProbabilityName();
    int cdmHash = pv.getCdmHash();
    long recordsPos = pv.getRecordsPos();
    int recordsLen = pv.getRecordsLen();
    int timeIdx = pv.getTimeIdx();
    int vertIdx = pv.getVertIdx();
    int ensIdx = pv.getEnsIdx();
    int tableVersion = pv.getTableVersion();

    return gc.makeVariableIndex(
        group,
        tableVersion,
        discipline,
        category,
        param,
        levelType,
        isLayer,
        intvType,
        ensDerivedType,
        probType,
        probabilityName,
        cdmHash,
        timeIdx,
        vertIdx,
        ensIdx,
        recordsPos,
        recordsLen);
  }

  ///////////////////////////////////////////////////////////////////////////////////
  // writing

  private class Group {
    public Grib2SectionGridDefinition gdss;
    public int gdsHash; // may have been modified
    public Grib2Rectilyser rect;
    public List<Grib2Record> records = new ArrayList<Grib2Record>();
    public String name;
    public Set<Integer>
        fileSet; // this is so we can show just the component files that are in this group

    private Group(Grib2SectionGridDefinition gdss, int gdsHash) {
      this.gdss = gdss;
      this.gdsHash = gdsHash;
      Grib2Gds gds = gdss.getGDS();
      name = gds.getNameShort() + "-" + gds.ny + "X" + gds.nx;
    }
  }

  ///////////////////////////////////////////////////
  // create the index

  private boolean createIndex(File indexFile, CollectionManager.Force ff, Formatter f)
      throws IOException {
    long start = System.currentTimeMillis();

    ArrayList<String> filenames = new ArrayList<String>();
    List<Group> groups = makeAggregatedGroups(filenames, ff, f);
    createIndex(indexFile, groups, filenames, f);

    long took = System.currentTimeMillis() - start;
    f.format("That took %d msecs%n", took);
    return true;
  }

  // read all records in all files,
  // divide into groups based on GDS hash
  // each group has an arraylist of all records that belong to it.
  // for each group, run rectlizer to derive the coordinates and variables
  public List<Group> makeAggregatedGroups(
      ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
    Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();

    f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
    int total = 0;
    int fileno = 0;
    for (CollectionManager dcm : collections) {
      // dcm.scanIfNeeded(); // LOOK ??
      f.format(" dcm= %s%n", dcm);
      Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash");

      for (MFile mfile : dcm.getFiles()) {
        // f.format("%3d: %s%n", fileno, mfile.getPath());
        filenames.add(mfile.getPath());

        Grib2Index index = new Grib2Index();
        try {
          if (!index.readIndex(
              mfile.getPath(),
              mfile.getLastModified(),
              force)) { // heres where the index date is checked against the data file
            index.makeIndex(mfile.getPath(), f);
            f.format(
                "  Index written: %s == %d records %n",
                mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
          } else if (debug) {
            f.format(
                "  Index read: %s == %d records %n",
                mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
          }
        } catch (IOException ioe) {
          f.format(
              "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n  skipping %s%n",
              ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT);
          continue;
        }

        for (Grib2Record gr : index.getRecords()) {
          gr.setFile(fileno); // each record tracks which file it belongs to
          int gdsHash =
              gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
          if (gdsConvert != null
              && gdsConvert.get(gdsHash)
                  != null) { // allow external config to muck with gdsHash. Why? because of error in
            // encoding
            gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
          }

          Group g = gdsMap.get(gdsHash);
          if (g == null) {
            g = new Group(gr.getGDSsection(), gdsHash);
            gdsMap.put(gdsHash, g);
          }
          g.records.add(gr);
          total++;
        }
        fileno++;
      }
    }
    f.format(" total grib records= %d%n", total);

    Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter();
    List<Group> result = new ArrayList<Group>(gdsMap.values());
    for (Group g : result) {
      g.rect = new Grib2Rectilyser(g.records, g.gdsHash);
      f.format(" GDS hash %d == ", g.gdsHash);
      g.rect.make(f, c);
    }
    f.format(
        " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
        c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);

    return result;
  }

  /*
  MAGIC_START
  version
  sizeRecords
  VariableRecords (sizeRecords bytes)
  sizeIndex
  GribCollectionIndex (sizeIndex bytes)
  */

  private void createIndex(
      File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
      throws IOException {
    Grib2Record first = null; // take global metadata from here

    if (indexFile.exists()) indexFile.delete(); // replace it
    f.format(" createIndex for %s%n", indexFile.getPath());

    RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;
      for (Group g : groups) {
        g.fileSet = new HashSet<Integer>();
        for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
          if (first == null) first = vb.first;
          GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.recordMap.length;
        }
      }
      long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
      f.format(
          "  write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
          countBytes, countRecords, bytesPerRecord);

      if (first == null) {
        logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
        throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(gc.getName());

      for (String fn : filenames) indexBuilder.addFiles(fn);

      for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));

      /* int count = 0;
      for (DatasetCollectionManager dcm : collections) {
        indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
        count++;
      } */

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("  write GribCollectionIndex= %d bytes%n", b.length);

    } finally {
      f.format("  file size =  %d bytes%n", raf.length());
      raf.close();
      if (raf != null) raf.close();
    }
  }

  /* private void createIndexForGroup(Group group, ArrayList<String> filenames) throws IOException {
    Grib2Record first = null; // take global metadata from here

    File file = new File(gc.getDirectory(), group.name + GribCollection.IDX_EXT);
    if (file.exists()) file.delete(); // replace it

    RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      String magic = gc.getMagicBytes();
      raf.write(magic.getBytes("UTF-8"));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;
      group.fileSet = new HashSet<Integer>();

      for (Rectilyser.VariableBag vb : group.rect.getGribvars()) {
        if (first == null) first = vb.first;
        GribCollectionProto.VariableRecords vr = makeRecordsProto(vb, group.fileSet);
        byte[] b = vr.toByteArray();
        vb.pos = raf.getFilePointer();
        vb.length = b.length;
        raf.write(b);
        countBytes += b.length;
      }
      countRecords += group.records.size();
      if (countRecords == 0) countRecords = 1;
      long bytesPerRecord = countBytes / countRecords;
      logger.debug("VariableRecords: bytes = {} record = {} bytesPerRecord={}", new Object[] {countBytes, countRecords, bytesPerRecord});

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder = GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(group.name);

      for (String fn : filenames)
        indexBuilder.addFiles(fn);

      indexBuilder.addGroups(makeGroupProto(group));

      int count = 0;
      for (CollectionManager dcm : collections) {
        indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.toString())));
        count++;
      }

      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b);  // message  - all in one gulp
      logger.debug("GribCollectionIndex= {} bytes%n", b.length);

    } finally {
      logger.debug("file size =  {} bytes%n", raf.length());
      raf.close();
      if (raf != null) raf.close();
    }
  } */

  private GribCollectionProto.VariableRecords writeRecordsProto(
      Grib2Rectilyser.VariableBag vb, Set<Integer> fileSet) throws IOException {
    GribCollectionProto.VariableRecords.Builder b =
        GribCollectionProto.VariableRecords.newBuilder();
    b.setCdmHash(vb.first.cdmVariableHash(0));
    for (Grib2Rectilyser.Record ar : vb.recordMap) {
      GribCollectionProto.Record.Builder br = GribCollectionProto.Record.newBuilder();

      if (ar == null || ar.gr == null) {
        br.setFileno(0);
        br.setPos(0); // missing : ok to use 0 since drsPos > 0

      } else {
        br.setFileno(ar.gr.getFile());
        fileSet.add(ar.gr.getFile());
        Grib2SectionDataRepresentation drs = ar.gr.getDataRepresentationSection();
        br.setPos(drs.getStartingPosition());
      }
      b.addRecords(br);
    }
    return b.build();
  }

  private GribCollectionProto.Group writeGroupProto(Group g) throws IOException {
    GribCollectionProto.Group.Builder b = GribCollectionProto.Group.newBuilder();

    b.setGds(ByteString.copyFrom(g.gdss.getRawBytes()));

    for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars())
      b.addVariables(writeVariableProto(vb));

    List<TimeCoord> timeCoords = g.rect.getTimeCoords();
    for (int i = 0; i < timeCoords.size(); i++)
      b.addTimeCoords(writeCoordProto(timeCoords.get(i), i));

    List<VertCoord> vertCoords = g.rect.getVertCoords();
    for (int i = 0; i < vertCoords.size(); i++)
      b.addVertCoords(writeCoordProto(vertCoords.get(i), i));

    List<EnsCoord> ensCoords = g.rect.getEnsCoords();
    for (int i = 0; i < ensCoords.size(); i++) b.addEnsCoords(writeCoordProto(ensCoords.get(i), i));

    for (Integer aFileSet : g.fileSet) b.addFileno(aFileSet);

    return b.build();
  }

  private GribCollectionProto.Variable writeVariableProto(Grib2Rectilyser.VariableBag vb)
      throws IOException {
    GribCollectionProto.Variable.Builder b = GribCollectionProto.Variable.newBuilder();

    b.setDiscipline(vb.first.getDiscipline());
    Grib2Pds pds = vb.first.getPDS();
    b.setCategory(pds.getParameterCategory());
    b.setParameter(pds.getParameterNumber());
    b.setLevelType(pds.getLevelType1());
    b.setIsLayer(Grib2Utils.isLayer(vb.first));
    b.setIntervalType(pds.getStatisticalProcessType());
    b.setCdmHash(vb.first.cdmVariableHash(0));
    b.setRecordsPos(vb.pos);
    b.setRecordsLen(vb.length);
    b.setTimeIdx(vb.timeCoordIndex);
    if (vb.vertCoordIndex >= 0) b.setVertIdx(vb.vertCoordIndex);
    if (vb.ensCoordIndex >= 0) b.setEnsIdx(vb.ensCoordIndex);

    if (pds.isEnsembleDerived()) {
      Grib2Pds.PdsEnsembleDerived pdsDerived = (Grib2Pds.PdsEnsembleDerived) pds;
      b.setEnsDerivedType(pdsDerived.getDerivedForecastType()); // derived type (table 4.7)
    }

    if (pds.isProbability()) {
      Grib2Pds.PdsProbability pdsProb = (Grib2Pds.PdsProbability) pds;
      b.setProbabilityName(pdsProb.getProbabilityName());
      b.setProbabilityType(pdsProb.getProbabilityType());
    }

    return b.build();
  }

  protected GribCollectionProto.Parameter writeParamProto(Parameter param) throws IOException {
    GribCollectionProto.Parameter.Builder b = GribCollectionProto.Parameter.newBuilder();

    b.setName(param.getName());
    if (param.isString()) b.setSdata(param.getStringValue());
    else {
      for (int i = 0; i < param.getLength(); i++) b.addData(param.getNumericValue(i));
    }

    return b.build();
  }

  protected GribCollectionProto.Coord writeCoordProto(TimeCoord tc, int index) throws IOException {
    GribCollectionProto.Coord.Builder b = GribCollectionProto.Coord.newBuilder();
    b.setCode(index);
    b.setUnit(tc.getUnits());
    float scale =
        (float) tc.getTimeUnitScale(); // deal with, eg, "6 hours" by multiplying values by 6
    if (tc.isInterval()) {
      for (TimeCoord.Tinv tinv : tc.getIntervals()) {
        b.addValues(tinv.getBounds1() * scale);
        b.addBound(tinv.getBounds2() * scale);
      }
    } else {
      for (int value : tc.getCoords()) b.addValues(value * scale);
    }
    return b.build();
  }

  protected GribCollectionProto.Coord writeCoordProto(VertCoord vc, int index) throws IOException {
    GribCollectionProto.Coord.Builder b = GribCollectionProto.Coord.newBuilder();
    b.setCode(vc.getCode());
    b.setUnit(vc.getUnits());
    for (VertCoord.Level coord : vc.getCoords()) {
      if (vc.isLayer()) {
        b.addValues((float) coord.getValue1());
        b.addBound((float) coord.getValue2());
      } else {
        b.addValues((float) coord.getValue1());
      }
    }
    return b.build();
  }

  protected GribCollectionProto.Coord writeCoordProto(EnsCoord ec, int index) throws IOException {
    GribCollectionProto.Coord.Builder b = GribCollectionProto.Coord.newBuilder();
    b.setCode(0);
    b.setUnit("");
    for (EnsCoord.Coord coord : ec.getCoords()) {
      b.addValues((float) coord.getCode());
      b.addValues((float) coord.getEnsMember());
    }
    return b.build();
  }
}
Exemplo n.º 18
0
/*
 * Processes Queries for the RadarServer  Spring Framework
 */
public class QueryRadarServerController extends AbstractController {

  private org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(getClass());

  private TdsContext tdsContext;
  private HtmlWriter htmlWriter;
  private boolean htmlView;
  private boolean releaseDataset = false;

  public void setTdsContext(TdsContext tdsContext) {
    this.tdsContext = tdsContext;
  }

  public void setHtmlWriter(HtmlWriter htmlWriter) {
    this.htmlWriter = htmlWriter;
  }

  public boolean isHtmlView() {
    return htmlView;
  }

  public void setHtmlView(boolean htmlView) {
    this.htmlView = htmlView;
  }

  public boolean isReleaseDataset() {
    return releaseDataset;
  }

  public void setReleaseDataset(boolean releaseDataset) {
    this.releaseDataset = releaseDataset;
  }

  /** The view to forward to in case a bad query. */
  private static final String CREATE_VIEW = "forward:badquery.htm";

  /** The model key used to retrieve the message from the model. */
  private static final String MODEL_KEY = "message";

  /** The unique key for retrieving the text associated with this message. */
  private static final String MSG_CODE = "message.bad.query";

  /*
   * why calculate over and over again  1970-01-01T00:00:00
   */
  private static DateType epicDateType;

  static {
    try {
      epicDateType = new DateType(RadarServerUtil.epic, null, null);
    } catch (java.text.ParseException e) {
    }
  }

  static SimpleDateFormat dateFormat;

  static {
    dateFormat = new SimpleDateFormat("yyyyMMdd", Locale.US);
    dateFormat.setTimeZone(TimeZone.getTimeZone("GMT"));
  }

  /**
   * Query RadarServer controller for Spring Framework
   *
   * @param request HttpServletRequest
   * @param response HttpServletResponse
   * @return ModelAndView
   * @throws Exception
   */
  protected ModelAndView handleRequestInternal(
      HttpServletRequest request, HttpServletResponse response) throws Exception {
    try {
      // Gather diagnostics for logging request.
      log.info("handleRequestInternal(): " + UsageLog.setupRequestContext(request));
      // catch rogue invalid request here
      if (request.getQueryString() == null) {
        log.info("Invalid dataset url reference " + request.getPathInfo());
        throw new RadarServerException("Invalid dataset url reference " + request.getPathInfo());
      }
      // Query results in model
      Map<String, Object> model = new HashMap<String, Object>();
      radarQuery(request, response, model);
      if (model == null || model.size() == 0) {
        ModelAndView mav = new ModelAndView(CREATE_VIEW);
        mav.addObject(MODEL_KEY, MSG_CODE);
        return mav;
      } else {
        return new ModelAndView("queryXml", model);
      }
    } catch (RadarServerException e) {
      throw e; // pass it onto Spring exceptionResolver
    } catch (Throwable e) {
      log.error("handleRequestInternal(): Problem handling request.", e);
      log.info(
          "handleRequestInternal(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
      throw new RadarServerException("handleRequestInternal(): Problem handling request.");
    }
  }

  // get/check/process query
  public void radarQuery(HttpServletRequest req, HttpServletResponse res, Map<String, Object> model)
      throws ServletException, IOException, RadarServerException {

    //      long  startms = System.currentTimeMillis();
    //      long  endms;
    DatasetRepository.RadarType radarType = DatasetRepository.RadarType.nexrad;
    // need to extract data according to the (dataset) given
    String pathInfo = req.getPathInfo();
    if (pathInfo == null) pathInfo = "";
    if (pathInfo.startsWith("/")) pathInfo = pathInfo.substring(1);
    try {
      String rt = pathInfo.substring(0, pathInfo.indexOf('/', 1));
      radarType = DatasetRepository.RadarType.valueOf(rt);
    } catch (Exception e) {
      log.info("Invalid dataset url reference " + pathInfo);
      throw new RadarServerException("Invalid dataset url reference " + pathInfo);
    }
    Boolean level2 = pathInfo.contains("level2");

    // parse the input
    QueryParams qp = new QueryParams();
    if (!qp.parseQuery(
        req,
        res,
        new String[] {QueryParams.XML, QueryParams.HTML, QueryParams.RAW, QueryParams.NETCDF})) {
      // log.error( "parseQuery Failed "+ qp.errs.toString() + req.getQueryString() );
      // throw new RadarServerException( qp.errs.toString() );//+ req.getQueryString() );
      return; // TODO: uncomment above 2 lines when QueryParams exception is fixed
    }
    //      endms = System.currentTimeMillis();
    //      System.out.println( "after QueryParams "+ (endms - startms));
    //      startms = System.currentTimeMillis();
    // check Query Params
    if (!checkQueryParms(radarType, qp, level2)) {
      log.error("checkQueryParms Failed " + qp.errs.toString() + req.getQueryString());
      throw new RadarServerException(qp.errs.toString()); // + req.getQueryString() );
    }
    //      endms = System.currentTimeMillis();
    //      System.out.println( "after checkQueryParms "+ (endms - startms));
    //      startms = System.currentTimeMillis();

    // check type of output wanted XML html
    qp.acceptType = qp.acceptType.replaceFirst(".*/", "");

    // creates first part of catalog
    if (!createHeader(radarType, qp, pathInfo, model)) {
      log.error("Write Header Failed " + qp.errs.toString() + req.getQueryString());
      throw new RadarServerException(qp.errs.toString()); // req.getQueryString() );
    }
    //      endms = System.currentTimeMillis();
    //      System.out.println( "after writeHeader "+ (endms - startms));
    //      startms = System.currentTimeMillis();
    // gets products according to stations, time, and variables
    boolean dataFound = false;
    List<DatasetEntry> entries = new ArrayList<DatasetEntry>();
    if (qp.vars == null) {
      dataFound = processQuery(pathInfo, qp, null, entries);
      if (releaseDataset) DatasetRepository.removeRadarDatasetCollection(pathInfo, null);
    } else {
      int count = 0;
      for (String var : qp.vars) {
        dataFound = processQuery(pathInfo, qp, var, entries);
        if (dataFound) count++;
        if (releaseDataset) DatasetRepository.removeRadarDatasetCollection(pathInfo, var);
      }
      if (count > 0) dataFound = true;
    }
    // save entries
    model.put("datasets", entries);
    if (dataFound) {
      model.put("documentation", Integer.toString(entries.size()) + " datasets found for query");
    } else if (qp.errs.length() > 0) {
      model.put("documentation", qp.errs.toString());
    } else {
      model.put("documentation", "No data available for station(s) and time range");
    }

    //      endms = System.currentTimeMillis();
    //      System.out.println( "after radarQuery "+ (endms - startms));
    //      startms = System.currentTimeMillis();

  } // end radarNexradQuery

  // check that parms have valid stations, vars and times
  private Boolean checkQueryParms(
      DatasetRepository.RadarType radarType, QueryParams qp, Boolean level2) throws IOException {
    if (qp.hasBB) {
      if (radarType.equals(DatasetRepository.RadarType.nexrad))
        qp.stns = RadarServerUtil.getStationNames(qp.getBB(), DatasetRepository.nexradList);
      else qp.stns = RadarServerUtil.getStationNames(qp.getBB(), DatasetRepository.terminalList);

      if (qp.stns.size() == 0) {
        qp.errs.append("Bounding Box contains no stations ");
        return false;
      }
      if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
    }

    if (qp.hasStns) {
      if (RadarServerUtil.isStationListEmpty(qp.stns, radarType)) {
        qp.errs.append("No valid stations specified, need 1 ");
        return false;
      } else if (level2) {
        for (String stn : qp.stns) {
          if (stn.length() == 3) {
            qp.errs.append("Need 4 character station names ");
            return false;
          }
        }
      } else if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
    }

    if (qp.hasLatlonPoint) {
      qp.stns = new ArrayList<String>();
      if (radarType.equals(DatasetRepository.RadarType.nexrad))
        qp.stns.add(
            RadarServerUtil.findClosestStation(qp.lat, qp.lon, DatasetRepository.nexradList));
      else
        qp.stns.add(
            RadarServerUtil.findClosestStation(qp.lat, qp.lon, DatasetRepository.terminalList));
      if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
    } else if (qp.fatal) {
      qp.errs.append("No valid stations specified 2 ");
      return false;
    }

    if (qp.stns == null || qp.stns.size() == 0) {
      qp.errs.append("No valid stations specified, need 1 ");
      return false;
    }
    boolean useAllStations = (qp.stns.get(0).toUpperCase().equals("ALL"));
    if (useAllStations) {
      if (radarType.equals(DatasetRepository.RadarType.nexrad))
        qp.stns =
            RadarServerUtil.getStationNames(DatasetRepository.nexradList); // need station names
      else
        qp.stns =
            RadarServerUtil.getStationNames(DatasetRepository.terminalList); // need station names
      if (!level2) qp.stns = RadarServerUtil.convert4to3stations(qp.stns);
    }

    if (qp.hasTimePoint) {
      if (qp.time.isPresent()) {
        try {
          qp.time_end = new DateType("present", null, null);
          qp.time_start = epicDateType;
        } catch (java.text.ParseException e) {
          qp.errs.append("Illegal param= 'time' must be valid ISO Duration");
          return false;
        }
      } else {
        qp.time_end = qp.time;
        qp.time_start = qp.time;
      }
    } else if (qp.hasDateRange) {
      DateRange dr = qp.getCalendarDateRange().toDateRange();
      qp.time_start = dr.getStart();
      qp.time_end = dr.getEnd();
    } else { // get all times
      qp.time_latest = 1;
      // qp.hasDateRange = true;
      try {
        qp.time = new DateType("present", null, null);
        qp.time_end = new DateType("present", null, null);
        qp.time_start = epicDateType;
      } catch (java.text.ParseException e) {
        qp.errs.append("Illegal param= 'time' must be valid ISO Duration ");
        return false;
      }
    }

    if (level2) {
      qp.vars = null; // level2 can't select vars
    } else if (qp.vars == null) { // level 3 with no vars
      qp.errs.append("No vars selected ");
      return false;
    } else if (qp.vars.get(0).contains("/")) { // remove desc from vars
      ArrayList<String> tmp = new ArrayList<String>();
      for (String var : qp.vars) {
        tmp.add(var.replaceFirst("/.*", ""));
      }
      qp.vars = tmp;
    }
    return true;
  }

  // create catalog Header
  private Boolean createHeader(
      DatasetRepository.RadarType radarType,
      QueryParams qp,
      String pathInfo,
      Map<String, Object> model)
      throws IOException {

    Boolean level2 = pathInfo.contains("level2");
    int level = (level2) ? 2 : 3;
    StringBuffer str = new StringBuffer();
    str.append("Radar Level").append(level).append(" datasets in near real time");
    model.put("name", str.toString());
    str.setLength(0);
    str.append("/thredds/dodsC/").append(pathInfo).append("/");
    model.put("base", str.toString());
    str.setLength(0);
    str.append("RadarLevel").append(level).append(" datasets for available stations and times");
    model.put("dname", str.toString());
    str.setLength(0);
    str.append("accept=").append(qp.acceptType).append("&");
    if (!level2 && qp.vars != null) { // add vars
      str.append("var=");
      for (int i = 0; i < qp.vars.size(); i++) {
        str.append(qp.vars.get(i));
        if (i < qp.vars.size() - 1) {
          str.append(",");
        }
      }
      str.append("&");
    }
    // use all stations
    if (qp.stns.get(0).toUpperCase().equals("ALL")) {
      str.append("stn=ALL&");
    } else if (qp.hasStns) {
      for (String station : qp.stns) {
        str.append("stn=").append(station).append("&");
      }
    } else if (qp.hasBB) {
      str.append("south=").append(qp.south).append("&north=").append(qp.north).append("&");
      str.append("west=").append(qp.west).append("&east=").append(qp.east).append("&");
    }

    // no time given
    if (qp.time_latest == 1) {
      // str.deleteCharAt( str.length() -1);
      str.append("time=present");
    } else if (qp.hasDateRange) {
      if (qp.time_start.getDate() == null
          || qp.time_start.isBlank()
          || qp.time_end.getDate() == null
          || qp.time_end.isBlank()) {
        str.append("time_start=").append(qp.time_start.toString());
        str.append("&time_end=").append(qp.time_end.toString());
        qp.errs.append("need ISO time format ");
        return false;
      } else {
        str.append("time_start=").append(qp.time_start.toDateTimeStringISO());
        str.append("&time_end=").append(qp.time_end.toDateTimeStringISO());
      }
    } else if (qp.time.isPresent()) {
      str.append("time=present");
    } else if (qp.hasTimePoint) {
      if (qp.time.getDate() == null || qp.time.isBlank()) {
        str.append("time=").append(qp.time.toString());
        qp.errs.append("need ISO time format ");
        return false;
      } else {
        str.append("time=").append(qp.time.toDateTimeStringISO());
      }
    }
    model.put("ID", str.toString());

    if (level2) {
      model.put("type", "NEXRAD2");
    } else if (radarType.equals(DatasetRepository.RadarType.nexrad)) {
      model.put("type", "NIDS");
    } else {
      model.put("type", "TDWR");
    }

    // at this point must have stations
    if (RadarServerUtil.isStationListEmpty(qp.stns, radarType)) {
      qp.errs.append("No station(s) meet query criteria ");
      return false;
    }
    return true;
  }

  /*
      Final Output format, save information in DatasetEntry de
       <dataset name="Level2_KFTG_20100121_0000.ar2v" ID="735519521"
          urlPath="KFTG/20100121/Level2_KFTG_20100121_0000.ar2v">
          <date type="start of ob">2010-01-21T00:00:00</date>
        </dataset>
  */
  private Boolean processQuery(
      String dataset, QueryParams qp, String var, List<DatasetEntry> entries)
      throws RadarServerException {

    Boolean getAllTimes = true;
    String yyyymmddStart = null;
    String yyyymmddEnd = null;
    String dateStart = null;
    String dateEnd = null;
    try {
      if (!qp.time_start.equals(epicDateType)) {
        getAllTimes = false;
        yyyymmddStart = qp.time_start.toDateString();
        yyyymmddStart = yyyymmddStart.replace("-", "");
        yyyymmddEnd = qp.time_end.toDateString();
        yyyymmddEnd = yyyymmddEnd.replace("-", "");
        dateStart = yyyymmddStart + "_" + RadarServerUtil.hhmm(qp.time_start.toDateTimeString());
        dateEnd = yyyymmddEnd + "_" + RadarServerUtil.hhmm(qp.time_end.toDateTimeString());
      }

      RadarDatasetCollection rdc = DatasetRepository.getRadarDatasetCollection(dataset, var);
      if (rdc == null) {
        qp.errs.append("Invalid dataset =").append(dataset);
        qp.errs.append(" or var =").append(var);
        return false;
      }
      StringBuffer time = new StringBuffer();
      StringBuffer product = new StringBuffer();
      StringBuffer url = new StringBuffer();
      boolean isLevel2 = dataset.contains("level2");
      String type = (isLevel2 ? "Level2" : "Level3");
      String suffix = (isLevel2 ? ".ar2v" : ".nids");
      Calendar cal = Calendar.getInstance(java.util.TimeZone.getTimeZone("GMT"));
      Date now = cal.getTime();
      String currentDay = dateFormat.format(now);

      for (String stn : qp.stns) {
        RadarStationCollection rsc = rdc.queryStation(stn, currentDay);
        if (rsc == null) continue;
        for (String day : rsc.getDays()) {
          // check for valid day
          if (!getAllTimes && !RadarServerUtil.isValidDay(day, yyyymmddStart, yyyymmddEnd))
            continue;
          ArrayList<String> tal;
          if (rdc.isCaseStudy()) { //
            tal = rsc.getHourMinute("all");
            for (String prod : tal) {
              // check times
              if (!getAllTimes && !RadarServerUtil.isValidDate(prod, dateStart, dateEnd)) continue;
              // save this entry
              DatasetEntry de = new DatasetEntry();
              int idx = prod.indexOf('/');
              if (idx > 0) {
                de.setName(prod.substring(idx + 1));
              } else {
                de.setName(prod);
              }
              de.setID(Integer.toString(prod.hashCode()));
              url.setLength(0);
              url.append(stn).append("/");
              if (var != null) {
                url.append(var).append("/");
              }
              url.append(prod);
              de.setUrlPath(url.toString());
              de.setDate(RadarServerUtil.getObTimeISO(prod));
              entries.add(de);
            }
            continue;
          } else {
            tal = rsc.getHourMinute(day);
          }
          if (tal == null) continue;
          for (String hm : tal) {
            time.setLength(0);
            time.append(day).append("_").append(hm);
            if (!getAllTimes && !RadarServerUtil.isValidDate(time.toString(), dateStart, dateEnd))
              continue;

            // save this entry
            DatasetEntry de = new DatasetEntry();

            product.setLength(0);
            product.append(type).append("_").append(rsc.getStnName()).append("_");
            if (!isLevel2) product.append(var).append("_");
            product.append(day).append("_").append(hm).append(suffix);

            de.setName(product.toString());
            de.setID(Integer.toString(product.toString().hashCode()));
            url.setLength(0);
            if (!isLevel2) {
              url.append(var).append("/");
            }
            url.append(rsc.getStnName())
                .append("/")
                .append(day)
                .append("/")
                .append(product.toString());
            de.setUrlPath(url.toString());
            de.setDate(RadarServerUtil.getObTimeISO(product.toString()));
            entries.add(de);
            if (qp.hasTimePoint) break;
          }
          if (qp.hasTimePoint) break;
        }
      }
      return true;
    } catch (Throwable e) {
      log.error("Invalid dataset =" + dataset + " or var =" + var, e);
      log.info(
          "handleRequestInternal(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, -1));
      throw new RadarServerException("Invalid dataset =" + dataset + " or var =" + var);
    }
  }

  /*
   * Used to store the information about a dataset
   */
  public class DatasetEntry {

    private String name;

    private String ID;

    private String urlPath;

    private String date;

    public String getName() {
      return name;
    }

    public void setName(String name) {
      this.name = name;
    }

    public String getID() {
      return ID;
    }

    public void setID(String ID) {
      this.ID = ID;
    }

    public String getUrlPath() {
      return urlPath;
    }

    public void setUrlPath(String urlPath) {
      this.urlPath = urlPath;
    }

    public String getDate() {
      return date;
    }

    public void setDate(String date) {
      this.date = date;
    }
  }
}
Exemplo n.º 19
0
public class ServletUtil {

  public static final String CONTENT_TEXT = "text/plain; charset=utf-8";

  // bogus status returns for our logging
  public static final int STATUS_CLIENT_ABORT = 1000;
  public static final int STATUS_FORWARDED = 1001;
  public static final int STATUS_FORWARD_FAILURE = 1002;

  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(ServletUtil.class);
  private static boolean isDebugInit = false;

  private static String contextPath = null;
  private static String rootPath = null;
  private static String contentPath = null;

  /**
   * @param context the Servlet context.
   * @deprecated Now handled in TdsContext.init().
   */
  public static void initContext(ServletContext context) {
    //    setContextPath(context);
    if (contextPath == null) {
      // Servlet 2.5 allows the following.
      // contextPath = servletContext.getContextPath();
      String tmpContextPath =
          context.getInitParameter("ContextPath"); // cannot be overridden in the ThreddsConfig file
      if (tmpContextPath == null) tmpContextPath = "thredds";
      contextPath = "/" + tmpContextPath;
    }
    //    setRootPath(context);
    if (rootPath == null) {
      rootPath = context.getRealPath("/");
      rootPath = rootPath.replace('\\', '/');
    }

    //    setContentPath();
    if (contentPath == null) {
      String tmpContentPath = "../../content" + getContextPath() + "/";
      File cf = new File(getRootPath(), tmpContentPath);
      try {
        contentPath = cf.getCanonicalPath() + "/";
        contentPath = contentPath.replace('\\', '/');
      } catch (IOException e) {
        throw new RuntimeException(e.getMessage());
      }
    }

    //    initDebugging(context);
    initDebugging(context);
  }

  public static void setContextPath(String newContextPath) {
    contextPath = newContextPath;
  }

  public static void setRootPath(String newRootPath) {
    rootPath = newRootPath;
  }

  public static void setContentPath(String newContentPath) {
    contentPath = newContentPath;
    if (!contentPath.endsWith("/")) contentPath = contentPath + "/";
  }

  public static void initDebugging(ServletContext webapp) {
    if (isDebugInit) return;
    isDebugInit = true;

    String debugOn = webapp.getInitParameter("DebugOn");
    if (debugOn != null) {
      StringTokenizer toker = new StringTokenizer(debugOn);
      while (toker.hasMoreTokens()) Debug.set(toker.nextToken(), true);
    }
  }

  /**
   * Return the real path on the servers file system that corresponds to the root document ("/") on
   * the given servlet.
   *
   * @return the real path on the servers file system that corresponds to the root document ("/") on
   *     the given servlet.
   */
  public static String getRootPath() {
    return rootPath;
  }

  /**
   * Return the context path for the given servlet. Note - ToDo: Why not just use
   * ServletContext.getServletContextName()?
   *
   * @return the context path for the given servlet.
   */
  public static String getContextPath() {
    return contextPath;
  }

  /**
   * Return the content path for the given servlet.
   *
   * @return the content path for the given servlet.
   */
  public static String getContentPath() {
    return contentPath;
  }

  /**
   * Return the default/initial content path for the given servlet. The content of which is copied
   * to the content path when the web app is first installed.
   *
   * @return the default/initial content path for the given servlet.
   */
  public static String getInitialContentPath() {
    return getRootPath() + "/WEB-INF/altContent/startup/";
  }

  /**
   * Return the file path dealing with leading and trailing path seperators (which must be a slash
   * ("/")) for the given directory and file paths.
   *
   * <p>Note: Dealing with path strings is fragile. ToDo: Switch from using path strings to
   * java.io.Files.
   *
   * @param dirPath the directory path.
   * @param filePath the file path.
   * @return a full file path with the given directory and file paths.
   */
  public static String formFilename(String dirPath, String filePath) {
    if ((dirPath == null) || (filePath == null)) return null;

    if (filePath.startsWith("/")) filePath = filePath.substring(1);

    return dirPath.endsWith("/") ? dirPath + filePath : dirPath + "/" + filePath;
  }

  /**
   * Handle a request for a raw/static file (i.e., not a catalog or dataset request).
   *
   * <p>Look in the content (user) directory then the root (distribution) directory for a file that
   * matches the given path and, if found, return it as the content of the HttpServletResponse. If
   * the file is forbidden (i.e., the path contains a "..", "WEB-INF", or "META-INF" directory),
   * send a HttpServletResponse.SC_FORBIDDEN response. If no file matches the request (including an
   * "index.html" file if the path ends in "/"), send an HttpServletResponse.SC_NOT_FOUND..
   *
   * <p>
   *
   * <ol>
   *   <li>Make sure the path does not contain ".." directories.
   *   <li>Make sure the path does not contain "WEB-INF" or "META-INF".
   *   <li>Check for requested file in the content directory (if the path is a directory, make sure
   *       the path ends with "/" and check for an "index.html" file).
   *   <li>Check for requested file in the root directory (if the path is a directory, make sure the
   *       path ends with "/" and check for an "index.html" file). </ol
   *
   * @param path the requested path
   * @param servlet the servlet handling the request
   * @param req the HttpServletRequest
   * @param res the HttpServletResponse
   * @throws IOException if can't complete request due to IO problems.
   */
  public static void handleRequestForRawFile(
      String path, HttpServlet servlet, HttpServletRequest req, HttpServletResponse res)
      throws IOException {
    // Don't allow ".." directories in path.
    if (path.indexOf("/../") != -1
        || path.equals("..")
        || path.startsWith("../")
        || path.endsWith("/..")) {
      res.sendError(HttpServletResponse.SC_FORBIDDEN, "Path cannot contain \"..\" directory.");
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, -1));
      return;
    }

    // Don't allow access to WEB-INF or META-INF directories.
    String upper = path.toUpperCase();
    if (upper.indexOf("WEB-INF") != -1 || upper.indexOf("META-INF") != -1) {
      res.sendError(
          HttpServletResponse.SC_FORBIDDEN, "Path cannot contain \"WEB-INF\" or \"META-INF\".");
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, -1));
      return;
    }

    // Find a regular file
    File regFile = null;
    // Look in content directory for regular file.
    File cFile = new File(ServletUtil.formFilename(getContentPath(), path));
    if (cFile.exists()) {
      if (cFile.isDirectory()) {
        if (!path.endsWith("/")) {
          String newPath = req.getRequestURL().append("/").toString();
          ServletUtil.sendPermanentRedirect(newPath, req, res);
        }
        // If request path is a directory, check for index.html file.
        cFile = new File(cFile, "index.html");
        if (cFile.exists() && !cFile.isDirectory()) regFile = cFile;
      }
      // If not a directory, use this file.
      else regFile = cFile;
    }

    if (regFile == null) {
      // Look in root directory.
      File rFile = new File(ServletUtil.formFilename(getRootPath(), path));
      if (rFile.exists()) {
        if (rFile.isDirectory()) {
          if (!path.endsWith("/")) {
            String newPath = req.getRequestURL().append("/").toString();
            ServletUtil.sendPermanentRedirect(newPath, req, res);
          }
          rFile = new File(rFile, "index.html");
          if (rFile.exists() && !rFile.isDirectory()) regFile = rFile;
        } else regFile = rFile;
      }
    }

    if (regFile == null) {
      res.sendError(HttpServletResponse.SC_NOT_FOUND); // 404
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
      return;
    }

    ServletUtil.returnFile(servlet, req, res, regFile, null);
  }

  /**
   * Handle an explicit request for a content directory file (path must start with "/content/".
   *
   * <p>Note: As these requests will show the configuration files for the server, these requests
   * should be covered by security constraints.
   *
   * <p>
   *
   * <ol>
   *   <li>Make sure the path does not contain ".." directories.
   *   <li>Check for the requested file in the content directory. </ol
   *
   * @param path the requested path (must start with "/content/")
   * @param servlet the servlet handling the request
   * @param req the HttpServletRequest
   * @param res the HttpServletResponse
   * @throws IOException if can't complete request due to IO problems.
   */
  public static void handleRequestForContentFile(
      String path, HttpServlet servlet, HttpServletRequest req, HttpServletResponse res)
      throws IOException {
    handleRequestForContentOrRootFile("/content/", path, servlet, req, res);
  }

  /**
   * Handle an explicit request for a root directory file (path must start with "/root/".
   *
   * <p>Note: As these requests will show the configuration files for the server, these requests
   * should be covered by security constraints.
   *
   * <p>
   *
   * <ol>
   *   <li>Make sure the path does not contain ".." directories.
   *   <li>Check for the requested file in the root directory. </ol
   *
   * @param path the requested path (must start with "/root/")
   * @param servlet the servlet handling the request
   * @param req the HttpServletRequest
   * @param res the HttpServletResponse
   * @throws IOException if can't complete request due to IO problems.
   */
  public static void handleRequestForRootFile(
      String path, HttpServlet servlet, HttpServletRequest req, HttpServletResponse res)
      throws IOException {
    handleRequestForContentOrRootFile("/root/", path, servlet, req, res);
  }

  /**
   * Convenience routine used by handleRequestForContentFile() and handleRequestForRootFile().
   *
   * @param pathPrefix
   * @param path
   * @param servlet
   * @param req request
   * @param res response
   * @throws IOException on IO error
   */
  private static void handleRequestForContentOrRootFile(
      String pathPrefix,
      String path,
      HttpServlet servlet,
      HttpServletRequest req,
      HttpServletResponse res)
      throws IOException {
    if (!pathPrefix.equals("/content/") && !pathPrefix.equals("/root/")) {
      log.error(
          "handleRequestForContentFile(): The path prefix <"
              + pathPrefix
              + "> must be \"/content/\" or \"/root/\".");
      throw new IllegalArgumentException("Path prefix must be \"/content/\" or \"/root/\".");
    }

    if (!path.startsWith(pathPrefix)) {
      log.error(
          "handleRequestForContentFile(): path <"
              + path
              + "> must start with \""
              + pathPrefix
              + "\".");
      throw new IllegalArgumentException("Path must start with \"" + pathPrefix + "\".");
    }

    // Don't allow ".." directories in path.
    if (path.indexOf("/../") != -1
        || path.equals("..")
        || path.startsWith("../")
        || path.endsWith("/..")) {
      res.sendError(HttpServletResponse.SC_FORBIDDEN, "Path cannot contain \"..\" directory.");
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, -1));
      return;
    }

    // Find the requested file.
    File file =
        new File(
            ServletUtil.formFilename(getContentPath(), path.substring(pathPrefix.length() - 1)));
    if (file.exists()) {
      // Do not allow request for a directory.
      if (file.isDirectory()) {
        if (!path.endsWith("/")) {
          String redirectPath = req.getRequestURL().append("/").toString();
          ServletUtil.sendPermanentRedirect(redirectPath, req, res);
          return;
        }

        int i = HtmlWriter.getInstance().writeDirectory(res, file, path);
        int status = i == 0 ? HttpServletResponse.SC_NOT_FOUND : HttpServletResponse.SC_OK;
        log.info(UsageLog.closingMessageForRequestContext(status, i));

        return;
      }

      // Return the requested file.
      ServletUtil.returnFile(servlet, req, res, file, null);
    } else {
      // Requested file not found.
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, -1));
      res.sendError(HttpServletResponse.SC_NOT_FOUND); // 404
    }
  }

  /**
   * Send a permanent redirect (HTTP status 301 "Moved Permanently") response with the given target
   * path.
   *
   * <p>The given target path may be relative or absolute. If it is relative, it will be resolved
   * against the request URL.
   *
   * @param targetPath the path to which the client is redirected.
   * @param req the HttpServletRequest
   * @param res the HttpServletResponse
   * @throws IOException if can't write the response.
   */
  public static void sendPermanentRedirect(
      String targetPath, HttpServletRequest req, HttpServletResponse res) throws IOException {
    // Absolute URL needed so resolve the target path against the request URL.
    URI uri;
    try {
      uri = new URI(req.getRequestURL().toString());
    } catch (URISyntaxException e) {
      log.error(
          "sendPermanentRedirect(): Bad syntax on request URL <" + req.getRequestURL() + ">.", e);
      log.info(
          "sendPermanentRedirect(): "
              + UsageLog.closingMessageForRequestContext(
                  HttpServletResponse.SC_INTERNAL_SERVER_ERROR, 0));
      if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
      return;
    }
    String absolutePath = uri.resolve(targetPath).toString();
    absolutePath = res.encodeRedirectURL(absolutePath);

    res.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY);
    res.addHeader("Location", absolutePath);

    String title = "Permanently Moved - 301";
    String body =
        new StringBuilder()
            .append("<p>")
            .append("The requested URL <")
            .append(req.getRequestURL())
            .append("> has been permanently moved (HTTP status code 301).")
            .append(" Instead, please use the following URL: <a href=\"")
            .append(absolutePath)
            .append("\">")
            .append(absolutePath)
            .append("</a>.")
            .append("</p>")
            .toString();
    String htmlResp =
        new StringBuilder()
            .append(HtmlWriter.getInstance().getHtmlDoctypeAndOpenTag())
            .append("<head><title>")
            .append(title)
            .append("</title></head><body>")
            .append("<h1>")
            .append(title)
            .append("</h1>")
            .append(body)
            .append("</body></html>")
            .toString();

    log.info("sendPermanentRedirect(): redirect to " + absolutePath);
    log.info(
        "sendPermanentRedirect(): "
            + UsageLog.closingMessageForRequestContext(
                HttpServletResponse.SC_MOVED_PERMANENTLY, htmlResp.length()));

    // Write the catalog out.
    PrintWriter out = res.getWriter();
    res.setContentType("text/html");
    out.print(htmlResp);
    out.flush();
  }

  /**
   * Write a file to the response stream.
   *
   * @param servlet called from here
   * @param contentPath file root path
   * @param path file path reletive to the root
   * @param req the request
   * @param res the response
   * @param contentType content type, or null
   * @throws IOException on write error
   */
  public static void returnFile(
      HttpServlet servlet,
      String contentPath,
      String path,
      HttpServletRequest req,
      HttpServletResponse res,
      String contentType)
      throws IOException {

    String filename = ServletUtil.formFilename(contentPath, path);

    log.debug("returnFile(): returning file <" + filename + ">.");
    // No file, nothing to view
    if (filename == null) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      res.sendError(HttpServletResponse.SC_NOT_FOUND);
      return;
    }

    // dontallow ..
    if (filename.indexOf("..") != -1) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
      res.sendError(HttpServletResponse.SC_FORBIDDEN);
      return;
    }

    // dont allow access to WEB-INF or META-INF
    String upper = filename.toUpperCase();
    if (upper.indexOf("WEB-INF") != -1 || upper.indexOf("META-INF") != -1) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_FORBIDDEN, 0));
      res.sendError(HttpServletResponse.SC_FORBIDDEN);
      return;
    }

    returnFile(servlet, req, res, new File(filename), contentType);
  }

  private static FileCacheRaf fileCacheRaf;

  public static void setFileCache(FileCacheRaf fileCache) {
    fileCacheRaf = fileCache;
  }

  public static FileCacheRaf getFileCache() {
    return fileCacheRaf;
  }

  /**
   * Write a file to the response stream. Handles Range requests.
   *
   * @param servlet called from here
   * @param req the request
   * @param res the response
   * @param file to serve
   * @param contentType content type, if null, will try to guess
   * @throws IOException on write error
   */
  public static void returnFile(
      HttpServlet servlet,
      HttpServletRequest req,
      HttpServletResponse res,
      File file,
      String contentType)
      throws IOException {

    // No file, nothing to view
    if (file == null) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      res.sendError(HttpServletResponse.SC_NOT_FOUND);
      return;
    }

    // check that it exists
    if (!file.exists()) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      res.sendError(HttpServletResponse.SC_NOT_FOUND);
      return;
    }

    // not a directory
    if (!file.isFile()) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_BAD_REQUEST, 0));
      res.sendError(HttpServletResponse.SC_BAD_REQUEST);
      return;
    }

    // Set the type of the file
    String filename = file.getPath();
    if (null == contentType) {
      if (filename.endsWith(".html")) contentType = "text/html; charset=iso-8859-1";
      else if (filename.endsWith(".xml")) contentType = "text/xml; charset=iso-8859-1";
      else if (filename.endsWith(".txt") || (filename.endsWith(".log"))) contentType = CONTENT_TEXT;
      else if (filename.indexOf(".log.") > 0) contentType = CONTENT_TEXT;
      else if (filename.endsWith(".nc")) contentType = "application/x-netcdf";
      else contentType = servlet.getServletContext().getMimeType(filename);

      if (contentType == null) contentType = "application/octet-stream";
    }

    returnFile(req, res, file, contentType);
  }

  /**
   * Write a file to the response stream. Handles Range requests.
   *
   * @param req request
   * @param res response
   * @param file must exists and not be a directory
   * @param contentType must not be null
   * @throws IOException or error
   */
  public static void returnFile(
      HttpServletRequest req, HttpServletResponse res, File file, String contentType)
      throws IOException {
    res.setContentType(contentType);

    // see if its a Range Request
    boolean isRangeRequest = false;
    long startPos = 0, endPos = Long.MAX_VALUE;
    String rangeRequest = req.getHeader("Range");
    if (rangeRequest != null) { // bytes=12-34 or bytes=12-
      int pos = rangeRequest.indexOf("=");
      if (pos > 0) {
        int pos2 = rangeRequest.indexOf("-");
        if (pos2 > 0) {
          String startString = rangeRequest.substring(pos + 1, pos2);
          String endString = rangeRequest.substring(pos2 + 1);
          startPos = Long.parseLong(startString);
          if (endString.length() > 0) endPos = Long.parseLong(endString) + 1;
          isRangeRequest = true;
        }
      }
    }

    // set content length
    long fileSize = file.length();
    long contentLength = fileSize;
    if (isRangeRequest) {
      endPos = Math.min(endPos, fileSize);
      contentLength = endPos - startPos;
    }

    if (contentLength > Integer.MAX_VALUE)
      res.addHeader(
          "Content-Length", Long.toString(contentLength)); // allow content length > MAX_INT
    else res.setContentLength((int) contentLength); // note HEAD only allows this

    String filename = file.getPath();
    boolean debugRequest = Debug.isSet("returnFile");
    if (debugRequest)
      log.debug(
          "returnFile(): filename = "
              + filename
              + " contentType = "
              + contentType
              + " contentLength = "
              + contentLength);

    // indicate we allow Range Requests
    res.addHeader("Accept-Ranges", "bytes");

    if (req.getMethod().equals("HEAD")) {
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, 0));
      return;
    }

    try {

      if (isRangeRequest) {
        // set before content is sent
        res.addHeader("Content-Range", "bytes " + startPos + "-" + (endPos - 1) + "/" + fileSize);
        res.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);

        FileCacheRaf.Raf craf = null;
        try {
          craf = fileCacheRaf.acquire(filename);
          IO.copyRafB(
              craf.getRaf(), startPos, contentLength, res.getOutputStream(), new byte[60000]);
          log.info(
              "returnFile(): "
                  + UsageLog.closingMessageForRequestContext(
                      HttpServletResponse.SC_PARTIAL_CONTENT, contentLength));
          return;
        } finally {
          if (craf != null) fileCacheRaf.release(craf);
        }
      }

      // Return the file
      ServletOutputStream out = res.getOutputStream();
      IO.copyFileB(file, out, 60000);
      res.flushBuffer();
      out.close();
      if (debugRequest) log.debug("returnFile(): returnFile ok = " + filename);
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, contentLength));
    }

    // @todo Split up this exception handling: those from file access vs those from dealing with
    // response
    //       File access: catch and res.sendError()
    //       response: don't catch (let bubble up out of doGet() etc)
    catch (FileNotFoundException e) {
      log.error("returnFile(): FileNotFoundException= " + filename);
      log.info(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_NOT_FOUND);
    } catch (java.net.SocketException e) {
      log.info("returnFile(): SocketException sending file: " + filename + " " + e.getMessage());
      log.info("returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
    } catch (IOException e) {
      String eName =
          e.getClass().getName(); // dont want compile time dependency on ClientAbortException
      if (eName.equals("org.apache.catalina.connector.ClientAbortException")) {
        log.info(
            "returnFile(): ClientAbortException while sending file: "
                + filename
                + " "
                + e.getMessage());
        log.info(
            "returnFile(): " + UsageLog.closingMessageForRequestContext(STATUS_CLIENT_ABORT, 0));
        return;
      }

      log.error("returnFile(): IOException (" + e.getClass().getName() + ") sending file ", e);
      log.error(
          "returnFile(): "
              + UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      if (!res.isCommitted())
        res.sendError(HttpServletResponse.SC_NOT_FOUND, "Problem sending file: " + e.getMessage());
    }
  }

  /**
   * Send given content string as the HTTP response.
   *
   * @param contents the string to return as the HTTP response.
   * @param res the HttpServletResponse
   * @throws IOException if an I/O error occurs while writing the response.
   */
  public static void returnString(String contents, HttpServletResponse res) throws IOException {

    try {
      ServletOutputStream out = res.getOutputStream();
      IO.copy(new ByteArrayInputStream(contents.getBytes()), out);
      log.info(
          UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_OK, contents.length()));
    } catch (IOException e) {
      log.error(" IOException sending string: ", e);
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_NOT_FOUND, 0));
      res.sendError(HttpServletResponse.SC_NOT_FOUND, "Problem sending string: " + e.getMessage());
    }
  }

  /**
   * Return the request URL relative to the server (i.e., starting with the context path).
   *
   * @param req request
   * @return URL relative to the server
   */
  public static String getReletiveURL(HttpServletRequest req) {
    return req.getContextPath() + req.getServletPath() + req.getPathInfo();
  }

  /**
   * Forward this request to the CatalogServices servlet ("/catalog.html").
   *
   * @param req request
   * @param res response
   * @throws IOException on IO error
   * @throws ServletException other error
   */
  public static void forwardToCatalogServices(HttpServletRequest req, HttpServletResponse res)
      throws IOException, ServletException {

    String reqs = "catalog=" + getReletiveURL(req);
    String query = req.getQueryString();
    if (query != null) reqs = reqs + "&" + query;
    log.info("forwardToCatalogServices(): request string = \"/catalog.html?" + reqs + "\"");

    // dispatch to CatalogHtml servlet
    RequestForwardUtils.forwardRequestRelativeToCurrentContext("/catalog.html?" + reqs, req, res);
  }

  public static boolean saveFile(
      HttpServlet servlet,
      String contentPath,
      String path,
      HttpServletRequest req,
      HttpServletResponse res) {

    // @todo Need to use logServerAccess() below here.
    boolean debugRequest = Debug.isSet("SaveFile");
    if (debugRequest) log.debug(" saveFile(): path= " + path);

    String filename = contentPath + path; // absolute path
    File want = new File(filename);

    // backup current version if it exists
    int version = getBackupVersion(want.getParent(), want.getName());
    String fileSave = filename + "~" + version;
    File file = new File(filename);
    if (file.exists()) {
      try {
        IO.copyFile(filename, fileSave);
      } catch (IOException e) {
        log.error(
            "saveFile(): Unable to save copy of file "
                + filename
                + " to "
                + fileSave
                + "\n"
                + e.getMessage());
        return false;
      }
    }

    // save new file
    try {
      OutputStream out = new BufferedOutputStream(new FileOutputStream(filename));
      IO.copy(req.getInputStream(), out);
      out.close();
      if (debugRequest) log.debug("saveFile(): ok= " + filename);
      res.setStatus(HttpServletResponse.SC_CREATED);
      log.info(UsageLog.closingMessageForRequestContext(HttpServletResponse.SC_CREATED, -1));
      return true;
    } catch (IOException e) {
      log.error(
          "saveFile(): Unable to PUT file " + filename + " to " + fileSave + "\n" + e.getMessage());
      return false;
    }
  }

  private static int getBackupVersion(String dirName, String fileName) {
    int maxN = 0;
    File dir = new File(dirName);
    if (!dir.exists()) return -1;

    String[] files = dir.list();
    if (null == files) return -1;

    for (String name : files) {
      if (name.indexOf(fileName) < 0) continue;
      int pos = name.indexOf('~');
      if (pos < 0) continue;
      String ver = name.substring(pos + 1);
      int n = 0;
      try {
        n = Integer.parseInt(ver);
      } catch (NumberFormatException e) {
        log.error("Format Integer error on backup filename= " + ver);
      }
      maxN = Math.max(n, maxN);
    }
    return maxN + 1;
  }

  public static boolean copyDir(String fromDir, String toDir) throws IOException {
    File contentFile = new File(toDir + ".INIT");
    if (!contentFile.exists()) {
      IO.copyDirTree(fromDir, toDir);
      contentFile.createNewFile();
      return true;
    }
    return false;
  }

  /**
   * ************************************************************************ Sends an error to the
   * client.
   *
   * @param t The exception that caused the problem.
   * @param res The <code>HttpServletResponse</code> for the client.
   */
  public static void handleException(Throwable t, HttpServletResponse res) {
    try {
      String message = t.getMessage();
      if (message == null) message = "NULL message " + t.getClass().getName();
      if (Debug.isSet("trustedMode")) { // security issue: only show stack if trusted
        ByteArrayOutputStream bs = new ByteArrayOutputStream();
        PrintStream ps = new PrintStream(bs);
        t.printStackTrace(ps);
        message = new String(bs.toByteArray());
      }
      log.info(
          UsageLog.closingMessageForRequestContext(
              HttpServletResponse.SC_BAD_REQUEST, message.length()));
      log.error("handleException", t);
      t.printStackTrace(); // debugging - log.error not showing stack trace !!
      if (!res.isCommitted()) res.sendError(HttpServletResponse.SC_BAD_REQUEST, message);

    } catch (Throwable e) {
      log.error("handleException() had problem reporting Exception", e);
      t.printStackTrace();
    }
  }

  public static void showServerInfo(PrintStream out) {
    out.println("Server Info");
    out.println(
        " getDocumentBuilderFactoryVersion(): "
            + XMLEntityResolver.getDocumentBuilderFactoryVersion());
    out.println();

    Properties sysp = System.getProperties();
    Enumeration e = sysp.propertyNames();
    List<String> list = Collections.list(e);
    Collections.sort(list);

    out.println("System Properties:");
    for (String name : list) {
      String value = System.getProperty(name);
      out.println("  " + name + " = " + value);
    }
    out.println();
  }

  public static void showServletInfo(HttpServlet servlet, PrintStream out) {
    out.println("Servlet Info");
    out.println(" getServletName(): " + servlet.getServletName());
    out.println(" getRootPath(): " + getRootPath());
    out.println(" Init Parameters:");
    Enumeration params = servlet.getInitParameterNames();
    while (params.hasMoreElements()) {
      String name = (String) params.nextElement();
      out.println("  " + name + ": " + servlet.getInitParameter(name));
    }
    out.println();

    ServletContext context = servlet.getServletContext();
    out.println("Context Info");

    try {
      out.println(" context.getResource('/'): " + context.getResource("/"));
    } catch (java.net.MalformedURLException e) {
    } // cant happen
    out.println(" context.getServerInfo(): " + context.getServerInfo());
    out.println("  name: " + getServerInfoName(context.getServerInfo()));
    out.println("  version: " + getServerInfoVersion(context.getServerInfo()));

    out.println(" context.getInitParameterNames():");
    params = context.getInitParameterNames();
    while (params.hasMoreElements()) {
      String name = (String) params.nextElement();
      out.println("  " + name + ": " + context.getInitParameter(name));
    }

    out.println(" context.getAttributeNames():");
    params = context.getAttributeNames();
    while (params.hasMoreElements()) {
      String name = (String) params.nextElement();
      out.println("  context.getAttribute(\"" + name + "\"): " + context.getAttribute(name));
    }

    out.println();
  }

  /**
   * Show the pieces of the request, for debugging
   *
   * @param req the HttpServletRequest
   * @return parsed request
   */
  public static String getRequestParsed(HttpServletRequest req) {
    return req.getRequestURI()
        + " = "
        + req.getContextPath()
        + "(context), "
        + req.getServletPath()
        + "(servletPath), "
        + req.getPathInfo()
        + "(pathInfo), "
        + req.getQueryString()
        + "(query)";
  }

  /**
   * This is the server part, eg http://motherlode:8080
   *
   * @param req the HttpServletRequest
   * @return request server
   */
  public static String getRequestServer(HttpServletRequest req) {
    return req.getScheme() + "://" + req.getServerName() + ":" + req.getServerPort();
  }

  /**
   * This is everything except the query string
   *
   * @param req the HttpServletRequest
   * @return parsed request base
   */
  public static String getRequestBase(HttpServletRequest req) {
    // return "http://"+req.getServerName()+":"+ req.getServerPort()+req.getRequestURI();
    return req.getRequestURL().toString();
  }

  /**
   * The request base as a URI
   *
   * @param req the HttpServletRequest
   * @return parsed request as a URI
   */
  public static URI getRequestURI(HttpServletRequest req) {
    try {
      return new URI(getRequestBase(req));
    } catch (URISyntaxException e) {
      e.printStackTrace();
      return null;
    }
  }

  /**
   * servletPath + pathInfo
   *
   * @param req the HttpServletRequest
   * @return parsed request servletPath + pathInfo
   */
  public static String getRequestPath(HttpServletRequest req) {
    StringBuffer buff = new StringBuffer();
    if (req.getServletPath() != null) buff.append(req.getServletPath());
    if (req.getPathInfo() != null) buff.append(req.getPathInfo());
    return buff.toString();
  }

  /**
   * The entire request including query string
   *
   * @param req the HttpServletRequest
   * @return entire parsed request
   */
  public static String getRequest(HttpServletRequest req) {
    String query = req.getQueryString();
    return getRequestBase(req) + (query == null ? "" : "?" + query);
  }

  /**
   * Return the value of the given parameter for the given request. Should only be used if the
   * parameter is known to only have one value. If used on a multi-valued parameter, the first value
   * is returned.
   *
   * @param req the HttpServletRequest
   * @param paramName the name of the parameter to find.
   * @return the value of the given parameter for the given request.
   */
  public static String getParameterIgnoreCase(HttpServletRequest req, String paramName) {
    Enumeration e = req.getParameterNames();
    while (e.hasMoreElements()) {
      String s = (String) e.nextElement();
      if (s.equalsIgnoreCase(paramName)) return req.getParameter(s);
    }
    return null;
  }

  /**
   * Return the values of the given parameter (ignoring case) for the given request.
   *
   * @param req the HttpServletRequest
   * @param paramName the name of the parameter to find.
   * @return the values of the given parameter for the given request.
   */
  public static String[] getParameterValuesIgnoreCase(HttpServletRequest req, String paramName) {
    Enumeration e = req.getParameterNames();
    while (e.hasMoreElements()) {
      String s = (String) e.nextElement();
      if (s.equalsIgnoreCase(paramName)) return req.getParameterValues(s);
    }
    return null;
  }

  public static String getFileURL(String filename) {
    filename = filename.replace('\\', '/');
    filename = StringUtil.replace(filename, ' ', "+");
    return "file:" + filename;
  }

  /**
   * Show details about the request
   *
   * @param servlet used to get teh servlet context, may be null
   * @param req the request
   * @return string showing the details of the request.
   */
  public static String showRequestDetail(HttpServlet servlet, HttpServletRequest req) {
    StringBuilder sbuff = new StringBuilder();

    sbuff.append("Request Info\n");
    sbuff.append(" req.getServerName(): ").append(req.getServerName()).append("\n");
    sbuff.append(" req.getServerPort(): ").append(req.getServerPort()).append("\n");
    sbuff.append(" req.getContextPath:").append(req.getContextPath()).append("\n");
    sbuff.append(" req.getServletPath:").append(req.getServletPath()).append("\n");
    sbuff.append(" req.getPathInfo:").append(req.getPathInfo()).append("\n");
    sbuff.append(" req.getQueryString:").append(req.getQueryString()).append("\n");
    sbuff
        .append(" getQueryStringDecoded:")
        .append(EscapeStrings.urlDecode(req.getQueryString()))
        .append("\n");
    /*try {
      sbuff.append(" getQueryStringDecoded:").append(URLDecoder.decode(req.getQueryString(), "UTF-8")).append("\n");
    } catch (UnsupportedEncodingException e1) {
      e1.printStackTrace();
    }*/
    sbuff.append(" req.getRequestURI:").append(req.getRequestURI()).append("\n");
    sbuff.append(" getRequestBase:").append(getRequestBase(req)).append("\n");
    sbuff.append(" getRequestServer:").append(getRequestServer(req)).append("\n");
    sbuff.append(" getRequest:").append(getRequest(req)).append("\n");
    sbuff.append("\n");

    sbuff.append(" req.getPathTranslated:").append(req.getPathTranslated()).append("\n");
    String path = req.getPathTranslated();
    if ((path != null) && (servlet != null)) {
      ServletContext context = servlet.getServletContext();
      sbuff.append(" getMimeType:").append(context.getMimeType(path)).append("\n");
    }
    sbuff.append("\n");
    sbuff.append(" req.getScheme:").append(req.getScheme()).append("\n");
    sbuff.append(" req.getProtocol:").append(req.getProtocol()).append("\n");
    sbuff.append(" req.getMethod:").append(req.getMethod()).append("\n");
    sbuff.append("\n");
    sbuff.append(" req.getContentType:").append(req.getContentType()).append("\n");
    sbuff.append(" req.getContentLength:").append(req.getContentLength()).append("\n");

    sbuff.append(" req.getRemoteAddr():").append(req.getRemoteAddr());
    try {
      sbuff
          .append(" getRemoteHost():")
          .append(java.net.InetAddress.getByName(req.getRemoteHost()).getHostName())
          .append("\n");
    } catch (java.net.UnknownHostException e) {
      sbuff.append(" getRemoteHost():").append(e.getMessage()).append("\n");
    }
    sbuff.append(" getRemoteUser():").append(req.getRemoteUser()).append("\n");

    sbuff.append("\n");
    sbuff.append("Request Parameters:\n");
    Enumeration params = req.getParameterNames();
    while (params.hasMoreElements()) {
      String name = (String) params.nextElement();
      String values[] = req.getParameterValues(name);
      if (values != null) {
        for (int i = 0; i < values.length; i++) {
          sbuff
              .append("  ")
              .append(name)
              .append("  (")
              .append(i)
              .append("): ")
              .append(values[i])
              .append("\n");
        }
      }
    }
    sbuff.append("\n");

    sbuff.append("Request Headers:\n");
    Enumeration names = req.getHeaderNames();
    while (names.hasMoreElements()) {
      String name = (String) names.nextElement();
      Enumeration values = req.getHeaders(name); // support multiple values
      if (values != null) {
        while (values.hasMoreElements()) {
          String value = (String) values.nextElement();
          sbuff.append("  ").append(name).append(": ").append(value).append("\n");
        }
      }
    }
    sbuff.append(" ------------------\n");

    return sbuff.toString();
  }

  public static String showRequestHeaders(HttpServletRequest req) {
    StringBuilder sbuff = new StringBuilder();
    sbuff.append("Request Headers:\n");
    Enumeration names = req.getHeaderNames();
    while (names.hasMoreElements()) {
      String name = (String) names.nextElement();
      Enumeration values = req.getHeaders(name); // support multiple values
      if (values != null) {
        while (values.hasMoreElements()) {
          String value = (String) values.nextElement();
          sbuff.append("  ").append(name).append(": ").append(value).append("\n");
        }
      }
    }
    return sbuff.toString();
  }

  public static void showSession(HttpServletRequest req, HttpServletResponse res, PrintStream out) {

    // res.setContentType("text/html");

    // Get the current session object, create one if necessary
    HttpSession session = req.getSession();

    // Increment the hit count for this page. The value is saved
    // in this client's session under the name "snoop.count".
    Integer count = (Integer) session.getAttribute("snoop.count");
    if (count == null) {
      count = 1;
    } else count = count + 1;
    session.setAttribute("snoop.count", count);

    out.println(HtmlWriter.getInstance().getHtmlDoctypeAndOpenTag());
    out.println("<HEAD><TITLE>SessionSnoop</TITLE></HEAD>");
    out.println("<BODY><H1>Session Snoop</H1>");

    // Display the hit count for this page
    out.println(
        "You've visited this page " + count + ((!(count.intValue() != 1)) ? " time." : " times."));

    out.println("<P>");

    out.println("<H3>Here is your saved session data:</H3>");
    Enumeration atts = session.getAttributeNames();
    while (atts.hasMoreElements()) {
      String name = (String) atts.nextElement();
      out.println(name + ": " + session.getAttribute(name) + "<BR>");
    }

    out.println("<H3>Here are some vital stats on your session:</H3>");
    out.println("Session id: " + session.getId() + " <I>(keep it secret)</I><BR>");
    out.println("New session: " + session.isNew() + "<BR>");
    out.println("Timeout: " + session.getMaxInactiveInterval());
    out.println("<I>(" + session.getMaxInactiveInterval() / 60 + " minutes)</I><BR>");
    out.println("Creation time: " + session.getCreationTime());
    out.println("<I>(" + new Date(session.getCreationTime()) + ")</I><BR>");
    out.println("Last access time: " + session.getLastAccessedTime());
    out.println("<I>(" + new Date(session.getLastAccessedTime()) + ")</I><BR>");

    out.println(
        "Requested session ID from cookie: " + req.isRequestedSessionIdFromCookie() + "<BR>");
    out.println("Requested session ID from URL: " + req.isRequestedSessionIdFromURL() + "<BR>");
    out.println("Requested session ID valid: " + req.isRequestedSessionIdValid() + "<BR>");

    out.println("<H3>Test URL Rewriting</H3>");
    out.println("Click <A HREF=\"" + res.encodeURL(req.getRequestURI()) + "\">here</A>");
    out.println("to test that session tracking works via URL");
    out.println("rewriting even when cookies aren't supported.");

    out.println("</BODY></HTML>");
  }

  public static void showSession(HttpServletRequest req, PrintStream out) {

    // res.setContentType("text/html");

    // Get the current session object, create one if necessary
    HttpSession session = req.getSession();

    out.println("Session id: " + session.getId());
    out.println(" session.isNew(): " + session.isNew());
    out.println(" session.getMaxInactiveInterval(): " + session.getMaxInactiveInterval() + " secs");
    out.println(
        " session.getCreationTime(): "
            + session.getCreationTime()
            + " ("
            + new Date(session.getCreationTime())
            + ")");
    out.println(
        " session.getLastAccessedTime(): "
            + session.getLastAccessedTime()
            + " ("
            + new Date(session.getLastAccessedTime())
            + ")");
    out.println(" req.isRequestedSessionIdFromCookie: " + req.isRequestedSessionIdFromCookie());
    out.println(" req.isRequestedSessionIdFromURL: " + req.isRequestedSessionIdFromURL());
    out.println(" req.isRequestedSessionIdValid: " + req.isRequestedSessionIdValid());

    out.println("Saved session Attributes:");
    Enumeration atts = session.getAttributeNames();
    while (atts.hasMoreElements()) {
      String name = (String) atts.nextElement();
      out.println(" " + name + ": " + session.getAttribute(name) + "<BR>");
    }
  }

  public static String showSecurity(HttpServletRequest req, String role) {
    StringBuilder sbuff = new StringBuilder();

    sbuff.append("Security Info\n");
    sbuff.append(" req.getRemoteUser(): ").append(req.getRemoteUser()).append("\n");
    sbuff.append(" req.getUserPrincipal(): ").append(req.getUserPrincipal()).append("\n");
    sbuff
        .append(" req.isUserInRole(")
        .append(role)
        .append("):")
        .append(req.isUserInRole(role))
        .append("\n");
    sbuff.append(" ------------------\n");

    return sbuff.toString();
  }

  /* from luca / ageci code, portResolver, portMapper not known
  static public void getSecureRedirect(HttpServletRequest req) {
      String pathInfo = req.getPathInfo();
      String queryString = req.getQueryString();
      String contextPath = req.getContextPath();
      String destination = req.getServletPath() + ((pathInfo ==   null) ? "" : pathInfo)
          + ((queryString == null) ? "" : ("?" + queryString));
      String redirectUrl = contextPath;

      Integer httpPort = new Integer(portResolver.getServerPort(req));
      Integer httpsPort = portMapper.lookupHttpsPort(httpPort);
      if (httpsPort != null) {
          boolean includePort = true;
          if (httpsPort.intValue() == 443) {
              includePort = false;
          }
          redirectUrl = "https://" + req.getServerName() +   ((includePort) ? (":" + httpsPort) : "") + contextPath
              + destination;
      }
  } */

  private static String getServerInfoName(String serverInfo) {
    int slash = serverInfo.indexOf('/');
    if (slash == -1) return serverInfo;
    else return serverInfo.substring(0, slash);
  }

  private static String getServerInfoVersion(String serverInfo) {
    // Version info is everything between the slash and the space
    int slash = serverInfo.indexOf('/');
    if (slash == -1) return null;
    int space = serverInfo.indexOf(' ', slash);
    if (space == -1) space = serverInfo.length();
    return serverInfo.substring(slash + 1, space);
  }

  public static void showThreads(PrintStream pw) {
    Thread current = Thread.currentThread();
    ThreadGroup group = current.getThreadGroup();
    while (true) {
      if (group.getParent() == null) break;
      group = group.getParent();
    }
    showThreads(pw, group, current);
  }

  private static void showThreads(PrintStream pw, ThreadGroup g, Thread current) {
    int nthreads = g.activeCount();
    pw.println("\nThread Group = " + g.getName() + " activeCount= " + nthreads);
    Thread[] tarray = new Thread[nthreads];
    int n = g.enumerate(tarray, false);

    for (int i = 0; i < n; i++) {
      Thread thread = tarray[i];
      ClassLoader loader = thread.getContextClassLoader();
      String loaderName = (loader == null) ? "Default" : loader.getClass().getName();
      Thread.State state = thread.getState();
      long id = thread.getId();
      pw.print("   " + id + " " + thread.getName() + " " + state + " " + loaderName);
      if (thread == current) pw.println(" **** CURRENT ***");
      else pw.println();
    }

    int ngroups = g.activeGroupCount();
    ThreadGroup[] garray = new ThreadGroup[ngroups];
    int ng = g.enumerate(garray, false);
    for (int i = 0; i < ng; i++) {
      ThreadGroup nested = garray[i];
      showThreads(pw, nested, current);
    }
  }
}
Exemplo n.º 20
0
/**
 * A class containing static methods which deliver descriptions and names of parameters, levels and
 * units for byte codes from GRIB records.
 *
 * <p>Performs operations related to loading parameter tables stored in files. Through a lookup
 * table (see readParameterTableLookup) all of the supported Parameter Tables are known. An actual
 * table is not loaded until a parameter from that center/subcenter/table is loaded. see <a
 * href="../../../Parameters.txt">Parameters.txt</a>
 *
 * <p>For now, the lookup table name is hard coded to "resources/grib/tables/tablelookup.lst"
 *
 * @author Capt Richard D. Gonzalez modified by Robb Kambic threadsafe 9/25/08 jcaron see
 *     http://www.ibm.com/developerworks/java/library/j-hashmap.html
 */
public final class GribPDSParamTable {
  private static org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(GribPDSParamTable.class);

  private static final String RESOURCE_PATH = "resources/grib/tables";
  private static final String TABLE_LIST = "tablelookup.lst";

  private static final Pattern valid = Pattern.compile("^[a-zA-Z_][a-zA-Z0-9_@:\\.\\-\\+]*$");
  private static final Pattern numberFirst = Pattern.compile("^[0-9]");

  /**
   * Added by Richard D. Gonzalez static Array with parameter tables used by the GRIB file (should
   * only be one, but not actually limited to that - this allows GRIB files to be read that have
   * more than one center's information in it)
   */
  private static volatile GribPDSParamTable[] paramTables = null;

  private static Object lock = new Object();

  private static boolean debug = false;
  private static int wmoTable;

  /**
   * This is a mapping from (center,subcenter,number)-> Param table for any data that has been
   * loaded
   */
  private static Map<String, GribPDSParamTable> tableMap =
      new ConcurrentHashMap<String, GribPDSParamTable>();

  static {
    try {
      ArrayList<GribPDSParamTable> tables = new ArrayList<GribPDSParamTable>();
      String resourceName = RESOURCE_PATH + "/" + TABLE_LIST;
      readTableEntries(resourceName, tables);
      paramTables = (GribPDSParamTable[]) tables.toArray(new GribPDSParamTable[tables.size()]);

    } catch (IOException ioe) {
      throw new RuntimeException(ioe);
    }
  }

  /**
   * _more_
   *
   * @param aTableList _more_
   * @param aTables _more_
   * @return Was read successful
   * @throws IOException On badness
   */
  private static boolean readTableEntries(String aTableList, ArrayList<GribPDSParamTable> aTables)
      throws IOException {
    InputStream inputStream = GribResourceReader.getInputStream(aTableList);
    if (inputStream == null) {
      logger.debug("Could not open table file:" + aTableList);
      return false;
    }
    return readTableEntries(inputStream, aTableList, aTables);
  }

  /**
   * Read the table list contained in the input stream
   *
   * @param is The input stream
   * @param aTableList The name of the table list file
   * @param aTables The list to add the tables into
   * @return Was successful
   * @throws IOException On badness
   */
  private static boolean readTableEntries(
      InputStream is, String aTableList, ArrayList<GribPDSParamTable> aTables) throws IOException {
    if (is == null) return false;

    InputStreamReader isr = new InputStreamReader(is);
    BufferedReader br = new BufferedReader(isr);

    String line;
    while ((line = br.readLine()) != null) {
      line = line.trim();
      if ((line.length() == 0) || line.startsWith("#")) {
        continue;
      }
      GribPDSParamTable table = new GribPDSParamTable();
      String[] tableDefArr = line.split(":");

      table.center_id = Integer.parseInt(tableDefArr[0].trim());
      table.subcenter_id = Integer.parseInt(tableDefArr[1].trim());
      table.table_number = Integer.parseInt(tableDefArr[2].trim());
      table.filename = tableDefArr[3].trim();
      if (table.filename.startsWith("/")
          || table.filename.startsWith("\\")
          || table.filename.startsWith("file:")
          || table.filename.startsWith("http://")) {
        table.path = table.filename;
      } else if (aTableList != null) {
        table.path = GribResourceReader.getFileRoot(aTableList);
        if (table.path.equals(aTableList)) {
          table.path = table.filename;
        } else {
          table.path = table.path + "/" + table.filename;
        }
        table.path = table.path.replace('\\', '/');
      }
      aTables.add(table);
    }
    is.close();
    return true;
  }

  public static GribPDSParamTable[] getParameterTables() {
    return paramTables;
  }

  /**
   * Reads in the list of tables available and stores them. Does not actually open the parameter
   * tables files, nor store the list of parameters, but just stores the file names of the parameter
   * tables. Parameters for a table are read in when the table is requested (in the
   * getParameterTable method).
   *
   * @param is UserGribTabList as a InputStream
   * @throws IOException or read error
   */
  public static void addParameterUserLookup(InputStream is) throws IOException {

    // leave out of lock since it does IO
    ArrayList<GribPDSParamTable> tables = new ArrayList<GribPDSParamTable>();
    if (!readTableEntries(is, null, tables)) {
      return;
    }

    synchronized (lock) {
      // tmp table stores new user defined tables plus tablelookup.lst table entries
      GribPDSParamTable[] tmp = new GribPDSParamTable[paramTables.length + tables.size()];
      for (int idx = 0; idx < paramTables.length + tables.size(); idx++) {
        if (idx < tables.size()) {
          tmp[idx] = (GribPDSParamTable) tables.get(idx);
          // System.out.println( "usrlookup tables = " + tmp[ idx ].path );
        } else {
          tmp[idx] = paramTables[idx - tables.size()]; // tablelookup.lst entries
        }
      }
      paramTables = tmp; // new copy of the data structure
    }
  }

  /**
   * Reads in the list of tables available and stores them. Does not actually open the parameter
   * tables files, nor store the list of parameters, but just stores the file names of the parameter
   * tables. Parameters for a table are read in when the table is requested (in the
   * getParameterTable method).
   *
   * @param userGribTabList string of userlookup file
   * @throws IOException if file found but read error
   * @return true if read ok, false if file not found
   */
  public static boolean addParameterUserLookup(String userGribTabList) throws IOException {

    // leave out of lock since it does IO
    ArrayList<GribPDSParamTable> tables = new ArrayList<GribPDSParamTable>();
    if (!readTableEntries(userGribTabList, tables)) {
      // logger.error("could not read:" + userGribTabList);
      return false;
    }

    synchronized (lock) {
      // tmp table stores new user defined tables plus tablelookup.lst table entries
      GribPDSParamTable[] tmp = new GribPDSParamTable[paramTables.length + tables.size()];
      for (int idx = 0; idx < paramTables.length + tables.size(); idx++) {
        if (idx < tables.size()) {
          tmp[idx] = tables.get(idx); // new stuff first
        } else {
          tmp[idx] = paramTables[idx - tables.size()]; // old stuff
        }
      }
      paramTables = tmp; // new copy of the data structure
    }

    return true;
  }

  /**
   * Looks for the parameter table which matches the center, subcenter and table version from the
   * tables array. If this is the first time asking for this table, then the parameters for this
   * table have not been read in yet, so this is done as well.
   *
   * @param center - integer from PDS octet 5, representing Center.
   * @param subcenter - integer from PDS octet 26, representing Subcenter
   * @param tableVersion - integer from PDS octet 4, representing Parameter Table Version
   * @return GribPDSParamTable matching center, subcenter, and number
   * @throws NotSupportedException no table found
   */
  public static GribPDSParamTable getParameterTable(int center, int subcenter, int tableVersion)
      throws NotSupportedException {

    String key = center + "_" + subcenter + "_" + tableVersion;
    if (center == -1) { // non existent table
      logger.error("GribPDSParamTable: non existent table for center, subcenter, table = " + key);
      return null;
    }

    GribPDSParamTable table = tableMap.get(key);
    if (table != null) return table;

    table = readParameterTable(center, subcenter, tableVersion, true);

    if (table == null) {
      logger.error("GribPDSParamTable: cannot find table for center, subcenter, table " + key);
      throw new NotSupportedException(
          "Could not find a table entry for GRIB file with center: "
              + center
              + " subCenter: "
              + subcenter
              + " number: "
              + tableVersion);
    }

    tableMap.put(key, table);
    return table;
  }

  /**
   * Looks for the parameter table which matches the center, subcenter and table version from the
   * tables array. If this is the first time asking for this table, then the parameters for this
   * table have not been read in yet, so this is done as well.
   *
   * @param center - integer from PDS octet 5, representing Center.
   * @param subcenter - integer from PDS octet 26, representing Subcenter
   * @param number - integer from PDS octet 4, representing Parameter Table Version
   * @param firstTry - Is this the first call or are we trying the wild cards
   * @return GribPDSParamTable matching center, subcenter, and number
   */
  private static GribPDSParamTable readParameterTable(
      int center, int subcenter, int number, boolean firstTry) {

    if (firstTry) wmoTable = number;

    GribPDSParamTable[] localCopy = paramTables; // thread safe

    for (GribPDSParamTable table : localCopy) {

      if (center == table.center_id) {
        if ((table.subcenter_id == -1) || (subcenter == table.subcenter_id)) {
          if (number == table.table_number) {
            // now that this table is being used, check to see if the
            //   parameters for this table have been read in yet.
            // If not, initialize table and read them in now.
            if (table.parameters == null) {
              if (!firstTry) {
                logger.warn(
                    "GribPDSParamTable: Using default table:"
                        + table.path
                        + " ("
                        + table.center_id
                        + ":"
                        + table.subcenter_id
                        + ":"
                        + table.table_number
                        + ")");
              }
              table.readParameterTable();
              if (table.parameters
                  == null) // failed - maybe theres another entry table in paramTables
              continue;

              // success - initialize other tables parameters with the same name
              for (int j = 0; j < paramTables.length; j++) {
                GribPDSParamTable tab = paramTables[j];
                if (tab.path.equals(table.path)) {
                  tab.parameters = table.parameters;
                }
              }
            }
            return table;
          }
        }
      }
    }

    // Try with the  wild cards
    if (number != -1) {
      return readParameterTable(center, subcenter, -1, false);

    } else if (subcenter != -1) {
      logger.warn(
          "GribPDSParamTable: Could not find table for center:"
              + center
              + " subcenter:"
              + subcenter
              + " number:"
              + wmoTable);
      return readParameterTable(center, -1, -1, false);

    } else if (center != -1) {
      // return readParameterTable(-1, -1, -1, false);
      return readParameterTable(-1, -1, wmoTable, false);
    }

    return null;
  }

  /**
   * Munge a description to make it suitable as variable name
   *
   * @param description start with this
   * @return Valid Description
   */
  private static String makeValidDesc(String description) {
    description = description.replaceAll("\\s+", "_");
    if (valid.matcher(description).find()) return description;
    // else check for special characters
    if (numberFirst.matcher(description).find()) description = "N" + description;
    return description.replaceAll("\\)|\\(|=|,|;|\\[|\\]", "");
  }

  //////////////////////////////////////////////////////////////////////////

  /** Identification of center e.g. 88 for Oslo */
  private int center_id;

  /** Identification of center defined sub-center - not fully implemented yet. */
  private int subcenter_id;

  /** Identification of parameter table version number. */
  private int table_number;

  /** Stores the name of the file containing this table - not opened unless required for lookup. */
  private String filename = null;

  /** path of filename containing this table. Opened if required for lookup. */
  private String path = null;

  /** Map ids to GridParameter objects */
  private Map<String, GridParameter> parameters = null;

  private GribPDSParamTable() {}

  public int getCenter_id() {
    return center_id;
  }

  public int getSubcenter_id() {
    return subcenter_id;
  }

  public int getTable_number() {
    return table_number;
  }

  public String getPath() {
    return path;
  }

  public String getFilename() {
    return filename;
  }

  public Map<String, GridParameter> getParameters() {
    if (parameters == null) readParameterTable();
    return parameters;
  }

  /** Read parameter table. */
  private void readParameterTable() {
    if (path == null) {
      logger.error("GribPDSParamTable: unknown path for " + this);
      return;
    }

    try {
      InputStream is = GribResourceReader.getInputStream(path);
      if (is == null) {
        logger.error("GribPDSParamTable: error getInputStream on " + this);
        return;
      }
      BufferedReader br = new BufferedReader(new InputStreamReader(is));

      // Read first line that has center, subcenter, and version of table
      String line = br.readLine();
      if (debug) System.out.println(line);
      String[] tableDefArr = line.split(":");

      /*  LOOK - why not test values ?
      center    = Integer.parseInt(tableDefArr[1].trim());
      subcenter = Integer.parseInt(tableDefArr[2].trim());
      number    = Integer.parseInt(tableDefArr[3].trim());
      if ((center != center_id) && (subcenter != subcenter_id)
              && (number != table_number)) {
          throw new java.io.IOException(
              "parameter table header values do not "
              + " match values in GRIB file.  Possible error in lookup table.");
      }
      */

      HashMap<String, GridParameter> tmpParameters =
          new HashMap<String, GridParameter>(); // thread safe - temp hash

      // rdg - added the 0 line length check to cover the case of blank lines at
      //       the end of the parameter table file.
      while ((line = br.readLine()) != null) {
        if ((line.length() == 0) || line.startsWith("#")) {
          continue;
        }
        GridParameter parameter = new GridParameter();
        tableDefArr = line.split(":");
        parameter.setNumber(Integer.parseInt(tableDefArr[0].trim()));
        parameter.setName(tableDefArr[1].trim());
        // check to see if unit defined, if not, parameter is undefined
        if (tableDefArr[2].indexOf('[') == -1) {
          // Undefined unit
          parameter.setDescription(tableDefArr[2].trim());
          parameter.setUnit(tableDefArr[2].trim());
        } else {
          String[] arr2 = tableDefArr[2].split("\\[");
          parameter.setDescription(makeValidDesc(arr2[0].trim()));
          // System.out.println( "Desc ="+ parameter.getDescription());
          // Remove "]"
          parameter.setUnit(arr2[1].substring(0, arr2[1].lastIndexOf(']')).trim());
        }
        tmpParameters.put(Integer.toString(parameter.getNumber()), parameter);
        if (debug)
          System.out.println(
              parameter.getNumber() + " " + parameter.getDescription() + " " + parameter.getUnit());
      }

      this.parameters = tmpParameters; // thread safe

    } catch (IOException ioError) {
      logger.warn(
          "An error occurred in GribPDSParamTable while trying to open the parameter table "
              + filename
              + " : "
              + ioError);
    }
  }

  /**
   * Get the parameter with id <tt>id</tt>.
   *
   * @param id the parameter id
   * @return the GridParameter
   */
  public GridParameter getParameter(int id) {
    GridParameter p = parameters.get(Integer.toString(id));
    if (p != null) return p;

    logger.warn(
        "GribPDSParamTable: Could not find parameter "
            + id
            + " for center:"
            + center_id
            + " subcenter:"
            + subcenter_id
            + " number:"
            + table_number
            + " table "
            + filename);
    String unknown = "UnknownParameter_" + Integer.toString(id) + "_table_" + filename;
    return new GridParameter(id, unknown, unknown, "Unknown");
  }

  @Override
  public String toString() {
    return "GribPDSParamTable{"
        + "center_id="
        + center_id
        + ", subcenter_id="
        + subcenter_id
        + ", table_number="
        + table_number
        + ", filename='"
        + filename
        + '\''
        + ", path='"
        + path
        + '\''
        + '}';
  }

  public static void main(String[] args) throws IOException {
    debug = true;
    addParameterUserLookup(
        "C:/dev/tds4.2/thredds/grib/resources/resources/grib/tables/zagreb_221_1.tab");
  }
}
Exemplo n.º 21
0
/**
 * A Vertical Coordinate variable for a Grid variable.
 *
 * @author caron
 * @version $Revision: 63 $ $Date: 2006-07-12 15:50:51 -0600 (Wed, 12 Jul 2006) $
 */
public class GridVertCoord implements Comparable {

  /** logger */
  private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(GridVertCoord.class);

  /** typical record for this vertical coordinate */
  private GridRecord typicalRecord;

  /** level name */
  private String levelName;

  /** lookup table */
  private GridTableLookup lookup;

  /** sequence # */
  private int seq = 0;

  /** coord values */
  private double[] coordValues = null;

  /** uses bounds flag */
  boolean usesBounds = false;

  /** don't use vertical flag */
  boolean dontUseVertical = false;

  /** vertical pressure factors */
  double[] factors = null;

  /** positive direction */
  String positive = "up";

  /** units */
  String units;

  /** levels */
  private List<LevelCoord> levels = new ArrayList<LevelCoord>(); // LevelCoord

  /**
   * Create a new GridVertCoord with the given name
   *
   * @param name name
   */
  GridVertCoord(String name) {
    this.levelName = name;
    dontUseVertical = true;
  }

  /**
   * Create a new GridVertCoord with the appropriate params
   *
   * @param records list of GridRecords that make up this coord
   * @param levelName the name of the level
   * @param lookup the lookup table
   * @param hcs Horizontal coordinate
   */
  GridVertCoord(
      List<GridRecord> records, String levelName, GridTableLookup lookup, GridHorizCoordSys hcs) {
    this.typicalRecord = records.get(0);
    this.levelName = levelName;
    this.lookup = lookup;

    dontUseVertical = !lookup.isVerticalCoordinate(typicalRecord);
    positive = lookup.isPositiveUp(typicalRecord) ? "up" : "down";
    units = lookup.getLevelUnit(typicalRecord);

    usesBounds = lookup.isLayer(this.typicalRecord);
    addLevels(records);

    if (typicalRecord.getLevelType1() == 109 && lookup instanceof Grib1GridTableLookup)
      checkForPressureLevels(records, hcs);

    if (GridServiceProvider.debugVert) {
      System.out.println(
          "GribVertCoord: "
              + getVariableName()
              + "("
              + typicalRecord.getLevelType1()
              + ") useVertical= "
              + (!dontUseVertical)
              + " positive="
              + positive
              + " units="
              + units);
    }
  }

  /**
   * Create a new GridVertCoord for a layer
   *
   * @param record layer record
   * @param levelName name of this level
   * @param lookup lookup table
   * @param level1 level 1
   * @param level2 level 2
   */
  GridVertCoord(
      GridRecord record,
      String levelName,
      GridTableLookup lookup,
      double[] level1,
      double[] level2) {
    this.typicalRecord = record;
    this.levelName = levelName;
    this.lookup = lookup;

    // dontUseVertical    = !lookup.isVerticalCoordinate(record);
    positive = lookup.isPositiveUp(record) ? "up" : "down";
    units = lookup.getLevelUnit(record);
    usesBounds = lookup.isLayer(this.typicalRecord);

    levels = new ArrayList<LevelCoord>(level1.length);
    for (int i = 0; i < level1.length; i++) {
      levels.add(new LevelCoord(level1[i], (level2 == null) ? 0.0 : level2[i]));
    }

    Collections.sort(levels);
    if (positive.equals("down")) {
      Collections.reverse(levels);
    }
    dontUseVertical = (levels.size() == 1);
  }

  /**
   * Set the sequence number
   *
   * @param seq the sequence number
   */
  void setSequence(int seq) {
    this.seq = seq;
  }

  /**
   * Set the level name
   *
   * @return the level name
   */
  String getLevelName() {
    return levelName;
  }

  /**
   * Get the variable name
   *
   * @return the variable name
   */
  String getVariableName() {
    return (seq == 0) ? levelName : levelName + seq; // more than one with same levelName
  }

  /**
   * Get the number of levels
   *
   * @return number of levels
   */
  int getNLevels() {
    return dontUseVertical ? 1 : levels.size();
  }

  /**
   * Add levels
   *
   * @param records GridRecords, one for each level
   */
  void addLevels(List<GridRecord> records) {
    for (int i = 0; i < records.size(); i++) {
      GridRecord record = records.get(i);

      if (coordIndex(record) < 0) {
        levels.add(new LevelCoord(record.getLevel1(), record.getLevel2()));
        if (dontUseVertical && (levels.size() > 1)) {
          if (GridServiceProvider.debugVert) {
            logger.warn(
                "GribCoordSys: unused level coordinate has > 1 levels = "
                    + levelName
                    + " "
                    + record.getLevelType1()
                    + " "
                    + levels.size());
          }
        }
      }
    }
    Collections.sort(levels);
    if (positive.equals("down")) {
      Collections.reverse(levels);
    }
  }

  /**
   * Match levels
   *
   * @param records records to match
   * @return true if they have the same levels
   */
  boolean matchLevels(List<GridRecord> records) {

    // first create a new list
    List<LevelCoord> levelList = new ArrayList<LevelCoord>(records.size());
    for (GridRecord record : records) {
      LevelCoord lc = new LevelCoord(record.getLevel1(), record.getLevel2());
      if (!levelList.contains(lc)) {
        levelList.add(lc);
      }
    }

    Collections.sort(levelList);
    if (positive.equals("down")) {
      Collections.reverse(levelList);
    }

    // gotta equal existing list
    return levelList.equals(levels);
  }

  /** check for Sigma Pressure Levels */
  boolean checkForPressureLevels(List<GridRecord> records, GridHorizCoordSys hcs) {
    GridDefRecord gdr = hcs.getGds();
    Grib1GDSVariables g1dr = (Grib1GDSVariables) gdr.getGdsv();
    if (g1dr == null || !g1dr.hasVerticalPressureLevels()) return false;

    // add hybrid numbers
    coordValues = new double[levels.size()];
    for (int i = 0; i < levels.size(); i++) {
      LevelCoord lc = levels.get(i);
      coordValues[i] = lc.value1;
    }
    int NV = g1dr.getNV();
    // add new variables
    if (NV > 2 && NV < 255) { // Some data doesn't add Pressure Level values
      factors = g1dr.getVerticalPressureLevels();
    }
    return true;
  }

  /**
   * Add this coord as a dimension to the netCDF file
   *
   * @param ncfile file to add to
   * @param g group in the file
   */
  void addDimensionsToNetcdfFile(NetcdfFile ncfile, Group g) {
    if (dontUseVertical) {
      return;
    }
    int nlevs = levels.size();
    if (coordValues != null) nlevs = coordValues.length;
    ncfile.addDimension(g, new Dimension(getVariableName(), nlevs, true));
  }

  /**
   * Add this coord as a variable in the netCDF file
   *
   * @param ncfile netCDF file to add to
   * @param g group in file
   */
  void addToNetcdfFile(NetcdfFile ncfile, Group g) {
    if (dontUseVertical) {
      typicalRecord = null;
      return;
    }

    if (g == null) {
      g = ncfile.getRootGroup();
    }

    // coordinate axis
    Variable v = new Variable(ncfile, g, null, getVariableName());
    v.setDataType(DataType.DOUBLE);

    String desc = lookup.getLevelDescription(typicalRecord);
    if (lookup instanceof Grib2GridTableLookup && usesBounds) {
      desc = "Layer between " + desc;
    }

    v.addAttribute(new Attribute("long_name", desc));
    v.addAttribute(new Attribute("units", lookup.getLevelUnit(typicalRecord)));

    // positive attribute needed for CF-1 Height and Pressure
    if (positive != null) {
      v.addAttribute(new Attribute("positive", positive));
    }

    if (units != null) {
      AxisType axisType;
      if (SimpleUnit.isCompatible("millibar", units)) {
        axisType = AxisType.Pressure;
      } else if (SimpleUnit.isCompatible("m", units)) {
        axisType = AxisType.Height;
      } else {
        axisType = AxisType.GeoZ;
      }

      if (lookup instanceof Grib2GridTableLookup || lookup instanceof Grib1GridTableLookup) {
        v.addAttribute(
            new Attribute("GRIB_level_type", Integer.toString(typicalRecord.getLevelType1())));
      } else {
        v.addAttribute(
            new Attribute("level_type", Integer.toString(typicalRecord.getLevelType1())));
      }
      v.addAttribute(new Attribute(_Coordinate.AxisType, axisType.toString()));
    }

    if (coordValues == null) {
      coordValues = new double[levels.size()];
      for (int i = 0; i < levels.size(); i++) {
        LevelCoord lc = (LevelCoord) levels.get(i);
        coordValues[i] = lc.mid;
      }
    }
    Array dataArray = Array.factory(DataType.DOUBLE, new int[] {coordValues.length}, coordValues);

    v.setDimensions(getVariableName());
    v.setCachedData(dataArray, true);

    ncfile.addVariable(g, v);

    if (usesBounds) {
      String boundsDimName = "bounds_dim";
      if (g.findDimension(boundsDimName) == null) {
        ncfile.addDimension(g, new Dimension(boundsDimName, 2, true));
      }

      String bname = getVariableName() + "_bounds";
      v.addAttribute(new Attribute("bounds", bname));
      v.addAttribute(new Attribute(_Coordinate.ZisLayer, "true"));

      Variable b = new Variable(ncfile, g, null, bname);
      b.setDataType(DataType.DOUBLE);
      b.setDimensions(getVariableName() + " " + boundsDimName);
      b.addAttribute(new Attribute("long_name", "bounds for " + v.getName()));
      b.addAttribute(new Attribute("units", lookup.getLevelUnit(typicalRecord)));

      Array boundsArray = Array.factory(DataType.DOUBLE, new int[] {coordValues.length, 2});
      ucar.ma2.Index ima = boundsArray.getIndex();
      for (int i = 0; i < coordValues.length; i++) {
        LevelCoord lc = (LevelCoord) levels.get(i);
        boundsArray.setDouble(ima.set(i, 0), lc.value1);
        boundsArray.setDouble(ima.set(i, 1), lc.value2);
      }
      b.setCachedData(boundsArray, true);

      ncfile.addVariable(g, b);
    }

    if (factors != null) {
      // check if already created
      if (g == null) {
        g = ncfile.getRootGroup();
      }
      if (g.findVariable("hybrida") != null) return;
      v.addAttribute(new Attribute("standard_name", "atmosphere_hybrid_sigma_pressure_coordinate"));
      v.addAttribute(new Attribute("formula_terms", "ap: hybrida b: hybridb ps: Pressure"));
      // create  hybrid factor variables
      // add hybrida variable
      Variable ha = new Variable(ncfile, g, null, "hybrida");
      ha.setDataType(DataType.DOUBLE);
      ha.addAttribute(new Attribute("long_name", "level_a_factor"));
      ha.addAttribute(new Attribute("units", ""));
      ha.setDimensions(getVariableName());
      // add data
      int middle = factors.length / 2;
      double[] adata;
      double[] bdata;
      if (levels.size() < middle) { // only partial data wanted
        adata = new double[levels.size()];
        bdata = new double[levels.size()];
      } else {
        adata = new double[middle];
        bdata = new double[middle];
      }
      for (int i = 0; i < middle && i < levels.size(); i++) adata[i] = factors[i];
      Array haArray = Array.factory(DataType.DOUBLE, new int[] {adata.length}, adata);
      ha.setCachedData(haArray, true);
      ncfile.addVariable(g, ha);

      // add hybridb variable
      Variable hb = new Variable(ncfile, g, null, "hybridb");
      hb.setDataType(DataType.DOUBLE);
      hb.addAttribute(new Attribute("long_name", "level_b_factor"));
      hb.addAttribute(new Attribute("units", ""));
      hb.setDimensions(getVariableName());
      // add data
      for (int i = 0; i < middle && i < levels.size(); i++) bdata[i] = factors[i + middle];
      Array hbArray = Array.factory(DataType.DOUBLE, new int[] {bdata.length}, bdata);
      hb.setCachedData(hbArray, true);
      ncfile.addVariable(g, hb);

      /*  // TODO: delete next time modifying code
      double[] adata = new double[ middle ];
      for( int i = 0; i < middle; i++ )
        adata[ i ] = factors[ i ];
      Array haArray = Array.factory(DataType.DOUBLE, new int[]{adata.length}, adata);
      ha.setCachedData(haArray, true);
      ncfile.addVariable(g, ha);

      // add hybridb variable
      Variable hb = new Variable(ncfile, g, null, "hybridb");
      hb.setDataType(DataType.DOUBLE);
      hb.addAttribute(new Attribute("long_name",  "level_b_factor" ));
      //hb.addAttribute(new Attribute("standard_name", "atmosphere_hybrid_sigma_pressure_coordinate" ));
      hb.addAttribute(new Attribute("units", ""));
      hb.setDimensions(getVariableName());
      // add data
      double[] bdata = new double[ middle ];
      for( int i = 0; i < middle; i++ )
        bdata[ i ] = factors[ i + middle ];
      Array hbArray = Array.factory(DataType.DOUBLE, new int[]{bdata.length}, bdata);
      hb.setCachedData(hbArray, true);
      ncfile.addVariable(g, hb);
      */
    }
  }

  void empty() {
    // let all references to Index go, to reduce retained size
    typicalRecord = null;
  }

  /**
   * Get the index of the particular record
   *
   * @param record record in question
   * @return the index or -1 if not found
   */
  int getIndex(GridRecord record) {
    if (dontUseVertical) {
      return 0;
    }
    return coordIndex(record);
  }

  /**
   * Compare this to another
   *
   * @param o the other GridVertCoord
   * @return the comparison
   */
  public int compareTo(Object o) {
    GridVertCoord gv = (GridVertCoord) o;
    return getLevelName().compareToIgnoreCase(gv.getLevelName());
  }

  /**
   * A level coordinate
   *
   * @author IDV Development Team
   * @version $Revision: 1.3 $
   */
  private class LevelCoord implements Comparable {

    /** midpoint */
    double mid;

    /** top/bottom values */
    double value1, value2;

    /**
     * Create a new LevelCoord
     *
     * @param value1 top
     * @param value2 bottom
     */
    LevelCoord(double value1, double value2) {
      this.value1 = value1;
      this.value2 = value2;
      if (usesBounds && (value1 > value2)) {
        this.value1 = value2;
        this.value2 = value1;
      }
      mid = usesBounds ? (value1 + value2) / 2 : value1;
    }

    /**
     * Compare to another LevelCoord
     *
     * @param o another LevelCoord
     * @return the comparison
     */
    public int compareTo(Object o) {
      LevelCoord other = (LevelCoord) o;
      // if (closeEnough(value1, other.value1) && closeEnough(value2, other.value2)) return 0;
      if (mid < other.mid) {
        return -1;
      }
      if (mid > other.mid) {
        return 1;
      }
      return 0;
    }

    /**
     * Check for equality
     *
     * @param oo object in question
     * @return true if equal
     */
    public boolean equals(Object oo) {
      if (this == oo) {
        return true;
      }
      if (!(oo instanceof LevelCoord)) {
        return false;
      }
      LevelCoord other = (LevelCoord) oo;
      return (ucar.nc2.util.Misc.closeEnough(value1, other.value1)
          && ucar.nc2.util.Misc.closeEnough(value2, other.value2));
    }

    /**
     * Generate a hashcode
     *
     * @return the hashcode
     */
    public int hashCode() {
      return (int) (value1 * 100000 + value2 * 100);
    }
  }

  /**
   * Get the coordinate index for the record
   *
   * @param record record in question
   * @return index or -1 if not found
   */
  private int coordIndex(GridRecord record) {
    double val = record.getLevel1();
    double val2 = record.getLevel2();
    if (usesBounds && (val > val2)) {
      val = record.getLevel2();
      val2 = record.getLevel1();
    }

    for (int i = 0; i < levels.size(); i++) {
      LevelCoord lc = (LevelCoord) levels.get(i);
      if (usesBounds) {
        if (ucar.nc2.util.Misc.closeEnough(lc.value1, val)
            && ucar.nc2.util.Misc.closeEnough(lc.value2, val2)) {
          return i;
        }
      } else {
        if (ucar.nc2.util.Misc.closeEnough(lc.value1, val)) {
          return i;
        }
      }
    }
    return -1;
  }
}
Exemplo n.º 22
0
/**
 * Read and Write Grib1 index (gbx9). Hides Grib1IndexProto
 *
 * <p>sample use:
 *
 * <pre>
 * Grib1Index index = new Grib1Index();
 * if (!index.readIndex(path))
 * index.makeIndex(path);
 *
 * for (Grib1SectionGridDefinition gds : index.getGds()) {
 * if (gdsSet.get(gds.calcCRC()) == null)
 * gdsSet.put(gds.calcCRC(), gds);
 * }
 *
 * for (Grib1Record gr : index.getRecords()) {
 * gr.setFile(fileno);
 *
 * Grib1Pds pds = gr.getPDSsection().getPDS();
 * int discipline = gr.getDiscipline();
 *
 * int id = gr.cdmVariableHash();
 * Grib1ParameterBean bean = pdsSet.get(id);
 * if (bean == null) {
 * bean = new Grib1ParameterBean(gr);
 * pdsSet.put(id, bean);
 * params.add(bean);
 * }
 * bean.addRecord(gr);
 * }
 * </pre>
 *
 * @author John
 * @since 9/5/11
 */
public class Grib1Index extends GribIndex {
  private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(Grib1Index.class);

  public static final String MAGIC_START = "Grib1Index";
  private static final int version = 5;
  private static final boolean debug = false;

  ////////////////////////////////////////////////////////////////////////////////////////////////

  private List<Grib1SectionGridDefinition> gdsList;
  private List<Grib1Record> records;

  public List<Grib1SectionGridDefinition> getGds() {
    return gdsList;
  }

  public List<Grib1Record> getRecords() {
    return records;
  }

  @Override
  public int getNRecords() {
    return records.size();
  }

  public boolean readIndex(String filename, long gribLastModified) throws IOException {
    return readIndex(filename, gribLastModified, CollectionManager.Force.test);
  }

  public boolean readIndex(String filename, long gribLastModified, CollectionManager.Force force)
      throws IOException {
    File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
    if (!idxFile.exists()) return false;
    long idxModified = idxFile.lastModified();
    if ((force != CollectionManager.Force.nocheck) && (idxModified < gribLastModified))
      return false;
    // force new index if file was updated

    FileInputStream fin =
        new FileInputStream(idxFile); // LOOK need DiskCache for non-writeable directories

    try {
      //// check header is ok
      if (!NcStream.readAndTest(fin, MAGIC_START.getBytes())) {
        log.info("Bad magic number of grib index, on file" + idxFile);
        return false;
      }

      int v = NcStream.readVInt(fin);
      if (v != version) {
        if ((v == 0) || (v > version))
          throw new IOException(
              "Grib1Index found version " + v + ", want version " + version + " on " + filename);
        if (log.isDebugEnabled())
          log.debug(
              "Grib1Index found version " + v + ", want version " + version + " on " + filename);
        return false;
      }

      int size = NcStream.readVInt(fin);
      if (size <= 0 || size > 100 * 1000 * 1000) { // try to catch garbage
        log.warn("Grib1Index bad size = {} for {} ", size, filename);
        return false;
      }

      byte[] m = new byte[size];
      NcStream.readFully(fin, m);

      Grib1IndexProto.Grib1Index proto = Grib1IndexProto.Grib1Index.parseFrom(m);
      String fname = proto.getFilename();
      if (debug) System.out.printf("%s for %s%n", fname, filename);

      gdsList = new ArrayList<Grib1SectionGridDefinition>(proto.getGdsListCount());
      for (Grib1IndexProto.Grib1GdsSection pgds : proto.getGdsListList()) {
        Grib1SectionGridDefinition gds = readGds(pgds);
        gdsList.add(gds);
      }
      if (debug) System.out.printf(" read %d gds%n", gdsList.size());

      records = new ArrayList<Grib1Record>(proto.getRecordsCount());
      for (Grib1IndexProto.Grib1Record precord : proto.getRecordsList()) {
        records.add(readRecord(precord));
      }
      if (debug) System.out.printf(" read %d records%n", records.size());

    } catch (java.lang.NegativeArraySizeException e) {
      log.error("GribIndex failed on " + filename, e);
      return false;

    } catch (IOException e) {
      log.error("GribIndex failed on " + filename, e);
      return false;

    } finally {
      fin.close();
    }

    return true;
  }

  // deserialize the Grib1Record object
  private Grib1Record readRecord(Grib1IndexProto.Grib1Record p) {
    Grib1SectionIndicator is =
        new Grib1SectionIndicator(p.getGribMessageStart(), p.getGribMessageLength());
    Grib1SectionProductDefinition pds = new Grib1SectionProductDefinition(p.getPds().toByteArray());

    Grib1SectionGridDefinition gds =
        pds.gdsExists() ? gdsList.get(p.getGdsIdx()) : new Grib1SectionGridDefinition(pds);
    Grib1SectionBitMap bms = pds.bmsExists() ? new Grib1SectionBitMap(p.getBmsPos()) : null;

    Grib1SectionBinaryData dataSection = new Grib1SectionBinaryData(p.getDataPos(), p.getDataLen());
    return new Grib1Record(p.getHeader().toByteArray(), is, gds, pds, bms, dataSection);
  }

  private Grib1SectionGridDefinition readGds(Grib1IndexProto.Grib1GdsSection proto) {
    ByteString bytes = proto.getGds();
    return new Grib1SectionGridDefinition(bytes.toByteArray());
  }

  ////////////////////////////////////////////////////////////////////////////////

  // LOOK what about extending an index ??
  public boolean makeIndex(String filename, RandomAccessFile dataRaf, Formatter f)
      throws IOException {
    File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
    FileOutputStream fout = new FileOutputStream(idxFile);
    RandomAccessFile raf = null;
    try {
      //// header message
      fout.write(MAGIC_START.getBytes("UTF-8"));
      NcStream.writeVInt(fout, version);

      Map<Long, Integer> gdsMap = new HashMap<Long, Integer>();
      gdsList = new ArrayList<Grib1SectionGridDefinition>();
      records = new ArrayList<Grib1Record>(200);

      Grib1IndexProto.Grib1Index.Builder rootBuilder = Grib1IndexProto.Grib1Index.newBuilder();
      rootBuilder.setFilename(filename);

      if (dataRaf == null) {
        raf = new RandomAccessFile(filename, "r");
        dataRaf = raf;
      }

      Grib1RecordScanner scan = new Grib1RecordScanner(dataRaf);
      while (scan.hasNext()) {
        Grib1Record r = scan.next();
        if (r == null) break; // done
        records.add(r);

        Grib1SectionGridDefinition gds = r.getGDSsection();
        Integer index = gdsMap.get(gds.calcCRC());
        if (gds.getPredefinedGridDefinition()
            >= 0) // skip predefined gds - they dont have raw bytes
        index = 0;
        else if (index == null) {
          gdsList.add(gds);
          index = gdsList.size() - 1;
          gdsMap.put(gds.calcCRC(), index);
          rootBuilder.addGdsList(makeGdsProto(gds));
        }
        rootBuilder.addRecords(makeRecordProto(r, index));
      }

      ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Index index = rootBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(fout, b.length); // message size
      fout.write(b); // message  - all in one gulp
      f.format("  made gbx9 index for %s size=%d%n", filename, b.length);
      return true;

    } finally {
      fout.close();
      if (raf != null) raf.close();
    }
  }

  private ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Record makeRecordProto(
      Grib1Record r, int gdsIndex) throws IOException {
    Grib1IndexProto.Grib1Record.Builder b = Grib1IndexProto.Grib1Record.newBuilder();

    b.setHeader(ByteString.copyFrom(r.getHeader()));

    b.setGribMessageStart(r.getIs().getStartPos());
    b.setGribMessageLength(r.getIs().getMessageLength());

    b.setGdsIdx(gdsIndex);
    Grib1SectionProductDefinition pds = r.getPDSsection();
    b.setPds(ByteString.copyFrom(pds.getRawBytes()));

    if (pds.bmsExists()) {
      Grib1SectionBitMap bms = r.getBitMapSection();
      b.setBmsPos(bms.getStartingPosition());
    }

    Grib1SectionBinaryData ds = r.getDataSection();
    b.setDataPos(ds.getStartingPosition());
    b.setDataLen(ds.getLength());

    return b.build();
  }

  private Grib1IndexProto.Grib1GdsSection makeGdsProto(Grib1SectionGridDefinition gds)
      throws IOException {
    Grib1IndexProto.Grib1GdsSection.Builder b = Grib1IndexProto.Grib1GdsSection.newBuilder();
    b.setGds(ByteString.copyFrom(gds.getRawBytes()));
    return b.build();
  }

  public static void main(String args[]) throws IOException {
    String filename =
        "G:/tigge/uv/z_tigge_c_kwbc_20110605120000_glob_prod_cf_HGHT_0000_000_10_uv.grib";
    // String filename = "G:/mlode/ndfdProb/extract.Grib1";
    new Grib1Index().makeIndex(filename, null, new Formatter(System.out));
  }
}
Exemplo n.º 23
0
/**
 * A pool of precompiled XSLT stylesheets ({@link Templates}). Caching can be disabled via
 * constructor parameter or via setting a system property:
 *
 * <pre>
 * template.caching
 * </pre>
 *
 * to <code>false</code>.
 */
public final class TemplatesPool {
  private static final Logger logger = org.slf4j.LoggerFactory.getLogger(TemplatesPool.class);

  /**
   * Global system property disabling template caching. This property can also be set at runtime
   * (after the pool is initialized).
   */
  public static final String TEMPLATE_CACHING_PROPERTY = "template.caching";

  /** A set of used XSLT processors. */
  private static final Set<String> reportedProcessors =
      Collections.synchronizedSet(new HashSet<String>());

  /** A map of precompiled stylesheets ({@link Templates} objects). */
  private volatile HashMap<String, Templates> stylesheets = new HashMap<String, Templates>();

  /**
   * If <code>true</code> the templates will not be cached until the application shuts down. This
   * speeds up the application, but may be annoying, especially during development.
   */
  private final boolean templateCaching;

  /** {@link SAXTransformerFactory} capable of producing SAX-based transformers. */
  public final SAXTransformerFactory tFactory;

  /** Creates a {@link TemplatesPool} with caching enabled. */
  public TemplatesPool() throws Exception {
    this(true);
  }

  /** Check for required facilities. If not available, an exception will be thrown. */
  public TemplatesPool(boolean templateCaching) throws Exception {
    final TransformerFactory tFactory = TransformerFactory.newInstance();
    final String processorClass = tFactory.getClass().getName();

    /*
     * Only report XSLT processor class once.
     */
    if (!reportedProcessors.contains(processorClass)) {
      logger.info("XSLT transformer factory: " + processorClass);
      reportedProcessors.add(processorClass);
    }

    if (!tFactory.getFeature(SAXSource.FEATURE) || !tFactory.getFeature(SAXResult.FEATURE)) {
      throw new Exception("Required source types not supported by the transformer factory.");
    }

    if (!tFactory.getFeature(SAXResult.FEATURE) || !tFactory.getFeature(StreamResult.FEATURE)) {
      throw new Exception("Required result types not supported by the transformer factory.");
    }

    if (!(tFactory instanceof SAXTransformerFactory)) {
      throw new Exception(
          "TransformerFactory not an instance of SAXTransformerFactory: "
              + tFactory.getClass().getName());
    }

    this.tFactory = ((SAXTransformerFactory) tFactory);
    this.tFactory.setErrorListener(new StylesheetErrorListener());
    this.templateCaching = templateCaching;
  }

  /** @return returns the identity transformer handler. */
  public TransformerHandler getIdentityTransformerHandler()
      throws TransformerConfigurationException {
    return tFactory.newTransformerHandler();
  }

  /** Retrieves a previously stored template, if available. */
  public Templates getTemplate(String key) {
    if (!isCaching()) {
      return null;
    }

    return stylesheets.get(key);
  }

  /**
   * Add a new template to the pool. Addition is quite costly as it replaces the internal {@link
   * #stylesheets} {@link HashMap}.
   */
  public void addTemplate(String key, Templates template) {
    if (!isCaching()) {
      return;
    }

    /*
     * Copy-on-write.
     */
    synchronized (this) {
      final HashMap<String, Templates> newMap = new HashMap<String, Templates>(this.stylesheets);
      newMap.put(key, template);
      this.stylesheets = newMap;
    }
  }

  /** @return <code>true</code> if template caching is enabled. */
  private boolean isCaching() {
    /*
     * Global override takes precedence.
     */
    final String global = System.getProperty(TEMPLATE_CACHING_PROPERTY);
    if (global != null) {
      return Boolean.parseBoolean(global);
    }

    return templateCaching;
  }

  /**
   * Compile a {@link Templates} from a given system identifier. The template is not added to the
   * pool, a manual call to {@link #addTemplate(String, Templates)} is required.
   */
  public Templates compileTemplate(String systemId) throws SAXException {
    final StreamSource source = new StreamSource(systemId);
    try {
      return tFactory.newTemplates(source);
    } catch (Exception e) {
      throw new SAXException("Could not compile stylesheet: " + systemId, e);
    }
  }

  /**
   * Compile a {@link Templates} from a given stream. The template is not added to the pool
   * automatically.
   */
  public Templates compileTemplate(InputStream stream) throws SAXException {
    final StreamSource source = new StreamSource(stream);
    try {
      return tFactory.newTemplates(source);
    } catch (Exception e) {
      throw new SAXException("Could not compile stylesheet.", e);
    }
  }

  /**
   * Return a new {@link TransformerHandler} based on a given precompiled {@link Templates}. The
   * handler {@link Transformer}'s {@link ErrorListener} is set to {@link TransformerErrorListener}
   * to raise exceptions and give proper warnings.
   */
  public TransformerHandler newTransformerHandler(Templates template)
      throws TransformerConfigurationException {
    final TransformerHandler handler = this.tFactory.newTransformerHandler(template);

    /*
     * We want to raise transformer exceptions on <xml:message terminate="true">, so
     * we add a custom listener. Also, various XSLT processors react in different ways
     * to transformation errors -- some of them report error as recoverable, some of
     * them report error as unrecoverable.
     */
    handler.getTransformer().setErrorListener(new TransformerErrorListener());
    return handler;
  }

  /**
   * Return a new {@link Transformer}.
   *
   * @see #newTransformerHandler(Templates)
   */
  public Transformer newTransformer(Templates t) throws TransformerConfigurationException {
    return newTransformerHandler(t).getTransformer();
  }
}
Exemplo n.º 24
0
/** @author pepijn */
public class WPTileProvider
    implements org.pepsoft.util.swing.TileProvider, Dimension.Listener, Tile.Listener {
  public WPTileProvider(
      Dimension dimension,
      ColourScheme colourScheme,
      BiomeScheme biomeScheme,
      CustomBiomeManager customBiomeManager,
      Collection<Layer> hiddenLayers,
      boolean contourLines,
      int contourSeparation,
      TileRenderer.LightOrigin lightOrigin,
      boolean showBorder,
      org.pepsoft.util.swing.TileProvider surroundingTileProvider,
      boolean active) {
    tileProvider = dimension;
    this.colourScheme = colourScheme;
    this.biomeScheme = biomeScheme;
    this.hiddenLayers = (hiddenLayers != null) ? new HashSet<>(hiddenLayers) : null;
    this.contourLines = contourLines;
    this.contourSeparation = contourSeparation;
    this.lightOrigin = lightOrigin;
    this.active = active;
    this.customBiomeManager = customBiomeManager;
    this.surroundingTileProvider = surroundingTileProvider;
    this.showBorder = showBorder;
    tileRendererRef = createNewTileRendererRef();
  }

  public WPTileProvider(
      TileProvider tileProvider,
      ColourScheme colourScheme,
      BiomeScheme biomeScheme,
      CustomBiomeManager customBiomeManager,
      Collection<Layer> hiddenLayers,
      boolean contourLines,
      int contourSeparation,
      TileRenderer.LightOrigin lightOrigin,
      boolean showBorder,
      org.pepsoft.util.swing.TileProvider surroundingTileProvider) {
    this.tileProvider = tileProvider;
    this.colourScheme = colourScheme;
    this.biomeScheme = biomeScheme;
    this.hiddenLayers = (hiddenLayers != null) ? new HashSet<>(hiddenLayers) : null;
    this.contourLines = contourLines;
    this.contourSeparation = contourSeparation;
    this.lightOrigin = lightOrigin;
    active = false;
    this.customBiomeManager = customBiomeManager;
    this.surroundingTileProvider = surroundingTileProvider;
    this.showBorder = showBorder;
    tileRendererRef = createNewTileRendererRef();
  }

  public synchronized void addHiddenLayer(Layer layer) {
    hiddenLayers.add(layer);
    tileRendererRef = createNewTileRendererRef();
  }

  public synchronized void removeHiddenLayer(Layer layer) {
    hiddenLayers.remove(layer);
    tileRendererRef = createNewTileRendererRef();
  }

  @Override
  public int getTileSize() {
    return TILE_SIZE;
  }

  @Override
  public boolean isTilePresent(int x, int y) {
    if (zoom == 0) {
      return getUnzoomedTileType(x, y) != TileType.SURROUNDS
          || ((surroundingTileProvider != null) && surroundingTileProvider.isTilePresent(x, y));
    } else {
      final int scale = 1 << -zoom;
      for (int dx = 0; dx < scale; dx++) {
        for (int dy = 0; dy < scale; dy++) {
          switch (getUnzoomedTileType(x * scale + dx, y * scale + dy)) {
            case WORLD:
            case BORDER:
            case WALL:
              return true;
            case SURROUNDS:
              if ((surroundingTileProvider != null)
                  && surroundingTileProvider.isTilePresent(x, y)) {
                return true;
              }
              break;
          }
        }
      }
      return false;
    }
  }

  @Override
  public void paintTile(
      final Image tileImage, final int x, final int y, final int imageX, final int imageY) {
    try {
      if (zoom == 0) {
        paintUnzoomedTile(tileImage, x, y, imageX, imageY);
      } else {
        Graphics2D g2 = (Graphics2D) tileImage.getGraphics();
        try {
          BufferedImage surroundingTileImage = null;
          final Color waterColour = new Color(colourScheme.getColour(BLK_WATER));
          final Color lavaColour = new Color(colourScheme.getColour(BLK_LAVA));
          final Color voidColour = new Color(VoidRenderer.getColour());
          final Color bedrockColour = new Color(colourScheme.getColour(BLK_BEDROCK));
          final int scale = 1 << -zoom;
          final int subSize = TILE_SIZE / scale;
          for (int dx = 0; dx < scale; dx++) {
            for (int dy = 0; dy < scale; dy++) {
              TileType tileType = getUnzoomedTileType(x * scale + dx, y * scale + dy);
              switch (tileType) {
                case WORLD:
                  Tile tile = tileProvider.getTile(x * scale + dx, y * scale + dy);
                  if (tile.hasLayer(NotPresent.INSTANCE)) {
                    if (surroundingTileProvider != null) {
                      if (surroundingTileImage == null) {
                        surroundingTileImage =
                            new BufferedImage(TILE_SIZE, TILE_SIZE, BufferedImage.TYPE_INT_ARGB);
                        surroundingTileProvider.paintTile(surroundingTileImage, x, y, 0, 0);
                      }
                      g2.drawImage(
                          surroundingTileImage,
                          imageX + dx * subSize,
                          imageY + dy * subSize,
                          imageX + (dx + 1) * subSize,
                          imageY + (dy + 1) * subSize,
                          imageX + dx * subSize,
                          imageY + dy * subSize,
                          imageX + (dx + 1) * subSize,
                          imageY + (dy + 1) * subSize,
                          null);
                    } else {
                      g2.setColor(voidColour);
                      g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, subSize);
                    }
                  }
                  TileRenderer tileRenderer = tileRendererRef.get();
                  tileRenderer.setTile(tile);
                  tileRenderer.renderTile(tileImage, dx * subSize, dy * subSize);
                  break;
                case BORDER:
                  Color colour;
                  switch (((Dimension) tileProvider).getBorder()) {
                    case WATER:
                      colour = waterColour;
                      break;
                    case LAVA:
                      colour = lavaColour;
                      break;
                    case VOID:
                      colour = voidColour;
                      break;
                    default:
                      throw new InternalError();
                  }
                  g2.setColor(colour);
                  g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, subSize);

                  // Draw border lines
                  g2.setColor(Color.BLACK);
                  g2.setStroke(
                      new BasicStroke(
                          2,
                          BasicStroke.CAP_BUTT,
                          BasicStroke.JOIN_BEVEL,
                          0.0f,
                          new float[] {4.0f, 4.0f},
                          0.0f));
                  if (tileProvider.isTilePresent(x * scale + dx, y * scale + dy - 1)) {
                    g2.drawLine(
                        imageX + dx * subSize,
                        imageY + dy * subSize,
                        imageX + (dx + 1) * subSize - 1,
                        imageY + dy * subSize);
                  }
                  if (tileProvider.isTilePresent(x * scale + dx + 1, y * scale + dy)) {
                    g2.drawLine(
                        imageX + (dx + 1) * subSize - 1,
                        imageY + dy * subSize,
                        imageX + (dx + 1) * subSize - 1,
                        imageY + (dy + 1) * subSize - 1);
                  }
                  if (tileProvider.isTilePresent(x * scale + dx, y * scale + dy + 1)) {
                    g2.drawLine(
                        imageX + dx * subSize,
                        imageY + (dy + 1) * subSize - 1,
                        imageX + (dx + 1) * subSize - 1,
                        imageY + (dy + 1) * subSize - 1);
                  }
                  if (tileProvider.isTilePresent(x * scale + dx - 1, y * scale + dy)) {
                    g2.drawLine(
                        imageX + dx * subSize,
                        imageY + dy * subSize,
                        imageX + dx * subSize,
                        imageY + (dy + 1) * subSize - 1);
                  }
                  break;
                case SURROUNDS:
                case WALL:
                  if (surroundingTileProvider != null) {
                    if (surroundingTileImage == null) {
                      surroundingTileImage =
                          new BufferedImage(TILE_SIZE, TILE_SIZE, BufferedImage.TYPE_INT_ARGB);
                      surroundingTileProvider.paintTile(surroundingTileImage, x, y, 0, 0);
                    }
                    g2.drawImage(
                        surroundingTileImage,
                        imageX + dx * subSize,
                        imageY + dy * subSize,
                        imageX + (dx + 1) * subSize,
                        imageY + (dy + 1) * subSize,
                        imageX + dx * subSize,
                        imageY + dy * subSize,
                        imageX + (dx + 1) * subSize,
                        imageY + (dy + 1) * subSize,
                        null);
                  } else {
                    g2.setColor(voidColour);
                    g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, subSize);
                  }
                  if (tileType == TileType.WALL) {
                    g2.setColor(bedrockColour);
                    TileType neighbourType =
                        getUnzoomedTileType(x * scale + dx, y * scale + dy - 1);
                    int wallWidth = Math.max(subSize / 8, 1);
                    if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
                      g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, subSize, wallWidth);
                    }
                    neighbourType = getUnzoomedTileType(x * scale + dx + 1, y * scale + dy);
                    if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
                      g2.fillRect(
                          imageX + (dx + 1) * subSize - wallWidth,
                          imageY + dy * subSize,
                          wallWidth,
                          subSize);
                    }
                    neighbourType = getUnzoomedTileType(x * scale + dx, y * scale + dy + 1);
                    if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
                      g2.fillRect(
                          imageX + dx * subSize,
                          imageY + (dy + 1) * subSize - wallWidth,
                          subSize,
                          wallWidth);
                    }
                    neighbourType = getUnzoomedTileType(x * scale + dx - 1, y * scale + dy);
                    if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
                      g2.fillRect(imageX + dx * subSize, imageY + dy * subSize, wallWidth, subSize);
                    }
                  }
                  break;
              }
            }
          }
        } finally {
          g2.dispose();
        }
      }
    } catch (Throwable e) {
      // Log at debug level because this tends to happen when zooming in
      // and out, probably due to some state getting out of sync. It
      // doesn't so far appear to have any visible consequences.
      logger.error("Exception while generating image for tile at " + x + ", " + y, e);
    }
  }

  @Override
  public int getTilePriority(int x, int y) {
    if (zoom == 0) {
      return (getUnzoomedTileType(x, y) == TileType.WORLD) ? 1 : 0;
    } else {
      final int scale = 1 << -zoom;
      for (int dx = 0; dx < scale; dx++) {
        for (int dy = 0; dy < scale; dy++) {
          if (getUnzoomedTileType(x * scale + dx, y * scale + dy) == TileType.WORLD) {
            return 1;
          }
        }
      }
      return 0;
    }
  }

  @Override
  public Rectangle getExtent() {
    Rectangle sourceExtent = tileProvider.getExtent();
    if (sourceExtent != null) {
      if (zoom == 0) {
        return sourceExtent;
      } else if (zoom < 0) {
        return new Rectangle(
            sourceExtent.x >> -zoom,
            sourceExtent.y >> -zoom,
            sourceExtent.width >> -zoom,
            sourceExtent.height >> -zoom);
      } else {
        return new Rectangle(
            sourceExtent.x << zoom,
            sourceExtent.y << zoom,
            sourceExtent.width << zoom,
            sourceExtent.height << zoom);
      }
    } else {
      return null;
    }
  }

  @Override
  public void addTileListener(TileListener tileListener) {
    if (active && listeners.isEmpty()) {
      ((Dimension) tileProvider).addDimensionListener(this);
      for (Tile tile : ((Dimension) tileProvider).getTiles()) {
        tile.addListener(this);
      }
    }
    if (!listeners.contains(tileListener)) {
      listeners.add(tileListener);
    }
  }

  @Override
  public void removeTileListener(TileListener tileListener) {
    listeners.remove(tileListener);
    if (active && listeners.isEmpty()) {
      for (Tile tile : ((Dimension) tileProvider).getTiles()) {
        tile.removeListener(this);
      }
      ((Dimension) tileProvider).removeDimensionListener(this);
    }
  }

  @Override
  public boolean isZoomSupported() {
    return true;
  }

  @Override
  public int getZoom() {
    return zoom;
  }

  @Override
  public void setZoom(int zoom) {
    if (zoom != this.zoom) {
      if (zoom > 0) {
        throw new UnsupportedOperationException("Zooming in not supported");
      }
      this.zoom = zoom;
      tileRendererRef = createNewTileRendererRef();
      if (surroundingTileProvider != null) {
        surroundingTileProvider.setZoom(zoom);
      }
    }
  }

  // Dimension.Listener

  @Override
  public void tilesAdded(Dimension dimension, Set<Tile> tiles) {
    for (Tile tile : tiles) {
      tile.addListener(this);
    }
    fireTilesChangedIncludeBorder(tiles);
  }

  @Override
  public void tilesRemoved(Dimension dimension, Set<Tile> tiles) {
    for (Tile tile : tiles) {
      tile.removeListener(this);
    }
    fireTilesChangedIncludeBorder(tiles);
  }

  // Tile.Listener

  @Override
  public void heightMapChanged(Tile tile) {
    fireTileChanged(tile);
  }

  @Override
  public void terrainChanged(Tile tile) {
    fireTileChanged(tile);
  }

  @Override
  public void waterLevelChanged(Tile tile) {
    fireTileChanged(tile);
  }

  @Override
  public void layerDataChanged(Tile tile, Set<Layer> changedLayers) {
    fireTileChanged(tile);
  }

  @Override
  public void allBitLayerDataChanged(Tile tile) {
    fireTileChanged(tile);
  }

  @Override
  public void allNonBitlayerDataChanged(Tile tile) {
    fireTileChanged(tile);
  }

  @Override
  public void seedsChanged(Tile tile) {
    fireTileChanged(tile);
  }

  private TileType getUnzoomedTileType(int x, int y) {
    if (tileProvider.isTilePresent(x, y)) {
      return TileType.WORLD;
    } else if (showBorder && (tileProvider instanceof Dimension)) {
      Dimension dimension = (Dimension) tileProvider;
      if (dimension.isBorderTile(x, y)) {
        return TileType.BORDER;
      } else if (dimension.isBedrockWall()
          && ((dimension.getBorder() != null)
              ? (dimension.isBorderTile(x - 1, y)
                  || dimension.isBorderTile(x, y - 1)
                  || dimension.isBorderTile(x + 1, y)
                  || dimension.isBorderTile(x, y + 1))
              : (tileProvider.isTilePresent(x - 1, y)
                  || tileProvider.isTilePresent(x, y - 1)
                  || tileProvider.isTilePresent(x + 1, y)
                  || tileProvider.isTilePresent(x, y + 1)))) {
        return TileType.WALL;
      }
    }
    return TileType.SURROUNDS;
  }

  private void paintUnzoomedTile(
      final Image tileImage, final int x, final int y, final int dx, final int dy) {
    TileType tileType = getUnzoomedTileType(x, y);
    switch (tileType) {
      case WORLD:
        Tile tile = tileProvider.getTile(x, y);
        if (tile.hasLayer(NotPresent.INSTANCE) && (surroundingTileProvider != null)) {
          surroundingTileProvider.paintTile(tileImage, x, y, dx, dy);
        }
        TileRenderer tileRenderer = tileRendererRef.get();
        tileRenderer.setTile(tile);
        tileRenderer.renderTile(tileImage, dx, dy);
        break;
      case BORDER:
        int colour;
        switch (((Dimension) tileProvider).getBorder()) {
          case WATER:
            colour = colourScheme.getColour(BLK_WATER);
            break;
          case LAVA:
            colour = colourScheme.getColour(BLK_LAVA);
            break;
          case VOID:
            colour = VoidRenderer.getColour();
            break;
          default:
            throw new InternalError();
        }
        Graphics2D g2 = (Graphics2D) tileImage.getGraphics();
        try {
          g2.setColor(new Color(colour));
          g2.fillRect(dx, dy, TILE_SIZE, TILE_SIZE);

          // Draw border lines
          g2.setColor(Color.BLACK);
          g2.setStroke(
              new BasicStroke(
                  2,
                  BasicStroke.CAP_BUTT,
                  BasicStroke.JOIN_BEVEL,
                  0.0f,
                  new float[] {4.0f, 4.0f},
                  0.0f));
          if (tileProvider.isTilePresent(x, y - 1)) {
            g2.drawLine(dx + 1, dy + 1, dx + TILE_SIZE - 1, dy + 1);
          }
          if (tileProvider.isTilePresent(x + 1, y)) {
            g2.drawLine(dx + TILE_SIZE - 1, dy + 1, dx + TILE_SIZE - 1, dy + TILE_SIZE - 1);
          }
          if (tileProvider.isTilePresent(x, y + 1)) {
            g2.drawLine(dx + 1, dy + TILE_SIZE - 1, dx + TILE_SIZE - 1, dy + TILE_SIZE - 1);
          }
          if (tileProvider.isTilePresent(x - 1, y)) {
            g2.drawLine(dx + 1, dy + 1, dx + 1, dy + TILE_SIZE - 1);
          }
        } finally {
          g2.dispose();
        }
        break;
      case WALL:
        if (surroundingTileProvider != null) {
          surroundingTileProvider.paintTile(tileImage, x, y, dx, dy);
        }
        g2 = (Graphics2D) tileImage.getGraphics();
        try {
          if (surroundingTileProvider == null) {
            // A surrounding tile provider would have completely
            // filled the image, but since there isn't one we have
            // to make sure of that ourselves
            g2.setColor(new Color(VoidRenderer.getColour()));
            g2.fillRect(dx, dy, TILE_SIZE, TILE_SIZE);
          }
          g2.setColor(new Color(colourScheme.getColour(BLK_BEDROCK)));
          TileType neighbourType = getUnzoomedTileType(x, y - 1);
          if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
            g2.fillRect(dx, dy, TILE_SIZE, 16);
          }
          neighbourType = getUnzoomedTileType(x + 1, y);
          if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
            g2.fillRect(dx + TILE_SIZE - 16, dy, 16, TILE_SIZE);
          }
          neighbourType = getUnzoomedTileType(x, y + 1);
          if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
            g2.fillRect(dx, dy + TILE_SIZE - 16, TILE_SIZE, 16);
          }
          neighbourType = getUnzoomedTileType(x - 1, y);
          if ((neighbourType == TileType.WORLD) || (neighbourType == TileType.BORDER)) {
            g2.fillRect(dx, dy, 16, TILE_SIZE);
          }
        } finally {
          g2.dispose();
        }
        break;
      case SURROUNDS:
        if (surroundingTileProvider != null) {
          surroundingTileProvider.paintTile(tileImage, x, y, dx, dy);
        }
        break;
      default:
        throw new InternalError();
    }
  }

  private void fireTileChanged(Tile tile) {
    Point coords = getTileCoordinates(tile);
    for (TileListener listener : listeners) {
      listener.tileChanged(this, coords.x, coords.y);
    }
  }

  private void fireTilesChangedIncludeBorder(Set<Tile> tiles) {
    if (showBorder
        && (tileProvider instanceof Dimension)
        && (((Dimension) tileProvider).getDim() == DIM_NORMAL)
        && (((Dimension) tileProvider).getBorder() != null)) {
      final Set<Point> coordSet = new HashSet<>();
      for (Tile tile : tiles) {
        final int tileX = tile.getX(),
            tileY = tile.getY(),
            borderSize = ((Dimension) tileProvider).getBorderSize();
        for (int dx = -borderSize; dx <= borderSize; dx++) {
          for (int dy = -borderSize; dy <= borderSize; dy++) {
            coordSet.add(getTileCoordinates(tileX + dx, tileY + dy));
          }
        }
      }
      for (TileListener listener : listeners) {
        listener.tilesChanged(this, coordSet);
      }
    } else {
      Set<Point> coords = tiles.stream().map(this::getTileCoordinates).collect(Collectors.toSet());
      for (TileListener listener : listeners) {
        listener.tilesChanged(this, coords);
      }
    }
  }

  /**
   * Convert the actual tile coordinates to zoom-corrected (tile provider coordinate system)
   * coordinates.
   *
   * @param tile The tile of which to convert the coordinates.
   * @return The coordinates of the tile in the tile provider coordinate system (corrected for
   *     zoom).
   */
  private Point getTileCoordinates(Tile tile) {
    return getTileCoordinates(tile.getX(), tile.getY());
  }

  /**
   * Convert the actual tile coordinates to zoom-corrected (tile provider coordinate system)
   * coordinates.
   *
   * @param tileX The X tile coordinate to convert.
   * @param tileY The Y tile coordinate to convert.
   * @return The coordinates of the tile in the tile provider coordinate system (corrected for
   *     zoom).
   */
  private Point getTileCoordinates(final int tileX, final int tileY) {
    if (zoom == 0) {
      return new Point(tileX, tileY);
    } else if (zoom < 0) {
      return new Point(tileX >> -zoom, tileY >> -zoom);
    } else {
      return new Point(tileX << zoom, tileY << zoom);
    }
  }

  @NotNull
  private ThreadLocal<TileRenderer> createNewTileRendererRef() {
    return new ThreadLocal<TileRenderer>() {
      @Override
      protected TileRenderer initialValue() {
        TileRenderer tileRenderer =
            new TileRenderer(tileProvider, colourScheme, biomeScheme, customBiomeManager, zoom);
        synchronized (WPTileProvider.this) {
          if (hiddenLayers != null) {
            tileRenderer.addHiddenLayers(hiddenLayers);
          }
        }
        tileRenderer.setContourLines(contourLines);
        tileRenderer.setContourSeparation(contourSeparation);
        tileRenderer.setLightOrigin(lightOrigin);
        return tileRenderer;
      }
    };
  }

  private final TileProvider tileProvider;
  private final ColourScheme colourScheme;
  private final BiomeScheme biomeScheme;
  private final Set<Layer> hiddenLayers;
  private final boolean contourLines, active, showBorder;
  private final int contourSeparation;
  private final TileRenderer.LightOrigin lightOrigin;
  private final List<TileListener> listeners = new ArrayList<>();
  private final CustomBiomeManager customBiomeManager;
  private final org.pepsoft.util.swing.TileProvider surroundingTileProvider;
  private int zoom = 0;
  private volatile ThreadLocal<TileRenderer> tileRendererRef;

  private static final org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(WPTileProvider.class);

  private enum TileType {
    /** The tile is part of the WorldPainter world. */
    WORLD,
    /** The tile is part of the WorldPainter border. */
    BORDER,
    /** The tile contains no WorldPainter-generated chunks. */
    SURROUNDS,
    /** The tile is outside the WorldPainter world and border but does contain part of a wall. */
    WALL
  }
}
Exemplo n.º 25
0
/**
 * A georeferencing "gridded" VariableEnhanced, that has a GridCoordSys. In VisAD data model, it is
 * a sampled Field. The dimension are put into canonical order: (rt, e, t, z, y, x).
 *
 * <p>
 *
 * <p>Implementation note: If the Horizontal axes are 2D, the x and y dimensions are arbitrarily
 * chosen to be gcs.getXHorizAxis().getDimension(1), gcs.getXHorizAxis().getDimension(0)
 * respectively.
 *
 * <p>
 *
 * @author caron
 */
public class GeoGrid implements NamedObject, ucar.nc2.dt.GridDatatype {
  private static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(GeoGrid.class);
  private static final boolean debugArrayShape = false;

  private final GridDataset dataset;
  private final GridCoordSys gcs;
  private final VariableDS vs;
  private int xDimOrgIndex = -1,
      yDimOrgIndex = -1,
      zDimOrgIndex = -1,
      tDimOrgIndex = -1,
      eDimOrgIndex = -1,
      rtDimOrgIndex = -1;
  private int xDimNewIndex = -1,
      yDimNewIndex = -1,
      zDimNewIndex = -1,
      tDimNewIndex = -1,
      eDimNewIndex = -1,
      rtDimNewIndex = -1;
  private final List<Dimension> mydims;

  /**
   * Constructor.
   *
   * @param dataset belongs to this dataset
   * @param dsvar wraps this Variable
   * @param gcs has this grid coordinate system
   */
  public GeoGrid(GridDataset dataset, VariableDS dsvar, GridCoordSys gcs) {
    this.dataset = dataset;
    this.vs = dsvar;
    this.gcs = gcs;

    CoordinateAxis xaxis = gcs.getXHorizAxis();
    if (xaxis instanceof CoordinateAxis1D) {
      xDimOrgIndex = findDimension(gcs.getXHorizAxis().getDimension(0));
      yDimOrgIndex = findDimension(gcs.getYHorizAxis().getDimension(0));

    } else { // 2D case
      yDimOrgIndex = findDimension(gcs.getXHorizAxis().getDimension(0));
      xDimOrgIndex = findDimension(gcs.getXHorizAxis().getDimension(1));
    }

    if (gcs.getVerticalAxis() != null)
      zDimOrgIndex = findDimension(gcs.getVerticalAxis().getDimension(0));
    if (gcs.getTimeAxis() != null) {
      if (gcs.getTimeAxis1D() != null)
        tDimOrgIndex = findDimension(gcs.getTimeAxis1D().getDimension(0));
      else tDimOrgIndex = findDimension(gcs.getTimeAxis().getDimension(1));
    }
    if (gcs.getEnsembleAxis() != null)
      eDimOrgIndex = findDimension(gcs.getEnsembleAxis().getDimension(0));
    if (gcs.getRunTimeAxis() != null)
      rtDimOrgIndex = findDimension(gcs.getRunTimeAxis().getDimension(0));

    // construct canonical dimension list
    int count = 0;
    this.mydims = new ArrayList<Dimension>();
    if ((rtDimOrgIndex >= 0) && (rtDimOrgIndex != tDimOrgIndex)) {
      mydims.add(dsvar.getDimension(rtDimOrgIndex));
      rtDimNewIndex = count++;
    }
    if (eDimOrgIndex >= 0) {
      mydims.add(dsvar.getDimension(eDimOrgIndex));
      eDimNewIndex = count++;
    }
    if (tDimOrgIndex >= 0) {
      mydims.add(dsvar.getDimension(tDimOrgIndex));
      tDimNewIndex = count++;
    }
    if (zDimOrgIndex >= 0) {
      mydims.add(dsvar.getDimension(zDimOrgIndex));
      zDimNewIndex = count++;
    }
    if (yDimOrgIndex >= 0) {
      mydims.add(dsvar.getDimension(yDimOrgIndex));
      yDimNewIndex = count++;
    }
    if (xDimOrgIndex >= 0) {
      mydims.add(dsvar.getDimension(xDimOrgIndex));
      xDimNewIndex = count;
    }
  }

  private int findDimension(Dimension want) {
    java.util.List dims = vs.getDimensions();
    for (int i = 0; i < dims.size(); i++) {
      Dimension d = (Dimension) dims.get(i);
      if (d.equals(want)) return i;
    }
    return -1;
  }

  /**
   * Returns an ArrayList containing the dimensions used by this geoGrid. The dimension are put into
   * canonical order: (rt, e, t, z, y, x). Note that the z and t dimensions are optional. If the
   * Horizontal axes are 2D, the x and y dimensions are arbitrarily chosen to be
   * gcs.getXHorizAxis().getDimension(1), gcs.getXHorizAxis().getDimension(0), respectively.
   *
   * @return List with objects of type Dimension, in canonical order.
   */
  public java.util.List<Dimension> getDimensions() {
    return new ArrayList<Dimension>(mydims);
  }

  /**
   * get the ith dimension
   *
   * @param i : which dimension
   * @return ith Dimension
   */
  public Dimension getDimension(int i) {
    if ((i < 0) || (i >= mydims.size())) return null;
    return mydims.get(i);
  }

  /** get the time Dimension, if it exists */
  public Dimension getTimeDimension() {
    return tDimNewIndex < 0 ? null : getDimension(tDimNewIndex);
  }

  /** get the z Dimension, if it exists */
  public Dimension getZDimension() {
    return zDimNewIndex < 0 ? null : getDimension(zDimNewIndex);
  }

  /** get the y Dimension, if it exists */
  public Dimension getYDimension() {
    return yDimNewIndex < 0 ? null : getDimension(yDimNewIndex);
  }

  /** get the x Dimension, if it exists */
  public Dimension getXDimension() {
    return xDimNewIndex < 0 ? null : getDimension(xDimNewIndex);
  }

  /** get the ensemble Dimension, if it exists */
  public Dimension getEnsembleDimension() {
    return eDimNewIndex < 0 ? null : getDimension(eDimNewIndex);
  }

  /** get the run time Dimension, if it exists */
  public Dimension getRunTimeDimension() {
    return rtDimNewIndex < 0 ? null : getDimension(rtDimNewIndex);
  }

  /** get the time Dimension index in the geogrid (canonical order), or -1 if none */
  public int getTimeDimensionIndex() {
    return tDimNewIndex;
  }

  /** get the z Dimension index in the geogrid (canonical order), or -1 if none */
  public int getZDimensionIndex() {
    return zDimNewIndex;
  }

  /** get the y Dimension index in the geogrid (canonical order) */
  public int getYDimensionIndex() {
    return yDimNewIndex;
  }

  /** get the x Dimension index in the geogrid (canonical order) */
  public int getXDimensionIndex() {
    return xDimNewIndex;
  }

  /** get the ensemble Dimension index in the geogrid (canonical order) */
  public int getEnsembleDimensionIndex() {
    return eDimNewIndex;
  }

  /** get the runtime Dimension index in the geogrid (canonical order) */
  public int getRunTimeDimensionIndex() {
    return rtDimNewIndex;
  }

  /**
   * Convenience function; lookup Attribute by name.
   *
   * @param name the name of the attribute
   * @return the attribute, or null if not found
   */
  public Attribute findAttributeIgnoreCase(String name) {
    return vs.findAttributeIgnoreCase(name);
  }

  /**
   * Convenience function; lookup Attribute value by name. Must be String valued
   *
   * @param attName name of the attribute
   * @param defaultValue if not found, use this as the default
   * @return Attribute string value, or default if not found.
   */
  public String findAttValueIgnoreCase(String attName, String defaultValue) {
    return dataset.getNetcdfDataset().findAttValueIgnoreCase((Variable) vs, attName, defaultValue);
  }

  // implementation of GridDatatype interface

  /** get the rank */
  public int getRank() {
    return mydims.size();
  }

  /** get the shape */
  public int[] getShape() {
    int[] shape = new int[mydims.size()];
    for (int i = 0; i < mydims.size(); i++) {
      Dimension d = mydims.get(i);
      shape[i] = d.getLength();
    }
    return shape;
  }

  /** get the data type */
  public DataType getDataType() {
    return vs.getDataType();
  }

  public List<Attribute> getAttributes() {
    return vs.getAttributes();
  }

  public VariableDS getVariable() {
    return vs;
  }

  public String getFullName() {
    return vs.getFullName();
  }

  public String getName() {
    return vs.getFullName();
  }

  public String getShortName() {
    return vs.getShortName();
  }

  /** get the GridCoordSys for this GeoGrid. */
  public GridCoordSystem getCoordinateSystem() {
    return gcs;
  }

  /** get the Projection. */
  public ProjectionImpl getProjection() {
    return gcs.getProjection();
  }

  /** @return ArrayList of thredds.util.NamedObject, from the GridCoordSys. */
  public List<NamedObject> getLevels() {
    return gcs.getLevels();
  }

  /** @return ArrayList of thredds.util.NamedObject, from the GridCoordSys. */
  public List<NamedObject> getTimes() {
    return gcs.getTimes();
  }

  /** get the standardized description */
  public String getDescription() {
    return vs.getDescription();
  }

  /** get the unit as a string */
  public String getUnitsString() {
    String units = vs.getUnitsString();
    return (units == null) ? "" : units;
  }

  /**
   * @return getUnitsString()
   * @deprecated use getUnitsString()
   */
  public java.lang.String getUnitString() {
    return getUnitsString();
  }

  // public ucar.unidata.geoloc.ProjectionImpl getProjection() { return gcs.getProjection(); }

  /** true if there may be missing data, see VariableDS.hasMissing() */
  public boolean hasMissingData() {
    return vs.hasMissing();
  }

  /** if val is missing data, see VariableDS.isMissingData() */
  public boolean isMissingData(double val) {
    return vs.isMissing(val);
  }

  /**
   * Convert (in place) all values in the given array that are considered as "missing" to Float.NaN,
   * according to isMissingData(val).
   *
   * @param values input array
   * @return input array, with missing values converted to NaNs.
   */
  public float[] setMissingToNaN(float[] values) {
    if (!vs.hasMissing()) return values;
    final int length = values.length;
    for (int i = 0; i < length; i++) {
      double value = values[i];
      if (vs.isMissing(value)) values[i] = Float.NaN;
    }
    return values;
  }

  /**
   * Get the minimum and the maximum data value of the previously read Array, skipping missing
   * values as defined by isMissingData(double val).
   *
   * @param a Array to get min/max values
   * @return both min and max value.
   */
  public MAMath.MinMax getMinMaxSkipMissingData(Array a) {
    if (!hasMissingData()) return MAMath.getMinMax(a);

    IndexIterator iter = a.getIndexIterator();
    double max = -Double.MAX_VALUE;
    double min = Double.MAX_VALUE;
    while (iter.hasNext()) {
      double val = iter.getDoubleNext();
      if (isMissingData(val)) continue;
      if (val > max) max = val;
      if (val < min) min = val;
    }
    return new MAMath.MinMax(min, max);
  }

  /**
   * Reads in the data "volume" at the given time index. If its a product set, put into canonical
   * order (z-y-x). If not a product set, reorder to (z,i,j), where i, j are from the original
   *
   * @param t time index; ignored if no time axis.
   * @return data[z,y,x] or data[y,x] if no z axis.
   */
  public Array readVolumeData(int t) throws java.io.IOException {
    // if (gcs.isProductSet())
    return readDataSlice(t, -1, -1, -1);
    /* else { // 2D XY
      int rank = vs.getRank();
      int[] shape = vs.getShape();
      int [] start = new int[rank];

      CoordinateAxis taxis = gcs.getTimeAxis();
      if (taxis != null) {
        if ((t >= 0) && (t < taxis.getSize()))
          shape[ tDim] = 1;  // fix t
          start[ tDim] = t;
      }

      if (debugArrayShape) {
        System.out.println("getDataVolume shape = ");
        for (int i=0; i<rank; i++)
          System.out.println("   start = "+start[i]+" shape = "+ shape[i]);
      }

      Array dataVolume;
      try {
        dataVolume = vs.read( start, shape);
      } catch (Exception e) {
        System.out.println("Exception: GeoGridImpl.getdataSlice() on dataset "+getName());
        e.printStackTrace();
        throw new java.io.IOException(e.getMessage());
      }

      // no reordering FIX
      return dataVolume.reduce();
    } */
  }

  /**
   * Reads a Y-X "horizontal slice" at the given time and vertical index. If its a product set, put
   * into canonical order (y-x).
   *
   * @param t time index; ignored if no time axis.
   * @param z vertical index; ignored if no z axis.
   * @return data[y,x]
   * @throws java.io.IOException on read error
   */
  public Array readYXData(int t, int z) throws java.io.IOException {
    return readDataSlice(t, z, -1, -1);
  }

  /**
   * Reads a Z-Y "vertical slice" at the given time and x index. If its a product set, put into
   * canonical order (z-y).
   *
   * @param t time index; ignored if no time axis.
   * @param x x index; ignored if no x axis.
   * @return data[z,y]
   * @throws java.io.IOException on read error
   */
  public Array readZYData(int t, int x) throws java.io.IOException {
    return readDataSlice(t, -1, -1, x);
  }

  /**
   * @throws java.io.IOException on read error
   * @deprecated use readDataSlice
   */
  public Array getDataSlice(int t, int z, int y, int x) throws java.io.IOException {
    return readDataSlice(t, z, y, x);
  }

  /**
   * This reads an arbitrary data slice, returning the data in canonical order (t-z-y-x). If any
   * dimension does not exist, ignore it.
   *
   * @param t if < 0, get all of time dim; if valid index, fix slice to that value.
   * @param z if < 0, get all of z dim; if valid index, fix slice to that value.
   * @param y if < 0, get all of y dim; if valid index, fix slice to that value.
   * @param x if < 0, get all of x dim; if valid index, fix slice to that value.
   * @return data[t,z,y,x], eliminating missing or fixed dimension.
   */
  public Array readDataSlice(int t, int z, int y, int x) throws java.io.IOException {
    return readDataSlice(0, 0, t, z, y, x);
  }

  /**
   * This reads an arbitrary data slice, returning the data in canonical order (rt-e-t-z-y-x). If
   * any dimension does not exist, ignore it.
   *
   * @param rt if < 0, get all of runtime dim; if valid index, fix slice to that value.
   * @param e if < 0, get all of ensemble dim; if valid index, fix slice to that value.
   * @param t if < 0, get all of time dim; if valid index, fix slice to that value.
   * @param z if < 0, get all of z dim; if valid index, fix slice to that value.
   * @param y if < 0, get all of y dim; if valid index, fix slice to that value.
   * @param x if < 0, get all of x dim; if valid index, fix slice to that value.
   * @return data[rt,e,t,z,y,x], eliminating missing or fixed dimension.
   */
  public Array readDataSlice(int rt, int e, int t, int z, int y, int x) throws java.io.IOException {

    int rank = vs.getRank();
    int[] start = new int[rank];
    int[] shape = new int[rank];
    for (int i = 0; i < rank; i++) {
      start[i] = 0;
      shape[i] = 1;
    }
    Dimension xdim = getXDimension();
    Dimension ydim = getYDimension();
    Dimension zdim = getZDimension();
    Dimension tdim = getTimeDimension();
    Dimension edim = getEnsembleDimension();
    Dimension rtdim = getRunTimeDimension();

    // construct the shape of the data volume to be read
    if (rtdim != null) {
      if ((rt >= 0) && (rt < rtdim.getLength())) start[rtDimOrgIndex] = rt; // fix rt
      else {
        shape[rtDimOrgIndex] = rtdim.getLength(); // all of rt
      }
    }

    if (edim != null) {
      if ((e >= 0) && (e < edim.getLength())) start[eDimOrgIndex] = e; // fix e
      else {
        shape[eDimOrgIndex] = edim.getLength(); // all of e
      }
    }

    if (tdim != null) {
      if ((t >= 0) && (t < tdim.getLength())) start[tDimOrgIndex] = t; // fix t
      else {
        shape[tDimOrgIndex] = tdim.getLength(); // all of t
      }
    }

    if (zdim != null) {
      if ((z >= 0) && (z < zdim.getLength())) start[zDimOrgIndex] = z; // fix z
      else {
        shape[zDimOrgIndex] = zdim.getLength(); // all of z
      }
    }

    if (ydim != null) {
      if ((y >= 0) && (y < ydim.getLength())) start[yDimOrgIndex] = y; // fix y
      else {
        shape[yDimOrgIndex] = ydim.getLength(); // all of y
      }
    }

    if (xdim != null) {
      if ((x >= 0) && (x < xdim.getLength())) // all of x
      start[xDimOrgIndex] = x; // fix x
      else {
        shape[xDimOrgIndex] = xdim.getLength(); // all of x
      }
    }

    if (debugArrayShape) {
      System.out.println("read shape from org variable = ");
      for (int i = 0; i < rank; i++)
        System.out.println(
            "   start = "
                + start[i]
                + " shape = "
                + shape[i]
                + " name = "
                + vs.getDimension(i).getName());
    }

    // read it
    Array dataVolume;
    try {
      dataVolume = vs.read(start, shape);
    } catch (Exception ex) {
      log.error(
          "GeoGrid.getdataSlice() on dataset " + getFullName() + " " + dataset.getLocation(), ex);
      throw new java.io.IOException(ex.getMessage());
    }

    // LOOK: the real problem is the lack of named dimensions in the Array object
    // figure out correct permutation for canonical ordering for permute
    List<Dimension> oldDims = new ArrayList<Dimension>(vs.getDimensions());
    int[] permuteIndex = new int[dataVolume.getRank()];
    int count = 0;
    if (oldDims.contains(rtdim)) permuteIndex[count++] = oldDims.indexOf(rtdim);
    if (oldDims.contains(edim)) permuteIndex[count++] = oldDims.indexOf(edim);
    if (oldDims.contains(tdim)) permuteIndex[count++] = oldDims.indexOf(tdim);
    if (oldDims.contains(zdim)) permuteIndex[count++] = oldDims.indexOf(zdim);
    if (oldDims.contains(ydim)) permuteIndex[count++] = oldDims.indexOf(ydim);
    if (oldDims.contains(xdim)) permuteIndex[count] = oldDims.indexOf(xdim);

    if (debugArrayShape) {
      System.out.println("oldDims = ");
      for (Dimension oldDim : oldDims) System.out.println("   oldDim = " + oldDim.getName());
      System.out.println("permute dims = ");
      for (int aPermuteIndex : permuteIndex)
        System.out.println("   oldDim index = " + aPermuteIndex);
    }

    // check to see if we need to permute
    boolean needPermute = false;
    for (int i = 0; i < permuteIndex.length; i++) {
      if (i != permuteIndex[i]) needPermute = true;
    }

    // permute to the order rt,e,t,z,y,x
    if (needPermute) dataVolume = dataVolume.permute(permuteIndex);

    // eliminate fixed dimensions, but not all dimensions of length 1.
    count = 0;
    if (rtdim != null) {
      if (rt >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (edim != null) {
      if (e >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (tdim != null) {
      if (t >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (zdim != null) {
      if (z >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (ydim != null) {
      if (y >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (xdim != null) {
      if (x >= 0) dataVolume = dataVolume.reduce(count);
    }

    return dataVolume;
  }

  //////////////////////////////////

  /**
   * Create a new GeoGrid that is a logical subset of this GeoGrid.
   *
   * @param t_range subset the time dimension, or null if you want all of it
   * @param z_range subset the vertical dimension, or null if you want all of it
   * @param bbox a lat/lon bounding box, or null if you want all x,y
   * @param z_stride use only if z_range is null, then take all z with this stride (1 means all)
   * @param y_stride use this stride on the y coordinate (1 means all)
   * @param x_stride use this stride on the x coordinate (1 means all)
   * @return subsetted GeoGrid
   * @throws InvalidRangeException if bbox does not intersect GeoGrid
   */
  public GeoGrid subset(
      Range t_range, Range z_range, LatLonRect bbox, int z_stride, int y_stride, int x_stride)
      throws InvalidRangeException {

    if ((z_range == null) && (z_stride > 1)) {
      Dimension zdim = getZDimension();
      if (zdim != null) z_range = new Range(0, zdim.getLength() - 1, z_stride);
    }

    Range y_range = null, x_range = null;
    if (bbox != null) {
      List yx_ranges = gcs.getRangesFromLatLonRect(bbox);
      y_range = (Range) yx_ranges.get(0);
      x_range = (Range) yx_ranges.get(1);
    }

    if (y_stride > 1) {
      if (y_range == null) {
        Dimension ydim = getYDimension();
        y_range = new Range(0, ydim.getLength() - 1, y_stride);
      } else {
        y_range = new Range(y_range.first(), y_range.last(), y_stride);
      }
    }

    if (x_stride > 1) {
      if (x_range == null) {
        Dimension xdim = getXDimension();
        x_range = new Range(0, xdim.getLength() - 1, x_stride);
      } else {
        x_range = new Range(x_range.first(), x_range.last(), x_stride);
      }
    }

    return subset(t_range, z_range, y_range, x_range);
  }

  public GridDatatype makeSubset(
      Range t_range, Range z_range, LatLonRect bbox, int z_stride, int y_stride, int x_stride)
      throws InvalidRangeException {
    return subset(t_range, z_range, bbox, z_stride, y_stride, x_stride);
  }

  /**
   * Create a new GeoGrid that is a logical subset of this GeoGrid.
   *
   * @param t_range subset the time dimension, or null if you want all of it
   * @param z_range subset the vertical dimension, or null if you want all of it
   * @param y_range subset the y dimension, or null if you want all of it
   * @param x_range subset the x dimension, or null if you want all of it
   * @return subsetted GeoGrid
   * @throws InvalidRangeException if any of the ranges are invalid
   */
  public GeoGrid subset(Range t_range, Range z_range, Range y_range, Range x_range)
      throws InvalidRangeException {
    return (GeoGrid) makeSubset(null, null, t_range, z_range, y_range, x_range);
  }

  public GridDatatype makeSubset(
      Range rt_range, Range e_range, Range t_range, Range z_range, Range y_range, Range x_range)
      throws InvalidRangeException {
    // get the ranges list
    int rank = getRank();
    Range[] ranges = new Range[rank];
    if (null != getXDimension()) ranges[xDimOrgIndex] = x_range;
    if (null != getYDimension()) ranges[yDimOrgIndex] = y_range;
    if (null != getZDimension()) ranges[zDimOrgIndex] = z_range;
    if (null != getTimeDimension()) ranges[tDimOrgIndex] = t_range;
    if (null != getRunTimeDimension()) ranges[rtDimOrgIndex] = rt_range;
    if (null != getEnsembleDimension()) ranges[eDimOrgIndex] = e_range;
    List<Range> rangesList = Arrays.asList(ranges);

    // subset the variable
    VariableDS v_section = (VariableDS) vs.section(new Section(rangesList));
    List<Dimension> dims = v_section.getDimensions();
    for (Dimension dim : dims) {
      dim.setShared(true); // make them shared (section will make them unshared)
    }

    // subset the axes in the GridCoordSys
    GridCoordSys gcs_section =
        new GridCoordSys(gcs, rt_range, e_range, t_range, z_range, y_range, x_range);

    // now we can make the geogrid
    return new GeoGrid(dataset, v_section, gcs_section);
  }

  /////////////////////////////////////////////////////////////////////////////////
  /** Instances which have same name and coordinate system are equal. */
  public boolean equals(Object oo) {
    if (this == oo) return true;
    if (!(oo instanceof GeoGrid)) return false;

    GeoGrid d = (GeoGrid) oo;
    if (!getFullName().equals(d.getFullName())) return false;
    if (!getCoordinateSystem().equals(d.getCoordinateSystem())) return false;

    return true;
  }

  /** Override Object.hashCode() to be consistent with equals. */
  public int hashCode() {
    if (hashCode == 0) {
      int result = 17;
      // result = 37*result + dataset.getName().hashCode();
      result = 37 * result + getFullName().hashCode();
      result = 37 * result + getCoordinateSystem().hashCode();
      hashCode = result;
    }
    return hashCode;
  }

  private int hashCode = 0; // Bloch, item 8

  /** string representation */
  public String toString() {
    return getFullName();
  }

  /** nicely formatted information */
  public String getInfo() {
    StringBuilder buf = new StringBuilder(200);
    buf.setLength(0);
    buf.append(getFullName());
    Format.tab(buf, 30, true);
    buf.append(getUnitsString());
    Format.tab(buf, 60, true);
    buf.append(hasMissingData());
    Format.tab(buf, 66, true);
    buf.append(getDescription());
    return buf.toString();
  }

  public int compareTo(GridDatatype g) {
    return getFullName().compareTo(g.getFullName());
  }
}
Exemplo n.º 26
0
/**
 * Jahia specific wrapper around <code>javax.jcr.Session</code> to be able to inject Jahia specific
 * actions and to manage sessions to multiple repository providers in the backend.
 *
 * <p>Jahia services should use this wrapper rather than the original session interface to ensure
 * that we manipulate wrapped nodes and not the ones from the underlying implementation.
 *
 * @author toto
 */
public class JCRSessionWrapper implements Session {
  private static Logger logger = org.slf4j.LoggerFactory.getLogger(JCRSessionWrapper.class);
  public static final String DEREF_SEPARATOR = "@/";

  private JCRSessionFactory sessionFactory;
  private JahiaUser user;
  private Credentials credentials;
  private JCRWorkspaceWrapper workspace;
  private boolean isLive = true;
  private Locale locale;
  private List<String> tokens = new ArrayList<String>();

  private Map<JCRStoreProvider, Session> sessions = new HashMap<JCRStoreProvider, Session>();

  private Map<String, JCRNodeWrapper> sessionCacheByPath = new HashMap<String, JCRNodeWrapper>();
  private Map<String, JCRNodeWrapper> sessionCacheByIdentifier =
      new HashMap<String, JCRNodeWrapper>();
  private Map<String, JCRNodeWrapper> newNodes = new HashMap<String, JCRNodeWrapper>();

  private Map<String, String> nsToPrefix = new HashMap<String, String>();
  private Map<String, String> prefixToNs = new HashMap<String, String>();

  private Map<String, String> uuidMapping = new HashMap<String, String>();
  private Map<String, String> pathMapping = new LinkedHashMap<String, String>();

  private boolean isSystem;
  private Date versionDate;

  private Locale fallbackLocale;
  private String versionLabel;

  private static AtomicLong activeSessions = new AtomicLong(0L);

  public JCRSessionWrapper(
      JahiaUser user,
      Credentials credentials,
      boolean isSystem,
      String workspace,
      Locale locale,
      JCRSessionFactory sessionFactory,
      Locale fallbackLocale) {
    this.user = user;
    this.credentials = credentials;
    this.isSystem = isSystem;
    this.versionDate = null;
    this.versionLabel = null;
    if (workspace == null) {
      this.workspace = new JCRWorkspaceWrapper("default", this, sessionFactory);
    } else {
      this.workspace = new JCRWorkspaceWrapper(workspace, this, sessionFactory);
    }
    this.locale = locale;
    this.fallbackLocale = fallbackLocale;
    this.sessionFactory = sessionFactory;
    activeSessions.incrementAndGet();
  }

  public JCRNodeWrapper getRootNode() throws RepositoryException {
    JCRStoreProvider provider = sessionFactory.getProvider("/");
    return provider.getNodeWrapper(getProviderSession(provider).getRootNode(), "/", null, this);
  }

  public Repository getRepository() {
    return sessionFactory;
  }

  public String getUserID() {
    return ((SimpleCredentials) credentials).getUserID();
  }

  public boolean isSystem() {
    return isSystem;
  }

  public Object getAttribute(String s) {
    return null; // To change body of implemented methods use File | Settings | File Templates.
  }

  public String[] getAttributeNames() {
    return new String
        [0]; // To change body of implemented methods use File | Settings | File Templates.
  }

  public JCRWorkspaceWrapper getWorkspace() {
    return workspace;
  }

  public Locale getLocale() {
    return locale;
  }

  //    public void setInterceptorsEnabled(boolean interceptorsEnabled) {
  //        this.interceptorsEnabled = interceptorsEnabled;
  //    }

  public Session impersonate(Credentials credentials) throws LoginException, RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  public JCRNodeWrapper getNodeByUUID(String uuid)
      throws ItemNotFoundException, RepositoryException {
    return getNodeByUUID(uuid, true);
  }

  public JCRNodeWrapper getNodeByUUID(final String uuid, final boolean checkVersion)
      throws ItemNotFoundException, RepositoryException {

    if (sessionCacheByIdentifier.containsKey(uuid)) {
      return sessionCacheByIdentifier.get(uuid);
    }
    RepositoryException originalEx = null;
    for (JCRStoreProvider provider : sessionFactory.getProviderList()) {
      if (!provider.isInitialized()) {
        logger.debug(
            "Provider "
                + provider.getKey()
                + " / "
                + provider.getClass().getName()
                + " is not yet initialized, skipping...");
        continue;
      }
      if (provider instanceof JackrabbitStoreProvider && JCRContentUtils.isNotJcrUuid(uuid)) {
        // not a valid UUID, probably a VFS node
        continue;
      }
      try {
        Session session = getProviderSession(provider);
        if (session instanceof JahiaSessionImpl
            && getUser() != null
            && sessionFactory.getCurrentAliasedUser() != null
            && sessionFactory.getCurrentAliasedUser().equals(getUser())) {
          ((JahiaSessionImpl) session).toggleThisSessionAsAliased();
        }
        Node n = session.getNodeByIdentifier(uuid);
        JCRNodeWrapper wrapper = provider.getNodeWrapper(n, this);
        if (getUser() != null
            && sessionFactory.getCurrentAliasedUser() != null
            && !sessionFactory.getCurrentAliasedUser().equals(getUser())) {
          JCRTemplate.getInstance()
              .doExecuteWithUserSession(
                  sessionFactory.getCurrentAliasedUser().getUsername(),
                  session.getWorkspace().getName(),
                  getLocale(),
                  new JCRCallback<Object>() {
                    public Object doInJCR(JCRSessionWrapper session) throws RepositoryException {
                      return session.getNodeByUUID(uuid, checkVersion);
                    }
                  });
        }
        if (checkVersion && (versionDate != null || versionLabel != null)) {
          wrapper = getFrozenVersionAsRegular(n, provider);
        }
        sessionCacheByIdentifier.put(uuid, wrapper);
        sessionCacheByPath.put(wrapper.getPath(), wrapper);

        return wrapper;
      } catch (ItemNotFoundException ee) {
        // All good
        if (originalEx == null) {
          originalEx = ee;
        }
      } catch (UnsupportedRepositoryOperationException uso) {
        logger.debug(
            "getNodeByUUID unsupported by : "
                + provider.getKey()
                + " / "
                + provider.getClass().getName());
        if (originalEx == null) {
          originalEx = uso;
        }
      } catch (RepositoryException ex) {
        if (originalEx == null) {
          originalEx = ex;
        }
        logger.warn(
            "repository exception : "
                + provider.getKey()
                + " / "
                + provider.getClass().getName()
                + " : "
                + ex.getMessage());
      }
    }
    if (originalEx != null) {
      if (originalEx instanceof ItemNotFoundException) {
        throw originalEx;
      } else {
        throw new ItemNotFoundException(uuid, originalEx);
      }
    }

    throw new ItemNotFoundException(uuid);
  }

  public JCRNodeWrapper getNodeByUUID(String providerKey, String uuid)
      throws ItemNotFoundException, RepositoryException {
    JCRStoreProvider provider = sessionFactory.getProviders().get(providerKey);
    if (provider == null) {
      throw new ItemNotFoundException(uuid);
    }
    Session session = getProviderSession(provider);
    Node n = session.getNodeByIdentifier(uuid);
    return provider.getNodeWrapper(n, this);
  }

  public JCRItemWrapper getItem(String path) throws PathNotFoundException, RepositoryException {
    return getItem(path, true);
  }

  public JCRItemWrapper getItem(String path, final boolean checkVersion)
      throws PathNotFoundException, RepositoryException {
    if (sessionCacheByPath.containsKey(path)) {
      return sessionCacheByPath.get(path);
    }
    if (path.contains(DEREF_SEPARATOR)) {
      JCRNodeWrapper parent =
          (JCRNodeWrapper)
              getItem(StringUtils.substringBeforeLast(path, DEREF_SEPARATOR), checkVersion);
      return dereference(parent, StringUtils.substringAfterLast(path, DEREF_SEPARATOR));
    }
    Map<String, JCRStoreProvider> dynamicMountPoints = sessionFactory.getDynamicMountPoints();
    for (Map.Entry<String, JCRStoreProvider> mp : dynamicMountPoints.entrySet()) {
      if (path.startsWith(mp.getKey() + "/")) {
        String localPath = path.substring(mp.getKey().length());
        JCRStoreProvider provider = mp.getValue();
        Item item = getProviderSession(provider).getItem(provider.getRelativeRoot() + localPath);
        if (item.isNode()) {
          return provider.getNodeWrapper((Node) item, localPath, null, this);
        } else {
          return provider.getPropertyWrapper((Property) item, this);
        }
      }
    }
    Map<String, JCRStoreProvider> mountPoints = sessionFactory.getMountPoints();
    for (Map.Entry<String, JCRStoreProvider> mp : mountPoints.entrySet()) {
      String key = mp.getKey();
      if (key.equals("/") || path.equals(key) || path.startsWith(key + "/")) {
        String localPath = path;
        if (!key.equals("/")) {
          localPath = localPath.substring(key.length());
        }
        JCRStoreProvider provider = mp.getValue();
        if (localPath.equals("")) {
          localPath = "/";
        }
        //                Item item = getProviderSession(provider).getItem(localPath);
        Session session = getProviderSession(provider);
        if (session instanceof JahiaSessionImpl
            && getUser() != null
            && sessionFactory.getCurrentAliasedUser() != null
            && sessionFactory.getCurrentAliasedUser().equals(getUser())) {
          ((JahiaSessionImpl) session).toggleThisSessionAsAliased();
        }
        Item item = session.getItem(provider.getRelativeRoot() + localPath);
        if (item.isNode()) {
          final Node node = (Node) item;
          JCRNodeWrapper wrapper = provider.getNodeWrapper(node, localPath, null, this);
          if (getUser() != null
              && sessionFactory.getCurrentAliasedUser() != null
              && !sessionFactory.getCurrentAliasedUser().equals(getUser())) {
            final JCRNodeWrapper finalWrapper = wrapper;
            JCRTemplate.getInstance()
                .doExecuteWithUserSession(
                    sessionFactory.getCurrentAliasedUser().getUsername(),
                    getWorkspace().getName(),
                    getLocale(),
                    new JCRCallback<Object>() {
                      public Object doInJCR(JCRSessionWrapper session) throws RepositoryException {
                        return session.getNodeByUUID(finalWrapper.getIdentifier(), checkVersion);
                      }
                    });
          }
          if (checkVersion
              && (versionDate != null || versionLabel != null)
              && node.isNodeType("mix:versionable")) {
            wrapper = getFrozenVersionAsRegular(node, provider);
          }
          sessionCacheByPath.put(path, wrapper);
          sessionCacheByIdentifier.put(wrapper.getIdentifier(), wrapper);

          return wrapper;
        } else {
          return provider.getPropertyWrapper((Property) item, this);
        }
      }
    }
    throw new PathNotFoundException(path);
  }

  private JCRNodeWrapper dereference(JCRNodeWrapper parent, String refPath)
      throws RepositoryException {
    JCRStoreProvider provider = parent.getProvider();
    JCRNodeWrapper wrapper;
    Node referencedNode = parent.getRealNode().getProperty("j:node").getNode();
    String fullPath = parent.getPath() + DEREF_SEPARATOR + refPath;
    if (parent.getPath().startsWith(referencedNode.getPath())) {
      throw new PathNotFoundException(fullPath);
    }
    String refRootName = StringUtils.substringBefore(refPath, "/");
    if (!referencedNode.getName().equals(refRootName)) {
      throw new PathNotFoundException(fullPath);
    }
    refPath = StringUtils.substringAfter(refPath, "/");
    if (refPath.equals("")) {
      wrapper = provider.getNodeWrapper(referencedNode, fullPath, parent, this);
    } else {
      Node node = referencedNode.getNode(refPath);
      wrapper = provider.getNodeWrapper(node, fullPath, null, this);
    }
    sessionCacheByPath.put(fullPath, wrapper);
    return wrapper;
  }

  public JCRNodeWrapper getNode(String path) throws PathNotFoundException, RepositoryException {
    return getNode(path, true);
  }

  public JCRNodeWrapper getNode(String path, boolean checkVersion)
      throws PathNotFoundException, RepositoryException {
    JCRItemWrapper item = getItem(path, checkVersion);
    if (item.isNode()) {
      return (JCRNodeWrapper) item;
    } else {
      throw new PathNotFoundException();
    }
  }

  public boolean itemExists(String path) throws RepositoryException {
    try {
      getItem(path);
      return true;
    } catch (RepositoryException e) {
      return false;
    }
  }

  public void move(String source, String dest)
      throws ItemExistsException, PathNotFoundException, VersionException,
          ConstraintViolationException, LockException, RepositoryException {
    getWorkspace().move(source, dest, true);
    if (sessionCacheByPath.containsKey(source)) {
      JCRNodeWrapper n = sessionCacheByPath.get(source);
      if (n instanceof JCRNodeDecorator) {
        n = ((JCRNodeDecorator) n).getDecoratedNode();
      }
      ((JCRNodeWrapperImpl) n).localPath = dest;
      ((JCRNodeWrapperImpl) n).localPathInProvider = dest;
    }
    flushCaches();
  }

  public void save()
      throws AccessDeniedException, ItemExistsException, ConstraintViolationException,
          InvalidItemStateException, VersionException, LockException, NoSuchNodeTypeException,
          RepositoryException {
    save(JCRObservationManager.SESSION_SAVE);
  }

  void registerNewNode(JCRNodeWrapper node) {
    newNodes.put(node.getPath(), node);
  }

  void unregisterNewNode(JCRNodeWrapper node) {
    if (!newNodes.isEmpty()) {
      newNodes.remove(node.getPath());
      try {
        if (node.hasNodes()) {
          NodeIterator it = node.getNodes();
          while (it.hasNext()) {
            unregisterNewNode((JCRNodeWrapper) it.next());
          }
        }
      } catch (RepositoryException e) {
        logger.warn("Error unregistering new nodes", e);
      }
    }
  }

  public void save(final int operationType)
      throws AccessDeniedException, ItemExistsException, ConstraintViolationException,
          InvalidItemStateException, VersionException, LockException, NoSuchNodeTypeException,
          RepositoryException {
    if (!isSystem() && getLocale() != null) {
      for (JCRNodeWrapper node : newNodes.values()) {
        for (String s : node.getNodeTypes()) {
          ExtendedPropertyDefinition[] propDefs =
              NodeTypeRegistry.getInstance().getNodeType(s).getPropertyDefinitions();
          for (ExtendedPropertyDefinition propDef : propDefs) {
            if (propDef.isMandatory()
                && propDef.getRequiredType() != PropertyType.WEAKREFERENCE
                && propDef.getRequiredType() != PropertyType.REFERENCE
                && !propDef.isProtected()
                && !node.hasProperty(propDef.getName())) {
              throw new ConstraintViolationException("Mandatory field : " + propDef.getName());
            }
          }
        }
      }
    }
    newNodes.clear();

    JCRObservationManager.doWorkspaceWriteCall(
        this,
        operationType,
        new JCRCallback<Object>() {
          public Object doInJCR(JCRSessionWrapper thisSession) throws RepositoryException {
            for (Session session : sessions.values()) {
              session.save();
            }
            return null;
          }
        });
  }

  public void refresh(boolean b) throws RepositoryException {
    for (Session session : sessions.values()) {
      session.refresh(b);
    }
  }

  public boolean hasPendingChanges() throws RepositoryException {
    for (Session session : sessions.values()) {
      if (session.hasPendingChanges()) {
        return true;
      }
    }
    return false;
  }

  public ValueFactory getValueFactory() {
    return ValueFactoryImpl.getInstance();
  }

  /**
   * Normally determines whether this <code>Session</code> has permission to perform the specified
   * actions at the specified <code>absPath</code>. This method is not supported.
   *
   * @param absPath an absolute path.
   * @param actions a comma separated list of action strings.
   * @throws UnsupportedRepositoryOperationException as long as Jahia doesn't support it
   */
  public void checkPermission(String absPath, String actions)
      throws AccessControlException, RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  public ContentHandler getImportContentHandler(String s, int i)
      throws PathNotFoundException, ConstraintViolationException, VersionException, LockException,
          RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  public void importXML(String path, InputStream inputStream, int uuidBehavior)
      throws IOException, PathNotFoundException, ItemExistsException, ConstraintViolationException,
          VersionException, InvalidSerializedDataException, LockException, RepositoryException {
    importXML(path, inputStream, uuidBehavior, DocumentViewImportHandler.ROOT_BEHAVIOUR_REPLACE);
  }

  public void importXML(String path, InputStream inputStream, int uuidBehavior, int rootBehavior)
      throws IOException, InvalidSerializedDataException, RepositoryException {
    importXML(path, inputStream, uuidBehavior, rootBehavior, null);
  }

  public void importXML(
      String path,
      InputStream inputStream,
      int uuidBehavior,
      int rootBehavior,
      Map<String, String> replacements)
      throws IOException, InvalidSerializedDataException, RepositoryException {
    JCRNodeWrapper node = getNode(path);
    try {
      if (!node.isCheckedOut()) {
        checkout(node);
      }
    } catch (UnsupportedRepositoryOperationException ex) {
      // versioning not supported
    }

    DocumentViewImportHandler documentViewImportHandler = new DocumentViewImportHandler(this, path);
    documentViewImportHandler.setRootBehavior(rootBehavior);
    documentViewImportHandler.setUuidBehavior(uuidBehavior);
    documentViewImportHandler.setReplacements(replacements);
    try {
      SAXParserFactory factory;

      factory = new SAXParserFactoryImpl();

      factory.setNamespaceAware(true);
      factory.setValidating(false);
      factory.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false);

      SAXParser parser = factory.newSAXParser();

      parser.parse(inputStream, documentViewImportHandler);
    } catch (SAXParseException e) {
      logger.error("Cannot import - File is not a valid XML", e);
    } catch (Exception e) {
      logger.error("Cannot import", e);
    }
  }

  /**
   * Applies the namespace prefix to the appropriate sessions, including the underlying provider
   * sessions.
   *
   * @param prefix
   * @param uri
   * @throws NamespaceException
   * @throws RepositoryException
   */
  public void setNamespacePrefix(String prefix, String uri)
      throws NamespaceException, RepositoryException {
    nsToPrefix.put(uri, prefix);
    prefixToNs.put(prefix, uri);
    for (Session s : sessions.values()) {
      s.setNamespacePrefix(prefix, uri);
      try {
        NamespaceRegistry nsReg = s.getWorkspace().getNamespaceRegistry();
        if (nsReg != null) {
          nsReg.registerNamespace(prefix, uri);
        }
      } catch (RepositoryException e) {
        if (logger.isDebugEnabled()) {
          logger.debug(
              "Prefix/uri could not be registered in workspace's registry- " + prefix + "/" + uri,
              e);
        }
      }
    }
  }

  public String[] getNamespacePrefixes() throws RepositoryException {
    Set<String> wsPrefixes =
        new HashSet<String>(Arrays.asList(getWorkspace().getNamespaceRegistry().getPrefixes()));
    wsPrefixes.addAll(prefixToNs.keySet());
    return wsPrefixes.toArray(new String[wsPrefixes.size()]);
  }

  public String getNamespaceURI(String prefix) throws NamespaceException, RepositoryException {
    if (prefixToNs.containsKey(prefix)) {
      return prefixToNs.get(prefix);
    }
    return getWorkspace().getNamespaceRegistry().getURI(prefix);
  }

  public String getNamespacePrefix(String uri) throws NamespaceException, RepositoryException {
    if (nsToPrefix.containsKey(uri)) {
      return nsToPrefix.get(uri);
    }
    return getWorkspace().getNamespaceRegistry().getPrefix(uri);
  }

  public void logout() {
    for (Session session : sessions.values()) {
      session.logout();
    }
    if (credentials instanceof SimpleCredentials) {
      SimpleCredentials simpleCredentials = (SimpleCredentials) credentials;
      JahiaLoginModule.removeToken(
          simpleCredentials.getUserID(), new String(simpleCredentials.getPassword()));
    }
    isLive = false;
    activeSessions.decrementAndGet();
  }

  public boolean isLive() {
    return isLive;
  }

  /**
   * Adds the specified lock token to the wrapped sessions. Holding a lock token makes the <code>
   * Session</code> the owner of the lock specified by that particular lock token.
   *
   * @param token a lock token (a string).
   * @deprecated As of JCR 2.0, {@link LockManager#addLockToken(String)} should be used instead.
   */
  public void addLockToken(String token) {
    tokens.add(token);
    for (Session session : sessions.values()) {
      session.addLockToken(token);
    }
  }

  public String[] getLockTokens() {
    List<String> allTokens = new ArrayList<String>(tokens);
    for (Session session : sessions.values()) {
      String[] tokens = session.getLockTokens();
      for (String token : tokens) {
        if (!allTokens.contains(token)) {
          allTokens.add(token);
        }
      }
    }
    return allTokens.toArray(new String[allTokens.size()]);
  }

  public void removeLockToken(String token) {
    tokens.remove(token);
    for (Session session : sessions.values()) {
      session.removeLockToken(token);
    }
  }

  /**
   * Get sessions from all providers used in this wrapper.
   *
   * @return a <code>Collection</code> of <code>JCRSessionWrapper</code> objects
   */
  public Collection<Session> getAllSessions() {
    return sessions.values();
  }

  public Session getProviderSession(JCRStoreProvider provider) throws RepositoryException {
    if (sessions.get(provider) == null) {
      Session s = null;

      if (credentials instanceof SimpleCredentials) {
        SimpleCredentials simpleCredentials = (SimpleCredentials) credentials;
        JahiaLoginModule.Token t =
            JahiaLoginModule.getToken(
                simpleCredentials.getUserID(), new String(simpleCredentials.getPassword()));

        s = provider.getSession(credentials, workspace.getName());

        credentials =
            JahiaLoginModule.getCredentials(
                simpleCredentials.getUserID(), t != null ? t.deniedPath : null);
      } else {
        s = provider.getSession(credentials, workspace.getName());
      }

      sessions.put(provider, s);
      for (String token : tokens) {
        s.addLockToken(token);
      }

      NamespaceRegistry namespaceRegistryWrapper = getWorkspace().getNamespaceRegistry();
      NamespaceRegistry providerNamespaceRegistry = s.getWorkspace().getNamespaceRegistry();

      if (providerNamespaceRegistry != null) {
        for (String prefix : namespaceRegistryWrapper.getPrefixes()) {
          try {
            providerNamespaceRegistry.getURI(prefix);
          } catch (NamespaceException ne) {
            providerNamespaceRegistry.registerNamespace(
                prefix, namespaceRegistryWrapper.getURI(prefix));
          }
        }
      }

      for (String prefix : prefixToNs.keySet()) {
        s.setNamespacePrefix(prefix, prefixToNs.get(prefix));
      }
    }
    return sessions.get(provider);
  }

  public JahiaUser getUser() {
    return user;
  }

  public JahiaUser getAliasedUser() {
    return sessionFactory.getCurrentAliasedUser();
  }

  public Calendar getPreviewDate() {
    return sessionFactory.getCurrentPreviewDate();
  }
  /**
   * Generates a document view export using a {@link
   * org.apache.jackrabbit.commons.xml.DocumentViewExporter} instance.
   *
   * @param path of the node to be exported
   * @param handler handler for the SAX events of the export
   * @param skipBinary whether binary values should be skipped
   * @param noRecurse whether to export just the identified node
   * @throws PathNotFoundException if a node at the given path does not exist
   * @throws SAXException if the SAX event handler failed
   * @throws RepositoryException if another error occurs
   */
  public void exportDocumentView(
      String path, ContentHandler handler, boolean skipBinary, boolean noRecurse)
      throws PathNotFoundException, SAXException, RepositoryException {
    DocumentViewExporter exporter = new DocumentViewExporter(this, handler, skipBinary, noRecurse);
    Item item = getItem(path);
    if (item.isNode()) {
      exporter.export((JCRNodeWrapper) item);
    } else {
      throw new PathNotFoundException("XML export is not defined for properties: " + path);
    }
  }

  /**
   * Generates a system view export using a {@link
   * org.apache.jackrabbit.commons.xml.SystemViewExporter} instance.
   *
   * @param path of the node to be exported
   * @param handler handler for the SAX events of the export
   * @param skipBinary whether binary values should be skipped
   * @param noRecurse whether to export just the identified node
   * @throws PathNotFoundException if a node at the given path does not exist
   * @throws SAXException if the SAX event handler failed
   * @throws RepositoryException if another error occurs
   */
  public void exportSystemView(
      String path, ContentHandler handler, boolean skipBinary, boolean noRecurse)
      throws PathNotFoundException, SAXException, RepositoryException {

    // todo implement our own system view .. ?
    SystemViewExporter exporter = new SystemViewExporter(this, handler, !noRecurse, !skipBinary);
    Item item = getItem(path);
    if (item.isNode()) {
      exporter.export((JCRNodeWrapper) item);
    } else {
      throw new PathNotFoundException("XML export is not defined for properties: " + path);
    }
  }

  /**
   * Calls {@link Session#exportDocumentView(String, ContentHandler, boolean, boolean)} with the
   * given arguments and a {@link ContentHandler} that serializes SAX events to the given output
   * stream.
   *
   * @param absPath passed through
   * @param out output stream to which the SAX events are serialized
   * @param skipBinary passed through
   * @param noRecurse passed through
   * @throws IOException if the SAX serialization failed
   * @throws RepositoryException if another error occurs
   */
  public void exportDocumentView(
      String absPath, OutputStream out, boolean skipBinary, boolean noRecurse)
      throws IOException, RepositoryException {
    try {
      ContentHandler handler = getExportContentHandler(out);
      exportDocumentView(absPath, handler, skipBinary, noRecurse);
    } catch (SAXException e) {
      Exception exception = e.getException();
      if (exception instanceof RepositoryException) {
        throw (RepositoryException) exception;
      } else if (exception instanceof IOException) {
        throw (IOException) exception;
      } else {
        throw new RepositoryException("Error serializing document view XML", e);
      }
    }
  }

  /**
   * Calls {@link Session#exportSystemView(String, ContentHandler, boolean, boolean)} with the given
   * arguments and a {@link ContentHandler} that serializes SAX events to the given output stream.
   *
   * @param absPath passed through
   * @param out output stream to which the SAX events are serialized
   * @param skipBinary passed through
   * @param noRecurse passed through
   * @throws IOException if the SAX serialization failed
   * @throws RepositoryException if another error occurs
   */
  public void exportSystemView(
      String absPath, OutputStream out, boolean skipBinary, boolean noRecurse)
      throws IOException, RepositoryException {
    try {
      ContentHandler handler = getExportContentHandler(out);
      exportSystemView(absPath, handler, skipBinary, noRecurse);
    } catch (SAXException e) {
      Exception exception = e.getException();
      if (exception instanceof RepositoryException) {
        throw (RepositoryException) exception;
      } else if (exception instanceof IOException) {
        throw (IOException) exception;
      } else {
        throw new RepositoryException("Error serializing system view XML", e);
      }
    }
  }

  /**
   * Creates a {@link ContentHandler} instance that serializes the received SAX events to the given
   * output stream.
   *
   * @param stream output stream to which the SAX events are serialized
   * @return SAX content handler
   * @throws RepositoryException if an error occurs
   */
  private ContentHandler getExportContentHandler(OutputStream stream) throws RepositoryException {
    try {
      SAXTransformerFactory stf = (SAXTransformerFactory) SAXTransformerFactory.newInstance();
      TransformerHandler handler = stf.newTransformerHandler();

      Transformer transformer = handler.getTransformer();
      transformer.setOutputProperty(OutputKeys.METHOD, "xml");
      transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8");
      transformer.setOutputProperty(OutputKeys.INDENT, "no");

      handler.setResult(new StreamResult(stream));
      return handler;
    } catch (TransformerFactoryConfigurationError e) {
      throw new RepositoryException("SAX transformer implementation not available", e);
    } catch (TransformerException e) {
      throw new RepositoryException("Error creating an XML export content handler", e);
    }
  }

  public JCRNodeWrapper getNodeByIdentifier(String id)
      throws ItemNotFoundException, RepositoryException {
    return getNodeByUUID(id);
  }

  public Property getProperty(String absPath) throws PathNotFoundException, RepositoryException {
    return (Property) getItem(absPath);
  }

  public boolean nodeExists(String absPath) throws RepositoryException {
    return itemExists(absPath);
  }

  public boolean propertyExists(String absPath) throws RepositoryException {
    return itemExists(absPath);
  }

  public void removeItem(String absPath)
      throws VersionException, LockException, ConstraintViolationException, AccessDeniedException,
          RepositoryException {
    JCRItemWrapper item = getItem(absPath);
    boolean flushNeeded = false;
    if (item.isNode()) {
      JCRNodeWrapper node = (JCRNodeWrapper) item;
      unregisterNewNode(node);
      if (node.hasNodes()) {
        flushNeeded = true;
      }
    }
    item.remove();
    if (flushNeeded) {
      flushCaches();
    } else {
      removeFromCache(item);
    }
  }

  void removeFromCache(JCRItemWrapper item) throws RepositoryException {
    sessionCacheByPath.remove(item.getPath());
    if (item instanceof JCRNodeWrapper) {
      sessionCacheByIdentifier.remove(((JCRNodeWrapper) item).getIdentifier());
    }
  }

  public boolean hasPermission(String absPath, String actions) throws RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  public boolean hasCapability(String s, Object o, Object[] objects) throws RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  /**
   * Returns the access control manager for this <code>Session</code>.
   *
   * <p>Jahia throws an <code>UnsupportedRepositoryOperationException</code>.
   *
   * @return the access control manager for this <code>Session</code>
   * @throws UnsupportedRepositoryOperationException if access control is not supported.
   * @since JCR 2.0
   */
  public AccessControlManager getAccessControlManager()
      throws UnsupportedRepositoryOperationException, RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  public RetentionManager getRetentionManager()
      throws UnsupportedRepositoryOperationException, RepositoryException {
    throw new UnsupportedRepositoryOperationException();
  }

  public Map<String, String> getStoredPasswordsProviders() {
    Map<String, String> results = new HashMap<String, String>();
    results.put(null, user.getUsername());
    for (JCRStoreProvider provider : sessionFactory.getProviders().values()) {
      if ("storedPasswords".equals(provider.getAuthenticationType())) {
        results.put(provider.getKey(), user.getProperty("storedUsername_" + provider.getKey()));
      }
    }
    return results;
  }

  public void storePasswordForProvider(String providerKey, String username, String password) {
    if (username == null) {
      user.removeProperty("storedUsername_" + providerKey);
    } else {
      user.setProperty("storedUsername_" + providerKey, username);
    }
    if (password == null) {
      user.removeProperty("storedPassword_" + providerKey);
    } else {
      user.setProperty("storedPassword_" + providerKey, password);
    }
  }

  /**
   * Performs check out of the specified node.
   *
   * @param node the node to perform the check out
   * @see VersionManager#checkout(String) for details
   */
  public void checkout(Node node)
      throws UnsupportedRepositoryOperationException, LockException, RepositoryException {
    while (!node.isCheckedOut()) {
      if (!node.isNodeType("mix:versionable") && !node.isNodeType("mix:simpleVersionable")) {
        node = node.getParent();
      } else {
        String absPath = node.getPath();
        VersionManager versionManager = getWorkspace().getVersionManager();
        if (!versionManager.isCheckedOut(absPath)) {
          versionManager.checkout(absPath);
        }
        return;
      }
    }
  }

  public Map<String, String> getUuidMapping() {
    return uuidMapping;
  }

  public Map<String, String> getPathMapping() {
    return pathMapping;
  }

  public Locale getFallbackLocale() {
    return fallbackLocale;
  }

  public Date getVersionDate() {
    return versionDate;
  }

  public String getVersionLabel() {
    return versionLabel;
  }

  public void setVersionDate(Date versionDate) {
    if (this.versionDate == null) {
      this.versionDate = versionDate;
    } else {
      throw new RuntimeException("Should not change versionDate on a session in same thread");
    }
  }

  public void setVersionLabel(String versionLabel) {
    if (this.versionLabel == null) {
      if (versionLabel != null && !versionLabel.startsWith(getWorkspace().getName())) {
        throw new RuntimeException(
            "Cannot use label " + versionLabel + " in workspace " + getWorkspace().getName());
      }
      this.versionLabel = versionLabel;
    } else {
      throw new RuntimeException("Should not change versionLabel on a session in same thread");
    }
  }

  /** {@inheritDoc} */
  public JCRNodeWrapper getFrozenVersionAsRegular(Node objectNode, JCRStoreProvider provider)
      throws RepositoryException {
    try {
      VersionHistory vh =
          objectNode
              .getSession()
              .getWorkspace()
              .getVersionManager()
              .getVersionHistory(objectNode.getPath());

      Version v = null;
      if (versionLabel != null) {
        v = JCRVersionService.findVersionByLabel(vh, versionLabel);
      }
      if (v == null && versionDate != null) {
        v = JCRVersionService.findClosestVersion(vh, versionDate);
      }

      if (v == null) {
        throw new PathNotFoundException();
      }

      Node frozen = v.getNode(Constants.JCR_FROZENNODE);

      return provider.getNodeWrapper(frozen, this);
    } catch (UnsupportedRepositoryOperationException e) {
      if (getVersionDate() == null && getVersionLabel() == null) {
        logger.error("Error while retrieving frozen version", e);
      }
    }
    return null;
  }

  public static AtomicLong getActiveSessions() {
    return activeSessions;
  }

  protected void flushCaches() {
    sessionCacheByIdentifier.clear();
    sessionCacheByPath.clear();
  }

  protected JCRNodeWrapper getCachedNode(String uuid) {
    return sessionCacheByIdentifier.get(uuid);
  }
}
Exemplo n.º 27
0
/**
 * Created by 4535992 on 28/12/2015.
 * href:http://www.journaldev.com/2544/java-csv-parserwriter-example-using-opencsv-apache-commons-csv-and-supercsv.
 */
@SuppressWarnings("unused")
public class OpenCsvUtilities extends FileUtilities {

  private static final org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(OpenCsvUtilities.class);

  /**
   * Method to write a CSV Data List of Beans to a String.
   *
   * @param beans the List of Beans to convert.
   * @param separator the char separator.
   * @param <T> the generic variable.
   * @return the String content of the List of Beans.
   */
  public static <T> String writeCSVDataToStringWithBeans(List<T> beans, char separator) {
    try {
      Writer writer = new StringWriter();
      try (CSVWriter csvWriter = new CSVWriter(writer, separator)) {
        List<String[]> data = toStringArray(beans);
        csvWriter.writeAll(data);
      }
      return writer.toString();
    } catch (IOException e) {
      logger.error(
          "Can't write the CSV String from the Bean:"
              + beans.get(0).getClass().getName()
              + " -> "
              + e.getMessage(),
          e);
      return "";
    }
  }

  /**
   * Method to write a CSV Data List of Array of String to a String.
   *
   * @param content the List of Array of String to convert.
   * @param separator the char separator.
   * @return the String content of the List of Beans.
   */
  public static String writeCSVDataToString(List<String[]> content, char separator) {
    try {
      Writer writer = new StringWriter();
      CSVWriter csvWriter;
      if (StringUtilities.NULL_CHAR2 == separator) {
        csvWriter =
            new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
      } else {
        csvWriter = new CSVWriter(writer, separator, CSVWriter.NO_QUOTE_CHARACTER);
      }
      csvWriter.writeAll(content);
      csvWriter.close();
      return writer.toString();
    } catch (IOException e) {
      logger.error("Can't write the CSV String -> " + e.getMessage(), e);
      return "";
    }
  }

  /**
   * Method to write a CSV Data List of Array of String to a String.
   *
   * @param content the List of Array of String to convert.
   * @return the String content of the List of Beans.
   */
  public static String writeCSVDataToString(List<String[]> content) {
    return writeCSVDataToString(content, StringUtilities.NULL_CHAR2);
  }

  /**
   * Method to write a CSV File from List of Array of String.
   *
   * @param content the List of Array of String to convert.
   * @param separator the char separator.
   * @param fileOutputCsv the output File Csv to create.
   * @return the File Csv created.
   */
  public static File writeCSVDataToFile(
      List<String[]> content, char separator, File fileOutputCsv) {
    try {
      Writer writer =
          new FileWriter(fileOutputCsv, true); // the true value make append the result...
      CSVWriter csvWriter;
      if (StringUtilities.NULL_CHAR2 == separator) {
        csvWriter =
            new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
      } else {
        csvWriter = new CSVWriter(writer, separator, CSVWriter.NO_QUOTE_CHARACTER);
      }
      csvWriter.writeAll(content);
      csvWriter.close();
      return fileOutputCsv;
    } catch (IOException e) {
      logger.error("Can't write the CSV to File:" + fileOutputCsv + " -> " + e.getMessage(), e);
      return null;
    }
  }

  /**
   * Method to write a CSV File from List of Array of String.
   *
   * @param content the List of Array of String to convert.
   * @param fileOutputCsv the output File Csv to create.
   * @return the File Csv created.
   */
  public static File writeCSVDataToFile(List<String[]> content, File fileOutputCsv) {
    return writeCSVDataToFile(content, StringUtilities.NULL_CHAR2, fileOutputCsv);
  }

  /**
   * Method to write a CSV List of Array of String to System Console..
   *
   * @param content the List of Array of String to convert.
   * @param separator the char separator.
   */
  public static void writeCSVDataToConsole(List<String[]> content, char separator) {
    try {
      Writer writer = new OutputStreamWriter(System.out, StringUtilities.UTF_8);
      CSVWriter csvWriter;
      if (StringUtilities.NULL_CHAR2 == separator) {
        csvWriter =
            new CSVWriter(writer, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER);
      } else {
        csvWriter = new CSVWriter(writer, separator, CSVWriter.NO_QUOTE_CHARACTER);
      }
      csvWriter.writeAll(content, false);
      csvWriter.close();
    } catch (IOException e) {
      logger.error("Can't write the CSV to Console -> " + e.getMessage(), e);
    }
  }

  /**
   * Method to write a CSV List of Array of String to System Console..
   *
   * @param content the List of Array of String to convert.
   */
  public static void writeCSVDataToConsole(List<String[]> content) {
    writeCSVDataToConsole(content, StringUtilities.NULL_CHAR2);
  }

  /**
   * Method to convert a List of beans to a List of Array of Strings.
   *
   * @param beans the List of Beans.
   * @param <T> generic value.
   * @return the List of Array of String content of the csv.
   */
  public static <T> List<String[]> toStringArray(List<T> beans) {
    return toStringArray(beans, null);
  }

  /**
   * Method to convert a List of beans to a List of Array of Strings.
   *
   * @param beans the List of Beans.
   * @param addNewHeader the new String Array for the Header row.
   * @param <T> generic value.
   * @return the List of Array of String content of the csv.
   */
  @SuppressWarnings("unchecked")
  public static <T> List<String[]> toStringArray(List<T> beans, String[] addNewHeader) {
    List<String[]> records = new ArrayList<>();
    // add header record
    // records.add(new String[]{"ID","Name","Role","Salary"});
    if (addNewHeader != null) records.add(addNewHeader);
    for (T bean : beans) {
      // beans.stream().map((bean) -> {
      List<String> record = new ArrayList<>();
      // invoke getter method and convert to String
      Class<T> clazz = (Class<T>) bean.getClass();
      // T t = ReflectionUtilities.invokeConstructor(clazz);
      List<Method> getter = (List<Method>) ReflectionUtilities.findGetters(clazz, true);

      for (Method method : getter) {
        record.add(String.valueOf(ReflectionUtilities.invokeGetter(bean, method)));
      }
      // getter.stream().forEach((method) ->
      // record.add(String.valueOf(ReflectionUtilities.invokeGetter(bean,method))));
      // return record;
      // }).forEach((record) -> records.add(ListUtilities.toArray(record)));
      records.add(ListUtilities.toArray(record));
    }
    return records;
  }

  /**
   * Method use OpenCsv Library for
   *
   * @param columnMapping Map allow the user to pass the column Names to a Field Names of the Class.
   * @param fileInputCsv the File CSV to parse.
   * @param <T> the generic variable.
   * @return the List of Bean parsed from the CSV file.
   */
  public static <T> List<T> parseCSVFileAsList(
      Map<String, String> columnMapping, File fileInputCsv) {
    try {
      HeaderColumnNameTranslateMappingStrategy<T> beanStrategy =
          new HeaderColumnNameTranslateMappingStrategy<>();
      // beanStrategy.setType(clazz); //deprecated
      /*Map<String, String> columnMapping = new HashMap<>();
      columnMapping.put("ID", "id");
      columnMapping.put("Name", "name");
      columnMapping.put("Role", "role");*/
      beanStrategy.setColumnMapping(columnMapping);
      CsvToBean<T> csvToBean = new CsvToBean<>();
      CSVReader reader = new CSVReader(new FileReader(fileInputCsv));
      return csvToBean.parse(beanStrategy, reader);
    } catch (IOException e) {
      logger.error(
          "Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
          e);
      return new ArrayList<>();
    }
  }

  /**
   * Method use OpenCsv Library for
   *
   * @param clazz the Class of the Bean.
   * @param fileInputCsv the File CSV to parse.
   * @param separator the char separator.
   * @param <T> the generic variable.
   * @return the List of Bean parsed from the CSV file.
   */
  public static <T> List<T> parseCSVFileAsList(Class<T> clazz, File fileInputCsv, char separator) {
    try {
      List<T> beans;
      try ( // create CSVReader object
      CSVReader reader = new CSVReader(new FileReader(fileInputCsv), separator)) {
        beans = new ArrayList<>();
        // read line by line
        String[] record;
        // skip header row
        String[] headers = reader.readNext();
        // read content
        while ((record = reader.readNext()) != null) {
          T t = ReflectionUtilities.invokeConstructor(clazz);
          for (int i = 0; i < record.length; i++) {
            String nameMethod = "set" + org.apache.commons.lang3.StringUtils.capitalize(headers[i]);
            // invoke setter method
            if (ReflectionUtilities.checkMethod(clazz, nameMethod)) {
              ReflectionUtilities.invokeSetter(t, nameMethod, record[i]);
            } else {
              logger.warn(
                  "Not exists the Method with name:"
                      + nameMethod
                      + " on the Bean:"
                      + t.getClass().getName());
            }
          }
          beans.add(t);
        }
      }
      return beans;
    } catch (IOException e) {
      logger.error(
          "Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
          e);
      return new ArrayList<>();
    }
  }

  /**
   * Method use OpenCsv Library for
   *
   * @param fileInputCsv the File CSV to parse.
   * @param separator the char separator.
   * @return the List of Bean parsed from the CSV file.
   */
  public static List<String[]> parseCSVFileAsList(File fileInputCsv, char separator) {
    try {
      List<String[]> records;
      // read all lines at once
      try ( // create CSVReader object
      CSVReader reader = new CSVReader(new FileReader(fileInputCsv), separator)) {
        // read all lines at once
        records = reader.readAll();
        Iterator<String[]> iterator = records.iterator();
        records.clear();
        // skip header row
        iterator.next();
        while (iterator.hasNext()) {
          String[] record = iterator.next();
          records.add(record);
        }
      }
      return records;
    } catch (IOException e) {
      logger.error(
          "Can't parse the CSV file:" + fileInputCsv.getAbsolutePath() + " -> " + e.getMessage(),
          e);
      return new ArrayList<>();
    }
  }

  /**
   * Method to get the content of a comma separated file (.csv,.input,.txt)
   *
   * @param CSV the File comma separated.
   * @param noHeaders if true jump the first line of the content.
   * @return the List of Array of the content of the File comma separated.
   */
  public static List<String[]> parseCSVFileAsList(File CSV, boolean noHeaders) {
    List<String[]> content;
    try {
      CSVReader reader1 = new CSVReader(new FileReader(CSV));
      content = reader1.readAll();
      /* List<String[]> myDatas = reader1.readAll();
      String[] lineI = myDatas.get(i);
      for (String[] line : myDatas) {
          for (String value : line) {
              //do stuff with value
          }
      }*/
      if (noHeaders) content.remove(0);
      if (content.get(0).length <= 1) {
        logger.warn(
            "Attention: You haven't parsed correctly the CSV file with OpenCSV API try with Univocity Method");
      }
      return content;
    } catch (IOException e) {
      logger.error("Can't find the CSV File:" + e.getMessage(), e);
      return null;
    }
  }

  /**
   * Method to get the String array of the columns of a CSV File.
   *
   * @param fileCSV the File CSV.
   * @param hasFirstLine if true the first line of CSV File contains the columns name.
   * @return a String Array of the columns.
   */
  public static String[] getHeaders(File fileCSV, boolean hasFirstLine) {
    String[] columns = new String[0];
    try {
      CSVReader reader = new CSVReader(new FileReader(fileCSV));
      columns = reader.readNext(); // assuming first read
      if (!hasFirstLine) {
        int columnCount = 0;
        if (columns != null) columnCount = columns.length;
        columns = new String[columnCount];
        for (int i = 0; i < columnCount; i++) {
          columns[i] = "Column#" + i;
        }
      }
    } catch (IOException e) {
      logger.error("Can't find the Headers on the CSV File", e);
    }
    return columns;
  }

  /**
   * Method to get the String array of the columns of a CSV File.
   *
   * @param contentWithHeaders the {@link List} of {@link String} array of the content of the csv.
   * @return a String Array of the columns.
   */
  public static String[] getHeaders(List<String[]> contentWithHeaders) {
    String[] columns = new String[0];
    try {
      String[] headers = contentWithHeaders.get(0);
      if (headers.length <= 1) {
        throw new Exception("Can't find the delimiter with openCSV try with Univicity method.");
      }
    } catch (Exception e) {
      logger.error("Can't find the Headers on the content", e);
    }
    return columns;
  }

  /**
   * Method to get the String array of the columns of a CSV File.
   *
   * @param fileCSV the File CSV.
   * @param hasFirstLine if true the first line of CSV File contains the columns name.
   * @return a String Array of the columns.
   */
  public static String[] getHeadersWithUnivocity(File fileCSV, boolean hasFirstLine) {
    return UnivocityUtilities.getHeaders(fileCSV, hasFirstLine);
  }

  // ------------------------------------------------------------------------------

  /**
   * Method use OpenCsv Library for
   *
   * @param fileInputCsv the File CSV to parse.
   * @param noHeaders if true the headers are excluded from the cotent.
   * @return the List of Bean parsed from the CSV file.
   */
  public static List<String[]> parseCSVFileAsListWithUnivocity(
      File fileInputCsv, boolean noHeaders) {
    return UnivocityUtilities.parseCSVFileAsList(fileInputCsv, noHeaders);
  }

  /* public static void main(String[] args) throws IOException {

      List<Employee> emps = parseCSVFileLineByLine();
      System.out.println("**********");
      parseCSVFileAsList();
      System.out.println("**********");
      parseCSVToBeanList();
      System.out.println("**********");
      writeCSVData(emps);
  }*/

  public static String getFieldLatitude(String[] headers) {
    Pattern pattern = Pattern.compile("(L|l)(at)(itude)?", Pattern.CASE_INSENSITIVE);
    return getField(headers, pattern);
  }

  public static String getFieldLongitude(String[] headers) {
    Pattern pattern = Pattern.compile("(L|l)(on|ng)(gitude)?", Pattern.CASE_INSENSITIVE);
    return getField(headers, pattern);
  }

  public static String getField(String[] headers, Pattern pattern) {
    // String[] headers = CSVGetHeaders(headers,true);
    for (String s : headers) {
      if (pattern.matcher(String.valueOf(s)).matches()) {
        return s;
      }
    }
    return "NULL";
  }

  /**
   * Parse CSV file using OpenCSV library and load in given database table. href:
   * http://viralpatel.net/blogs/java-load-csv-file-to-database/. Modified by rammar:
   * https://github.com/BaderLab/pharmacogenomics/blob/master/src/CSVLoader/CSVLoader.java
   *
   * @param connection the {@link Connection} SQL.
   * @param separator the {@link Character} separator.
   * @param csvFile Input CSV InputStream
   * @param tableName Database table name to import data
   * @param truncateBeforeLoad Truncate the table before inserting new records.
   * @throws SQLException if any error is occurred with the SQL Connection.
   * @throws java.io.IOException if any error is occurred with the file.
   */
  public static void loadCSVToSQLTable(
      Connection connection,
      char separator,
      InputStream csvFile,
      String tableName,
      boolean truncateBeforeLoad)
      throws SQLException, IOException {

    String SQL_INSERT = "INSERT INTO ${table}(${keys}) VALUES(${values})";
    String TABLE_REGEX = "\\$\\{table\\}";
    String KEYS_REGEX = "\\$\\{keys\\}";
    String VALUES_REGEX = "\\$\\{values\\}";
    CSVReader csvReader;
    if (null == connection) {
      throw new SQLException("Not a valid connection.");
    }
    try {

      /* Modified by rammar.
       *
       * I was having issues with the CSVReader using the "\" to escape characters.
       * A MySQL CSV file contains quote-enclosed fields and non-quote-enclosed NULL
       * values written as "\N". The CSVReader was removing the "\". To detect "\N"
       * I must remove the escape character, and the only character you can replace
       * it with that you are pretty much guaranteed will not be used to escape
       * text is '\0'.
       * I read this on:
       * http://stackoverflow.com/questions/6008395/opencsv-in-java-ignores-backslash-in-a-field-value
       * based on:
       * http://sourceforge.net/p/opencsv/support-requests/5/
       */
      // PREVIOUS VERSION: csvReader = new CSVReader(new FileReader(csvFile), this.seprator);
      csvReader = new CSVReader(new InputStreamReader(csvFile), separator, '"', '\0');

    } catch (Exception e) {
      throw new IOException("Error occured while executing file. " + e.getMessage());
    }

    String[] headerRow = csvReader.readNext();

    if (null == headerRow) {
      throw new FileNotFoundException(
          "No columns defined in given CSV file." + "Please check the CSV file format.");
    }

    String questionmarks = StringUtils.repeat("?,", headerRow.length);
    questionmarks = (String) questionmarks.subSequence(0, questionmarks.length() - 1);

    /* NOTE from Ron: Header column names must match SQL table fields */
    String query = SQL_INSERT.replaceFirst(TABLE_REGEX, tableName);
    query = query.replaceFirst(KEYS_REGEX, StringUtils.join(headerRow, ","));
    query = query.replaceFirst(VALUES_REGEX, questionmarks);

    // System.out.println("Query: " + query); // Modified by rammar to suppress output

    String[] nextLine;
    Connection con = null;
    PreparedStatement ps = null;
    try {
      con = connection;
      con.setAutoCommit(false);
      ps = con.prepareStatement(query);

      if (truncateBeforeLoad) {
        // delete data from table before loading csv
        con.createStatement().execute("DELETE FROM " + tableName);
      }
      final int batchSize = 1000;
      int count = 0;
      Date date;
      while ((nextLine = csvReader.readNext()) != null) {
        int index = 1;
        for (String string : nextLine) {
          date = DateUtilities.convertToDate(string);
          if (null != date) {
            ps.setDate(index++, new java.sql.Date(date.getTime()));
          } else {

            /* Section modified by rammar to allow NULL values
             * to be input into the DB. */
            if (string.length() > 0 && !string.equals("\\N")) {
              ps.setString(index++, string);
            } else {
              ps.setNull(index++, Types.VARCHAR);
              // ps.setString(index++, null); // can use this syntax also - not sure which is better
            }
          }
        }
        ps.addBatch();
        if (++count % batchSize == 0) {
          ps.executeBatch();
        }
      }
      ps.executeBatch(); // insert remaining records
      logger.info(count + " records loaded into " + tableName + " DB table");
      con.commit();
    } catch (SQLException | IOException e) {
      con.rollback();
      throw new IOException(
          "Error occured while loading data from file to database." + e.getMessage());
    } finally {
      if (null != ps) ps.close();
      con.close();
      csvReader.close();
    }
  }
}
Exemplo n.º 28
0
/**
 * Builds Collections of Grib1 Time Partitioned.
 *
 * @author caron
 * @since 1/7/12
 */
public class Grib1TimePartitionBuilder extends Grib1CollectionBuilder {
  public static final String MAGIC_START = "Grib1Partition0Index";

  private static final org.slf4j.Logger logger =
      org.slf4j.LoggerFactory.getLogger(Grib1TimePartitionBuilder.class);
  // static private final int versionTP = 4;
  private static final boolean trace = false;

  // called by tdm
  public static boolean update(TimePartitionCollection tpc, Formatter f) throws IOException {
    Grib1TimePartitionBuilder builder =
        new Grib1TimePartitionBuilder(tpc.getCollectionName(), new File(tpc.getRoot()), tpc);
    if (!builder.needsUpdate()) return false;
    builder.readOrCreateIndex(CollectionManager.Force.always, f);
    builder.gc.close();
    return true;
  }

  // read in the index, create if it doesnt exist or is out of date
  public static Grib1TimePartition factory(
      TimePartitionCollection tpc, CollectionManager.Force force, Formatter f) throws IOException {
    Grib1TimePartitionBuilder builder =
        new Grib1TimePartitionBuilder(tpc.getCollectionName(), new File(tpc.getRoot()), tpc);
    builder.readOrCreateIndex(force, f);
    return builder.tp;
  }

  // read in the index, index raf already open
  public static Grib1TimePartition createFromIndex(
      String name, File directory, RandomAccessFile raf) throws IOException {
    Grib1TimePartitionBuilder builder = new Grib1TimePartitionBuilder(name, directory, null);
    if (builder.readIndex(raf)) {
      return builder.tp;
    }
    throw new IOException("Reading index failed");
  }

  /**
   * write new index if needed
   *
   * @param tpc use this collection
   * @param force force index
   * @param f put status messagess here
   * @return true if index was written
   * @throws IOException on error
   */
  public static boolean writeIndexFile(
      TimePartitionCollection tpc, CollectionManager.Force force, Formatter f) throws IOException {
    Grib1TimePartitionBuilder builder = null;
    try {
      builder =
          new Grib1TimePartitionBuilder(tpc.getCollectionName(), new File(tpc.getRoot()), tpc);
      return builder.readOrCreateIndex(force, f);

    } finally {
      if ((builder != null) && (builder.tp != null)) builder.tp.close();
    }
  }

  //////////////////////////////////////////////////////////////////////////////////

  private final TimePartitionCollection tpc; // defines the partition
  private final Grib1TimePartition tp; // build this object

  private Grib1TimePartitionBuilder(String name, File directory, TimePartitionCollection tpc) {
    FeatureCollectionConfig.GribConfig config =
        (tpc == null)
            ? null
            : (FeatureCollectionConfig.GribConfig)
                tpc.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG);
    this.tp = new Grib1TimePartition(name, directory, config);
    this.gc = tp;
    this.tpc = tpc;
  }

  private boolean readOrCreateIndex(CollectionManager.Force ff, Formatter f) throws IOException {
    File idx = gc.getIndexFile();

    // force new index or test for new index needed
    boolean force =
        ((ff == CollectionManager.Force.always)
            || (ff == CollectionManager.Force.test && needsUpdate(idx.lastModified(), f)));

    // otherwise, we're good as long as the index file exists and can be read
    if (force || !idx.exists() || !readIndex(idx.getPath())) {
      logger.info("TimePartitionBuilder createIndex {}", idx.getPath());
      createPartitionedIndex(f); // write out index
      readIndex(idx.getPath()); // read back in index
      return true;
    }
    return false;
  }

  private boolean needsUpdate(long collectionLastModified, Formatter f) throws IOException {
    CollectionManager.ChangeChecker cc = Grib1Index.getChangeChecker();
    for (CollectionManager dcm :
        tpc.makePartitions()) { // LOOK not really right, since we dont know if these files are the
      // same as in the index
      File idxFile = GribCollection.getIndexFile(dcm);
      if (!idxFile.exists()) return true;
      if (collectionLastModified < idxFile.lastModified()) return true;
      for (MFile mfile : dcm.getFiles()) {
        if (cc.hasChangedSince(mfile, idxFile.lastModified())) return true;
      }
    }
    return false;
  }

  ///////////////////////////////////////////////////
  // create the index

  private boolean createPartitionedIndex(Formatter f) throws IOException {
    long start = System.currentTimeMillis();

    // create partitions based on TimePartitionCollections object
    for (CollectionManager dcm : tpc.makePartitions()) {
      tp.addPartition(dcm);
    }

    List<TimePartition.Partition> bad = new ArrayList<TimePartition.Partition>();
    for (TimePartition.Partition dc : tp.getPartitions()) {
      try {
        dc.makeGribCollection(f); // ensure collection has been read successfully
        if (trace) f.format(" Open partition %s%n", dc.getDcm().getCollectionName());
      } catch (Throwable t) {
        logger.error(" Failed to open partition " + dc.getName(), t);
        f.format(" FAIL on partition %s (remove) %n", dc.getDcm().getCollectionName());
        bad.add(dc); // LOOK may be a file leak ?
      }
    }

    // remove ones that failed
    for (TimePartition.Partition p : bad) tp.removePartition(p);

    // choose the "canonical" partition, aka prototype
    int n = tp.getPartitions().size();
    if (n == 0) {
      logger.error(" Nothing in this partition = " + tp.getName());
      f.format(" FAIL Partition empty collection = %s%n", tp.getName());
      return false;
    }
    int idx = tpc.getProtoIndex(n);
    TimePartition.Partition canon = tp.getPartitions().get(idx);
    f.format(" Using canonical partition %s%n", canon.getDcm().getCollectionName());

    // check consistency across vert and ens coords
    if (!checkPartitions(canon, f)) {
      logger.error(
          " Partition check failed, index not written on {} message = {}",
          tp.getName(),
          f.toString());
      f.format(" FAIL Partition check collection = %s%n", tp.getName());
      return false;
    }

    // make the time coordinates, place results into canon
    createPartitionedTimeCoordinates(canon, f);

    // ready to write the index file
    writeIndex(canon, f);

    // close open gc's
    tp.cleanup();

    long took = System.currentTimeMillis() - start;
    f.format(" CreatePartitionedIndex took %d msecs%n", took);
    return true;
  }

  // consistency check on variables : compare each variable to corresponding one in proto
  // also set the groupno and partno for each partition
  private boolean checkPartitions(TimePartition.Partition canon, Formatter f) throws IOException {
    List<TimePartition.Partition> partitions = tp.getPartitions();
    int npart = partitions.size();
    boolean ok = true;

    // for each group in canonical Partition
    GribCollection canonGc = canon.makeGribCollection(f);
    for (GribCollection.GroupHcs firstGroup : canonGc.getGroups()) {
      String gname = firstGroup.getId();
      if (trace) f.format(" Check Group %s%n", gname);

      // hash proto variables for quick lookup
      Map<Integer, GribCollection.VariableIndex> check =
          new HashMap<Integer, GribCollection.VariableIndex>(firstGroup.varIndex.size());
      List<GribCollection.VariableIndex> varIndexP =
          new ArrayList<GribCollection.VariableIndex>(firstGroup.varIndex.size());
      for (GribCollection.VariableIndex vi : firstGroup.varIndex) {
        TimePartition.VariableIndexPartitioned vip = tp.makeVariableIndexPartitioned(vi, npart);
        varIndexP.add(vip);
        check.put(vi.cdmHash, vip); // replace with its evil twin
      }
      firstGroup.varIndex = varIndexP; // replace with its evil twin

      // for each partition
      for (int partno = 0; partno < npart; partno++) {
        TimePartition.Partition tpp = partitions.get(partno);
        if (trace) f.format(" Check Partition %s%n", tpp.getName());

        // get corresponding group
        GribCollection gc = tpp.makeGribCollection(f);
        int groupIdx = gc.findGroupIdxById(firstGroup.getId());
        if (groupIdx < 0) {
          f.format(" Cant find group %s in partition %s%n", gname, tpp.getName());
          ok = false;
          continue;
        }
        GribCollection.GroupHcs group = gc.getGroup(groupIdx);

        // for each variable in partition group
        for (int varIdx = 0; varIdx < group.varIndex.size(); varIdx++) {
          GribCollection.VariableIndex vi2 = group.varIndex.get(varIdx);
          if (trace) f.format(" Check variable %s%n", vi2);
          int flag = 0;

          GribCollection.VariableIndex vi1 = check.get(vi2.cdmHash); // compare with proto variable
          if (vi1 == null) {
            f.format(
                "   WARN Cant find variable %s from %s in proto - ignoring that variable%n",
                vi2, tpp.getName());
            continue; // we can tolerate this
          }

          // compare vert coordinates
          VertCoord vc1 = vi1.getVertCoord();
          VertCoord vc2 = vi2.getVertCoord();
          if ((vc1 == null) != (vc2 == null)) {
            f.format(
                "   ERR Vert coordinates existence on variable %s in %s doesnt match%n",
                vi2, tpp.getName());
            ok = false;
          } else if ((vc1 != null) && !vc1.equalsData(vc2)) {
            f.format(
                "   WARN Vert coordinates values on variable %s in %s dont match%n",
                vi2, tpp.getName());
            f.format("    canon vc = %s%n", vc1);
            f.format("    this vc = %s%n", vc2);
            flag |= TimePartition.VERT_COORDS_DIFFER;
          }

          // compare ens coordinates
          EnsCoord ec1 = vi1.getEnsCoord();
          EnsCoord ec2 = vi2.getEnsCoord();
          if ((ec1 == null) != (ec2 == null)) {
            f.format(
                "   ERR Ensemble coordinates existence on variable %s in %s doesnt match%n",
                vi2, tpp.getName());
            ok = false;
          } else if ((ec1 != null) && !ec1.equalsData(ec2)) {
            f.format(
                "   WARN Ensemble coordinates values on variable %s in %s dont match%n",
                vi2, tpp.getName());
            f.format("    canon ec = %s%n", ec1);
            f.format("    this ec = %s%n", ec2);
            flag |= TimePartition.ENS_COORDS_DIFFER;
          }

          ((TimePartition.VariableIndexPartitioned) vi1)
              .setPartitionIndex(partno, groupIdx, varIdx, flag);
        } // loop over variable
      } // loop over partition
    } // loop over group

    if (ok) f.format("  Partition check: vert, ens coords OK%n");
    return ok;
  }

  private class PartGroup {
    GribCollection.GroupHcs group;
    TimePartition.Partition tpp;

    private PartGroup(GribCollection.GroupHcs group, TimePartition.Partition tpp) {
      this.group = group;
      this.tpp = tpp;
    }
  }

  private boolean createPartitionedTimeCoordinates(TimePartition.Partition canon, Formatter f)
      throws IOException {
    List<TimePartition.Partition> partitions = tp.getPartitions();
    boolean ok = true;

    // for each group in canonical Partition
    for (GribCollection.GroupHcs firstGroup : canon.makeGribCollection(f).getGroups()) {
      String gname = firstGroup.getId();
      if (trace) f.format(" Check Group %s%n", gname);

      // get list of corresponding groups from all the time partition, so we dont have to keep
      // looking it up
      List<PartGroup> pgList = new ArrayList<PartGroup>(partitions.size());
      for (TimePartition.Partition dc : partitions) {
        GribCollection.GroupHcs gg = dc.makeGribCollection(f).findGroupById(gname);
        if (gg == null) logger.error(" Cant find group {} in partition {}", gname, dc.getName());
        else pgList.add(new PartGroup(gg, dc));
      }

      // unique time coordinate unions
      List<TimeCoordUnion> unionList = new ArrayList<TimeCoordUnion>();

      // for each variable in canonical Partition
      for (GribCollection.VariableIndex viCanon : firstGroup.varIndex) {
        if (trace) f.format(" Check variable %s%n", viCanon);
        TimeCoord tcCanon = viCanon.getTimeCoord();

        List<TimeCoord> tcPartitions = new ArrayList<TimeCoord>(pgList.size());

        // for each partition, get the time index
        for (PartGroup pg : pgList) {
          // get corresponding variable
          GribCollection.VariableIndex vi2 = pg.group.findVariableByHash(viCanon.cdmHash);
          if (vi2 == null) { // apparently not in the file
            f.format(
                "   WARN Cant find variable %s in partition %s / %s%n",
                viCanon, pg.tpp.getName(), pg.group.getId());
            tcPartitions.add(null);
          } else {
            if (vi2.timeIdx < 0 || vi2.timeIdx >= pg.group.timeCoords.size()) {
              logger.error(" timeIdx out of range var= {} on partition {}", vi2, pg.tpp.getName());
              tcPartitions.add(null);
            } else {
              TimeCoord tc2 = vi2.getTimeCoord();
              if (tc2.isInterval() != tcCanon.isInterval()) {
                logger.error(
                    " timeIdx wrong interval type var= {} on partition {}", vi2, pg.tpp.getName());
                tcPartitions.add(null);
              } else {
                tcPartitions.add(tc2);
              }
            }
          }
        }

        // union of time coordinates
        TimeCoordUnion union = new TimeCoordUnion(tcCanon.getCode(), tcPartitions, tcCanon);

        // store result in the first group
        viCanon.partTimeCoordIdx =
            TimeCoordUnion.findUnique(unionList, union); // this merges identical TimeCoordUnion
      }

      /* turn TimeIndex into TimeCoord
      for (int tidx = 0; tidx <unionList.size(); tidx++) {
        TimeCoordUnion union = unionList.get(tidx);
        f.format(" %s %d: timeIndexList=", firstGroup.hcs.getName(), tidx);
        for (int idx : union.) f.format("%d,",idx);
        f.format("%n");
      } */

      // store results in first group
      firstGroup.timeCoordPartitions = unionList;
    }

    return ok;
  }

  //////////////////////////////////////////////////////////

  @Override
  public String getMagicStart() {
    return MAGIC_START;
  }

  // writing ncx
  /*
  MAGIC_START
  version
  sizeRecords
  VariableRecords (sizeRecords bytes)
  sizeIndex
  GribCollectionIndex (sizeIndex bytes)
  */
  private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
    File file = tp.getIndexFile();
    if (file.exists()) {
      if (!file.delete()) logger.error("Cant delete " + file.getPath());
    }

    RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      raf.writeLong(0); // no record section

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(tp.getName());

      GribCollection canonGc = canon.makeGribCollection(f);
      for (GribCollection.GroupHcs g : canonGc.getGroups())
        indexBuilder.addGroups(writeGroupProto(g));

      indexBuilder.setCenter(canonGc.getCenter());
      indexBuilder.setSubcenter(canonGc.getSubcenter());
      indexBuilder.setMaster(canonGc.getMaster());
      indexBuilder.setLocal(canonGc.getLocal());

      for (TimePartition.Partition p : tp.getPartitions()) {
        indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
      }

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);

    } finally {
      f.format("file size =  %d bytes%n", raf.length());
      raf.close();
    }

    return true;
  }

  private GribCollectionProto.Group writeGroupProto(GribCollection.GroupHcs g) throws IOException {
    GribCollectionProto.Group.Builder b = GribCollectionProto.Group.newBuilder();

    b.setGds(ByteString.copyFrom(g.rawGds));
    b.setGdsHash(g.gdsHash);

    for (GribCollection.VariableIndex vb : g.varIndex)
      b.addVariables(writeVariableProto((TimePartition.VariableIndexPartitioned) vb));

    for (int i = 0; i < g.timeCoordPartitions.size(); i++)
      b.addTimeCoordUnions(writeTimeCoordUnionProto(g.timeCoordPartitions.get(i), i));

    List<VertCoord> vertCoords = g.vertCoords;
    for (int i = 0; i < vertCoords.size(); i++)
      b.addVertCoords(writeCoordProto(vertCoords.get(i), i));

    List<EnsCoord> ensCoords = g.ensCoords;
    for (int i = 0; i < ensCoords.size(); i++) b.addEnsCoords(writeCoordProto(ensCoords.get(i), i));
    return b.build();
  }

  private GribCollectionProto.Variable writeVariableProto(TimePartition.VariableIndexPartitioned v)
      throws IOException {
    GribCollectionProto.Variable.Builder b = GribCollectionProto.Variable.newBuilder();

    b.setDiscipline(v.discipline);
    b.setCategory(v.category);
    b.setParameter(v.parameter);
    b.setLevelType(v.levelType);
    b.setIsLayer(v.isLayer);
    b.setIntervalType(v.intvType);
    if (v.intvName != null) b.setIntvName(v.intvName);
    b.setCdmHash(v.cdmHash);
    b.setRecordsPos(0);
    b.setRecordsLen(0);
    b.setTimeIdx(v.partTimeCoordIdx); // note
    if (v.vertIdx >= 0) b.setVertIdx(v.vertIdx);
    if (v.ensIdx >= 0) b.setEnsIdx(v.ensIdx);
    if (v.ensDerivedType >= 0) b.setEnsDerivedType(v.ensDerivedType); // derived type (table 4.7)
    if (v.probabilityName != null) b.setProbabilityName(v.probabilityName);
    if (v.probType >= 0) b.setProbabilityType(v.probType);
    for (int idx : v.groupno) b.addGroupno(idx);
    for (int idx : v.varno) b.addVarno(idx);
    for (int idx : v.flag) b.addFlag(idx);

    return b.build();
  }

  protected GribCollectionProto.TimeCoordUnion writeTimeCoordUnionProto(
      TimeCoordUnion tcu, int index) throws IOException {
    GribCollectionProto.TimeCoordUnion.Builder b = GribCollectionProto.TimeCoordUnion.newBuilder();
    b.setCode(index);
    b.setUnit(tcu.getUnits());

    if (tcu.isInterval()) {
      for (TimeCoord.Tinv tinv : tcu.getIntervals()) {
        b.addValues((float) tinv.getBounds1());
        b.addBound((float) tinv.getBounds2());
      }
    } else {
      for (int value : tcu.getCoords()) b.addValues((float) value);
    }
    for (TimeCoordUnion.Val val : tcu.getValues()) {
      b.addPartition(val.getPartition());
      b.addIndex(val.getIndex());
    }

    return b.build();
  }

  private GribCollectionProto.Partition writePartitionProto(String name, TimePartition.Partition p)
      throws IOException {
    GribCollectionProto.Partition.Builder b = GribCollectionProto.Partition.newBuilder();

    b.setFilename(p.getIndexFilename());
    b.setName(name);

    return b.build();
  }

  ///////////////////////////////////////////////////////////////////////////
  // reading ncx

  @Override
  protected boolean readPartitions(GribCollectionProto.GribCollectionIndex proto) {
    for (int i = 0; i < proto.getPartitionsCount(); i++) {
      GribCollectionProto.Partition pp = proto.getPartitions(i);
      tp.addPartition(pp.getName(), pp.getFilename());
    }
    return proto.getPartitionsCount() > 0;
  }

  @Override
  protected void readTimePartitions(
      GribCollection.GroupHcs group, GribCollectionProto.Group proto) {
    List<TimeCoord> list = new ArrayList<TimeCoord>(proto.getTimeCoordUnionsCount());
    for (int i = 0; i < proto.getTimeCoordUnionsCount(); i++) {
      GribCollectionProto.TimeCoordUnion tpu = proto.getTimeCoordUnions(i);
      list.add(readTimePartition(tpu, i));
    }
    group.timeCoords = list;
  }

  /*
   protected TimeCoord readTimePartition(GribCollectionProto.TimeCoordUnion pc, int timeIndex) {
    int[] partition = new int[pc.getPartitionCount()];
    int[] index = new int[pc.getPartitionCount()];  // better be the same
    for (int i = 0; i < pc.getPartitionCount(); i++) {
      partition[i] = pc.getPartition(i);
      index[i] = pc.getIndex(i);
    }

    if (pc.getBoundCount() > 0) {  // its an interval
      List<TimeCoord.Tinv> coords = new ArrayList<TimeCoord.Tinv>(pc.getValuesCount());
      for (int i = 0; i < pc.getValuesCount(); i++)
        coords.add(new TimeCoord.Tinv((int) pc.getValues(i), (int) pc.getBound(i)));
      TimeCoordUnion tc =  new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
      return tc.setIndex( timeIndex);

    } else {
      List<Integer> coords = new ArrayList<Integer>(pc.getValuesCount());
      for (float value : pc.getValuesList())
        coords.add((int) value);
      TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
      return tc.setIndex( timeIndex);
    }
  }

   */

  protected TimeCoord readTimePartition(GribCollectionProto.TimeCoordUnion pc, int timeIndex) {
    int[] partition = new int[pc.getPartitionCount()];
    int[] index = new int[pc.getPartitionCount()]; // better be the same
    for (int i = 0; i < pc.getPartitionCount(); i++) {
      partition[i] = pc.getPartition(i);
      index[i] = pc.getIndex(i);
    }

    if (pc.getBoundCount() > 0) { // its an interval
      List<TimeCoord.Tinv> coords = new ArrayList<TimeCoord.Tinv>(pc.getValuesCount());
      for (int i = 0; i < pc.getValuesCount(); i++)
        coords.add(new TimeCoord.Tinv((int) pc.getValues(i), (int) pc.getBound(i)));
      TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
      return tc.setIndex(timeIndex);

    } else {
      List<Integer> coords = new ArrayList<Integer>(pc.getValuesCount());
      for (float value : pc.getValuesList()) coords.add((int) value);
      TimeCoordUnion tc = new TimeCoordUnion(pc.getCode(), pc.getUnit(), coords, partition, index);
      return tc.setIndex(timeIndex);
    }
  }

  @Override
  protected GribCollection.VariableIndex readVariable(
      GribCollectionProto.Variable pv, GribCollection.GroupHcs group) {
    int discipline = pv.getDiscipline();
    int category = pv.getCategory();
    int param = pv.getParameter();
    int levelType = pv.getLevelType();
    int intvType = pv.getIntervalType();
    String intvName = pv.getIntvName();
    boolean isLayer = pv.getIsLayer();
    int ensDerivedType = pv.getEnsDerivedType();
    int probType = pv.getProbabilityType();
    String probabilityName = pv.getProbabilityName();
    int cdmHash = pv.getCdmHash();
    long recordsPos = pv.getRecordsPos();
    int recordsLen = pv.getRecordsLen();
    int timeIdx = pv.getTimeIdx();
    int vertIdx = pv.getVertIdx();
    int ensIdx = pv.getEnsIdx();
    int tableVersion = pv.getTableVersion();
    List<Integer> groupnoList = pv.getGroupnoList();
    List<Integer> varnoList = pv.getVarnoList();
    List<Integer> flagList = pv.getFlagList();

    return tp.makeVariableIndex(
        group,
        tableVersion,
        discipline,
        category,
        param,
        levelType,
        isLayer,
        intvType,
        intvName,
        ensDerivedType,
        probType,
        probabilityName,
        -1,
        cdmHash,
        timeIdx,
        vertIdx,
        ensIdx,
        recordsPos,
        recordsLen,
        groupnoList,
        varnoList,
        flagList);
  }

  public static void main(String[] args) throws IOException {
    Formatter f = new Formatter();
    String indexName = (args.length > 0) ? args[0] : "F:/nomads/NOMADS-cfsrr-timeseries.ncx";
    RandomAccessFile raf = new RandomAccessFile(indexName, "r");
    Grib1TimePartition gtc = Grib1TimePartitionBuilder.createFromIndex("test", null, raf);
    gtc.showIndex(f);
    System.out.printf("%s%n", f);
  }
}
Exemplo n.º 29
0
/**
 * Forecast Model Run Collection, manages dynamic collections of GridDatasets. Fmrc represents a
 * virtual dataset. To instantiate, you obtain an FmrcInv "snapshot" from which you can call
 * getDatatset().
 *
 * <p>Assumes that we dont have multiple runtimes in the same file. Can handle different time steps
 * in different files. Can handle different grids in different files. However this creates problems
 * for the "typical dataset". Cannot handle different ensembles in different files. (LOOK fix)
 * Cannot handle different levels in different files. ok
 *
 * @author caron
 * @since Jan 11, 2010
 */
@ThreadSafe
public class Fmrc {
  private static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fmrc.class);

  /**
   * Factory method
   *
   * @param collection describes the collection. May be one of:
   *     <ol>
   *       <li>collection specification string
   *       <li>catalog:catalogURL
   *       <li>filename.ncml
   *       <li>
   *     </ol>
   *     collectionSpec date extraction is used to get rundates
   * @param errlog place error messages here
   * @return Fmrc or null on error
   * @throws IOException on read error
   * @see
   *     "http://www.unidata.ucar.edu/software/netcdf-java/reference/collections/CollectionSpecification.html"
   */
  public static Fmrc open(String collection, Formatter errlog) throws IOException {
    if (collection.startsWith(MFileCollectionManager.CATALOG)) {
      CatalogCollectionManager manager = new CatalogCollectionManager(collection);
      return new Fmrc(manager, new FeatureCollectionConfig());

    } else if (collection.endsWith(".ncml")) {
      NcmlCollectionReader ncmlCollection = NcmlCollectionReader.open(collection, errlog);
      if (ncmlCollection == null) return null;
      Fmrc fmrc = new Fmrc(ncmlCollection.getCollectionManager(), new FeatureCollectionConfig());
      fmrc.setNcml(ncmlCollection.getNcmlOuter(), ncmlCollection.getNcmlInner());
      return fmrc;
    }

    return new Fmrc(collection, errlog);
  }

  public static Fmrc open(FeatureCollectionConfig config, Formatter errlog) throws IOException {
    if (config.spec.startsWith(MFileCollectionManager.CATALOG)) {
      CatalogCollectionManager manager = new CatalogCollectionManager(config.spec);
      return new Fmrc(manager, config);
    }

    return new Fmrc(config, errlog);
  }

  ////////////////////////////////////////////////////////////////////////
  private final CollectionManager manager;
  private final FeatureCollectionConfig config;

  // should be final
  // private Element ncmlOuter, ncmlInner;

  // the current state - changing must be thread safe
  private final Object lock = new Object();
  private FmrcDataset fmrcDataset;
  private volatile boolean forceProto = false;
  private volatile long lastInvChanged;
  private volatile long lastProtoChanged;

  private Fmrc(String collectionSpec, Formatter errlog) throws IOException {
    this.manager = MFileCollectionManager.open(collectionSpec, null, errlog);
    this.config = new FeatureCollectionConfig();
    this.config.spec = collectionSpec;
  }

  private Fmrc(FeatureCollectionConfig config, Formatter errlog) {
    this.manager = new MFileCollectionManager(config, errlog);
    this.config = config;
  }

  // from AggregationFmrc
  public Fmrc(CollectionManager manager, FeatureCollectionConfig config) {
    this.manager = manager;
    this.config = config;
  }

  public void setNcml(Element outerNcml, Element innerNcml) {
    config.protoConfig.outerNcml = outerNcml;
    config.innerNcml = innerNcml;
  }

  public void close() {
    manager.close();
  }

  // exposed for debugging

  public CollectionManager getManager() {
    return manager;
  }

  public FmrcInv getFmrcInv(Formatter debug) throws IOException {
    return makeFmrcInv(debug);
  }

  /////////////////////////////////////////////////////////////////////////////////////////

  public CalendarDateRange getDateRangeForRun(CalendarDate run) {
    return fmrcDataset.getDateRangeForRun(run);
  }

  public CalendarDateRange getDateRangeForOffset(double offset) {
    return fmrcDataset.getDateRangeForOffset(offset);
  }

  public List<CalendarDate> getRunDates() throws IOException {
    checkNeeded(false); // ??
    return fmrcDataset.getRunDates();
  }

  public List<CalendarDate> getForecastDates() throws IOException {
    checkNeeded(false); // ??
    return fmrcDataset.getForecastDates();
  }

  // for making offset datasets
  public double[] getForecastOffsets() throws IOException {
    checkNeeded(false); // ??
    return fmrcDataset.getForecastOffsets();
  }

  // LOOK : all of these guys could use ehcache
  public GridDataset getDataset2D(NetcdfDataset result) throws IOException {
    checkNeeded(false);
    GridDataset gds = fmrcDataset.getNetcdfDataset2D(result);
    return gds;
  }

  public GridDataset getDatasetBest() throws IOException {
    checkNeeded(false);
    GridDataset gds = fmrcDataset.getBest();
    return gds;
  }

  public GridDataset getDatasetBest(FeatureCollectionConfig.BestDataset bd) throws IOException {
    checkNeeded(false);
    GridDataset gds = fmrcDataset.getBest(bd);
    return gds;
  }

  public GridDataset getRunTimeDataset(CalendarDate run) throws IOException {
    checkNeeded(false);
    GridDataset gds = fmrcDataset.getRunTimeDataset(run);
    return gds;
  }

  public GridDataset getConstantForecastDataset(CalendarDate time) throws IOException {
    checkNeeded(false);
    GridDataset gds = fmrcDataset.getConstantForecastDataset(time);
    return gds;
  }

  public GridDataset getConstantOffsetDataset(double hour) throws IOException {
    checkNeeded(false);
    GridDataset gds = fmrcDataset.getConstantOffsetDataset(hour);
    return gds;
  }

  /////////////////////////////////////////

  public void updateProto() {
    forceProto = true;
  }

  public void update() {
    synchronized (lock) {
      boolean forceProtoLocal = forceProto;

      if (fmrcDataset == null) {
        try {
          fmrcDataset = new FmrcDataset(config);
        } catch (Throwable t) {
          logger.error(config.spec + ": initial fmrcDataset creation failed", t);
          // throw new RuntimeException(t);
        }
      }

      try {
        FmrcInv fmrcInv = makeFmrcInv(null);
        fmrcDataset.setInventory(fmrcInv, forceProtoLocal);
        logger.debug(config.spec + ": make new Dataset, new proto = {}", forceProtoLocal);
        if (forceProtoLocal) forceProto = false;
        this.lastInvChanged = System.currentTimeMillis();
        if (forceProtoLocal) this.lastProtoChanged = this.lastInvChanged;

      } catch (Throwable t) {
        logger.error(config.spec + ": makeFmrcInv failed", t);
        // throw new RuntimeException(t);
      }
    }
  }

  // true if things have changed since given time
  public boolean checkInvState(long lastInvChange) throws IOException {
    return this.lastInvChanged > lastInvChange;
  }
  // true if things have changed since given time
  public boolean checkProtoState(long lastProtoChanged) throws IOException {
    return this.lastProtoChanged > lastProtoChanged;
  }

  public void checkNeeded(boolean force) {
    synchronized (lock) {
      if (fmrcDataset == null) {
        try {
          manager.scan(true);
          update();
          return;
        } catch (Throwable t) {
          logger.error(config.spec + ": rescan failed");
          throw new RuntimeException(t);
        }
      }

      if (!force && !manager.isScanNeeded()) return;
      try {
        if (!manager.scan(true)) return;
        update();
      } catch (Throwable t) {
        logger.error(config.spec + ": rescan failed");
        throw new RuntimeException(t);
      }
    }
  }

  // scan has been done, create FmrcInv
  private FmrcInv makeFmrcInv(Formatter debug) throws IOException {
    try {
      Map<CalendarDate, FmrInv> fmrMap =
          new HashMap<CalendarDate, FmrInv>(); // all files are grouped by run date in an FmrInv
      List<FmrInv> fmrList = new ArrayList<FmrInv>(); // an fmrc is a collection of fmr

      // get the inventory, sorted by path
      for (MFile f : manager.getFiles()) {
        if (logger.isDebugEnabled()) logger.debug("Fmrc: " + config.spec + ": file=" + f.getPath());

        GridDatasetInv inv = null;
        try {
          inv =
              GridDatasetInv.open(
                  manager, f, config.innerNcml); // inventory is discovered for each GDS
        } catch (IOException ioe) {
          logger.warn("Error opening " + f.getPath() + "(skipped)", ioe);
          continue; // skip
        }

        CalendarDate runDate = inv.getRunDate();
        if (debug != null)
          debug.format("  opened %s rundate = %s%n", f.getPath(), inv.getRunDateString());

        // add to fmr for that rundate
        FmrInv fmr = fmrMap.get(runDate);
        if (fmr == null) {
          fmr = new FmrInv(runDate);
          fmrMap.put(runDate, fmr);
          fmrList.add(fmr);
        }
        fmr.addDataset(inv, debug);
      }
      if (debug != null) debug.format("%n");

      // finish the FmrInv
      Collections.sort(fmrList);
      for (FmrInv fmr : fmrList) {
        fmr.finish();
        if (logger.isDebugEnabled())
          logger.debug(
              "Fmrc: spec="
                  + config.spec
                  + ": fmr rundate="
                  + fmr.getRunDate()
                  + " nfiles= "
                  + fmr.getFiles().size());
      }

      return new FmrcInv(
          "fmrc:" + manager.getCollectionName(), fmrList, config.fmrcConfig.regularize);

    } catch (Throwable t) {
      logger.error("makeFmrcInv", t);
      throw new RuntimeException(t);
    }
  }

  public void showDetails(Formatter out) throws IOException {
    checkNeeded(false);
    fmrcDataset.showDetails(out);
  }

  public static void main(String[] args) throws IOException {
    Formatter errlog = new Formatter();

    String spec1 = "/data/testdata/ncml/nc/nam_c20s/NAM_CONUS_20km_surface_#yyyyMMdd_HHmm#.grib1";
    String spec2 = "/data/testdata/grid/grib/grib1/data/agg/.*grb";
    String spec3 = "/data/testdata/ncml/nc/ruc_conus40/RUC_CONUS_40km_#yyyyMMdd_HHmm#.grib1";
    String spec4 = "/data/testdata/cdmUnitTest/rtmodels/.*_nmm\\.GrbF[0-9]{5}$";

    String cat1 =
        "catalog:http://motherlode.ucar.edu:8080/thredds/catalog/fmrc/NCEP/RUC2/CONUS_40km/files/catalog.xml";
    String cat2 =
        "catalog:http://motherlode.ucar.edu:8080/thredds/catalog/fmrc/NCEP/NDFD/CONUS_5km/files/catalog.xml";

    String specH = "C:/data/datasets/nogaps/US058GMET-GR1mdl.*air_temp";
    String specH2 = "C:/data/ft/grid/cg/.*nc$";
    String specH3 = "C:/data/ft/grid/namExtract/#yyyyMMdd_HHmm#.*nc$";
    Fmrc fmrc = new Fmrc(specH3, errlog);
    System.out.printf("errlog = %s%n", errlog);
  }
}