/** Process the links to downloadable files */
  private Element processDownloadLinks(
      ServiceContext context, String id, String siteURL, List elems, Element response) {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");

    for (Iterator<Object> iter = elems.iterator(); iter.hasNext(); ) {
      Object ob = iter.next();
      if (ob instanceof Element) {
        Element elem = (Element) ob;
        elem = (Element) elem.clone();

        // --- get file name and href
        String fname = elem.getAttributeValue("name");
        String href = elem.getAttributeValue("href");

        if (href != null) {
          boolean local = href.startsWith(siteURL);
          boolean found = false;
          long size = 0;
          String dateModified = "";

          String linkPieces[] = href.split("\\&");

          // local file (possibly)
          if (linkPieces.length > 0) {

            if (local) {
              String access = "private";
              for (int i = 0; i < linkPieces.length; i++) {
                String lp = linkPieces[i];
                if (lp.startsWith("access=")) {
                  access = lp.substring(lp.indexOf('=') + 1);
                }
              }

              File dir = new File(Lib.resource.getDir(context, access, id));
              File file = new File(dir, fname);
              if (file.exists()) {
                size = file.length();
                Date date = new Date(file.lastModified());
                dateModified = sdf.format(date);
                found = true;
              }
            }
          }
          elem.setAttribute("size", size + "");
          elem.setAttribute("datemodified", dateModified);
          elem.setAttribute("found", found + "");
          elem.setAttribute("local", local + "");
          elem.setAttribute("download", "true");
          response.addContent(elem);
        } else {
          context.info("Unknown download link: " + Xml.getString(elem));
        }
      }
    }

    return response;
  }
  private Element doSearch(XmlRequest request, Search s) throws OperationAbortedEx {
    request.setAddress(params.getServletPath() + "/srv/en/" + Geonet.Service.XML_SEARCH);

    try {
      log.info("Searching on : " + params.name);
      Element response = request.execute(s.createRequest());
      if (log.isDebugEnabled()) log.debug("Search results:\n" + Xml.getString(response));

      return response;
    } catch (Exception e) {
      log.warning("Raised exception when searching : " + e);
      throw new OperationAbortedEx("Raised exception when searching", e);
    }
  }
Example #3
0
  /**
   * Does CSW GetRecordById request. If validation is requested and the metadata does not validate,
   * null is returned.
   *
   * @param uuid uuid of metadata to request
   * @return metadata the metadata
   */
  private Element retrieveMetadata(String uuid) {
    request.clearIds();
    request.addId(uuid);

    try {
      if (log.isDebugEnabled())
        log.debug("Getting record from : " + request.getHost() + " (uuid:" + uuid + ")");
      Element response = request.execute();
      if (log.isDebugEnabled()) log.debug("Record got:\n" + Xml.getString(response));

      @SuppressWarnings("unchecked")
      List<Element> list = response.getChildren();

      // --- maybe the metadata has been removed

      if (list.size() == 0) return null;

      response = list.get(0);
      response = (Element) response.detach();

      // validate it here if requested
      if (params.validate) {
        if (!dataMan.validate(response)) {
          log.info("Ignoring invalid metadata with uuid " + uuid);
          result.doesNotValidate++;
          return null;
        }
      }

      if (params.rejectDuplicateResource) {
        if (foundDuplicateForResource(uuid, response)) {
          return null;
        }
      }

      return response;
    } catch (Exception e) {
      log.warning("Raised exception while getting record : " + e);
      e.printStackTrace();
      result.unretrievable++;

      // --- we don't raise any exception here. Just try to go on
      return null;
    }
  }
Example #4
0
  public Z3950ServerResults harvest() throws Exception {
    Set<String> newUuids = new HashSet<String>();

    int groupSize = 10;

    log.info("Retrieving remote metadata information:" + params.uuid);

    Z3950ServerResults serverResults = new Z3950ServerResults();

    // --- Clean all before harvest : Remove/Add mechanism
    localUuids = new UUIDMapper(dbms, params.uuid);

    // --- remove old metadata
    for (String uuid : localUuids.getUUIDs()) {
      String id = localUuids.getID(uuid);
      if (log.isDebugEnabled()) log.debug("  - Removing old metadata before update with id: " + id);
      dataMan.deleteMetadataGroup(context, dbms, id);
      serverResults.locallyRemoved++;
    }

    if (serverResults.locallyRemoved > 0) dbms.commit();

    // --- Search remote node
    MetaSearcher s = searchMan.newSearcher(SearchManager.Z3950, Geonet.File.SEARCH_Z3950_CLIENT);

    ServiceConfig config = new ServiceConfig();

    Element request = new Element("request");

    // --- Z39.50 servers from harvest params
    for (String id : params.getRepositories()) {
      request.addContent(new Element(Geonet.SearchResult.SERVERS).setText(id));
    }

    // --- Z39.50 query from harvest params
    request.addContent(new Element(Geonet.SearchResult.ZQUERY).setText(params.query));

    // --- don't get html presentations (get them later)
    request.addContent(new Element(Geonet.SearchResult.SERVERHTML).setText("off"));

    // --- set timeout to be 100 seconds
    request.addContent(new Element(Geonet.SearchResult.TIMEOUT).setText("100"));
    // --- set hitsPerPage
    request.addContent(new Element(Geonet.SearchResult.HITS_PER_PAGE).setText(groupSize + ""));

    // --- do the search
    s.search(context, request, config);

    if (s.getSize() == 0) {
      log.error("Search failed or returned 0 results, trying again");
      s.search(context, request, config);
      if (s.getSize() == 0) {
        throw new Exception("Bad luck, Search failed or returned 0 results");
      }
    }

    if (log.isDebugEnabled()) log.debug("Search returned " + s.getSize() + " hits");

    // -- process the hits in groups of groupSize
    int numberOfHits = Math.min(Integer.parseInt(params.maximumHits), s.getSize());
    // -- add from and to placeholders to request

    request.addContent(new Element("from"));
    request.addContent(new Element("to"));

    Element categories = Lib.local.retrieve(dbms, "Categories");
    if (log.isDebugEnabled()) log.debug("categories " + Xml.getString(categories));

    Element repositories = new Info().getZRepositories(context, settingMan);
    if (log.isDebugEnabled()) log.debug("repos " + Xml.getString(repositories));

    // -- build a map of collection code versus repository name for
    // -- assigning the categories
    Map<String, String> codes = new HashMap<String, String>();
    Map<String, String> catCodes = new HashMap<String, String>();

    // -- add new category for each repository
    boolean addcateg = false;
    for (String repo : params.getRepositories()) {
      Element repoElem = Xml.selectElement(repositories, "record[id='" + repo + "']");
      if (repoElem != null) {
        Element repoId = repoElem.getChild("id");
        String repoName = repoElem.getChildText("name");
        codes.put(
            repoId.getAttributeValue("serverCode") + ":" + repoId.getAttributeValue("code"),
            repoName);
        // create a result holder for this repository
        serverResults.getServerResult(repoName);

        // sanitize the name of the category
        String categName = repoName.replaceAll("[^\\w]", "");
        categName = categName.toLowerCase();
        catCodes.put(
            repoId.getAttributeValue("serverCode") + ":" + repoId.getAttributeValue("code"),
            categName);

        if (Xml.selectElement(categories, "record[name='" + categName + "']") == null) {
          int newId = context.getSerialFactory().getSerial(dbms, "Categories");
          dbms.execute("INSERT INTO Categories(id, name) VALUES (?, ?)", newId, categName);
          Lib.local.insert(dbms, "Categories", newId, repoName);
          addcateg = true;
        }
      }
    }

    if (addcateg) dbms.commit();

    // --- return only maximum hits as directed by the harvest params
    int nrGroups = (numberOfHits / groupSize) + 1;
    for (int i = 1; i <= nrGroups; i++) {
      int lower = ((i - 1) * groupSize) + 1;
      int upper = Math.min((i * groupSize), numberOfHits);
      request.getChild("from").setText("" + lower);
      request.getChild("to").setText("" + upper);

      // --- Loading results
      List<Document> list = s.presentDocuments(context, request, config);

      // --- Loading categories and groups
      localCateg = new CategoryMapper(dbms);
      localGroups = new GroupMapper(dbms);

      if (log.isDebugEnabled())
        log.debug(
            "There are "
                + (list.size() - 1)
                + " children in the results ("
                + lower
                + " to "
                + upper
                + ")");

      boolean transformIt = false;
      String thisXslt = context.getAppPath() + Geonet.Path.IMPORT_STYLESHEETS + "/";
      if (!params.importXslt.equals("none")) {
        thisXslt = thisXslt + params.importXslt;
        transformIt = true;
      }

      // --- For each record....
      for (Document doc : list) {
        Element md = doc.getRootElement();
        String eName = md.getQualifiedName();
        if (eName.equals("summary")) continue;

        // -- Remove existing geonet:info children as for example
        // -- GeoNetwork Z39.50 server return when full mode
        // -- an extra element with server info not needed
        // -- once harvested
        String colCode = "";
        Element info = md.getChild(Edit.RootChild.INFO, Edit.NAMESPACE);
        if (info != null) {
          String serverCode = info.getChildText("server");
          int colPos = serverCode.indexOf(':');
          if (colPos != -1) {
            colCode = serverCode.substring(0, colPos) + ":" + info.getChildText("collection");
          }
        }
        md.removeChildren(Edit.RootChild.INFO, Edit.NAMESPACE);
        String repoName = codes.get(colCode);
        if (log.isDebugEnabled()) log.debug("Processing record from server " + repoName);
        HarvestResult result = serverResults.getServerResult(repoName);
        result.totalMetadata++;

        if (eName.equals("error")) {
          log.error("JZKit could not retrieve record - returned " + Xml.getString(md));
          result.unretrievable++;
          continue;
        }

        // transform using importxslt if not none
        if (transformIt) {
          try {
            if (log.isDebugEnabled()) log.debug("Before transform: " + Xml.getString(md));
            md = Xml.transform(md, thisXslt);
            if (log.isDebugEnabled()) log.debug("After transform: " + Xml.getString(md));
          } catch (Exception e) {
            System.out.println("Cannot transform XML, ignoring. Error was: " + e.getMessage());
            result.badFormat++;
            continue; // skip this one
          }
        }

        // detect schema, extract uuid and add
        String schema = dataMan.autodetectSchema(md, null);
        if (schema == null) {
          log.warning("Skipping metadata with unknown schema.");
          result.unknownSchema++;
          continue;
        }

        String uuid = null;
        try {
          uuid = dataMan.extractUUID(schema, md);
        } catch (Exception e) {
          log.error("Unable to extract UUID: " + e.getMessage());
          e.printStackTrace();
        }

        if (uuid == null || uuid.equals("")) {
          log.warning("Skipping metadata due to failure extracting uuid (uuid null or empty).");
          result.unretrievable++;
          continue;
        }

        log.info("  - Adding metadata with " + uuid);

        // --- generate a new metadata id

        int id = context.getSerialFactory().getSerial(dbms, "Metadata");
        // TODO end confusion about datatypes
        String id$ = Integer.toString(id);

        String docType = "";
        if (!transformIt && (doc.getDocType() != null)) {
          docType = Xml.getString(doc.getDocType());
        }

        // --- check for duplicate uuid - violates constraints on metadata table
        // --- if we attempt insert
        boolean alreadyAdded = !newUuids.add(uuid);
        boolean alreadyInDb = (dataMan.getMetadataId(dbms, uuid) != null);
        if (alreadyAdded || alreadyInDb) {
          log.error("Uuid " + uuid + " already exists in this set/database - cannot insert");
          result.couldNotInsert++;
          continue;
        }

        //
        // insert metadata
        //
        try {
          String groupOwner = "1", isTemplate = "n", title = null;
          int owner = 1;
          String category = null, createDate = new ISODate().toString(), changeDate = createDate;
          boolean ufo = false, indexImmediate = false;
          dataMan.insertMetadata(
              context,
              dbms,
              schema,
              md,
              id,
              uuid,
              owner,
              groupOwner,
              params.uuid,
              isTemplate,
              docType,
              title,
              category,
              createDate,
              changeDate,
              ufo,
              indexImmediate);

        } catch (Exception e) {
          log.error("Unable to insert metadata " + e.getMessage());
          e.printStackTrace();
          result.couldNotInsert++;
          continue;
        }

        addPrivileges(id$, params.getPrivileges(), localGroups, dataMan, context, dbms, log);
        addCategories(
            id$,
            params.getCategories(),
            localCateg,
            dataMan,
            dbms,
            context,
            log,
            catCodes.get(colCode));

        dataMan.setTemplateExt(dbms, id, "n", null);
        dataMan.setHarvestedExt(dbms, id, params.uuid, params.name);

        // validate it here if requested
        if (params.validate) {
          Document docVal;
          if (!transformIt && (doc.getDocType() != null)) {
            docVal = new Document(md, (DocType) doc.getDocType().detach());
          } else {
            docVal = new Document(md);
          }

          if (!dataMan.doValidate(dbms, schema, id$, docVal, context.getLanguage())) {
            result.doesNotValidate++;
          }
        }

        dataMan.indexMetadata(dbms, id$);

        result.addedMetadata++;
      }
    }

    dbms.commit();
    return serverResults;
  }
 public String toString() {
   return Xml.getString(getQuery());
 }
  public InformationFragment[] getFragment(
      int startingFragment, int count, RecordFormatSpecification spec) throws IRResultSetException {
    if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
      Log.debug(
          Geonet.Z3950_SERVER,
          "Request for fragment start:" + startingFragment + ", count:" + count);

    InformationFragment fragment[] = new InformationFragment[count];

    ExplicitRecordFormatSpecification rec_spec =
        new ExplicitRecordFormatSpecification("xml", null, "f");

    try {
      // build fragment data
      int from = startingFragment;
      int to = startingFragment + count - 1;

      Element request = new Element("request");
      request.addContent(new Element("from").setText(from + ""));
      request.addContent(new Element("to").setText(to + ""));
      ServiceConfig config = new ServiceConfig();

      if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
        Log.debug(Geonet.Z3950_SERVER, "Search request:\n" + Xml.getString(request));
      // get result set
      Element result = this.metasearcher.present(this.srvxtx, request, config);

      if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
        Log.debug(Geonet.Z3950_SERVER, "Search result:\n" + Xml.getString(result));

      // remove summary
      result.removeChildren("summary");
      @SuppressWarnings("unchecked")
      List<Element> list = result.getChildren();

      if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
        Log.debug(Geonet.Z3950_SERVER, "Set name asked:" + spec);

      // save other records to fragment
      for (int i = 0; i < count; i++) {
        Element md = list.get(0);
        md.detach();

        if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
          Log.debug(Geonet.Z3950_SERVER, "Returning fragment:\n" + Xml.getString(md));

        // add metadata

        // fragment[i] = new DOMTree("geonetwork", "geonetwork", null,   getRecord(md),rec_spec );
        // fragment[i].setHitNo(startingFragment+i);

        DOMOutputter outputter = new DOMOutputter();
        Document doc = new Document(md);
        org.w3c.dom.Document doc2 = outputter.output(doc);

        fragment[i] =
            new InformationFragmentImpl(
                startingFragment + i, "geonetwork", "geonetwork", null, doc2, rec_spec);
        // fragment[i] = new
        // InformationFragmentImpl(startingFragment+i,"geonetwork","geonetwork",null,doc,rec_spec);

        // System.err.println(fragment[i]);

      }
      this.srvxtx.getResourceManager().close();
      if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
        Log.debug(Geonet.Z3950_SERVER, "Fragment returned");
    } catch (Throwable e) {
      try {
        this.srvxtx.getResourceManager().abort();
      } catch (Exception e2) {
        e2.printStackTrace();
      }

      if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
        Log.debug(Geonet.Z3950_SERVER, "Exception: " + e.getClass().getName() + " " + e);
      // e.printStackTrace();
    }

    return fragment;
  }