Ejemplo n.º 1
0
  public int evaluate(int timeout) {
    try {
      if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
        Log.debug(Geonet.Z3950_SERVER, "INCOMING XML QUERY:\n" + query);

      Element request = new Element("request");
      request.addContent(query.toGNXMLRep());

      List<String> categories = query.getCollections();
      for (String category : categories) {
        if (!category.equals("geonetwork") && !category.equals("Default"))
          request.addContent(new Element("category").setText(category));
      }

      ServiceConfig config = new ServiceConfig();

      // perform the search and save search results

      metasearcher.search(this.srvxtx, request, config);

      // System.out.println("summary:\n" + Xml.getString(s.getSummary()));
      // // DEBUG

      // Random number of records.. Set up the result set
      setFragmentCount(metasearcher.getSize());
      setTaskStatusCode(IRResultSetStatus.COMPLETE);

      this.srvxtx.getResourceManager().close();
    } catch (Throwable e) {
      Log.error(Geonet.Z3950_SERVER, "error evaluating query.." + e);
      e.printStackTrace();

      try {
        this.srvxtx.getResourceManager().abort();
      } catch (Exception e2) {
        e2.printStackTrace();
      }
    }
    return (getStatus());
  }
Ejemplo n.º 2
0
  public Z3950ServerResults harvest() throws Exception {
    Set<String> newUuids = new HashSet<String>();

    int groupSize = 10;

    log.info("Retrieving remote metadata information:" + params.uuid);

    Z3950ServerResults serverResults = new Z3950ServerResults();

    // --- Clean all before harvest : Remove/Add mechanism
    localUuids = new UUIDMapper(dbms, params.uuid);

    // --- remove old metadata
    for (String uuid : localUuids.getUUIDs()) {
      String id = localUuids.getID(uuid);
      if (log.isDebugEnabled()) log.debug("  - Removing old metadata before update with id: " + id);
      dataMan.deleteMetadataGroup(context, dbms, id);
      serverResults.locallyRemoved++;
    }

    if (serverResults.locallyRemoved > 0) dbms.commit();

    // --- Search remote node
    MetaSearcher s = searchMan.newSearcher(SearchManager.Z3950, Geonet.File.SEARCH_Z3950_CLIENT);

    ServiceConfig config = new ServiceConfig();

    Element request = new Element("request");

    // --- Z39.50 servers from harvest params
    for (String id : params.getRepositories()) {
      request.addContent(new Element(Geonet.SearchResult.SERVERS).setText(id));
    }

    // --- Z39.50 query from harvest params
    request.addContent(new Element(Geonet.SearchResult.ZQUERY).setText(params.query));

    // --- don't get html presentations (get them later)
    request.addContent(new Element(Geonet.SearchResult.SERVERHTML).setText("off"));

    // --- set timeout to be 100 seconds
    request.addContent(new Element(Geonet.SearchResult.TIMEOUT).setText("100"));
    // --- set hitsPerPage
    request.addContent(new Element(Geonet.SearchResult.HITS_PER_PAGE).setText(groupSize + ""));

    // --- do the search
    s.search(context, request, config);

    if (s.getSize() == 0) {
      log.error("Search failed or returned 0 results, trying again");
      s.search(context, request, config);
      if (s.getSize() == 0) {
        throw new Exception("Bad luck, Search failed or returned 0 results");
      }
    }

    if (log.isDebugEnabled()) log.debug("Search returned " + s.getSize() + " hits");

    // -- process the hits in groups of groupSize
    int numberOfHits = Math.min(Integer.parseInt(params.maximumHits), s.getSize());
    // -- add from and to placeholders to request

    request.addContent(new Element("from"));
    request.addContent(new Element("to"));

    Element categories = Lib.local.retrieve(dbms, "Categories");
    if (log.isDebugEnabled()) log.debug("categories " + Xml.getString(categories));

    Element repositories = new Info().getZRepositories(context, settingMan);
    if (log.isDebugEnabled()) log.debug("repos " + Xml.getString(repositories));

    // -- build a map of collection code versus repository name for
    // -- assigning the categories
    Map<String, String> codes = new HashMap<String, String>();
    Map<String, String> catCodes = new HashMap<String, String>();

    // -- add new category for each repository
    boolean addcateg = false;
    for (String repo : params.getRepositories()) {
      Element repoElem = Xml.selectElement(repositories, "record[id='" + repo + "']");
      if (repoElem != null) {
        Element repoId = repoElem.getChild("id");
        String repoName = repoElem.getChildText("name");
        codes.put(
            repoId.getAttributeValue("serverCode") + ":" + repoId.getAttributeValue("code"),
            repoName);
        // create a result holder for this repository
        serverResults.getServerResult(repoName);

        // sanitize the name of the category
        String categName = repoName.replaceAll("[^\\w]", "");
        categName = categName.toLowerCase();
        catCodes.put(
            repoId.getAttributeValue("serverCode") + ":" + repoId.getAttributeValue("code"),
            categName);

        if (Xml.selectElement(categories, "record[name='" + categName + "']") == null) {
          int newId = context.getSerialFactory().getSerial(dbms, "Categories");
          dbms.execute("INSERT INTO Categories(id, name) VALUES (?, ?)", newId, categName);
          Lib.local.insert(dbms, "Categories", newId, repoName);
          addcateg = true;
        }
      }
    }

    if (addcateg) dbms.commit();

    // --- return only maximum hits as directed by the harvest params
    int nrGroups = (numberOfHits / groupSize) + 1;
    for (int i = 1; i <= nrGroups; i++) {
      int lower = ((i - 1) * groupSize) + 1;
      int upper = Math.min((i * groupSize), numberOfHits);
      request.getChild("from").setText("" + lower);
      request.getChild("to").setText("" + upper);

      // --- Loading results
      List<Document> list = s.presentDocuments(context, request, config);

      // --- Loading categories and groups
      localCateg = new CategoryMapper(dbms);
      localGroups = new GroupMapper(dbms);

      if (log.isDebugEnabled())
        log.debug(
            "There are "
                + (list.size() - 1)
                + " children in the results ("
                + lower
                + " to "
                + upper
                + ")");

      boolean transformIt = false;
      String thisXslt = context.getAppPath() + Geonet.Path.IMPORT_STYLESHEETS + "/";
      if (!params.importXslt.equals("none")) {
        thisXslt = thisXslt + params.importXslt;
        transformIt = true;
      }

      // --- For each record....
      for (Document doc : list) {
        Element md = doc.getRootElement();
        String eName = md.getQualifiedName();
        if (eName.equals("summary")) continue;

        // -- Remove existing geonet:info children as for example
        // -- GeoNetwork Z39.50 server return when full mode
        // -- an extra element with server info not needed
        // -- once harvested
        String colCode = "";
        Element info = md.getChild(Edit.RootChild.INFO, Edit.NAMESPACE);
        if (info != null) {
          String serverCode = info.getChildText("server");
          int colPos = serverCode.indexOf(':');
          if (colPos != -1) {
            colCode = serverCode.substring(0, colPos) + ":" + info.getChildText("collection");
          }
        }
        md.removeChildren(Edit.RootChild.INFO, Edit.NAMESPACE);
        String repoName = codes.get(colCode);
        if (log.isDebugEnabled()) log.debug("Processing record from server " + repoName);
        HarvestResult result = serverResults.getServerResult(repoName);
        result.totalMetadata++;

        if (eName.equals("error")) {
          log.error("JZKit could not retrieve record - returned " + Xml.getString(md));
          result.unretrievable++;
          continue;
        }

        // transform using importxslt if not none
        if (transformIt) {
          try {
            if (log.isDebugEnabled()) log.debug("Before transform: " + Xml.getString(md));
            md = Xml.transform(md, thisXslt);
            if (log.isDebugEnabled()) log.debug("After transform: " + Xml.getString(md));
          } catch (Exception e) {
            System.out.println("Cannot transform XML, ignoring. Error was: " + e.getMessage());
            result.badFormat++;
            continue; // skip this one
          }
        }

        // detect schema, extract uuid and add
        String schema = dataMan.autodetectSchema(md, null);
        if (schema == null) {
          log.warning("Skipping metadata with unknown schema.");
          result.unknownSchema++;
          continue;
        }

        String uuid = null;
        try {
          uuid = dataMan.extractUUID(schema, md);
        } catch (Exception e) {
          log.error("Unable to extract UUID: " + e.getMessage());
          e.printStackTrace();
        }

        if (uuid == null || uuid.equals("")) {
          log.warning("Skipping metadata due to failure extracting uuid (uuid null or empty).");
          result.unretrievable++;
          continue;
        }

        log.info("  - Adding metadata with " + uuid);

        // --- generate a new metadata id

        int id = context.getSerialFactory().getSerial(dbms, "Metadata");
        // TODO end confusion about datatypes
        String id$ = Integer.toString(id);

        String docType = "";
        if (!transformIt && (doc.getDocType() != null)) {
          docType = Xml.getString(doc.getDocType());
        }

        // --- check for duplicate uuid - violates constraints on metadata table
        // --- if we attempt insert
        boolean alreadyAdded = !newUuids.add(uuid);
        boolean alreadyInDb = (dataMan.getMetadataId(dbms, uuid) != null);
        if (alreadyAdded || alreadyInDb) {
          log.error("Uuid " + uuid + " already exists in this set/database - cannot insert");
          result.couldNotInsert++;
          continue;
        }

        //
        // insert metadata
        //
        try {
          String groupOwner = "1", isTemplate = "n", title = null;
          int owner = 1;
          String category = null, createDate = new ISODate().toString(), changeDate = createDate;
          boolean ufo = false, indexImmediate = false;
          dataMan.insertMetadata(
              context,
              dbms,
              schema,
              md,
              id,
              uuid,
              owner,
              groupOwner,
              params.uuid,
              isTemplate,
              docType,
              title,
              category,
              createDate,
              changeDate,
              ufo,
              indexImmediate);

        } catch (Exception e) {
          log.error("Unable to insert metadata " + e.getMessage());
          e.printStackTrace();
          result.couldNotInsert++;
          continue;
        }

        addPrivileges(id$, params.getPrivileges(), localGroups, dataMan, context, dbms, log);
        addCategories(
            id$,
            params.getCategories(),
            localCateg,
            dataMan,
            dbms,
            context,
            log,
            catCodes.get(colCode));

        dataMan.setTemplateExt(dbms, id, "n", null);
        dataMan.setHarvestedExt(dbms, id, params.uuid, params.name);

        // validate it here if requested
        if (params.validate) {
          Document docVal;
          if (!transformIt && (doc.getDocType() != null)) {
            docVal = new Document(md, (DocType) doc.getDocType().detach());
          } else {
            docVal = new Document(md);
          }

          if (!dataMan.doValidate(dbms, schema, id$, docVal, context.getLanguage())) {
            result.doesNotValidate++;
          }
        }

        dataMan.indexMetadata(dbms, id$);

        result.addedMetadata++;
      }
    }

    dbms.commit();
    return serverResults;
  }