/**
   * Queries the {@link #rdfEndpoint(String)} with each of the {@link #rdfQueries} and harvests the
   * results of the query.
   */
  private void harvestFromEndpoint() {

    Query query;
    QueryExecution qExec;

    for (String rdfQuery : rdfQueries) {
      if (closed) break;

      logger.info(
          "Harvesting with query: [{}] on index [{}] and type [{}]", rdfQuery, indexName, typeName);

      try {
        query = QueryFactory.create(rdfQuery);
      } catch (QueryParseException qpe) {
        logger.error("Could not parse [{}]. Please provide a relevant query. {}", rdfQuery, qpe);
        continue;
      }

      qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);

      try {
        harvest(qExec);
      } catch (Exception e) {
        logger.error("Exception [{}] occurred while harvesting", e.getLocalizedMessage());
      } finally {
        qExec.close();
      }
    }
  }
Esempio n. 2
0
  void runTestSelect(Query query, QueryExecution qe) throws Exception {
    // Do the query!
    ResultSetRewindable resultsActual = ResultSetFactory.makeRewindable(qe.execSelect());

    qe.close();

    if (results == null) return;

    // Assumes resultSetCompare can cope with full isomorphism possibilities.
    ResultSetRewindable resultsExpected;
    if (results.isResultSet())
      resultsExpected = ResultSetFactory.makeRewindable(results.getResultSet());
    else if (results.isModel())
      resultsExpected = ResultSetFactory.makeRewindable(results.getModel());
    else {
      fail("Wrong result type for SELECT query");
      resultsExpected = null; // Keep the compiler happy
    }

    if (query.isReduced()) {
      // Reduced - best we can do is DISTINCT
      resultsExpected = unique(resultsExpected);
      resultsActual = unique(resultsActual);
    }

    // Hack for CSV : tests involving bNodes need manually checking.
    if (testItem.getResultFile().endsWith(".csv")) {
      resultsActual = convertToStrings(resultsActual);
      resultsActual.reset();

      int nActual = ResultSetFormatter.consume(resultsActual);
      int nExpected = ResultSetFormatter.consume(resultsExpected);
      resultsActual.reset();
      resultsExpected.reset();
      assertEquals("CSV: Different number of rows", nExpected, nActual);
      boolean b = resultSetEquivalent(query, resultsExpected, resultsActual);
      if (!b) System.out.println("Manual check of CSV results required: " + testItem.getName());
      return;
    }

    boolean b = resultSetEquivalent(query, resultsExpected, resultsActual);

    if (!b) {
      resultsExpected.reset();
      resultsActual.reset();
      boolean b2 = resultSetEquivalent(query, resultsExpected, resultsActual);
      printFailedResultSetTest(query, qe, resultsExpected, resultsActual);
    }
    assertTrue("Results do not match: " + testItem.getName(), b);

    return;
  }
Esempio n. 3
0
  @Override
  protected void runTestForReal() throws Throwable {
    Query query = null;
    try {
      try {
        query = queryFromTestItem(testItem);
      } catch (QueryException qEx) {
        query = null;
        qEx.printStackTrace(System.err);
        fail("Parse failure: " + qEx.getMessage());
        throw qEx;
      }

      Dataset dataset = setUpDataset(query, testItem);
      if (dataset == null && !doesQueryHaveDataset(query)) fail("No dataset for query");

      QueryExecution qe = null;

      if (dataset == null) qe = QueryExecutionFactory.create(query, queryFileManager);
      else qe = QueryExecutionFactory.create(query, dataset);

      try {
        if (query.isSelectType()) runTestSelect(query, qe);
        else if (query.isConstructType()) runTestConstruct(query, qe);
        else if (query.isDescribeType()) runTestDescribe(query, qe);
        else if (query.isAskType()) runTestAsk(query, qe);
      } finally {
        qe.close();
      }
    } catch (IOException ioEx) {
      // log.debug("IOException: ",ioEx) ;
      fail("IOException: " + ioEx.getMessage());
      throw ioEx;
    } catch (NullPointerException ex) {
      throw ex;
    } catch (Exception ex) {
      ex.printStackTrace(System.err);
      fail("Exception: " + ex.getClass().getName() + ": " + ex.getMessage());
    }
  }
  /**
   * Returns the string value of the first of the properties in the uriDescriptionList for the given
   * resource (as an URI). In case the resource does not have any of the properties mentioned, its
   * URI is returned. The value is obtained by querying the endpoint and the endpoint is queried
   * repeatedly until it gives a response (value or the lack of it)
   *
   * <p>It is highly recommended that the list contains properties like labels or titles, with test
   * values.
   *
   * @param uri - the URI for which a label is required
   * @return a String value, either a label for the parameter or its value if no label is obtained
   *     from the endpoint
   */
  private String getLabelForUri(String uri) {
    String result;

    if (uriLabelCache.containsKey(uri)) {
      return uriLabelCache.get(uri);
    }

    for (String prop : uriDescriptionList) {
      String innerQuery = "SELECT ?r WHERE {<" + uri + "> <" + prop + "> ?r } LIMIT 1";

      try {
        Query query = QueryFactory.create(innerQuery);
        QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
        boolean keepTrying = true;
        while (keepTrying) {
          keepTrying = false;
          try {
            ResultSet results = qExec.execSelect();

            if (results.hasNext()) {
              QuerySolution sol = results.nextSolution();
              result = EEASettings.parseForJson(sol.getLiteral("r").getLexicalForm());
              if (!result.isEmpty()) {
                uriLabelCache.put(uri, result);
                return result;
              }
            }
          } catch (Exception e) {
            keepTrying = true;
            logger.warn("Could not get label for uri {}. Retrying.", uri);
          } finally {
            qExec.close();
          }
        }
      } catch (QueryParseException qpe) {
        logger.error("Exception for query {}. The label cannot be obtained", innerQuery);
      }
    }
    return uri;
  }
  /**
   * Get a set of unique queryObjName returned from a select query
   *
   * <p>Used to retrieve sets of modified objects used in sync
   *
   * @param rdfQuery query to execute
   * @param queryObjName name of the object returned
   * @return set of values for queryObjectName in the rdfQuery result
   */
  HashSet<String> executeSyncQuery(String rdfQuery, String queryObjName) {
    HashSet<String> rdfUrls = new HashSet<String>();

    Query query;
    try {
      query = QueryFactory.create(rdfQuery);
    } catch (QueryParseException qpe) {
      logger.warn(
          "Could not parse [{}]. Please provide a relevant query. {}",
          rdfQuery,
          qpe.getLocalizedMessage());
      return null;
    }

    QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
    try {
      ResultSet results = qExec.execSelect();

      while (results.hasNext()) {
        QuerySolution sol = results.nextSolution();
        try {
          String value = sol.getResource(queryObjName).toString();
          rdfUrls.add(value);
        } catch (NoSuchElementException e) {
          logger.error("Encountered a NoSuchElementException: " + e.getLocalizedMessage());
          return null;
        }
      }
    } catch (Exception e) {
      logger.error(
          "Encountered a [{}] while querying the endpoint for sync", e.getLocalizedMessage());
      return null;
    } finally {
      qExec.close();
    }

    return rdfUrls;
  }
  /**
   * Starts a harvester with predefined queries to synchronize with the changes from the SPARQL
   * endpoint
   */
  public boolean sync() {
    logger.info("Sync resources newer than {}", startTime);

    String rdfQueryTemplate =
        "PREFIX xsd:<http://www.w3.org/2001/XMLSchema#> "
            + "SELECT DISTINCT ?resource WHERE { "
            + " GRAPH ?graph { %s }"
            + " ?graph <%s> ?time .  %s "
            + " FILTER (?time > xsd:dateTime(\"%s\")) }";

    String queryStr =
        String.format(
            rdfQueryTemplate, syncConditions, syncTimeProp, graphSyncConditions, startTime);
    Set<String> syncUris = executeSyncQuery(queryStr, "resource");

    if (syncUris == null) {
      logger.error("Errors occurred during sync procedure. Aborting!");
      return false;
    }

    /**
     * If desired, query for old data that has the sync conditions modified
     *
     * <p>This option is useful in the case in which the application indexes resources that match
     * some conditions. In this case, if they are modified and no longer match the initial
     * conditions, they will not be synchronized. When syncOldData is True, the modified resources
     * that no longer match the conditions are deleted.
     */
    int deleted = 0;
    int count = 0;
    if (this.syncOldData) {
      SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
      queryStr =
          String.format(
              rdfQueryTemplate,
              syncConditions,
              syncTimeProp,
              graphSyncConditions,
              sdf.format(new Date(0)));

      HashSet<String> allIndexURIs = executeSyncQuery(queryStr, "resource");

      if (allIndexURIs == null) {
        logger.error("Errors occurred during modified content sync query. Aborting!");
        return false;
      }

      deleted = removeMissingUris(allIndexURIs);
    }

    /* Prepare a series of bulk uris to be described so we can make
     * a smaller number of calls to the SPARQL endpoint. */
    ArrayList<ArrayList<String>> bulks = new ArrayList<ArrayList<String>>();
    ArrayList<String> currentBulk = new ArrayList<String>();

    for (String uri : syncUris) {
      currentBulk.add(uri);

      if (currentBulk.size() == EEASettings.DEFAULT_BULK_SIZE) {
        bulks.add(currentBulk);
        currentBulk = new ArrayList<String>();
      }
    }

    if (currentBulk.size() > 0) {
      bulks.add(currentBulk);
    }

    /* Execute RDF queries for the resources in each bulk */
    for (ArrayList<String> bulk : bulks) {
      String syncQuery = getSyncQueryStr(bulk);

      try {
        Query query = QueryFactory.create(syncQuery);
        QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
        try {
          Model constructModel = ModelFactory.createDefaultModel();
          qExec.execConstruct(constructModel);
          BulkRequestBuilder bulkRequest = client.prepareBulk();

          /**
           * When adding the model to ES do not use toDescribeURIs as the query already returned the
           * correct labels.
           */
          addModelToES(constructModel, bulkRequest, false);
          count += bulk.size();
        } catch (Exception e) {
          logger.error("Error while querying for modified content. {}", e.getLocalizedMessage());
          return false;
        } finally {
          qExec.close();
        }
      } catch (QueryParseException qpe) {
        logger.warn(
            "Could not parse Sync query. Please provide a relevant query. {}",
            qpe.getLocalizedMessage());
        return false;
      }
    }
    logger.info(
        "Finished synchronisation: Deleted {}, Updated {}/{}", deleted, count, syncUris.size());
    return true;
  }