/**
   * Queries the {@link #rdfEndpoint(String)} with each of the {@link #rdfQueries} and harvests the
   * results of the query.
   */
  private void harvestFromEndpoint() {

    Query query;
    QueryExecution qExec;

    for (String rdfQuery : rdfQueries) {
      if (closed) break;

      logger.info(
          "Harvesting with query: [{}] on index [{}] and type [{}]", rdfQuery, indexName, typeName);

      try {
        query = QueryFactory.create(rdfQuery);
      } catch (QueryParseException qpe) {
        logger.error("Could not parse [{}]. Please provide a relevant query. {}", rdfQuery, qpe);
        continue;
      }

      qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);

      try {
        harvest(qExec);
      } catch (Exception e) {
        logger.error("Exception [{}] occurred while harvesting", e.getLocalizedMessage());
      } finally {
        qExec.close();
      }
    }
  }
Beispiel #2
0
  void runTestSelect(Query query, QueryExecution qe) throws Exception {
    // Do the query!
    ResultSetRewindable resultsActual = ResultSetFactory.makeRewindable(qe.execSelect());

    qe.close();

    if (results == null) return;

    // Assumes resultSetCompare can cope with full isomorphism possibilities.
    ResultSetRewindable resultsExpected;
    if (results.isResultSet())
      resultsExpected = ResultSetFactory.makeRewindable(results.getResultSet());
    else if (results.isModel())
      resultsExpected = ResultSetFactory.makeRewindable(results.getModel());
    else {
      fail("Wrong result type for SELECT query");
      resultsExpected = null; // Keep the compiler happy
    }

    if (query.isReduced()) {
      // Reduced - best we can do is DISTINCT
      resultsExpected = unique(resultsExpected);
      resultsActual = unique(resultsActual);
    }

    // Hack for CSV : tests involving bNodes need manually checking.
    if (testItem.getResultFile().endsWith(".csv")) {
      resultsActual = convertToStrings(resultsActual);
      resultsActual.reset();

      int nActual = ResultSetFormatter.consume(resultsActual);
      int nExpected = ResultSetFormatter.consume(resultsExpected);
      resultsActual.reset();
      resultsExpected.reset();
      assertEquals("CSV: Different number of rows", nExpected, nActual);
      boolean b = resultSetEquivalent(query, resultsExpected, resultsActual);
      if (!b) System.out.println("Manual check of CSV results required: " + testItem.getName());
      return;
    }

    boolean b = resultSetEquivalent(query, resultsExpected, resultsActual);

    if (!b) {
      resultsExpected.reset();
      resultsActual.reset();
      boolean b2 = resultSetEquivalent(query, resultsExpected, resultsActual);
      printFailedResultSetTest(query, qe, resultsExpected, resultsActual);
    }
    assertTrue("Results do not match: " + testItem.getName(), b);

    return;
  }
Beispiel #3
0
  private void setAnyTimeouts(QueryExecution qexec, HttpAction action) {
    //        if ( !(action.getDataService().allowTimeoutOverride) )
    //            return ;

    long desiredTimeout = Long.MAX_VALUE;
    String timeoutHeader = action.request.getHeader("Timeout");
    String timeoutParameter = action.request.getParameter("timeout");
    if (timeoutHeader != null) {
      try {
        desiredTimeout = (int) (Float.parseFloat(timeoutHeader) * 1000);
      } catch (NumberFormatException e) {
        throw new FusekiException("Timeout header must be a number", e);
      }
    } else if (timeoutParameter != null) {
      try {
        desiredTimeout = (int) (Float.parseFloat(timeoutParameter) * 1000);
      } catch (NumberFormatException e) {
        throw new FusekiException("timeout parameter must be a number", e);
      }
    }

    //        desiredTimeout = Math.min(action.getDataService().maximumTimeoutOverride,
    // desiredTimeout) ;
    if (desiredTimeout != Long.MAX_VALUE) qexec.setTimeout(desiredTimeout);
  }
  /**
   * Query SPARQL endpoint with a SELECT query
   *
   * @param qExec QueryExecution encapsulating the query
   * @return model retrieved by querying the endpoint
   */
  private Model getSelectModel(QueryExecution qExec) {
    Model model = ModelFactory.createDefaultModel();
    Graph graph = model.getGraph();
    ResultSet results = qExec.execSelect();

    while (results.hasNext()) {
      QuerySolution sol = results.next();
      String subject;
      String predicate;
      RDFNode object;

      try {
        subject = sol.getResource("s").toString();
        predicate = sol.getResource("p").toString();
        object = sol.get("o");
      } catch (NoSuchElementException e) {
        logger.error("SELECT query does not return a (?s ?p ?o) Triple");
        continue;
      }

      Node objNode;
      if (object.isLiteral()) {
        Literal obj = object.asLiteral();
        objNode = NodeFactory.createLiteral(obj.getString(), obj.getDatatype());
      } else {
        objNode = NodeFactory.createLiteral(object.toString());
      }

      graph.add(
          new Triple(NodeFactory.createURI(subject), NodeFactory.createURI(predicate), objNode));
    }

    return model;
  }
Beispiel #5
0
  /**
   * Perform the {@link QueryExecution} once.
   *
   * @param action
   * @param queryExecution
   * @param query
   * @param queryStringLog Informational string created from the initial query.
   * @return
   */
  protected SPARQLResult executeQuery(
      HttpAction action, QueryExecution queryExecution, Query query, String queryStringLog) {
    setAnyTimeouts(queryExecution, action);

    if (query.isSelectType()) {
      ResultSet rs = queryExecution.execSelect();

      // Force some query execution now.
      // If the timeout-first-row goes off, the output stream has not
      // been started so the HTTP error code is sent.

      rs.hasNext();

      // If we wanted perfect query time cancellation, we could consume
      // the result now to see if the timeout-end-of-query goes off.
      // rs = ResultSetFactory.copyResults(rs) ;

      // action.log.info(format("[%d] exec/select", action.id)) ;
      return new SPARQLResult(rs);
    }

    if (query.isConstructType()) {
      Dataset dataset = queryExecution.execConstructDataset();
      // action.log.info(format("[%d] exec/construct", action.id));
      return new SPARQLResult(dataset);
    }

    if (query.isDescribeType()) {
      Model model = queryExecution.execDescribe();
      // action.log.info(format("[%d] exec/describe", action.id)) ;
      return new SPARQLResult(model);
    }

    if (query.isAskType()) {
      boolean b = queryExecution.execAsk();
      // action.log.info(format("[%d] exec/ask", action.id)) ;
      return new SPARQLResult(b);
    }

    ServletOps.errorBadRequest("Unknown query type - " + queryStringLog);
    return null;
  }
Beispiel #6
0
  @Override
  protected void runTestForReal() throws Throwable {
    Query query = null;
    try {
      try {
        query = queryFromTestItem(testItem);
      } catch (QueryException qEx) {
        query = null;
        qEx.printStackTrace(System.err);
        fail("Parse failure: " + qEx.getMessage());
        throw qEx;
      }

      Dataset dataset = setUpDataset(query, testItem);
      if (dataset == null && !doesQueryHaveDataset(query)) fail("No dataset for query");

      QueryExecution qe = null;

      if (dataset == null) qe = QueryExecutionFactory.create(query, queryFileManager);
      else qe = QueryExecutionFactory.create(query, dataset);

      try {
        if (query.isSelectType()) runTestSelect(query, qe);
        else if (query.isConstructType()) runTestConstruct(query, qe);
        else if (query.isDescribeType()) runTestDescribe(query, qe);
        else if (query.isAskType()) runTestAsk(query, qe);
      } finally {
        qe.close();
      }
    } catch (IOException ioEx) {
      // log.debug("IOException: ",ioEx) ;
      fail("IOException: " + ioEx.getMessage());
      throw ioEx;
    } catch (NullPointerException ex) {
      throw ex;
    } catch (Exception ex) {
      ex.printStackTrace(System.err);
      fail("Exception: " + ex.getClass().getName() + ": " + ex.getMessage());
    }
  }
  /**
   * Returns the string value of the first of the properties in the uriDescriptionList for the given
   * resource (as an URI). In case the resource does not have any of the properties mentioned, its
   * URI is returned. The value is obtained by querying the endpoint and the endpoint is queried
   * repeatedly until it gives a response (value or the lack of it)
   *
   * <p>It is highly recommended that the list contains properties like labels or titles, with test
   * values.
   *
   * @param uri - the URI for which a label is required
   * @return a String value, either a label for the parameter or its value if no label is obtained
   *     from the endpoint
   */
  private String getLabelForUri(String uri) {
    String result;

    if (uriLabelCache.containsKey(uri)) {
      return uriLabelCache.get(uri);
    }

    for (String prop : uriDescriptionList) {
      String innerQuery = "SELECT ?r WHERE {<" + uri + "> <" + prop + "> ?r } LIMIT 1";

      try {
        Query query = QueryFactory.create(innerQuery);
        QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
        boolean keepTrying = true;
        while (keepTrying) {
          keepTrying = false;
          try {
            ResultSet results = qExec.execSelect();

            if (results.hasNext()) {
              QuerySolution sol = results.nextSolution();
              result = EEASettings.parseForJson(sol.getLiteral("r").getLexicalForm());
              if (!result.isEmpty()) {
                uriLabelCache.put(uri, result);
                return result;
              }
            }
          } catch (Exception e) {
            keepTrying = true;
            logger.warn("Could not get label for uri {}. Retrying.", uri);
          } finally {
            qExec.close();
          }
        }
      } catch (QueryParseException qpe) {
        logger.error("Exception for query {}. The label cannot be obtained", innerQuery);
      }
    }
    return uri;
  }
  /**
   * Get a set of unique queryObjName returned from a select query
   *
   * <p>Used to retrieve sets of modified objects used in sync
   *
   * @param rdfQuery query to execute
   * @param queryObjName name of the object returned
   * @return set of values for queryObjectName in the rdfQuery result
   */
  HashSet<String> executeSyncQuery(String rdfQuery, String queryObjName) {
    HashSet<String> rdfUrls = new HashSet<String>();

    Query query;
    try {
      query = QueryFactory.create(rdfQuery);
    } catch (QueryParseException qpe) {
      logger.warn(
          "Could not parse [{}]. Please provide a relevant query. {}",
          rdfQuery,
          qpe.getLocalizedMessage());
      return null;
    }

    QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
    try {
      ResultSet results = qExec.execSelect();

      while (results.hasNext()) {
        QuerySolution sol = results.nextSolution();
        try {
          String value = sol.getResource(queryObjName).toString();
          rdfUrls.add(value);
        } catch (NoSuchElementException e) {
          logger.error("Encountered a NoSuchElementException: " + e.getLocalizedMessage());
          return null;
        }
      }
    } catch (Exception e) {
      logger.error(
          "Encountered a [{}] while querying the endpoint for sync", e.getLocalizedMessage());
      return null;
    } finally {
      qExec.close();
    }

    return rdfUrls;
  }
Beispiel #9
0
  void runTestAsk(Query query, QueryExecution qe) throws Exception {
    boolean result = qe.execAsk();
    if (results != null) {
      if (results.isBoolean()) {
        boolean b = results.getBooleanResult();
        assertEquals("ASK test results do not match", b, result);
      } else {
        Model resultsAsModel = results.getModel();
        StmtIterator sIter =
            results.getModel().listStatements(null, RDF.type, ResultSetGraphVocab.ResultSet);
        if (!sIter.hasNext()) throw new QueryTestException("Can't find the ASK result");
        Statement s = sIter.nextStatement();
        if (sIter.hasNext()) throw new QueryTestException("Too many result sets in ASK result");
        Resource r = s.getSubject();
        Property p = resultsAsModel.createProperty(ResultSetGraphVocab.getURI() + "boolean");

        boolean x = r.getRequiredProperty(p).getBoolean();
        if (x != result) assertEquals("ASK test results do not match", x, result);
      }
    }
    return;
  }
Beispiel #10
0
 void runTestDescribe(Query query, QueryExecution qe) throws Exception {
   Model resultsActual = qe.execDescribe();
   compareGraphResults(resultsActual, query);
 }
Beispiel #11
0
 void runTestConstruct(Query query, QueryExecution qe) throws Exception {
   // Do the query!
   Model resultsActual = qe.execConstruct();
   compareGraphResults(resultsActual, query);
 }
 /**
  * Query SPARQL endpoint with a DESCRIBE query
  *
  * @param qExec QueryExecution encapsulating the query
  * @return model retrieved by querying the endpoint
  */
 private Model getDescribeModel(QueryExecution qExec) {
   return qExec.execDescribe(ModelFactory.createDefaultModel());
 }
 /**
  * Query SPARQL endpoint with a CONSTRUCT query
  *
  * @param qExec QueryExecution encapsulating the query
  * @return model retrieved by querying the endpoint
  */
 private Model getConstructModel(QueryExecution qExec) {
   return qExec.execConstruct(ModelFactory.createDefaultModel());
 }
  /**
   * Starts a harvester with predefined queries to synchronize with the changes from the SPARQL
   * endpoint
   */
  public boolean sync() {
    logger.info("Sync resources newer than {}", startTime);

    String rdfQueryTemplate =
        "PREFIX xsd:<http://www.w3.org/2001/XMLSchema#> "
            + "SELECT DISTINCT ?resource WHERE { "
            + " GRAPH ?graph { %s }"
            + " ?graph <%s> ?time .  %s "
            + " FILTER (?time > xsd:dateTime(\"%s\")) }";

    String queryStr =
        String.format(
            rdfQueryTemplate, syncConditions, syncTimeProp, graphSyncConditions, startTime);
    Set<String> syncUris = executeSyncQuery(queryStr, "resource");

    if (syncUris == null) {
      logger.error("Errors occurred during sync procedure. Aborting!");
      return false;
    }

    /**
     * If desired, query for old data that has the sync conditions modified
     *
     * <p>This option is useful in the case in which the application indexes resources that match
     * some conditions. In this case, if they are modified and no longer match the initial
     * conditions, they will not be synchronized. When syncOldData is True, the modified resources
     * that no longer match the conditions are deleted.
     */
    int deleted = 0;
    int count = 0;
    if (this.syncOldData) {
      SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss");
      queryStr =
          String.format(
              rdfQueryTemplate,
              syncConditions,
              syncTimeProp,
              graphSyncConditions,
              sdf.format(new Date(0)));

      HashSet<String> allIndexURIs = executeSyncQuery(queryStr, "resource");

      if (allIndexURIs == null) {
        logger.error("Errors occurred during modified content sync query. Aborting!");
        return false;
      }

      deleted = removeMissingUris(allIndexURIs);
    }

    /* Prepare a series of bulk uris to be described so we can make
     * a smaller number of calls to the SPARQL endpoint. */
    ArrayList<ArrayList<String>> bulks = new ArrayList<ArrayList<String>>();
    ArrayList<String> currentBulk = new ArrayList<String>();

    for (String uri : syncUris) {
      currentBulk.add(uri);

      if (currentBulk.size() == EEASettings.DEFAULT_BULK_SIZE) {
        bulks.add(currentBulk);
        currentBulk = new ArrayList<String>();
      }
    }

    if (currentBulk.size() > 0) {
      bulks.add(currentBulk);
    }

    /* Execute RDF queries for the resources in each bulk */
    for (ArrayList<String> bulk : bulks) {
      String syncQuery = getSyncQueryStr(bulk);

      try {
        Query query = QueryFactory.create(syncQuery);
        QueryExecution qExec = QueryExecutionFactory.sparqlService(rdfEndpoint, query);
        try {
          Model constructModel = ModelFactory.createDefaultModel();
          qExec.execConstruct(constructModel);
          BulkRequestBuilder bulkRequest = client.prepareBulk();

          /**
           * When adding the model to ES do not use toDescribeURIs as the query already returned the
           * correct labels.
           */
          addModelToES(constructModel, bulkRequest, false);
          count += bulk.size();
        } catch (Exception e) {
          logger.error("Error while querying for modified content. {}", e.getLocalizedMessage());
          return false;
        } finally {
          qExec.close();
        }
      } catch (QueryParseException qpe) {
        logger.warn(
            "Could not parse Sync query. Please provide a relevant query. {}",
            qpe.getLocalizedMessage());
        return false;
      }
    }
    logger.info(
        "Finished synchronisation: Deleted {}, Updated {}/{}", deleted, count, syncUris.size());
    return true;
  }