Example #1
0
 @Override
 protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
   QueryImpl query = new QueryImpl(filter);
   query.setRequestsTotalResultsCount(false);
   query.setPageSize(batchSize);
   query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
   QueryRequest queryRequest = new QueryRequestImpl(query);
   query.setStartIndex(startIndex);
   SourceResponse response;
   try {
     LOGGER.debug("Querying with startIndex: {}", startIndex);
     response = framework.query(queryRequest);
   } catch (UnsupportedQueryException e) {
     printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
     return null;
   } catch (SourceUnavailableException e) {
     printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
     return null;
   } catch (FederationException e) {
     printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
     return null;
   }
   if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
     for (SourceProcessingDetails details : response.getProcessingDetails()) {
       LOGGER.debug("Got Issues: {}", details.getWarnings());
     }
     return null;
   }
   return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
 }
  @Test
  public void testResultNumReturnedIsZero()
      throws WfsException, TransformerConfigurationException, UnsupportedQueryException {
    // Setup
    final String TITLE = "title";
    final String searchPhrase = "*";
    final int pageSize = 1;

    WfsSource source =
        getWfsSource(
            ONE_TEXT_PROPERTY_SCHEMA,
            MockWfsServer.getFilterCapabilities(),
            Wfs20Constants.EPSG_4326_URN,
            3,
            false,
            true,
            0);

    QueryImpl query =
        new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
    query.setPageSize(pageSize);
    SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
    query.setSortBy(sortBy);
    QueryRequestImpl queryReq = new QueryRequestImpl(query);

    // Perform test
    SourceResponse resp = source.query(queryReq);
    assertEquals(3, resp.getResults().size());
  }
  private CswRecordCollection buildCollection(
      SourceResponse sourceResponse, Map<String, Serializable> arguments) {

    CswRecordCollection recordCollection = new CswRecordCollection();

    recordCollection.setNumberOfRecordsMatched(sourceResponse.getHits());
    recordCollection.setNumberOfRecordsReturned(sourceResponse.getResults().size());
    recordCollection.setStartPosition(sourceResponse.getRequest().getQuery().getStartIndex());

    Object elementSetTypeArg = arguments.get(CswConstants.ELEMENT_SET_TYPE);
    if (elementSetTypeArg instanceof ElementSetType) {
      ElementSetType elementSetType = (ElementSetType) elementSetTypeArg;
      recordCollection.setElementSetType(elementSetType);
    }

    Object elementNamesArg = arguments.get(CswConstants.ELEMENT_NAMES);
    if (elementNamesArg instanceof QName[]) {
      QName[] qnames = (QName[]) elementNamesArg;
      if (qnames.length > 0) {
        List<QName> elementNames = new ArrayList();
        for (QName entry : qnames) {
          elementNames.add(entry);
        }
        recordCollection.setElementName(elementNames);
      }
    }

    Object isByIdQuery = arguments.get(CswConstants.IS_BY_ID_QUERY);
    if (isByIdQuery != null) {
      recordCollection.setById((Boolean) isByIdQuery);
    }

    Object arg = arguments.get((CswConstants.GET_RECORDS));
    if (arg != null && arg instanceof GetRecordsType) {
      recordCollection.setRequest((GetRecordsType) arg);
    }

    Object resultType = arguments.get(CswConstants.RESULT_TYPE_PARAMETER);
    if (resultType instanceof ResultType) {
      recordCollection.setResultType((ResultType) resultType);
    }

    Object outputSchema = arguments.get(CswConstants.OUTPUT_SCHEMA_PARAMETER);
    if (outputSchema instanceof String) {
      recordCollection.setOutputSchema((String) outputSchema);
    } else {
      recordCollection.setOutputSchema(CswConstants.CSW_OUTPUT_SCHEMA);
    }

    Object doWriteNamespaces = arguments.get(CswConstants.WRITE_NAMESPACES);
    if (doWriteNamespaces instanceof Boolean) {
      recordCollection.setDoWriteNamespaces((Boolean) doWriteNamespaces);
    }

    return recordCollection;
  }
  private void validateInput(SourceResponse sourceResponse, Map<String, Serializable> arguments) {

    if (null == sourceResponse) {
      throw new IllegalArgumentException("Null source response.");
    } else if (null == arguments) {
      throw new IllegalArgumentException("Null argument map.");
    } else if (null == sourceResponse.getResults()) {
      throw new IllegalArgumentException("Null results list.");
    } else if (!isByIdQuery(arguments)
        && null == arguments.get(CswConstants.RESULT_TYPE_PARAMETER)) {
      // An exception is thrown only if the query isn't by ID (i.e. it's not a GetRecordById
      // request) because GetRecordById does not use the ResultType attribute.
      throw new IllegalArgumentException("Null result type argument.");
    } else if (null == sourceResponse.getRequest()) {
      throw new IllegalArgumentException("Null source response query request.");
    } else if (null == sourceResponse.getRequest().getQuery()) {
      throw new IllegalArgumentException("Null source response query.");
    }
  }
  @Override
  public BinaryContent transform(SourceResponse sourceResponse, Map<String, Serializable> arguments)
      throws CatalogTransformerException {

    validateInput(sourceResponse, arguments);

    CswRecordCollection recordCollection = buildCollection(sourceResponse, arguments);

    ByteArrayInputStream bais;

    if (ResultType.VALIDATE.equals(recordCollection.getResultType())) {
      ByteArrayOutputStream baos = writeAcknowledgement(recordCollection.getRequest());
      bais = new ByteArrayInputStream(baos.toByteArray());
    } else {
      // "catches" recordCollection.getResultType() == null
      List<Result> results = sourceResponse.getResults();
      String xmlString = convert(recordCollection, results, arguments);
      bais = new ByteArrayInputStream(xmlString.getBytes(StandardCharsets.UTF_8));
    }

    BinaryContent transformedContent = new BinaryContentImpl(bais, new MimeType());
    return transformedContent;
  }
Example #6
0
  @Override
  protected Object doExecute() throws Exception {

    String formatString =
        "%1$-33s %2$-26s %3$-" + TITLE_MAX_LENGTH + "s %4$-" + EXCERPT_MAX_LENGTH + "s%n";

    CatalogFacade catalogProvider = getCatalog();

    Filter filter = null;
    if (cqlFilter != null) {
      filter = CQL.toFilter(cqlFilter);
    } else {
      if (searchPhrase == null) {
        searchPhrase = "*";
      }
      if (caseSensitive) {
        filter =
            getFilterBuilder()
                .attribute(Metacard.ANY_TEXT)
                .is()
                .like()
                .caseSensitiveText(searchPhrase);
      } else {
        filter = getFilterBuilder().attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase);
      }
    }

    QueryImpl query = new QueryImpl(filter);

    query.setRequestsTotalResultsCount(true);

    if (numberOfItems > -1) {
      query.setPageSize(numberOfItems);
    }

    long start = System.currentTimeMillis();

    SourceResponse response = catalogProvider.query(new QueryRequestImpl(query));

    long end = System.currentTimeMillis();

    int size = 0;
    if (response.getResults() != null) {
      size = response.getResults().size();
    }

    console.println();
    console.printf(
        " %d result(s) out of %s%d%s in %3.3f seconds",
        (size),
        Ansi.ansi().fg(Ansi.Color.CYAN).toString(),
        response.getHits(),
        Ansi.ansi().reset().toString(),
        (end - start) / MILLISECONDS_PER_SECOND);
    console.printf(formatString, "", "", "", "");
    printHeaderMessage(String.format(formatString, ID, DATE, TITLE, EXCERPT));

    for (Result result : response.getResults()) {
      Metacard metacard = result.getMetacard();

      String title = (metacard.getTitle() != null ? metacard.getTitle() : "N/A");
      String excerpt = "N/A";
      String modifiedDate = "";

      if (searchPhrase != null) {
        if (metacard.getMetadata() != null) {
          XPathHelper helper = new XPathHelper(metacard.getMetadata());
          String indexedText = helper.getDocument().getDocumentElement().getTextContent();
          indexedText = indexedText.replaceAll("\\r\\n|\\r|\\n", " ");

          String normalizedSearchPhrase = searchPhrase.replaceAll("\\*", "");

          int index = -1;

          if (caseSensitive) {
            index = indexedText.indexOf(normalizedSearchPhrase);
          } else {
            index = indexedText.toLowerCase().indexOf(normalizedSearchPhrase.toLowerCase());
          }

          if (index != -1) {
            int contextLength = (EXCERPT_MAX_LENGTH - normalizedSearchPhrase.length() - 8) / 2;
            excerpt = "..." + indexedText.substring(Math.max(index - contextLength, 0), index);
            excerpt = excerpt + Ansi.ansi().fg(Ansi.Color.GREEN).toString();
            excerpt =
                excerpt + indexedText.substring(index, index + normalizedSearchPhrase.length());
            excerpt = excerpt + Ansi.ansi().reset().toString();
            excerpt =
                excerpt
                    + indexedText.substring(
                        index + normalizedSearchPhrase.length(),
                        Math.min(
                            indexedText.length(),
                            index + normalizedSearchPhrase.length() + contextLength))
                    + "...";
          }
        }
      }

      if (metacard.getModifiedDate() != null) {
        modifiedDate =
            new DateTime(metacard.getModifiedDate().getTime()).toString(DATETIME_FORMATTER);
      }

      console.printf(
          formatString,
          metacard.getId(),
          modifiedDate,
          title.substring(0, Math.min(title.length(), TITLE_MAX_LENGTH)),
          excerpt);
    }

    return null;
  }
Example #7
0
  @Override
  protected Object executeWithSubject() throws Exception {

    List<CatalogProvider> providers = getCatalogProviders();

    if (providers.isEmpty() || providers.size() < 2) {
      console.println("Not enough CatalogProviders installed to migrate");
      return null;
    }

    console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
    CatalogProvider provider = providers.get(1);
    console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
    String answer = getInput("Do you wish to continue? (yes/no)");
    if (!"yes".equalsIgnoreCase(answer)) {
      console.println();
      console.println("Now exiting...");
      console.flush();
      return null;
    }

    ingestProvider = new Provider(provider);

    framework = getCatalog();

    start = System.currentTimeMillis();

    final Filter filter =
        (cqlFilter != null)
            ? CQL.toFilter(cqlFilter)
            : getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);

    QueryImpl query = new QueryImpl(filter);
    query.setRequestsTotalResultsCount(true);
    query.setPageSize(batchSize);
    query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
    QueryRequest queryRequest = new QueryRequestImpl(query);
    SourceResponse response;
    try {
      response = framework.query(queryRequest);
    } catch (FederationException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (SourceUnavailableException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (UnsupportedQueryException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    }
    final long totalHits = response.getHits();
    final long totalPossible;
    if (totalHits == 0) {
      console.println("No records were found to replicate.");
      return null;
    }

    // If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
    if (maxMetacards > 0 && maxMetacards <= totalHits) {
      totalPossible = maxMetacards;
    } else {
      totalPossible = totalHits;
    }

    console.println("Starting migration for " + totalPossible + " Records");

    if (multithreaded > 1 && totalPossible > batchSize) {
      BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
      RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
      final ExecutorService executorService =
          new ThreadPoolExecutor(
              multithreaded,
              multithreaded,
              0L,
              TimeUnit.MILLISECONDS,
              blockingQueue,
              rejectedExecutionHandler);
      console.printf("Running %d threads during replication.%n", multithreaded);

      do {
        LOGGER.debug("In loop at iteration {}", queryIndex.get());
        executorService.submit(
            () -> {
              int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
              printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
            });
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
      executorService.shutdown();

      while (!executorService.isTerminated()) {
        try {
          TimeUnit.SECONDS.sleep(1);
        } catch (InterruptedException e) {
          // ignore
        }
      }
    } else {
      do {
        int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
        printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
    }

    console.println();
    long end = System.currentTimeMillis();
    String completed =
        String.format(
            " %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
            ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
    LOGGER.info("Replication Complete: {}", completed);
    console.println(completed);

    return null;
  }
    @SuppressWarnings({"rawtypes", "unchecked"})
    @Override
    public void run() {
      String methodName = "run";
      logger.entry(methodName);

      SortBy sortBy = query.getSortBy();
      // Prepare the Comparators that we will use
      Comparator<Result> coreComparator = DEFAULT_COMPARATOR;

      if (sortBy != null && sortBy.getPropertyName() != null) {
        PropertyName sortingProp = sortBy.getPropertyName();
        String sortType = sortingProp.getPropertyName();
        SortOrder sortOrder =
            (sortBy.getSortOrder() == null) ? SortOrder.DESCENDING : sortBy.getSortOrder();
        logger.debug("Sorting by type: " + sortType);
        logger.debug("Sorting by Order: " + sortBy.getSortOrder());

        // Temporal searches are currently sorted by the effective time
        if (Metacard.EFFECTIVE.equals(sortType) || Result.TEMPORAL.equals(sortType)) {
          coreComparator = new TemporalResultComparator(sortOrder);
        } else if (Result.DISTANCE.equals(sortType)) {
          coreComparator = new DistanceResultComparator(sortOrder);
        } else if (Result.RELEVANCE.equals(sortType)) {
          coreComparator = new RelevanceResultComparator(sortOrder);
        }
      }

      List<Result> resultList = new ArrayList<Result>();
      long totalHits = 0;
      Set<ProcessingDetails> processingDetails = returnResults.getProcessingDetails();

      Map<String, Serializable> returnProperties = returnResults.getProperties();
      for (final Entry<Source, Future<SourceResponse>> entry : futures.entrySet()) {
        Source site = entry.getKey();
        SourceResponse sourceResponse = null;
        try {
          sourceResponse =
              query.getTimeoutMillis() < 1
                  ? entry.getValue().get()
                  : entry.getValue().get(getTimeRemaining(deadline), TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
          logger.warn(
              "Couldn't get results from completed federated query on site with ShortName "
                  + site.getId(),
              e);
          processingDetails.add(new ProcessingDetailsImpl(site.getId(), e));
        } catch (ExecutionException e) {
          logger.warn(
              "Couldn't get results from completed federated query on site " + site.getId(), e);
          if (logger.isDebugEnabled()) {
            logger.debug("Adding exception to response.");
          }
          processingDetails.add(new ProcessingDetailsImpl(site.getId(), e));
        } catch (TimeoutException e) {
          logger.warn("search timed out: " + new Date() + " on site " + site.getId());
          processingDetails.add(new ProcessingDetailsImpl(site.getId(), e));
        }
        if (sourceResponse != null) {
          List<Result> sourceResults = sourceResponse.getResults();
          resultList.addAll(sourceResults);
          long sourceHits = sourceResponse.getHits();

          totalHits += sourceHits;
          Map<String, Serializable> newSourceProperties = new HashMap<String, Serializable>();
          newSourceProperties.put(QueryResponse.TOTAL_HITS, sourceHits);
          newSourceProperties.put(QueryResponse.TOTAL_RESULTS_RETURNED, sourceResults.size());

          Map<String, Serializable> originalSourceProperties = sourceResponse.getProperties();
          if (originalSourceProperties != null) {
            Serializable object = originalSourceProperties.get(QueryResponse.ELAPSED_TIME);
            if (object != null && object instanceof Long) {
              newSourceProperties.put(QueryResponse.ELAPSED_TIME, (Long) object);
              originalSourceProperties.remove(QueryResponse.ELAPSED_TIME);
              logger.debug(
                  "Setting the ellapsedTime responseProperty to {} for source {}",
                  object,
                  site.getId());
            }

            // TODO: for now add all properties into outgoing response's properties.
            // this is not the best idea because we could get properties from records
            // that get eliminated by the max results enforcement done below.
            // See DDF-1183 for a possible solution.
            returnProperties.putAll(originalSourceProperties);
          }
          returnProperties.put(site.getId(), (Serializable) newSourceProperties);
          logger.debug("Setting the query responseProperties for site {}", site.getId());

          // Add a List of siteIds so endpoints know what sites got queried
          Serializable siteListObject = returnProperties.get(QueryResponse.SITE_LIST);
          if (siteListObject != null && siteListObject instanceof List<?>) {
            ((List) siteListObject).add(site.getId());
          } else {
            siteListObject = new ArrayList<String>();
            ((List) siteListObject).add(site.getId());
            returnProperties.put(QueryResponse.SITE_LIST, (Serializable) siteListObject);
          }
        }
      }
      logger.debug("all sites finished returning results: " + resultList.size());

      Collections.sort(resultList, coreComparator);

      returnResults.setHits(totalHits);
      int maxResults = query.getPageSize() > 0 ? query.getPageSize() : Integer.MAX_VALUE;

      returnResults.addResults(
          resultList.size() > maxResults ? resultList.subList(0, maxResults) : resultList, true);
    }