Example #1
0
 @Override
 protected List<Metacard> query(CatalogFacade framework, int startIndex, Filter filter) {
   QueryImpl query = new QueryImpl(filter);
   query.setRequestsTotalResultsCount(false);
   query.setPageSize(batchSize);
   query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
   QueryRequest queryRequest = new QueryRequestImpl(query);
   query.setStartIndex(startIndex);
   SourceResponse response;
   try {
     LOGGER.debug("Querying with startIndex: {}", startIndex);
     response = framework.query(queryRequest);
   } catch (UnsupportedQueryException e) {
     printErrorMessage(String.format("Received error from Framework: %s%n", e.getMessage()));
     return null;
   } catch (SourceUnavailableException e) {
     printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
     return null;
   } catch (FederationException e) {
     printErrorMessage(String.format("Received error from Frameworks: %s%n", e.getMessage()));
     return null;
   }
   if (response.getProcessingDetails() != null && !response.getProcessingDetails().isEmpty()) {
     for (SourceProcessingDetails details : response.getProcessingDetails()) {
       LOGGER.debug("Got Issues: {}", details.getWarnings());
     }
     return null;
   }
   return response.getResults().stream().map(Result::getMetacard).collect(Collectors.toList());
 }
  @Test
  public void testResultNumReturnedIsZero()
      throws WfsException, TransformerConfigurationException, UnsupportedQueryException {
    // Setup
    final String TITLE = "title";
    final String searchPhrase = "*";
    final int pageSize = 1;

    WfsSource source =
        getWfsSource(
            ONE_TEXT_PROPERTY_SCHEMA,
            MockWfsServer.getFilterCapabilities(),
            Wfs20Constants.EPSG_4326_URN,
            3,
            false,
            true,
            0);

    QueryImpl query =
        new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
    query.setPageSize(pageSize);
    SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
    query.setSortBy(sortBy);
    QueryRequestImpl queryReq = new QueryRequestImpl(query);

    // Perform test
    SourceResponse resp = source.query(queryReq);
    assertEquals(3, resp.getResults().size());
  }
  /**
   * Verify that the SortBy is NOT set. In this case, sorting is not supported in the capabilities.
   */
  @Test
  public void testSortingAscendingSortingNotSupported() throws Exception {
    // Setup
    final String searchPhrase = "*";
    final String mockTemporalFeatureProperty = "myTemporalFeatureProperty";
    final String mockFeatureType = "{http://example.com}" + SAMPLE_FEATURE_NAME + 0;
    final int pageSize = 1;

    // Set ImplementsSorting to FALSE (sorting not supported)
    FilterCapabilities mockCapabilitiesSortingNotSupported = MockWfsServer.getFilterCapabilities();
    ConformanceType conformance = mockCapabilitiesSortingNotSupported.getConformance();
    List<DomainType> domainTypes = conformance.getConstraint();
    for (DomainType domainType : domainTypes) {
      if (StringUtils.equals(domainType.getName(), "ImplementsSorting")) {
        ValueType valueType = new ValueType();
        valueType.setValue("FALSE");
        domainType.setDefaultValue(valueType);
        break;
      }
    }

    WfsSource source =
        getWfsSource(
            ONE_TEXT_PROPERTY_SCHEMA,
            mockCapabilitiesSortingNotSupported,
            Wfs20Constants.EPSG_4326_URN,
            1,
            false,
            false,
            0);
    MetacardMapper mockMetacardMapper = mock(MetacardMapper.class);
    when(mockMetacardMapper.getFeatureType()).thenReturn(mockFeatureType);
    when(mockMetacardMapper.getSortByTemporalFeatureProperty())
        .thenReturn(mockTemporalFeatureProperty);
    List<MetacardMapper> mappers = new ArrayList<MetacardMapper>(1);
    mappers.add(mockMetacardMapper);
    source.setMetacardToFeatureMapper(mappers);

    QueryImpl query =
        new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
    query.setPageSize(pageSize);
    SortBy sortBy = new SortByImpl(Result.TEMPORAL, SortOrder.ASCENDING);
    query.setSortBy(sortBy);

    // Perform Test
    GetFeatureType featureType = source.buildGetFeatureRequest(query);

    // Verify
    QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
    assertFalse(queryType.isSetAbstractSortingClause());
  }
  /**
   * Verify that the SortBy is set with the mapped Feature Property and a DESC sort order. In this
   * case, the incoming sort property of TEMPORAL is mapped to myTemporalFeatureProperty.
   *
   * <p><?xml version="1.0" encoding="UTF-8" standalone="yes"?> <ns5:GetFeature startIndex="1"
   * count="1" service="WFS" version="2.0.0" xmlns:ns2="http://www.opengis.net/ows/1.1"
   * xmlns="http://www.opengis.net/fes/2.0" xmlns:ns4="http://www.opengis.net/gml"
   * xmlns:ns3="http://www.w3.org/1999/xlink" xmlns:ns5="http://www.opengis.net/wfs/2.0"> <ns5:Query
   * typeNames="SampleFeature0" handle="SampleFeature0"> <Filter> <PropertyIsLike wildCard="*"
   * singleChar="?" escapeChar="!"> <Literal>*</Literal> <ValueReference>title</ValueReference>
   * </PropertyIsLike> </Filter> <SortBy> <SortProperty>
   * <ValueReference>myTemporalFeatureProperty</ValueReference> <SortOrder>DESC</SortOrder>
   * </SortProperty> </SortBy> </ns5:Query> </ns5:GetFeature>
   */
  @Test
  public void testSortingDescendingSortingSupported() throws Exception {
    // Setup
    final String searchPhrase = "*";
    final String mockTemporalFeatureProperty = "myTemporalFeatureProperty";
    final String mockFeatureType = "{http://example.com}" + SAMPLE_FEATURE_NAME + 0;
    final int pageSize = 1;

    WfsSource source =
        getWfsSource(
            ONE_TEXT_PROPERTY_SCHEMA,
            MockWfsServer.getFilterCapabilities(),
            Wfs20Constants.EPSG_4326_URN,
            1,
            false,
            false,
            0);
    MetacardMapper mockMetacardMapper = mock(MetacardMapper.class);
    when(mockMetacardMapper.getFeatureType()).thenReturn(mockFeatureType);
    when(mockMetacardMapper.getSortByTemporalFeatureProperty())
        .thenReturn(mockTemporalFeatureProperty);
    List<MetacardMapper> mappers = new ArrayList<MetacardMapper>(1);
    mappers.add(mockMetacardMapper);
    source.setMetacardToFeatureMapper(mappers);

    QueryImpl query =
        new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
    query.setPageSize(pageSize);
    SortBy sortBy = new SortByImpl(Result.TEMPORAL, SortOrder.DESCENDING);
    query.setSortBy(sortBy);

    // Perform Test
    GetFeatureType featureType = source.buildGetFeatureRequest(query);

    // Verify
    QueryType queryType = (QueryType) featureType.getAbstractQueryExpression().get(0).getValue();
    JAXBElement<?> abstractSortingClause = queryType.getAbstractSortingClause();
    SortByType sortByType = (SortByType) abstractSortingClause.getValue();
    assertThat(
        sortByType.getSortProperty().get(0).getValueReference(), is(mockTemporalFeatureProperty));
    assertThat(
        sortByType.getSortProperty().get(0).getSortOrder().name(), is(SortOrderType.DESC.value()));
  }
  @Test
  public void testTypeNameHasNoPrefix() throws WfsException, UnsupportedQueryException {

    // Setup
    final String TITLE = "title";
    final String searchPhrase = "*";
    final int pageSize = 1;

    WfsSource source =
        getWfsSource(
            ONE_TEXT_PROPERTY_SCHEMA,
            MockWfsServer.getFilterCapabilities(),
            Wfs20Constants.EPSG_4326_URN,
            3,
            false,
            false,
            3);

    QueryImpl query =
        new QueryImpl(builder.attribute(Metacard.ANY_TEXT).is().like().text(searchPhrase));
    query.setPageSize(pageSize);
    SortBy sortBy = new SortByImpl(TITLE, SortOrder.DESCENDING);
    query.setSortBy(sortBy);

    // Perform test
    GetFeatureType featureType = source.buildGetFeatureRequest(query);

    // Validate
    List<JAXBElement<?>> queryList = featureType.getAbstractQueryExpression();
    for (JAXBElement<?> queryType : queryList) {
      Object val = queryType.getValue();
      QueryType queryTypeVal = (QueryType) val;
      assertThat(queryTypeVal.getTypeNames().get(0), containsString("SampleFeature"));
      assertThat(queryTypeVal.getTypeNames().get(0), is(not(containsString("Prefix"))));
      assertThat(queryTypeVal.getTypeNames().get(0), is(not(containsString(":"))));
    }
  }
Example #6
0
  @Override
  protected Object executeWithSubject() throws Exception {

    List<CatalogProvider> providers = getCatalogProviders();

    if (providers.isEmpty() || providers.size() < 2) {
      console.println("Not enough CatalogProviders installed to migrate");
      return null;
    }

    console.println("The \"FROM\" provider is: " + providers.get(0).getClass().getSimpleName());
    CatalogProvider provider = providers.get(1);
    console.println("The \"TO\" provider is: " + provider.getClass().getSimpleName());
    String answer = getInput("Do you wish to continue? (yes/no)");
    if (!"yes".equalsIgnoreCase(answer)) {
      console.println();
      console.println("Now exiting...");
      console.flush();
      return null;
    }

    ingestProvider = new Provider(provider);

    framework = getCatalog();

    start = System.currentTimeMillis();

    final Filter filter =
        (cqlFilter != null)
            ? CQL.toFilter(cqlFilter)
            : getFilter(getFilterStartTime(start), start, Metacard.MODIFIED);

    QueryImpl query = new QueryImpl(filter);
    query.setRequestsTotalResultsCount(true);
    query.setPageSize(batchSize);
    query.setSortBy(new SortByImpl(Metacard.MODIFIED, SortOrder.DESCENDING));
    QueryRequest queryRequest = new QueryRequestImpl(query);
    SourceResponse response;
    try {
      response = framework.query(queryRequest);
    } catch (FederationException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (SourceUnavailableException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    } catch (UnsupportedQueryException e) {
      printErrorMessage("Error occurred while querying the Framework." + e.getMessage());
      return null;
    }
    final long totalHits = response.getHits();
    final long totalPossible;
    if (totalHits == 0) {
      console.println("No records were found to replicate.");
      return null;
    }

    // If the maxMetacards is set, restrict the totalPossible to the number of maxMetacards
    if (maxMetacards > 0 && maxMetacards <= totalHits) {
      totalPossible = maxMetacards;
    } else {
      totalPossible = totalHits;
    }

    console.println("Starting migration for " + totalPossible + " Records");

    if (multithreaded > 1 && totalPossible > batchSize) {
      BlockingQueue<Runnable> blockingQueue = new ArrayBlockingQueue<Runnable>(multithreaded);
      RejectedExecutionHandler rejectedExecutionHandler = new ThreadPoolExecutor.CallerRunsPolicy();
      final ExecutorService executorService =
          new ThreadPoolExecutor(
              multithreaded,
              multithreaded,
              0L,
              TimeUnit.MILLISECONDS,
              blockingQueue,
              rejectedExecutionHandler);
      console.printf("Running %d threads during replication.%n", multithreaded);

      do {
        LOGGER.debug("In loop at iteration {}", queryIndex.get());
        executorService.submit(
            () -> {
              int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
              printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
            });
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
      executorService.shutdown();

      while (!executorService.isTerminated()) {
        try {
          TimeUnit.SECONDS.sleep(1);
        } catch (InterruptedException e) {
          // ignore
        }
      }
    } else {
      do {
        int count = queryAndIngest(framework, ingestProvider, queryIndex.get(), filter);
        printProgressAndFlush(start, totalPossible, ingestCount.addAndGet(count));
      } while (queryIndex.addAndGet(batchSize) <= totalPossible);
    }

    console.println();
    long end = System.currentTimeMillis();
    String completed =
        String.format(
            " %d record(s) replicated; %d record(s) failed; completed in %3.3f seconds.",
            ingestCount.get(), failedCount.get(), (end - start) / MS_PER_SECOND);
    LOGGER.info("Replication Complete: {}", completed);
    console.println(completed);

    return null;
  }