/**
   * Push all new objects from the specified {@link Ref} to the remote.
   *
   * @param ref the local ref that points to new commit data
   * @param refspec the remote branch to push to
   */
  @Override
  public void pushNewData(Ref ref, String refspec, ProgressListener progress)
      throws SynchronizationException {
    Optional<Ref> remoteRef = HttpUtils.getRemoteRef(repositoryURL, refspec);
    checkPush(ref, remoteRef);
    beginPush();

    progress.setDescription("Uploading objects to " + refspec);
    progress.setProgress(0);

    CommitTraverser traverser = getPushTraverser(remoteRef);

    traverser.traverse(ref.getObjectId());

    List<ObjectId> toSend = new LinkedList<ObjectId>(traverser.commits);
    Collections.reverse(toSend);
    Set<ObjectId> have = new HashSet<ObjectId>(traverser.have);

    Deduplicator deduplicator = deduplicationService.createDeduplicator();
    try {
      sendPackedObjects(toSend, have, deduplicator, progress);
    } finally {
      deduplicator.release();
    }

    ObjectId originalRemoteRefValue = ObjectId.NULL;
    if (remoteRef.isPresent()) {
      originalRemoteRefValue = remoteRef.get().getObjectId();
    }

    String nameToSet =
        remoteRef.isPresent() ? remoteRef.get().getName() : Ref.HEADS_PREFIX + refspec;

    endPush(nameToSet, ref.getObjectId(), originalRemoteRefValue.toString());
  }
  /**
   * Fetch all new objects from the specified {@link Ref} from the remote.
   *
   * @param ref the remote ref that points to new commit data
   * @param fetchLimit the maximum depth to fetch
   */
  @Override
  public void fetchNewData(Ref ref, Optional<Integer> fetchLimit, ProgressListener progress) {

    CommitTraverser traverser = getFetchTraverser(fetchLimit);

    try {
      progress.setDescription("Fetching objects from " + ref.getName());
      traverser.traverse(ref.getObjectId());
      List<ObjectId> want = new LinkedList<ObjectId>();
      want.addAll(traverser.commits);
      Collections.reverse(want);
      Set<ObjectId> have = new HashSet<ObjectId>();
      have.addAll(traverser.have);
      while (!want.isEmpty()) {
        progress.setProgress(0);
        fetchMoreData(want, have, progress);
      }
    } catch (Exception e) {
      Throwables.propagate(e);
    }
  }
Beispiel #3
0
  /**
   * Executes the export operation using the parameters that have been specified.
   *
   * @return a FeatureCollection with the specified features
   */
  @Override
  protected SimpleFeatureStore _call() {
    final ObjectDatabase database = objectDatabase();
    if (filterFeatureTypeId != null) {
      RevObject filterType = database.getIfPresent(filterFeatureTypeId);
      checkArgument(
          filterType instanceof RevFeatureType, "Provided filter feature type is does not exist");
    }

    final SimpleFeatureStore targetStore = getTargetStore();

    final String refspec = resolveRefSpec();
    final String treePath = refspec.substring(refspec.indexOf(':') + 1);
    final RevTree rootTree = resolveRootTree(refspec);
    final NodeRef typeTreeRef = resolTypeTreeRef(refspec, treePath, rootTree);

    final ObjectId defaultMetadataId = typeTreeRef.getMetadataId();

    final RevTree typeTree = database.getTree(typeTreeRef.getObjectId());

    final ProgressListener progressListener = getProgressListener();

    progressListener.started();
    progressListener.setDescription(
        "Exporting from " + path + " to " + targetStore.getName().getLocalPart() + "... ");

    final Iterator<SimpleFeature> filtered;
    {
      final Iterator<SimpleFeature> plainFeatures =
          getFeatures(typeTree, database, defaultMetadataId, progressListener);

      Iterator<SimpleFeature> adaptedFeatures = adaptToArguments(plainFeatures, defaultMetadataId);

      Iterator<Optional<Feature>> transformed =
          Iterators.transform(adaptedFeatures, ExportOp.this.function);

      Iterator<SimpleFeature> result =
          Iterators.filter(
              Iterators.transform(
                  transformed,
                  new Function<Optional<Feature>, SimpleFeature>() {
                    @Override
                    public SimpleFeature apply(Optional<Feature> input) {
                      return (SimpleFeature) input.orNull();
                    }
                  }),
              Predicates.notNull());

      // check the resulting schema has something to contribute
      PeekingIterator<SimpleFeature> peekingIt = Iterators.peekingIterator(result);
      if (peekingIt.hasNext()) {
        Function<AttributeDescriptor, String> toString =
            new Function<AttributeDescriptor, String>() {
              @Override
              public String apply(AttributeDescriptor input) {
                return input.getLocalName();
              }
            };
        SimpleFeature peek = peekingIt.peek();
        Set<String> sourceAtts =
            new HashSet<String>(
                Lists.transform(peek.getFeatureType().getAttributeDescriptors(), toString));
        Set<String> targetAtts =
            new HashSet<String>(
                Lists.transform(targetStore.getSchema().getAttributeDescriptors(), toString));
        if (Sets.intersection(sourceAtts, targetAtts).isEmpty()) {
          throw new GeoToolsOpException(
              StatusCode.UNABLE_TO_ADD,
              "No common attributes between source and target feature types");
        }
      }

      filtered = peekingIt;
    }
    FeatureCollection<SimpleFeatureType, SimpleFeature> asFeatureCollection =
        new BaseFeatureCollection<SimpleFeatureType, SimpleFeature>() {

          @Override
          public FeatureIterator<SimpleFeature> features() {

            return new DelegateFeatureIterator<SimpleFeature>(filtered);
          }
        };

    // add the feature collection to the feature store
    final Transaction transaction;
    if (transactional) {
      transaction = new DefaultTransaction("create");
    } else {
      transaction = Transaction.AUTO_COMMIT;
    }
    try {
      targetStore.setTransaction(transaction);
      try {
        targetStore.addFeatures(asFeatureCollection);
        transaction.commit();
      } catch (final Exception e) {
        if (transactional) {
          transaction.rollback();
        }
        Throwables.propagateIfInstanceOf(e, GeoToolsOpException.class);
        throw new GeoToolsOpException(e, StatusCode.UNABLE_TO_ADD);
      } finally {
        transaction.close();
      }
    } catch (IOException e) {
      throw new GeoToolsOpException(e, StatusCode.UNABLE_TO_ADD);
    }

    progressListener.complete();

    return targetStore;
  }
  /**
   * Retrieve objects from the remote repository, and update have/want lists accordingly.
   * Specifically, any retrieved commits are removed from the want list and added to the have list,
   * and any parents of those commits are removed from the have list (it only represents the most
   * recent common commits.) Retrieved objects are added to the local repository, and the want/have
   * lists are updated in-place.
   *
   * @param want a list of ObjectIds that need to be fetched
   * @param have a list of ObjectIds that are in common with the remote repository
   * @param progress
   */
  private void fetchMoreData(
      final List<ObjectId> want, final Set<ObjectId> have, final ProgressListener progress) {
    final JsonObject message = createFetchMessage(want, have);
    final URL resourceURL;
    try {
      resourceURL = new URL(repositoryURL.toString() + "/repo/batchobjects");
    } catch (MalformedURLException e) {
      throw Throwables.propagate(e);
    }

    final HttpURLConnection connection;
    try {
      final Gson gson = new Gson();
      OutputStream out;
      final Writer writer;
      connection = (HttpURLConnection) resourceURL.openConnection();
      connection.setDoOutput(true);
      connection.setDoInput(true);
      connection.addRequestProperty("Accept-Encoding", "gzip");
      out = connection.getOutputStream();
      writer = new OutputStreamWriter(out);
      gson.toJson(message, writer);
      writer.flush();
      out.flush();
    } catch (IOException e) {
      throw Throwables.propagate(e);
    }

    final HttpUtils.ReportingInputStream in = HttpUtils.getResponseStream(connection);

    BinaryPackedObjects unpacker = new BinaryPackedObjects(localRepository.objectDatabase());
    BinaryPackedObjects.Callback callback =
        new BinaryPackedObjects.Callback() {
          @Override
          public void callback(Supplier<RevObject> supplier) {
            RevObject object = supplier.get();
            progress.setProgress(progress.getProgress() + 1);
            if (object instanceof RevCommit) {
              RevCommit commit = (RevCommit) object;
              want.remove(commit.getId());
              have.removeAll(commit.getParentIds());
              have.add(commit.getId());
            } else if (object instanceof RevTag) {
              RevTag tag = (RevTag) object;
              want.remove(tag.getId());
              have.remove(tag.getCommitId());
              have.add(tag.getId());
            }
          }
        };

    Stopwatch sw = Stopwatch.createStarted();
    IngestResults ingestResults = unpacker.ingest(in, callback);
    sw.stop();

    String msg =
        String.format(
            "Processed %,d objects. Inserted: %,d. Existing: %,d. Time: %s. Compressed size: %,d bytes. Uncompressed size: %,d bytes.",
            ingestResults.total(),
            ingestResults.getInserted(),
            ingestResults.getExisting(),
            sw,
            in.compressedSize(),
            in.unCompressedSize());
    LOGGER.info(msg);
    progress.setDescription(msg);
  }