コード例 #1
0
 private GcsPath getGcsPath(String path) {
   try {
     return GcsPath.fromUri(path);
   } catch (IllegalArgumentException e) {
     throw new IllegalArgumentException(
         String.format(
             "%s expected a valid 'gs://' path but was given '%s'",
             dataflowOptions.getRunner().getSimpleName(), path),
         e);
   }
 }
コード例 #2
0
  private GcsUtil buildMockGcsUtil() throws IOException {
    GcsUtil mockGcsUtil = Mockito.mock(GcsUtil.class);

    // Any request to open gets a new bogus channel
    Mockito.when(mockGcsUtil.open(Mockito.any(GcsPath.class)))
        .thenReturn(new EmptySeekableByteChannel());

    // Any request for expansion gets a single bogus URL
    // after we first run the expansion code (which will generally
    // return no results, which causes a crash we aren't testing)
    Mockito.when(mockGcsUtil.expand(Mockito.any(GcsPath.class)))
        .thenReturn(Arrays.asList(GcsPath.fromUri("gs://bucket/foo")));

    return mockGcsUtil;
  }
コード例 #3
0
  private void testBase(String[] ARGS) throws IOException, GeneralSecurityException {
    // Run the pipeline.
    VariantSimilarity.main(ARGS);

    // Download the pipeline results.
    List<GraphResult> results = Lists.newArrayList();
    for (GcsPath path : helper.gcsUtil.expand(GcsPath.fromUri(outputPrefix + "*"))) {
      BufferedReader reader = helper.openOutput(path.toString());
      for (String line = reader.readLine(); line != null; line = reader.readLine()) {
        results.add(GraphResult.fromString(line));
      }
    }

    // Check the pipeline results.
    assertEquals(helper.PLATINUM_GENOMES_NUMBER_OF_SAMPLES, results.size());

    assertThat(results, CoreMatchers.allOf(CoreMatchers.hasItems(EXPECTED_RESULT)));
  }
コード例 #4
0
ファイル: TfIdf.java プロジェクト: ssesha/DataflowJavaSDK
  /** Lists documents contained beneath the {@code options.input} prefix/directory. */
  public static Set<URI> listInputDocuments(Options options)
      throws URISyntaxException, IOException {
    URI baseUri = new URI(options.getInput());

    // List all documents in the directory or GCS prefix.
    URI absoluteUri;
    if (baseUri.getScheme() != null) {
      absoluteUri = baseUri;
    } else {
      absoluteUri =
          new URI(
              "file",
              baseUri.getAuthority(),
              baseUri.getPath(),
              baseUri.getQuery(),
              baseUri.getFragment());
    }

    Set<URI> uris = new HashSet<>();
    if (absoluteUri.getScheme().equals("file")) {
      File directory = new File(absoluteUri);
      for (String entry : directory.list()) {
        File path = new File(directory, entry);
        uris.add(path.toURI());
      }
    } else if (absoluteUri.getScheme().equals("gs")) {
      GcsUtil gcsUtil = options.as(GcsOptions.class).getGcsUtil();
      URI gcsUriGlob =
          new URI(
              absoluteUri.getScheme(),
              absoluteUri.getAuthority(),
              absoluteUri.getPath() + "*",
              absoluteUri.getQuery(),
              absoluteUri.getFragment());
      for (GcsPath entry : gcsUtil.expand(GcsPath.fromUri(gcsUriGlob))) {
        uris.add(entry.toUri());
      }
    }

    return uris;
  }
コード例 #5
0
 @After
 public void tearDown() throws Exception {
   for (GcsPath path : helper.gcsUtil.expand(GcsPath.fromUri(outputPrefix + "*"))) {
     helper.deleteOutput(path.toString());
   }
 }