private GcsPath getGcsPath(String path) {
try {
return GcsPath.fromUri(path);
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format(
"%s expected a valid 'gs://' path but was given '%s'",
dataflowOptions.getRunner().getSimpleName(), path),
e);
}
}
private GcsUtil buildMockGcsUtil() throws IOException {
GcsUtil mockGcsUtil = Mockito.mock(GcsUtil.class);
// Any request to open gets a new bogus channel
Mockito.when(mockGcsUtil.open(Mockito.any(GcsPath.class)))
.thenReturn(new EmptySeekableByteChannel());
// Any request for expansion gets a single bogus URL
// after we first run the expansion code (which will generally
// return no results, which causes a crash we aren't testing)
Mockito.when(mockGcsUtil.expand(Mockito.any(GcsPath.class)))
.thenReturn(Arrays.asList(GcsPath.fromUri("gs://bucket/foo")));
return mockGcsUtil;
}
private void testBase(String[] ARGS) throws IOException, GeneralSecurityException {
// Run the pipeline.
VariantSimilarity.main(ARGS);
// Download the pipeline results.
List<GraphResult> results = Lists.newArrayList();
for (GcsPath path : helper.gcsUtil.expand(GcsPath.fromUri(outputPrefix + "*"))) {
BufferedReader reader = helper.openOutput(path.toString());
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
results.add(GraphResult.fromString(line));
}
}
// Check the pipeline results.
assertEquals(helper.PLATINUM_GENOMES_NUMBER_OF_SAMPLES, results.size());
assertThat(results, CoreMatchers.allOf(CoreMatchers.hasItems(EXPECTED_RESULT)));
}
/** Lists documents contained beneath the {@code options.input} prefix/directory. */
public static Set<URI> listInputDocuments(Options options)
throws URISyntaxException, IOException {
URI baseUri = new URI(options.getInput());
// List all documents in the directory or GCS prefix.
URI absoluteUri;
if (baseUri.getScheme() != null) {
absoluteUri = baseUri;
} else {
absoluteUri =
new URI(
"file",
baseUri.getAuthority(),
baseUri.getPath(),
baseUri.getQuery(),
baseUri.getFragment());
}
Set<URI> uris = new HashSet<>();
if (absoluteUri.getScheme().equals("file")) {
File directory = new File(absoluteUri);
for (String entry : directory.list()) {
File path = new File(directory, entry);
uris.add(path.toURI());
}
} else if (absoluteUri.getScheme().equals("gs")) {
GcsUtil gcsUtil = options.as(GcsOptions.class).getGcsUtil();
URI gcsUriGlob =
new URI(
absoluteUri.getScheme(),
absoluteUri.getAuthority(),
absoluteUri.getPath() + "*",
absoluteUri.getQuery(),
absoluteUri.getFragment());
for (GcsPath entry : gcsUtil.expand(GcsPath.fromUri(gcsUriGlob))) {
uris.add(entry.toUri());
}
}
return uris;
}
@After
public void tearDown() throws Exception {
for (GcsPath path : helper.gcsUtil.expand(GcsPath.fromUri(outputPrefix + "*"))) {
helper.deleteOutput(path.toString());
}
}