public class ITBigQueryTest {
private static final Logger LOG = Logger.getLogger(ITBigQueryTest.class.getName());
private static final String DATASET = RemoteBigQueryHelper.generateDatasetName();
private static final String DESCRIPTION = "Test dataset";
private static final String OTHER_DATASET = RemoteBigQueryHelper.generateDatasetName();
private static final Field TIMESTAMP_FIELD_SCHEMA =
Field.builder("TimestampField", Field.Type.timestamp())
.mode(Field.Mode.NULLABLE)
.description("TimestampDescription")
.build();
private static final Field STRING_FIELD_SCHEMA =
Field.builder("StringField", Field.Type.string())
.mode(Field.Mode.NULLABLE)
.description("StringDescription")
.build();
private static final Field INTEGER_FIELD_SCHEMA =
Field.builder("IntegerField", Field.Type.integer())
.mode(Field.Mode.REPEATED)
.description("IntegerDescription")
.build();
private static final Field BOOLEAN_FIELD_SCHEMA =
Field.builder("BooleanField", Field.Type.bool())
.mode(Field.Mode.NULLABLE)
.description("BooleanDescription")
.build();
private static final Field RECORD_FIELD_SCHEMA =
Field.builder(
"RecordField",
Field.Type.record(
TIMESTAMP_FIELD_SCHEMA,
STRING_FIELD_SCHEMA,
INTEGER_FIELD_SCHEMA,
BOOLEAN_FIELD_SCHEMA))
.mode(Field.Mode.REQUIRED)
.description("RecordDescription")
.build();
private static final Schema TABLE_SCHEMA =
Schema.of(
TIMESTAMP_FIELD_SCHEMA,
STRING_FIELD_SCHEMA,
INTEGER_FIELD_SCHEMA,
BOOLEAN_FIELD_SCHEMA,
RECORD_FIELD_SCHEMA);
private static final Schema SIMPLE_SCHEMA = Schema.of(STRING_FIELD_SCHEMA);
private static final Schema QUERY_RESULT_SCHEMA =
Schema.builder()
.addField(
Field.builder("TimestampField", Field.Type.timestamp())
.mode(Field.Mode.NULLABLE)
.build())
.addField(
Field.builder("StringField", Field.Type.string()).mode(Field.Mode.NULLABLE).build())
.addField(
Field.builder("BooleanField", Field.Type.bool()).mode(Field.Mode.NULLABLE).build())
.build();
private static final String LOAD_FILE = "load.csv";
private static final String JSON_LOAD_FILE = "load.json";
private static final String EXTRACT_FILE = "extract.csv";
private static final String BUCKET = RemoteGcsHelper.generateBucketName();
private static final TableId TABLE_ID = TableId.of(DATASET, "testing_table");
private static final String CSV_CONTENT = "StringValue1\nStringValue2\n";
private static final String JSON_CONTENT =
"{"
+ "\"TimestampField\": \"2014-08-19 07:41:35.220 -05:00\","
+ "\"StringField\": \"stringValue\","
+ "\"IntegerField\": [\"0\", \"1\"],"
+ "\"BooleanField\": \"false\","
+ "\"RecordField\": {"
+ "\"TimestampField\": \"1969-07-20 20:18:04 UTC\","
+ "\"StringField\": null,"
+ "\"IntegerField\": [\"1\",\"0\"],"
+ "\"BooleanField\": \"true\""
+ "}"
+ "}\n"
+ "{"
+ "\"TimestampField\": \"2014-08-19 07:41:35.220 -05:00\","
+ "\"StringField\": \"stringValue\","
+ "\"IntegerField\": [\"0\", \"1\"],"
+ "\"BooleanField\": \"false\","
+ "\"RecordField\": {"
+ "\"TimestampField\": \"1969-07-20 20:18:04 UTC\","
+ "\"StringField\": null,"
+ "\"IntegerField\": [\"1\",\"0\"],"
+ "\"BooleanField\": \"true\""
+ "}"
+ "}";
private static BigQuery bigquery;
private static Storage storage;
@Rule public Timeout globalTimeout = Timeout.seconds(300);
@BeforeClass
public static void beforeClass() throws InterruptedException {
RemoteBigQueryHelper bigqueryHelper = RemoteBigQueryHelper.create();
RemoteGcsHelper gcsHelper = RemoteGcsHelper.create();
bigquery = bigqueryHelper.options().service();
storage = gcsHelper.options().service();
storage.create(BucketInfo.of(BUCKET));
storage.create(
BlobInfo.builder(BUCKET, LOAD_FILE).contentType("text/plain").build(),
CSV_CONTENT.getBytes(StandardCharsets.UTF_8));
storage.create(
BlobInfo.builder(BUCKET, JSON_LOAD_FILE).contentType("application/json").build(),
JSON_CONTENT.getBytes(StandardCharsets.UTF_8));
DatasetInfo info = DatasetInfo.builder(DATASET).description(DESCRIPTION).build();
bigquery.create(info);
LoadJobConfiguration configuration =
LoadJobConfiguration.builder(
TABLE_ID, "gs://" + BUCKET + "/" + JSON_LOAD_FILE, FormatOptions.json())
.createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
.schema(TABLE_SCHEMA)
.build();
Job job = bigquery.create(JobInfo.of(configuration));
while (!job.isDone()) {
Thread.sleep(1000);
}
assertNull(job.status().error());
}
@AfterClass
public static void afterClass() throws ExecutionException, InterruptedException {
if (bigquery != null) {
RemoteBigQueryHelper.forceDelete(bigquery, DATASET);
}
if (storage != null) {
boolean wasDeleted = RemoteGcsHelper.forceDelete(storage, BUCKET, 10, TimeUnit.SECONDS);
if (!wasDeleted && LOG.isLoggable(Level.WARNING)) {
LOG.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
}
}
}
@Test
public void testGetDataset() {
Dataset dataset = bigquery.getDataset(DATASET);
assertEquals(bigquery.options().projectId(), dataset.datasetId().project());
assertEquals(DATASET, dataset.datasetId().dataset());
assertEquals(DESCRIPTION, dataset.description());
assertNotNull(dataset.acl());
assertNotNull(dataset.etag());
assertNotNull(dataset.id());
assertNotNull(dataset.lastModified());
assertNotNull(dataset.selfLink());
}
@Test
public void testGetDatasetWithSelectedFields() {
Dataset dataset =
bigquery.getDataset(DATASET, DatasetOption.fields(DatasetField.CREATION_TIME));
assertEquals(bigquery.options().projectId(), dataset.datasetId().project());
assertEquals(DATASET, dataset.datasetId().dataset());
assertNotNull(dataset.creationTime());
assertNull(dataset.description());
assertNull(dataset.defaultTableLifetime());
assertNull(dataset.acl());
assertNull(dataset.etag());
assertNull(dataset.friendlyName());
assertNull(dataset.id());
assertNull(dataset.lastModified());
assertNull(dataset.location());
assertNull(dataset.selfLink());
}
@Test
public void testUpdateDataset() {
Dataset dataset =
bigquery.create(DatasetInfo.builder(OTHER_DATASET).description("Some Description").build());
assertNotNull(dataset);
assertEquals(bigquery.options().projectId(), dataset.datasetId().project());
assertEquals(OTHER_DATASET, dataset.datasetId().dataset());
assertEquals("Some Description", dataset.description());
Dataset updatedDataset =
bigquery.update(dataset.toBuilder().description("Updated Description").build());
assertEquals("Updated Description", updatedDataset.description());
assertTrue(dataset.delete());
}
@Test
public void testUpdateDatasetWithSelectedFields() {
Dataset dataset =
bigquery.create(DatasetInfo.builder(OTHER_DATASET).description("Some Description").build());
assertNotNull(dataset);
assertEquals(bigquery.options().projectId(), dataset.datasetId().project());
assertEquals(OTHER_DATASET, dataset.datasetId().dataset());
assertEquals("Some Description", dataset.description());
Dataset updatedDataset =
bigquery.update(
dataset.toBuilder().description("Updated Description").build(),
DatasetOption.fields(DatasetField.DESCRIPTION));
assertEquals("Updated Description", updatedDataset.description());
assertNull(updatedDataset.creationTime());
assertNull(updatedDataset.defaultTableLifetime());
assertNull(updatedDataset.acl());
assertNull(updatedDataset.etag());
assertNull(updatedDataset.friendlyName());
assertNull(updatedDataset.id());
assertNull(updatedDataset.lastModified());
assertNull(updatedDataset.location());
assertNull(updatedDataset.selfLink());
assertTrue(dataset.delete());
}
@Test
public void testGetNonExistingTable() {
assertNull(bigquery.getTable(DATASET, "test_get_non_existing_table"));
}
@Test
public void testCreateAndGetTable() {
String tableName = "test_create_and_get_table";
TableId tableId = TableId.of(DATASET, tableName);
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(tableName, createdTable.tableId().table());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);
assertTrue(remoteTable.definition() instanceof StandardTableDefinition);
assertEquals(createdTable.tableId(), remoteTable.tableId());
assertEquals(TableDefinition.Type.TABLE, remoteTable.definition().type());
assertEquals(TABLE_SCHEMA, remoteTable.definition().schema());
assertNotNull(remoteTable.creationTime());
assertNotNull(remoteTable.lastModifiedTime());
assertNotNull(remoteTable.<StandardTableDefinition>definition().numBytes());
assertNotNull(remoteTable.<StandardTableDefinition>definition().numRows());
assertTrue(remoteTable.delete());
}
@Test
public void testCreateAndGetTableWithSelectedField() {
String tableName = "test_create_and_get_selected_fields_table";
TableId tableId = TableId.of(DATASET, tableName);
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
Table createdTable = bigquery.create(TableInfo.of(tableId, tableDefinition));
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(tableName, createdTable.tableId().table());
Table remoteTable =
bigquery.getTable(DATASET, tableName, TableOption.fields(TableField.CREATION_TIME));
assertNotNull(remoteTable);
assertTrue(remoteTable.definition() instanceof StandardTableDefinition);
assertEquals(createdTable.tableId(), remoteTable.tableId());
assertEquals(TableDefinition.Type.TABLE, remoteTable.definition().type());
assertNotNull(remoteTable.creationTime());
assertNull(remoteTable.definition().schema());
assertNull(remoteTable.lastModifiedTime());
assertNull(remoteTable.<StandardTableDefinition>definition().numBytes());
assertNull(remoteTable.<StandardTableDefinition>definition().numRows());
assertTrue(remoteTable.delete());
}
@Test
public void testCreateExternalTable() throws InterruptedException {
String tableName = "test_create_external_table";
TableId tableId = TableId.of(DATASET, tableName);
ExternalTableDefinition externalTableDefinition =
ExternalTableDefinition.of(
"gs://" + BUCKET + "/" + JSON_LOAD_FILE, TABLE_SCHEMA, FormatOptions.json());
TableInfo tableInfo = TableInfo.of(tableId, externalTableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(tableName, createdTable.tableId().table());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);
assertTrue(remoteTable.definition() instanceof ExternalTableDefinition);
assertEquals(createdTable.tableId(), remoteTable.tableId());
assertEquals(TABLE_SCHEMA, remoteTable.definition().schema());
QueryRequest request =
QueryRequest.builder(
"SELECT TimestampField, StringField, IntegerField, BooleanField FROM "
+ DATASET
+ "."
+ tableName)
.defaultDataset(DatasetId.of(DATASET))
.maxWaitTime(60000L)
.maxResults(1000L)
.build();
QueryResponse response = bigquery.query(request);
while (!response.jobCompleted()) {
response = bigquery.getQueryResults(response.jobId());
Thread.sleep(1000);
}
long integerValue = 0;
int rowCount = 0;
for (List<FieldValue> row : response.result().values()) {
FieldValue timestampCell = row.get(0);
FieldValue stringCell = row.get(1);
FieldValue integerCell = row.get(2);
FieldValue booleanCell = row.get(3);
assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, integerCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute());
assertEquals(1408452095220000L, timestampCell.timestampValue());
assertEquals("stringValue", stringCell.stringValue());
assertEquals(integerValue, integerCell.longValue());
assertEquals(false, booleanCell.booleanValue());
integerValue = ~integerValue & 0x1;
rowCount++;
}
assertEquals(4, rowCount);
assertTrue(remoteTable.delete());
}
@Test
public void testCreateViewTable() throws InterruptedException {
String tableName = "test_create_view_table";
TableId tableId = TableId.of(DATASET, tableName);
ViewDefinition viewDefinition =
ViewDefinition.of(
"SELECT TimestampField, StringField, BooleanField FROM "
+ DATASET
+ "."
+ TABLE_ID.table());
TableInfo tableInfo = TableInfo.of(tableId, viewDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(tableName, createdTable.tableId().table());
Table remoteTable = bigquery.getTable(DATASET, tableName);
assertNotNull(remoteTable);
assertEquals(createdTable.tableId(), remoteTable.tableId());
assertTrue(remoteTable.definition() instanceof ViewDefinition);
Schema expectedSchema =
Schema.builder()
.addField(
Field.builder("TimestampField", Field.Type.timestamp())
.mode(Field.Mode.NULLABLE)
.build())
.addField(
Field.builder("StringField", Field.Type.string()).mode(Field.Mode.NULLABLE).build())
.addField(
Field.builder("BooleanField", Field.Type.bool()).mode(Field.Mode.NULLABLE).build())
.build();
assertEquals(expectedSchema, remoteTable.definition().schema());
QueryRequest request =
QueryRequest.builder("SELECT * FROM " + tableName)
.defaultDataset(DatasetId.of(DATASET))
.maxWaitTime(60000L)
.maxResults(1000L)
.build();
QueryResponse response = bigquery.query(request);
while (!response.jobCompleted()) {
response = bigquery.getQueryResults(response.jobId());
Thread.sleep(1000);
}
int rowCount = 0;
for (List<FieldValue> row : response.result().values()) {
FieldValue timestampCell = row.get(0);
FieldValue stringCell = row.get(1);
FieldValue booleanCell = row.get(2);
assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute());
assertEquals(1408452095220000L, timestampCell.timestampValue());
assertEquals("stringValue", stringCell.stringValue());
assertEquals(false, booleanCell.booleanValue());
rowCount++;
}
assertEquals(2, rowCount);
assertTrue(remoteTable.delete());
}
@Test
public void testListTables() {
String tableName = "test_list_tables";
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
Page<Table> tables = bigquery.listTables(DATASET);
boolean found = false;
Iterator<Table> tableIterator = tables.values().iterator();
while (tableIterator.hasNext() && !found) {
if (tableIterator.next().tableId().equals(createdTable.tableId())) {
found = true;
}
}
assertTrue(found);
assertTrue(createdTable.delete());
}
@Test
public void testUpdateTable() {
String tableName = "test_update_table";
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
Table updatedTable =
bigquery.update(tableInfo.toBuilder().description("newDescription").build());
assertEquals(DATASET, updatedTable.tableId().dataset());
assertEquals(tableName, updatedTable.tableId().table());
assertEquals(TABLE_SCHEMA, updatedTable.definition().schema());
assertEquals("newDescription", updatedTable.description());
assertTrue(updatedTable.delete());
}
@Test
public void testUpdateTableWithSelectedFields() {
String tableName = "test_update_with_selected_fields_table";
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
Table updatedTable =
bigquery.update(
tableInfo.toBuilder().description("newDescr").build(),
TableOption.fields(TableField.DESCRIPTION));
assertTrue(updatedTable.definition() instanceof StandardTableDefinition);
assertEquals(DATASET, updatedTable.tableId().dataset());
assertEquals(tableName, updatedTable.tableId().table());
assertEquals("newDescr", updatedTable.description());
assertNull(updatedTable.definition().schema());
assertNull(updatedTable.lastModifiedTime());
assertNull(updatedTable.<StandardTableDefinition>definition().numBytes());
assertNull(updatedTable.<StandardTableDefinition>definition().numRows());
assertTrue(createdTable.delete());
}
@Test
public void testUpdateNonExistingTable() {
TableInfo tableInfo =
TableInfo.of(
TableId.of(DATASET, "test_update_non_existing_table"),
StandardTableDefinition.of(SIMPLE_SCHEMA));
try {
bigquery.update(tableInfo);
fail("BigQueryException was expected");
} catch (BigQueryException e) {
BigQueryError error = e.error();
assertNotNull(error);
assertEquals("notFound", error.reason());
assertNotNull(error.message());
}
}
@Test
public void testDeleteNonExistingTable() {
assertFalse(bigquery.delete(DATASET, "test_delete_non_existing_table"));
}
@Test
public void testInsertAll() {
String tableName = "test_insert_all_table";
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition);
assertNotNull(bigquery.create(tableInfo));
InsertAllRequest request =
InsertAllRequest.builder(tableInfo.tableId())
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"2014-08-19 07:41:35.220 -05:00",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false,
"RecordField",
ImmutableMap.of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"IntegerField",
ImmutableList.of(1, 0),
"BooleanField",
true)))
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"2014-08-19 07:41:35.220 -05:00",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false,
"RecordField",
ImmutableMap.of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"IntegerField",
ImmutableList.of(1, 0),
"BooleanField",
true)))
.build();
InsertAllResponse response = bigquery.insertAll(request);
assertFalse(response.hasErrors());
assertEquals(0, response.insertErrors().size());
assertTrue(bigquery.delete(TableId.of(DATASET, tableName)));
}
@Test
public void testInsertAllWithSuffix() throws InterruptedException {
String tableName = "test_insert_all_with_suffix_table";
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition);
assertNotNull(bigquery.create(tableInfo));
InsertAllRequest request =
InsertAllRequest.builder(tableInfo.tableId())
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"2014-08-19 07:41:35.220 -05:00",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false,
"RecordField",
ImmutableMap.of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"IntegerField",
ImmutableList.of(1, 0),
"BooleanField",
true)))
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"2014-08-19 07:41:35.220 -05:00",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false,
"RecordField",
ImmutableMap.of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"IntegerField",
ImmutableList.of(1, 0),
"BooleanField",
true)))
.templateSuffix("_suffix")
.build();
InsertAllResponse response = bigquery.insertAll(request);
assertFalse(response.hasErrors());
assertEquals(0, response.insertErrors().size());
String newTableName = tableName + "_suffix";
Table suffixTable = bigquery.getTable(DATASET, newTableName, TableOption.fields());
// wait until the new table is created. If the table is never created the test will time-out
while (suffixTable == null) {
Thread.sleep(1000L);
suffixTable = bigquery.getTable(DATASET, newTableName, TableOption.fields());
}
assertTrue(bigquery.delete(TableId.of(DATASET, tableName)));
assertTrue(suffixTable.delete());
}
@Test
public void testInsertAllWithErrors() {
String tableName = "test_insert_all_with_errors_table";
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(TableId.of(DATASET, tableName), tableDefinition);
assertNotNull(bigquery.create(tableInfo));
InsertAllRequest request =
InsertAllRequest.builder(tableInfo.tableId())
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"2014-08-19 07:41:35.220 -05:00",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false,
"RecordField",
ImmutableMap.of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"IntegerField",
ImmutableList.of(1, 0),
"BooleanField",
true)))
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"invalidDate",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false,
"RecordField",
ImmutableMap.of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"IntegerField",
ImmutableList.of(1, 0),
"BooleanField",
true)))
.addRow(
ImmutableMap.<String, Object>of(
"TimestampField",
"1969-07-20 20:18:04 UTC",
"StringField",
"stringValue",
"IntegerField",
ImmutableList.of(0, 1),
"BooleanField",
false))
.skipInvalidRows(true)
.build();
InsertAllResponse response = bigquery.insertAll(request);
assertTrue(response.hasErrors());
assertEquals(2, response.insertErrors().size());
assertNotNull(response.errorsFor(1L));
assertNotNull(response.errorsFor(2L));
assertTrue(bigquery.delete(TableId.of(DATASET, tableName)));
}
@Test
public void testListAllTableData() {
Page<List<FieldValue>> rows = bigquery.listTableData(TABLE_ID);
int rowCount = 0;
for (List<FieldValue> row : rows.values()) {
FieldValue timestampCell = row.get(0);
FieldValue stringCell = row.get(1);
FieldValue integerCell = row.get(2);
FieldValue booleanCell = row.get(3);
FieldValue recordCell = row.get(4);
assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute());
assertEquals(FieldValue.Attribute.REPEATED, integerCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute());
assertEquals(FieldValue.Attribute.RECORD, recordCell.attribute());
assertEquals(1408452095220000L, timestampCell.timestampValue());
assertEquals("stringValue", stringCell.stringValue());
assertEquals(0, integerCell.repeatedValue().get(0).longValue());
assertEquals(1, integerCell.repeatedValue().get(1).longValue());
assertEquals(false, booleanCell.booleanValue());
assertEquals(-14182916000000L, recordCell.recordValue().get(0).timestampValue());
assertTrue(recordCell.recordValue().get(1).isNull());
assertEquals(1, recordCell.recordValue().get(2).repeatedValue().get(0).longValue());
assertEquals(0, recordCell.recordValue().get(2).repeatedValue().get(1).longValue());
assertEquals(true, recordCell.recordValue().get(3).booleanValue());
rowCount++;
}
assertEquals(2, rowCount);
}
@Test
public void testQuery() throws InterruptedException {
String query =
new StringBuilder()
.append("SELECT TimestampField, StringField, BooleanField FROM ")
.append(TABLE_ID.table())
.toString();
QueryRequest request =
QueryRequest.builder(query)
.defaultDataset(DatasetId.of(DATASET))
.maxWaitTime(60000L)
.maxResults(1000L)
.build();
QueryResponse response = bigquery.query(request);
while (!response.jobCompleted()) {
Thread.sleep(1000);
response = bigquery.getQueryResults(response.jobId());
}
assertEquals(QUERY_RESULT_SCHEMA, response.result().schema());
int rowCount = 0;
for (List<FieldValue> row : response.result().values()) {
FieldValue timestampCell = row.get(0);
FieldValue stringCell = row.get(1);
FieldValue booleanCell = row.get(2);
assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute());
assertEquals(1408452095220000L, timestampCell.timestampValue());
assertEquals("stringValue", stringCell.stringValue());
assertEquals(false, booleanCell.booleanValue());
rowCount++;
}
assertEquals(2, rowCount);
Job queryJob = bigquery.getJob(response.jobId());
JobStatistics.QueryStatistics statistics = queryJob.statistics();
assertNotNull(statistics.queryPlan());
}
@Test
public void testListJobs() {
Page<Job> jobs = bigquery.listJobs();
for (Job job : jobs.values()) {
assertNotNull(job.jobId());
assertNotNull(job.statistics());
assertNotNull(job.status());
assertNotNull(job.userEmail());
assertNotNull(job.id());
}
}
@Test
public void testListJobsWithSelectedFields() {
Page<Job> jobs = bigquery.listJobs(JobListOption.fields(JobField.USER_EMAIL));
for (Job job : jobs.values()) {
assertNotNull(job.jobId());
assertNotNull(job.status());
assertNotNull(job.userEmail());
assertNull(job.statistics());
assertNull(job.id());
}
}
@Test
public void testCreateAndGetJob() {
String sourceTableName = "test_create_and_get_job_source_table";
String destinationTableName = "test_create_and_get_job_destination_table";
TableId sourceTable = TableId.of(DATASET, sourceTableName);
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(sourceTableName, createdTable.tableId().table());
TableId destinationTable = TableId.of(DATASET, destinationTableName);
CopyJobConfiguration copyJobConfiguration =
CopyJobConfiguration.of(destinationTable, sourceTable);
Job createdJob = bigquery.create(JobInfo.of(copyJobConfiguration));
Job remoteJob = bigquery.getJob(createdJob.jobId());
assertEquals(createdJob.jobId(), remoteJob.jobId());
CopyJobConfiguration createdConfiguration = createdJob.configuration();
CopyJobConfiguration remoteConfiguration = remoteJob.configuration();
assertEquals(createdConfiguration.sourceTables(), remoteConfiguration.sourceTables());
assertEquals(createdConfiguration.destinationTable(), remoteConfiguration.destinationTable());
assertEquals(createdConfiguration.createDisposition(), remoteConfiguration.createDisposition());
assertEquals(createdConfiguration.writeDisposition(), remoteConfiguration.writeDisposition());
assertNotNull(remoteJob.etag());
assertNotNull(remoteJob.statistics());
assertNotNull(remoteJob.status());
assertEquals(createdJob.selfLink(), remoteJob.selfLink());
assertEquals(createdJob.userEmail(), remoteJob.userEmail());
assertTrue(createdTable.delete());
assertTrue(bigquery.delete(DATASET, destinationTableName));
}
@Test
public void testCreateAndGetJobWithSelectedFields() {
String sourceTableName = "test_create_and_get_job_with_selected_fields_source_table";
String destinationTableName = "test_create_and_get_job_with_selected_fields_destination_table";
TableId sourceTable = TableId.of(DATASET, sourceTableName);
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(sourceTableName, createdTable.tableId().table());
TableId destinationTable = TableId.of(DATASET, destinationTableName);
CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable);
Job createdJob = bigquery.create(JobInfo.of(configuration), JobOption.fields(JobField.ETAG));
CopyJobConfiguration createdConfiguration = createdJob.configuration();
assertNotNull(createdJob.jobId());
assertNotNull(createdConfiguration.sourceTables());
assertNotNull(createdConfiguration.destinationTable());
assertNotNull(createdJob.etag());
assertNull(createdJob.statistics());
assertNull(createdJob.status());
assertNull(createdJob.selfLink());
assertNull(createdJob.userEmail());
Job remoteJob = bigquery.getJob(createdJob.jobId(), JobOption.fields(JobField.ETAG));
CopyJobConfiguration remoteConfiguration = remoteJob.configuration();
assertEquals(createdJob.jobId(), remoteJob.jobId());
assertEquals(createdConfiguration.sourceTables(), remoteConfiguration.sourceTables());
assertEquals(createdConfiguration.destinationTable(), remoteConfiguration.destinationTable());
assertEquals(createdConfiguration.createDisposition(), remoteConfiguration.createDisposition());
assertEquals(createdConfiguration.writeDisposition(), remoteConfiguration.writeDisposition());
assertNotNull(remoteJob.etag());
assertNull(remoteJob.statistics());
assertNull(remoteJob.status());
assertNull(remoteJob.selfLink());
assertNull(remoteJob.userEmail());
assertTrue(createdTable.delete());
assertTrue(bigquery.delete(DATASET, destinationTableName));
}
@Test
public void testCopyJob() throws InterruptedException {
String sourceTableName = "test_copy_job_source_table";
String destinationTableName = "test_copy_job_destination_table";
TableId sourceTable = TableId.of(DATASET, sourceTableName);
StandardTableDefinition tableDefinition = StandardTableDefinition.of(TABLE_SCHEMA);
TableInfo tableInfo = TableInfo.of(sourceTable, tableDefinition);
Table createdTable = bigquery.create(tableInfo);
assertNotNull(createdTable);
assertEquals(DATASET, createdTable.tableId().dataset());
assertEquals(sourceTableName, createdTable.tableId().table());
TableId destinationTable = TableId.of(DATASET, destinationTableName);
CopyJobConfiguration configuration = CopyJobConfiguration.of(destinationTable, sourceTable);
Job remoteJob = bigquery.create(JobInfo.of(configuration));
while (!remoteJob.isDone()) {
Thread.sleep(1000);
}
assertNull(remoteJob.status().error());
Table remoteTable = bigquery.getTable(DATASET, destinationTableName);
assertNotNull(remoteTable);
assertEquals(destinationTable.dataset(), remoteTable.tableId().dataset());
assertEquals(destinationTableName, remoteTable.tableId().table());
assertEquals(TABLE_SCHEMA, remoteTable.definition().schema());
assertTrue(createdTable.delete());
assertTrue(remoteTable.delete());
}
@Test
public void testQueryJob() throws InterruptedException {
String tableName = "test_query_job_table";
String query =
new StringBuilder()
.append("SELECT TimestampField, StringField, BooleanField FROM ")
.append(TABLE_ID.table())
.toString();
TableId destinationTable = TableId.of(DATASET, tableName);
QueryJobConfiguration configuration =
QueryJobConfiguration.builder(query)
.defaultDataset(DatasetId.of(DATASET))
.destinationTable(destinationTable)
.build();
Job remoteJob = bigquery.create(JobInfo.of(configuration));
while (!remoteJob.isDone()) {
Thread.sleep(1000);
}
assertNull(remoteJob.status().error());
QueryResponse response = bigquery.getQueryResults(remoteJob.jobId());
while (!response.jobCompleted()) {
Thread.sleep(1000);
response = bigquery.getQueryResults(response.jobId());
}
assertFalse(response.hasErrors());
assertEquals(QUERY_RESULT_SCHEMA, response.result().schema());
int rowCount = 0;
for (List<FieldValue> row : response.result().values()) {
FieldValue timestampCell = row.get(0);
FieldValue stringCell = row.get(1);
FieldValue booleanCell = row.get(2);
assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute());
assertEquals(1408452095220000L, timestampCell.timestampValue());
assertEquals("stringValue", stringCell.stringValue());
assertEquals(false, booleanCell.booleanValue());
rowCount++;
}
assertEquals(2, rowCount);
assertTrue(bigquery.delete(DATASET, tableName));
Job queryJob = bigquery.getJob(remoteJob.jobId());
JobStatistics.QueryStatistics statistics = queryJob.statistics();
assertNotNull(statistics.queryPlan());
}
@Test
public void testExtractJob() throws InterruptedException {
String tableName = "test_export_job_table";
TableId destinationTable = TableId.of(DATASET, tableName);
LoadJobConfiguration configuration =
LoadJobConfiguration.builder(destinationTable, "gs://" + BUCKET + "/" + LOAD_FILE)
.schema(SIMPLE_SCHEMA)
.build();
Job remoteLoadJob = bigquery.create(JobInfo.of(configuration));
while (!remoteLoadJob.isDone()) {
Thread.sleep(1000);
}
assertNull(remoteLoadJob.status().error());
ExtractJobConfiguration extractConfiguration =
ExtractJobConfiguration.builder(destinationTable, "gs://" + BUCKET + "/" + EXTRACT_FILE)
.printHeader(false)
.build();
Job remoteExtractJob = bigquery.create(JobInfo.of(extractConfiguration));
while (!remoteExtractJob.isDone()) {
Thread.sleep(1000);
}
assertNull(remoteExtractJob.status().error());
assertEquals(
CSV_CONTENT,
new String(storage.readAllBytes(BUCKET, EXTRACT_FILE), StandardCharsets.UTF_8));
assertTrue(bigquery.delete(DATASET, tableName));
}
@Test
public void testCancelJob() throws InterruptedException {
String destinationTableName = "test_cancel_query_job_table";
String query = "SELECT TimestampField, StringField, BooleanField FROM " + TABLE_ID.table();
TableId destinationTable = TableId.of(DATASET, destinationTableName);
QueryJobConfiguration configuration =
QueryJobConfiguration.builder(query)
.defaultDataset(DatasetId.of(DATASET))
.destinationTable(destinationTable)
.build();
Job remoteJob = bigquery.create(JobInfo.of(configuration));
assertTrue(remoteJob.cancel());
while (!remoteJob.isDone()) {
Thread.sleep(1000);
}
assertNull(remoteJob.status().error());
}
@Test
public void testCancelNonExistingJob() {
assertFalse(bigquery.cancel("test_cancel_non_existing_job"));
}
@Test
public void testInsertFromFile() throws InterruptedException {
String destinationTableName = "test_insert_from_file_table";
TableId tableId = TableId.of(DATASET, destinationTableName);
WriteChannelConfiguration configuration =
WriteChannelConfiguration.builder(tableId)
.formatOptions(FormatOptions.json())
.createDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED)
.schema(TABLE_SCHEMA)
.build();
try (WriteChannel channel = bigquery.writer(configuration)) {
channel.write(ByteBuffer.wrap(JSON_CONTENT.getBytes(StandardCharsets.UTF_8)));
} catch (IOException e) {
fail("IOException was not expected");
}
// wait until the new table is created. If the table is never created the test will time-out
while (bigquery.getTable(tableId) == null) {
Thread.sleep(1000L);
}
Page<List<FieldValue>> rows = bigquery.listTableData(tableId);
int rowCount = 0;
for (List<FieldValue> row : rows.values()) {
FieldValue timestampCell = row.get(0);
FieldValue stringCell = row.get(1);
FieldValue integerCell = row.get(2);
FieldValue booleanCell = row.get(3);
FieldValue recordCell = row.get(4);
assertEquals(FieldValue.Attribute.PRIMITIVE, timestampCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, stringCell.attribute());
assertEquals(FieldValue.Attribute.REPEATED, integerCell.attribute());
assertEquals(FieldValue.Attribute.PRIMITIVE, booleanCell.attribute());
assertEquals(FieldValue.Attribute.RECORD, recordCell.attribute());
assertEquals(1408452095220000L, timestampCell.timestampValue());
assertEquals("stringValue", stringCell.stringValue());
assertEquals(0, integerCell.repeatedValue().get(0).longValue());
assertEquals(1, integerCell.repeatedValue().get(1).longValue());
assertEquals(false, booleanCell.booleanValue());
assertEquals(-14182916000000L, recordCell.recordValue().get(0).timestampValue());
assertTrue(recordCell.recordValue().get(1).isNull());
assertEquals(1, recordCell.recordValue().get(2).repeatedValue().get(0).longValue());
assertEquals(0, recordCell.recordValue().get(2).repeatedValue().get(1).longValue());
assertEquals(true, recordCell.recordValue().get(3).booleanValue());
rowCount++;
}
assertEquals(2, rowCount);
assertTrue(bigquery.delete(DATASET, destinationTableName));
}
}
public class ITStorageTest {
private static Storage storage;
private static final Logger log = Logger.getLogger(ITStorageTest.class.getName());
private static final String BUCKET = RemoteGcsHelper.generateBucketName();
private static final String CONTENT_TYPE = "text/plain";
private static final byte[] BLOB_BYTE_CONTENT = {0xD, 0xE, 0xA, 0xD};
private static final String BLOB_STRING_CONTENT = "Hello Google Cloud Storage!";
@BeforeClass
public static void beforeClass() {
RemoteGcsHelper gcsHelper = RemoteGcsHelper.create();
storage = gcsHelper.options().service();
storage.create(BucketInfo.of(BUCKET));
}
@AfterClass
public static void afterClass()
throws ExecutionException, TimeoutException, InterruptedException {
if (storage != null && !RemoteGcsHelper.forceDelete(storage, BUCKET, 5, TimeUnit.SECONDS)) {
if (log.isLoggable(Level.WARNING)) {
log.log(Level.WARNING, "Deletion of bucket {0} timed out, bucket is not empty", BUCKET);
}
}
}
@Test(timeout = 5000)
public void testListBuckets() throws InterruptedException {
Iterator<BucketInfo> bucketIterator =
storage
.list(Storage.BucketListOption.prefix(BUCKET), Storage.BucketListOption.fields())
.values()
.iterator();
while (!bucketIterator.hasNext()) {
Thread.sleep(500);
bucketIterator =
storage
.list(Storage.BucketListOption.prefix(BUCKET), Storage.BucketListOption.fields())
.values()
.iterator();
}
while (bucketIterator.hasNext()) {
BucketInfo remoteBucket = bucketIterator.next();
assertTrue(remoteBucket.name().startsWith(BUCKET));
assertNull(remoteBucket.createTime());
assertNull(remoteBucket.selfLink());
}
}
@Test
public void testGetBucketSelectedFields() {
BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.ID));
assertEquals(BUCKET, remoteBucket.name());
assertNull(remoteBucket.createTime());
assertNotNull(remoteBucket.id());
}
@Test
public void testGetBucketAllSelectedFields() {
BucketInfo remoteBucket =
storage.get(BUCKET, Storage.BucketGetOption.fields(BucketField.values()));
assertEquals(BUCKET, remoteBucket.name());
assertNotNull(remoteBucket.createTime());
assertNotNull(remoteBucket.selfLink());
}
@Test
public void testGetBucketEmptyFields() {
BucketInfo remoteBucket = storage.get(BUCKET, Storage.BucketGetOption.fields());
assertEquals(BUCKET, remoteBucket.name());
assertNull(remoteBucket.createTime());
assertNull(remoteBucket.selfLink());
}
@Test
public void testCreateBlob() {
String blobName = "test-create-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
BlobInfo remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT);
assertNotNull(remoteBlob);
assertEquals(blob.blobId(), remoteBlob.blobId());
byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
assertArrayEquals(BLOB_BYTE_CONTENT, readBytes);
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testCreateEmptyBlob() {
String blobName = "test-create-empty-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
BlobInfo remoteBlob = storage.create(blob);
assertNotNull(remoteBlob);
assertEquals(blob.blobId(), remoteBlob.blobId());
byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
assertArrayEquals(new byte[0], readBytes);
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testCreateBlobStream() throws UnsupportedEncodingException {
String blobName = "test-create-blob-stream";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
BlobInfo remoteBlob = storage.create(blob, stream);
assertNotNull(remoteBlob);
assertEquals(blob.blobId(), remoteBlob.blobId());
assertEquals(blob.contentType(), remoteBlob.contentType());
byte[] readBytes = storage.readAllBytes(BUCKET, blobName);
assertEquals(BLOB_STRING_CONTENT, new String(readBytes, UTF_8));
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testCreateBlobFail() {
String blobName = "test-create-blob-fail";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
try {
storage.create(
blob.toBuilder().generation(-1L).build(),
BLOB_BYTE_CONTENT,
Storage.BlobTargetOption.generationMatch());
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testCreateBlobMd5Fail() throws UnsupportedEncodingException {
String blobName = "test-create-blob-md5-fail";
BlobInfo blob =
BlobInfo.builder(BUCKET, blobName)
.contentType(CONTENT_TYPE)
.md5("O1R4G1HJSDUISJjoIYmVhQ==")
.build();
ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8));
try {
storage.create(blob, stream, Storage.BlobWriteOption.md5Match());
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
}
@Test
public void testGetBlobEmptySelectedFields() {
String blobName = "test-get-empty-selected-fields-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).build();
assertNotNull(storage.create(blob));
BlobInfo remoteBlob = storage.get(blob.blobId(), Storage.BlobGetOption.fields());
assertEquals(blob.blobId(), remoteBlob.blobId());
assertNull(remoteBlob.contentType());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testGetBlobSelectedFields() {
String blobName = "test-get-selected-fields-blob";
BlobInfo blob =
BlobInfo.builder(BUCKET, blobName)
.contentType(CONTENT_TYPE)
.metadata(ImmutableMap.of("k", "v"))
.build();
assertNotNull(storage.create(blob));
BlobInfo remoteBlob =
storage.get(blob.blobId(), Storage.BlobGetOption.fields(BlobField.METADATA));
assertEquals(blob.blobId(), remoteBlob.blobId());
assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata());
assertNull(remoteBlob.contentType());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testGetBlobAllSelectedFields() {
String blobName = "test-get-all-selected-fields-blob";
BlobInfo blob =
BlobInfo.builder(BUCKET, blobName)
.contentType(CONTENT_TYPE)
.metadata(ImmutableMap.of("k", "v"))
.build();
assertNotNull(storage.create(blob));
BlobInfo remoteBlob =
storage.get(blob.blobId(), Storage.BlobGetOption.fields(BlobField.values()));
assertEquals(blob.blobId(), remoteBlob.blobId());
assertEquals(ImmutableMap.of("k", "v"), remoteBlob.metadata());
assertNotNull(remoteBlob.id());
assertNotNull(remoteBlob.selfLink());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testListBlobsSelectedFields() {
String[] blobNames = {
"test-list-blobs-selected-fields-blob1", "test-list-blobs-selected-fields-blob2"
};
ImmutableMap metadata = ImmutableMap.of("k", "v");
BlobInfo blob1 =
BlobInfo.builder(BUCKET, blobNames[0]).contentType(CONTENT_TYPE).metadata(metadata).build();
BlobInfo blob2 =
BlobInfo.builder(BUCKET, blobNames[1]).contentType(CONTENT_TYPE).metadata(metadata).build();
assertNotNull(storage.create(blob1));
assertNotNull(storage.create(blob2));
Page<BlobInfo> page =
storage.list(
BUCKET,
Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"),
Storage.BlobListOption.fields(BlobField.METADATA));
int index = 0;
for (BlobInfo remoteBlob : page.values()) {
assertEquals(BUCKET, remoteBlob.bucket());
assertEquals(blobNames[index++], remoteBlob.name());
assertEquals(metadata, remoteBlob.metadata());
assertNull(remoteBlob.contentType());
}
assertTrue(storage.delete(BUCKET, blobNames[0]));
assertTrue(storage.delete(BUCKET, blobNames[1]));
}
@Test
public void testListBlobsEmptySelectedFields() {
String[] blobNames = {
"test-list-blobs-empty-selected-fields-blob1", "test-list-blobs-empty-selected-fields-blob2"
};
BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0]).contentType(CONTENT_TYPE).build();
BlobInfo blob2 = BlobInfo.builder(BUCKET, blobNames[1]).contentType(CONTENT_TYPE).build();
assertNotNull(storage.create(blob1));
assertNotNull(storage.create(blob2));
Page<BlobInfo> page =
storage.list(
BUCKET,
Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"),
Storage.BlobListOption.fields());
int index = 0;
for (BlobInfo remoteBlob : page.values()) {
assertEquals(BUCKET, remoteBlob.bucket());
assertEquals(blobNames[index++], remoteBlob.name());
assertNull(remoteBlob.contentType());
}
assertTrue(storage.delete(BUCKET, blobNames[0]));
assertTrue(storage.delete(BUCKET, blobNames[1]));
}
@Test
public void testUpdateBlob() {
String blobName = "test-update-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
BlobInfo updatedBlob = storage.update(blob.toBuilder().contentType(CONTENT_TYPE).build());
assertNotNull(updatedBlob);
assertEquals(blob.blobId(), updatedBlob.blobId());
assertEquals(CONTENT_TYPE, updatedBlob.contentType());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testUpdateBlobReplaceMetadata() {
String blobName = "test-update-blob-replace-metadata";
ImmutableMap<String, String> metadata = ImmutableMap.of("k1", "a");
ImmutableMap<String, String> newMetadata = ImmutableMap.of("k2", "b");
BlobInfo blob =
BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).metadata(metadata).build();
assertNotNull(storage.create(blob));
BlobInfo updatedBlob = storage.update(blob.toBuilder().metadata(null).build());
assertNotNull(updatedBlob);
assertNull(updatedBlob.metadata());
updatedBlob = storage.update(blob.toBuilder().metadata(newMetadata).build());
assertEquals(blob.blobId(), updatedBlob.blobId());
assertEquals(newMetadata, updatedBlob.metadata());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testUpdateBlobMergeMetadata() {
String blobName = "test-update-blob-merge-metadata";
ImmutableMap<String, String> metadata = ImmutableMap.of("k1", "a");
ImmutableMap<String, String> newMetadata = ImmutableMap.of("k2", "b");
ImmutableMap<String, String> expectedMetadata = ImmutableMap.of("k1", "a", "k2", "b");
BlobInfo blob =
BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).metadata(metadata).build();
assertNotNull(storage.create(blob));
BlobInfo updatedBlob = storage.update(blob.toBuilder().metadata(newMetadata).build());
assertNotNull(updatedBlob);
assertEquals(blob.blobId(), updatedBlob.blobId());
assertEquals(expectedMetadata, updatedBlob.metadata());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testUpdateBlobUnsetMetadata() {
String blobName = "test-update-blob-unset-metadata";
ImmutableMap<String, String> metadata = ImmutableMap.of("k1", "a", "k2", "b");
Map<String, String> newMetadata = new HashMap<>();
newMetadata.put("k1", "a");
newMetadata.put("k2", null);
ImmutableMap<String, String> expectedMetadata = ImmutableMap.of("k1", "a");
BlobInfo blob =
BlobInfo.builder(BUCKET, blobName).contentType(CONTENT_TYPE).metadata(metadata).build();
assertNotNull(storage.create(blob));
BlobInfo updatedBlob = storage.update(blob.toBuilder().metadata(newMetadata).build());
assertNotNull(updatedBlob);
assertEquals(blob.blobId(), updatedBlob.blobId());
assertEquals(expectedMetadata, updatedBlob.metadata());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testUpdateBlobFail() {
String blobName = "test-update-blob-fail";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
try {
storage.update(
blob.toBuilder().contentType(CONTENT_TYPE).generation(-1L).build(),
Storage.BlobTargetOption.generationMatch());
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testDeleteNonExistingBlob() {
String blobName = "test-delete-non-existing-blob";
assertTrue(!storage.delete(BUCKET, blobName));
}
@Test
public void testDeleteBlobFail() {
String blobName = "test-delete-blob-fail";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
try {
storage.delete(BUCKET, blob.name(), Storage.BlobSourceOption.generationMatch(-1L));
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, blob.name()));
}
@Test
public void testComposeBlob() {
String sourceBlobName1 = "test-compose-blob-source-1";
String sourceBlobName2 = "test-compose-blob-source-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1, BLOB_BYTE_CONTENT));
assertNotNull(storage.create(sourceBlob2, BLOB_BYTE_CONTENT));
String targetBlobName = "test-compose-blob-target";
BlobInfo targetBlob = BlobInfo.builder(BUCKET, targetBlobName).build();
Storage.ComposeRequest req =
Storage.ComposeRequest.of(ImmutableList.of(sourceBlobName1, sourceBlobName2), targetBlob);
BlobInfo remoteBlob = storage.compose(req);
assertNotNull(remoteBlob);
assertEquals(targetBlob.blobId(), remoteBlob.blobId());
byte[] readBytes = storage.readAllBytes(BUCKET, targetBlobName);
byte[] composedBytes = Arrays.copyOf(BLOB_BYTE_CONTENT, BLOB_BYTE_CONTENT.length * 2);
System.arraycopy(
BLOB_BYTE_CONTENT, 0, composedBytes, BLOB_BYTE_CONTENT.length, BLOB_BYTE_CONTENT.length);
assertArrayEquals(composedBytes, readBytes);
assertTrue(storage.delete(BUCKET, sourceBlobName1));
assertTrue(storage.delete(BUCKET, sourceBlobName2));
assertTrue(storage.delete(BUCKET, targetBlobName));
}
@Test
public void testComposeBlobFail() {
String sourceBlobName1 = "test-compose-blob-fail-source-1";
String sourceBlobName2 = "test-compose-blob-fail-source-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1));
assertNotNull(storage.create(sourceBlob2));
String targetBlobName = "test-compose-blob-fail-target";
BlobInfo targetBlob = BlobInfo.builder(BUCKET, targetBlobName).build();
Storage.ComposeRequest req =
Storage.ComposeRequest.builder()
.addSource(sourceBlobName1, -1L)
.addSource(sourceBlobName2, -1L)
.target(targetBlob)
.build();
try {
storage.compose(req);
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, sourceBlobName1));
assertTrue(storage.delete(BUCKET, sourceBlobName2));
}
@Test
public void testCopyBlob() {
String sourceBlobName = "test-copy-blob-source";
BlobId source = BlobId.of(BUCKET, sourceBlobName);
ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v");
BlobInfo blob = BlobInfo.builder(source).contentType(CONTENT_TYPE).metadata(metadata).build();
assertNotNull(storage.create(blob, BLOB_BYTE_CONTENT));
String targetBlobName = "test-copy-blob-target";
Storage.CopyRequest req = Storage.CopyRequest.of(source, BlobId.of(BUCKET, targetBlobName));
CopyWriter copyWriter = storage.copy(req);
assertEquals(BUCKET, copyWriter.result().bucket());
assertEquals(targetBlobName, copyWriter.result().name());
assertEquals(CONTENT_TYPE, copyWriter.result().contentType());
assertEquals(metadata, copyWriter.result().metadata());
assertTrue(copyWriter.isDone());
assertTrue(storage.delete(BUCKET, sourceBlobName));
assertTrue(storage.delete(BUCKET, targetBlobName));
}
@Test
public void testCopyBlobUpdateMetadata() {
String sourceBlobName = "test-copy-blob-update-metadata-source";
BlobId source = BlobId.of(BUCKET, sourceBlobName);
assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
String targetBlobName = "test-copy-blob-update-metadata-target";
ImmutableMap<String, String> metadata = ImmutableMap.of("k", "v");
BlobInfo target =
BlobInfo.builder(BUCKET, targetBlobName)
.contentType(CONTENT_TYPE)
.metadata(metadata)
.build();
Storage.CopyRequest req = Storage.CopyRequest.of(source, target);
CopyWriter copyWriter = storage.copy(req);
assertEquals(BUCKET, copyWriter.result().bucket());
assertEquals(targetBlobName, copyWriter.result().name());
assertEquals(CONTENT_TYPE, copyWriter.result().contentType());
assertEquals(metadata, copyWriter.result().metadata());
assertTrue(copyWriter.isDone());
assertTrue(storage.delete(BUCKET, sourceBlobName));
assertTrue(storage.delete(BUCKET, targetBlobName));
}
@Test
public void testCopyBlobFail() {
String sourceBlobName = "test-copy-blob-source-fail";
BlobId source = BlobId.of(BUCKET, sourceBlobName);
assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
String targetBlobName = "test-copy-blob-target-fail";
BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build();
Storage.CopyRequest req =
Storage.CopyRequest.builder()
.source(source)
.sourceOptions(Storage.BlobSourceOption.generationMatch(-1L))
.target(target)
.build();
try {
storage.copy(req);
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, sourceBlobName));
}
@Test
public void testBatchRequest() {
String sourceBlobName1 = "test-batch-request-blob-1";
String sourceBlobName2 = "test-batch-request-blob-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1));
assertNotNull(storage.create(sourceBlob2));
// Batch update request
BlobInfo updatedBlob1 = sourceBlob1.toBuilder().contentType(CONTENT_TYPE).build();
BlobInfo updatedBlob2 = sourceBlob2.toBuilder().contentType(CONTENT_TYPE).build();
BatchRequest updateRequest =
BatchRequest.builder().update(updatedBlob1).update(updatedBlob2).build();
BatchResponse updateResponse = storage.apply(updateRequest);
assertEquals(2, updateResponse.updates().size());
assertEquals(0, updateResponse.deletes().size());
assertEquals(0, updateResponse.gets().size());
BlobInfo remoteUpdatedBlob1 = updateResponse.updates().get(0).get();
BlobInfo remoteUpdatedBlob2 = updateResponse.updates().get(1).get();
assertEquals(sourceBlob1.blobId(), remoteUpdatedBlob1.blobId());
assertEquals(sourceBlob2.blobId(), remoteUpdatedBlob2.blobId());
assertEquals(updatedBlob1.contentType(), remoteUpdatedBlob1.contentType());
assertEquals(updatedBlob2.contentType(), remoteUpdatedBlob2.contentType());
// Batch get request
BatchRequest getRequest =
BatchRequest.builder().get(BUCKET, sourceBlobName1).get(BUCKET, sourceBlobName2).build();
BatchResponse getResponse = storage.apply(getRequest);
assertEquals(2, getResponse.gets().size());
assertEquals(0, getResponse.deletes().size());
assertEquals(0, getResponse.updates().size());
BlobInfo remoteBlob1 = getResponse.gets().get(0).get();
BlobInfo remoteBlob2 = getResponse.gets().get(1).get();
assertEquals(remoteUpdatedBlob1, remoteBlob1);
assertEquals(remoteUpdatedBlob2, remoteBlob2);
// Batch delete request
BatchRequest deleteRequest =
BatchRequest.builder()
.delete(BUCKET, sourceBlobName1)
.delete(BUCKET, sourceBlobName2)
.build();
BatchResponse deleteResponse = storage.apply(deleteRequest);
assertEquals(2, deleteResponse.deletes().size());
assertEquals(0, deleteResponse.gets().size());
assertEquals(0, deleteResponse.updates().size());
assertTrue(deleteResponse.deletes().get(0).get());
assertTrue(deleteResponse.deletes().get(1).get());
}
@Test
public void testBatchRequestFail() {
String blobName = "test-batch-request-blob-fail";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
BlobInfo updatedBlob = blob.toBuilder().generation(-1L).build();
BatchRequest batchRequest =
BatchRequest.builder()
.update(updatedBlob, Storage.BlobTargetOption.generationMatch())
.delete(BUCKET, blobName, Storage.BlobSourceOption.generationMatch(-1L))
.get(BUCKET, blobName, Storage.BlobGetOption.generationMatch(-1L))
.build();
BatchResponse updateResponse = storage.apply(batchRequest);
assertEquals(1, updateResponse.updates().size());
assertEquals(1, updateResponse.deletes().size());
assertEquals(1, updateResponse.gets().size());
assertTrue(updateResponse.updates().get(0).failed());
assertTrue(updateResponse.gets().get(0).failed());
assertTrue(updateResponse.deletes().get(0).failed());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testReadAndWriteChannels() throws IOException {
String blobName = "test-read-and-write-channels-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
byte[] stringBytes;
try (BlobWriteChannel writer = storage.writer(blob)) {
stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8);
writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT));
writer.write(ByteBuffer.wrap(stringBytes));
}
ByteBuffer readBytes;
ByteBuffer readStringBytes;
try (BlobReadChannel reader = storage.reader(blob.blobId())) {
readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length);
readStringBytes = ByteBuffer.allocate(stringBytes.length);
reader.read(readBytes);
reader.read(readStringBytes);
}
assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array());
assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8));
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testReadAndWriteCaptureChannels() throws IOException {
String blobName = "test-read-and-write-capture-channels-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
byte[] stringBytes;
BlobWriteChannel writer = storage.writer(blob);
stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8);
writer.write(ByteBuffer.wrap(BLOB_BYTE_CONTENT));
RestorableState<BlobWriteChannel> writerState = writer.capture();
BlobWriteChannel secondWriter = writerState.restore();
secondWriter.write(ByteBuffer.wrap(stringBytes));
secondWriter.close();
ByteBuffer readBytes;
ByteBuffer readStringBytes;
BlobReadChannel reader = storage.reader(blob.blobId());
reader.chunkSize(BLOB_BYTE_CONTENT.length);
readBytes = ByteBuffer.allocate(BLOB_BYTE_CONTENT.length);
reader.read(readBytes);
RestorableState<BlobReadChannel> readerState = reader.capture();
BlobReadChannel secondReader = readerState.restore();
readStringBytes = ByteBuffer.allocate(stringBytes.length);
secondReader.read(readStringBytes);
reader.close();
secondReader.close();
assertArrayEquals(BLOB_BYTE_CONTENT, readBytes.array());
assertEquals(BLOB_STRING_CONTENT, new String(readStringBytes.array(), UTF_8));
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testReadChannelFail() throws IOException {
String blobName = "test-read-channel-blob-fail";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
try (BlobReadChannel reader =
storage.reader(blob.blobId(), Storage.BlobSourceOption.metagenerationMatch(-1L))) {
reader.read(ByteBuffer.allocate(42));
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testWriteChannelFail() throws IOException {
String blobName = "test-write-channel-blob-fail";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).generation(-1L).build();
try {
try (BlobWriteChannel writer =
storage.writer(blob, Storage.BlobWriteOption.generationMatch())) {
writer.write(ByteBuffer.allocate(42));
}
fail("StorageException was expected");
} catch (StorageException ex) {
// expected
}
}
@Test
public void testWriteChannelExistingBlob() throws IOException {
String blobName = "test-write-channel-existing-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
BlobInfo remoteBlob = storage.create(blob);
byte[] stringBytes;
try (BlobWriteChannel writer = storage.writer(remoteBlob)) {
stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8);
writer.write(ByteBuffer.wrap(stringBytes));
}
assertArrayEquals(stringBytes, storage.readAllBytes(blob.blobId()));
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testGetSignedUrl() throws IOException {
String blobName = "test-get-signed-url-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob, BLOB_BYTE_CONTENT));
URL url = storage.signUrl(blob, 1, TimeUnit.HOURS);
URLConnection connection = url.openConnection();
byte[] readBytes = new byte[BLOB_BYTE_CONTENT.length];
try (InputStream responseStream = connection.getInputStream()) {
assertEquals(BLOB_BYTE_CONTENT.length, responseStream.read(readBytes));
assertArrayEquals(BLOB_BYTE_CONTENT, readBytes);
assertTrue(storage.delete(BUCKET, blobName));
}
}
@Test
public void testPostSignedUrl() throws IOException {
String blobName = "test-post-signed-url-blob";
BlobInfo blob = BlobInfo.builder(BUCKET, blobName).build();
assertNotNull(storage.create(blob));
URL url =
storage.signUrl(blob, 1, TimeUnit.HOURS, Storage.SignUrlOption.httpMethod(HttpMethod.POST));
URLConnection connection = url.openConnection();
connection.setDoOutput(true);
connection.connect();
BlobInfo remoteBlob = storage.get(BUCKET, blobName);
assertNotNull(remoteBlob);
assertEquals(blob.blobId(), remoteBlob.blobId());
assertTrue(storage.delete(BUCKET, blobName));
}
@Test
public void testGetBlobs() {
String sourceBlobName1 = "test-get-blobs-1";
String sourceBlobName2 = "test-get-blobs-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1));
assertNotNull(storage.create(sourceBlob2));
List<BlobInfo> remoteBlobs = storage.get(sourceBlob1.blobId(), sourceBlob2.blobId());
assertEquals(sourceBlob1.blobId(), remoteBlobs.get(0).blobId());
assertEquals(sourceBlob2.blobId(), remoteBlobs.get(1).blobId());
assertTrue(storage.delete(BUCKET, sourceBlobName1));
assertTrue(storage.delete(BUCKET, sourceBlobName2));
}
@Test
public void testGetBlobsFail() {
String sourceBlobName1 = "test-get-blobs-fail-1";
String sourceBlobName2 = "test-get-blobs-fail-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1));
List<BlobInfo> remoteBlobs = storage.get(sourceBlob1.blobId(), sourceBlob2.blobId());
assertEquals(sourceBlob1.blobId(), remoteBlobs.get(0).blobId());
assertNull(remoteBlobs.get(1));
assertTrue(storage.delete(BUCKET, sourceBlobName1));
}
@Test
public void testDeleteBlobs() {
String sourceBlobName1 = "test-delete-blobs-1";
String sourceBlobName2 = "test-delete-blobs-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1));
assertNotNull(storage.create(sourceBlob2));
List<Boolean> deleteStatus = storage.delete(sourceBlob1.blobId(), sourceBlob2.blobId());
assertTrue(deleteStatus.get(0));
assertTrue(deleteStatus.get(1));
}
@Test
public void testDeleteBlobsFail() {
String sourceBlobName1 = "test-delete-blobs-fail-1";
String sourceBlobName2 = "test-delete-blobs-fail-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
assertNotNull(storage.create(sourceBlob1));
List<Boolean> deleteStatus = storage.delete(sourceBlob1.blobId(), sourceBlob2.blobId());
assertTrue(deleteStatus.get(0));
assertTrue(!deleteStatus.get(1));
}
@Test
public void testUpdateBlobs() {
String sourceBlobName1 = "test-update-blobs-1";
String sourceBlobName2 = "test-update-blobs-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
BlobInfo remoteBlob1 = storage.create(sourceBlob1);
BlobInfo remoteBlob2 = storage.create(sourceBlob2);
assertNotNull(remoteBlob1);
assertNotNull(remoteBlob2);
List<BlobInfo> updatedBlobs =
storage.update(
remoteBlob1.toBuilder().contentType(CONTENT_TYPE).build(),
remoteBlob2.toBuilder().contentType(CONTENT_TYPE).build());
assertEquals(sourceBlob1.blobId(), updatedBlobs.get(0).blobId());
assertEquals(CONTENT_TYPE, updatedBlobs.get(0).contentType());
assertEquals(sourceBlob2.blobId(), updatedBlobs.get(1).blobId());
assertEquals(CONTENT_TYPE, updatedBlobs.get(1).contentType());
assertTrue(storage.delete(BUCKET, sourceBlobName1));
assertTrue(storage.delete(BUCKET, sourceBlobName2));
}
@Test
public void testUpdateBlobsFail() {
String sourceBlobName1 = "test-update-blobs-fail-1";
String sourceBlobName2 = "test-update-blobs-fail-2";
BlobInfo sourceBlob1 = BlobInfo.builder(BUCKET, sourceBlobName1).build();
BlobInfo sourceBlob2 = BlobInfo.builder(BUCKET, sourceBlobName2).build();
BlobInfo remoteBlob1 = storage.create(sourceBlob1);
assertNotNull(remoteBlob1);
List<BlobInfo> updatedBlobs =
storage.update(
remoteBlob1.toBuilder().contentType(CONTENT_TYPE).build(),
sourceBlob2.toBuilder().contentType(CONTENT_TYPE).build());
assertEquals(sourceBlob1.blobId(), updatedBlobs.get(0).blobId());
assertEquals(CONTENT_TYPE, updatedBlobs.get(0).contentType());
assertNull(updatedBlobs.get(1));
assertTrue(storage.delete(BUCKET, sourceBlobName1));
}
}