@Test
  public void testSimpleGetVersion() throws ServiceException, IOException {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);

    S3Object object0 = new S3Object();

    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));

    EasyMock.expect(s3Client.getObjectDetails(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);

    EasyMock.replay(s3Client);

    String version =
        puller.getVersion(URI.create(String.format("s3://%s/%s", bucket, object0.getKey())));

    EasyMock.verify(s3Client);

    Assert.assertEquals(String.format("%d", new Date(0).getTime()), version);
  }
  @Test
  public void testGZUncompressRetries()
      throws ServiceException, IOException, SegmentLoadingException {
    final String bucket = "bucket";
    final String keyPrefix = "prefix/dir/0";
    final RestS3Service s3Client = EasyMock.createStrictMock(RestS3Service.class);
    final byte[] value = bucket.getBytes("utf8");

    final File tmpFile = temporaryFolder.newFile("gzTest.gz");

    try (OutputStream outputStream = new GZIPOutputStream(new FileOutputStream(tmpFile))) {
      outputStream.write(value);
    }

    S3Object object0 = new S3Object();

    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModifiedDate(new Date(0));
    object0.setDataInputStream(new FileInputStream(tmpFile));

    File tmpDir = temporaryFolder.newFolder("gzTestDir");

    S3ServiceException exception = new S3ServiceException();
    exception.setErrorCode("NoSuchKey");
    exception.setResponseCode(404);
    EasyMock.expect(
            s3Client.getObjectDetails(
                EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
        .andReturn(null)
        .once();
    EasyMock.expect(
            s3Client.getObjectDetails(
                EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
        .andThrow(exception)
        .once();
    EasyMock.expect(
            s3Client.getObjectDetails(
                EasyMock.eq(object0.getBucketName()), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    EasyMock.expect(s3Client.getObject(EasyMock.eq(bucket), EasyMock.eq(object0.getKey())))
        .andReturn(object0)
        .once();
    S3DataSegmentPuller puller = new S3DataSegmentPuller(s3Client);

    EasyMock.replay(s3Client);
    FileUtils.FileCopyResult result =
        puller.getSegmentFiles(new S3DataSegmentPuller.S3Coords(bucket, object0.getKey()), tmpDir);
    EasyMock.verify(s3Client);

    Assert.assertEquals(value.length, result.size());
    File expected = new File(tmpDir, "renames-0");
    Assert.assertTrue(expected.exists());
    Assert.assertEquals(value.length, expected.length());
  }
Ejemplo n.º 3
0
    public void endElement(String uri, String name, String qName) {
      String elementText = this.currText.toString();
      // Listing details
      if (name.equals("Name")) {
        bucketName = elementText;
        if (log.isDebugEnabled()) {
          log.debug("Examining listing for bucket: " + bucketName);
        }
      } else if (!insideCommonPrefixes && name.equals("Prefix")) {
        requestPrefix = elementText;
      } else if (name.equals("Marker")) {
        requestMarker = elementText;
      } else if (name.equals("NextMarker")) {
        nextMarker = elementText;
      } else if (name.equals("MaxKeys")) {
        requestMaxKeys = Long.parseLong(elementText);
      } else if (name.equals("IsTruncated")) {
        String isTruncatedStr = elementText.toLowerCase(Locale.getDefault());
        if (isTruncatedStr.startsWith("false")) {
          listingTruncated = false;
        } else if (isTruncatedStr.startsWith("true")) {
          listingTruncated = true;
        } else {
          throw new RuntimeException("Invalid value for IsTruncated field: " + isTruncatedStr);
        }
      }
      // Object details.
      else if (name.equals("Contents")) {
        objects.add(currentObject);
        if (log.isDebugEnabled()) {
          log.debug("Created new S3Object from listing: " + currentObject);
        }
      } else if (name.equals("Key")) {
        currentObject.setKey(elementText);
        lastKey = elementText;
      } else if (name.equals("LastModified")) {
        try {
          currentObject.setLastModifiedDate(ServiceUtils.parseIso8601Date(elementText));
        } catch (ParseException e) {
          throw new RuntimeException("Unexpected date format in list bucket output", e);
        }
      } else if (name.equals("ETag")) {
        currentObject.setETag(elementText);
      } else if (name.equals("Size")) {
        currentObject.setContentLength(Long.parseLong(elementText));
      } else if (name.equals("StorageClass")) {
        currentObject.setStorageClass(elementText);
      }
      // Owner details.
      else if (name.equals("ID")) {
        // Work-around to support Eucalyptus responses, which do not
        // contain Owner elements.
        if (currentOwner == null) {
          currentOwner = new S3Owner();
          currentObject.setOwner(currentOwner);
        }

        currentOwner.setId(elementText);
      } else if (name.equals("DisplayName")) {
        currentOwner.setDisplayName(elementText);
      }
      // Common prefixes.
      else if (insideCommonPrefixes && name.equals("Prefix")) {
        commonPrefixes.add(elementText);
      } else if (name.equals("CommonPrefixes")) {
        insideCommonPrefixes = false;
      }

      this.currText = new StringBuffer();
    }