public boolean shouldRetryRequest(HttpCommand command, HttpResponse response) {
if (command.getFailureCount() > retryCountLimit) return false;
if (response.getStatusCode() == 404
&& command.getCurrentRequest().getMethod().equals("DELETE")) {
command.incrementFailureCount();
return true;
} else if (response.getStatusCode() == 409) {
byte[] content = HttpUtils.closeClientButKeepContentStream(response);
// Content can be null in the case of HEAD requests
if (content != null) {
try {
AtmosError error =
utils.parseAtmosErrorFromContent(command, response, new String(content));
if (error.getCode() == 1006) {
return backoffHandler.shouldRetryRequest(command, response);
}
// don't increment count before here, since backoff handler does already
command.incrementFailureCount();
} catch (HttpException e) {
logger.warn(e, "error parsing response: %s", new String(content));
}
} else {
command.incrementFailureCount();
}
return true;
}
return false;
}
@Override
public URI deserialize(JsonElement jsonElement, Type type, JsonDeserializationContext context)
throws JsonParseException {
String toParse = jsonElement.getAsJsonPrimitive().getAsString();
URI toReturn = HttpUtils.createUri(toParse);
return toReturn;
}
@Inject
public JavaUrlHttpCommandExecutorService(
HttpUtils utils,
ContentMetadataCodec contentMetadataCodec,
@Named(Constants.PROPERTY_IO_WORKER_THREADS) ListeningExecutorService ioExecutor,
DelegatingRetryHandler retryHandler,
IOExceptionRetryHandler ioRetryHandler,
DelegatingErrorHandler errorHandler,
HttpWire wire,
@Named("untrusted") HostnameVerifier verifier,
@Named("untrusted") Supplier<SSLContext> untrustedSSLContextProvider,
Function<URI, Proxy> proxyForURI)
throws SecurityException, NoSuchFieldException {
super(
utils, contentMetadataCodec, ioExecutor, retryHandler, ioRetryHandler, errorHandler, wire);
if (utils.getMaxConnections() > 0)
System.setProperty(
"http.maxConnections", String.valueOf(checkNotNull(utils, "utils").getMaxConnections()));
this.untrustedSSLContextProvider =
checkNotNull(untrustedSSLContextProvider, "untrustedSSLContextProvider");
this.verifier = checkNotNull(verifier, "verifier");
this.proxyForURI = checkNotNull(proxyForURI, "proxyForURI");
this.methodField = HttpURLConnection.class.getDeclaredField("method");
this.methodField.setAccessible(true);
}
public void filter(HttpRequest request) throws HttpException {
String toSign =
replaceUIDHeader(request)
.removeOldSignature(request)
.replaceDateHeader(request)
.createStringToSign(request);
calculateAndReplaceAuthHeader(request, toSign);
utils.logRequest(signatureLog, request, "<<");
}
private void appendPayloadMetadata(HttpRequest request, StringBuilder buffer) {
buffer
.append(
utils.valueOrEmpty(
request.getPayload() == null
? null
: request.getPayload().getContentMetadata().getContentType()))
.append("\n");
}
public MutableBlobPropertiesImpl(BlobProperties from) {
this.contentMetadata = new BaseMutableContentMetadata();
this.name = from.getName();
this.container = from.getContainer();
this.url = from.getUrl();
this.lastModified = from.getLastModified();
this.eTag = from.getETag();
this.metadata.putAll(from.getMetadata());
HttpUtils.copy(from.getContentMetadata(), this.contentMetadata);
}
/**
* parses the http response headers to create a new {@link
* org.jclouds.aws.s3.domain.internal.MutableObjectMetadata} object.
*/
public MutableObjectMetadata apply(HttpResponse from) {
BlobMetadata base = blobMetadataParser.apply(from);
MutableObjectMetadata to = blobToObjectMetadata.apply(base);
addETagTo(from, to);
to.setContentMD5(HttpUtils.fromHexString(to.getETag().replaceAll("\"", "")));
to.setCacheControl(from.getFirstHeaderOrNull(HttpHeaders.CACHE_CONTROL));
to.setContentDisposition(from.getFirstHeaderOrNull("Content-Disposition"));
to.setContentEncoding(from.getFirstHeaderOrNull(HttpHeaders.CONTENT_ENCODING));
return to;
}
@VisibleForTesting
void appendHttpHeaders(HttpRequest request, StringBuilder toSign) {
// Only the value is used, not the header
// name. If a request does not include the header, this is an empty string.
for (String header : new String[] {"Range"})
toSign
.append(utils.valueOrEmpty(request.getHeaders().get(header)).toLowerCase())
.append("\n");
// Standard HTTP header, in UTC format. Only the date value is used, not the header name.
toSign.append(request.getHeaders().get(HttpHeaders.DATE).iterator().next()).append("\n");
}
public String createStringToSign(HttpRequest request) {
utils.logRequest(signatureLog, request, ">>");
StringBuilder buffer = new StringBuilder();
// re-sign the request
appendMethod(request, buffer);
appendPayloadMetadata(request, buffer);
appendHttpHeaders(request, buffer);
appendCanonicalizedResource(request, buffer);
appendCanonicalizedHeaders(request, buffer);
if (signatureWire.enabled()) signatureWire.output(buffer.toString());
return buffer.toString();
}
public HttpRequest filter(HttpRequest input) throws HttpException {
HttpRequest request =
input.toBuilder().endpoint(input.getEndpoint().toString().replace("%3F", "?")).build();
String contentHash = hashBody(request.getPayload());
Multimap<String, String> headers = ArrayListMultimap.create();
headers.put("X-Ops-Content-Hash", contentHash);
String timestamp = timeStampProvider.get();
String toSign =
createStringToSign(
request.getMethod(), hashPath(request.getEndpoint().getPath()), contentHash, timestamp);
headers.put("X-Ops-Userid", creds.get().identity);
headers.put("X-Ops-Sign", SIGNING_DESCRIPTION);
request = calculateAndReplaceAuthorizationHeaders(request, toSign);
headers.put("X-Ops-Timestamp", timestamp);
utils.logRequest(signatureLog, request, "<<");
return request.toBuilder().replaceHeaders(headers).build();
}
public static void main(String... args) {
if (args.length != 1) throw new IllegalArgumentException(INVALID_SYNTAX);
URI location;
try {
location = HttpUtils.createUri(args[0]);
checkArgument(location.getScheme().equals("blobstore"), "wrong scheme");
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(String.format("%s%n%s", e.getMessage(), INVALID_SYNTAX));
}
try {
(new BlobStoreShell(args[0])).go();
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
}
System.exit(0);
}
public MutableBlobMetadata apply(AtmosObject from) {
if (from == null) return null;
MutableBlobMetadata to = new MutableBlobMetadataImpl();
to.setId(from.getSystemMetadata().getObjectID());
to.setLastModified(from.getSystemMetadata().getLastUserDataModification());
HttpUtils.copy(from.getContentMetadata(), to.getContentMetadata());
to.setName(objectName.apply(from));
if (from.getSystemMetadata().getType() == FileType.DIRECTORY) {
to.setType(StorageType.FOLDER);
} else {
to.setType(StorageType.BLOB);
}
Map<String, String> lowerKeyMetadata = Maps.newHashMap();
for (Entry<String, String> entry : from.getUserMetadata().getMetadata().entrySet()) {
String key = entry.getKey().toLowerCase();
if (!systemMetadata.contains(key)) lowerKeyMetadata.put(key, entry.getValue());
}
to.setUserMetadata(lowerKeyMetadata);
return to;
}
public HttpResponse apply(Response nativeResponse) {
InputStream in = null;
try {
in =
BaseHttpCommandExecutorService.consumeOnClose(nativeResponse.getResponseBodyAsStream());
} catch (IOException e) {
Closeables.closeQuietly(in);
propagate(e);
assert false : "should have propagated exception";
}
Payload payload = in != null ? Payloads.newInputStreamPayload(in) : null;
HttpResponse response =
new HttpResponse(nativeResponse.getStatusCode(), nativeResponse.getStatusText(), payload);
Multimap<String, String> headers = LinkedHashMultimap.create();
for (Entry<String, List<String>> header : nativeResponse.getHeaders()) {
headers.putAll(header.getKey(), header.getValue());
}
utils.setPayloadPropertiesFromHeaders(headers, response);
return response;
}
public void endElement(String uri, String name, String qName) {
if (inMetadata && !qName.equals("Metadata")) {
currentMetadata.put(qName, currentText.toString().trim());
} else if (qName.equals("Metadata")) {
inMetadata = false;
} else if (qName.equals("MaxResults")) {
maxResults = Integer.parseInt(currentText.toString().trim());
} else if (qName.equals("Marker")) {
marker = currentText.toString().trim();
marker = (marker.equals("")) ? null : marker;
} else if (qName.equals("Prefix")) {
prefix = currentText.toString().trim();
prefix = (prefix.equals("")) ? null : prefix;
} else if (qName.equals("Delimiter")) {
delimiter = currentText.toString().trim();
delimiter = (delimiter.equals("")) ? null : delimiter;
} else if (qName.equals("NextMarker")) {
nextMarker = currentText.toString().trim();
nextMarker = (nextMarker.equals("")) ? null : nextMarker;
} else if (qName.equals("BlobType")) {
currentBlobType = BlobType.fromValue(currentText.toString().trim());
} else if (qName.equals("LeaseStatus")) {
currentLeaseStatus = LeaseStatus.fromValue(currentText.toString().trim());
} else if (qName.equals("Blob")) {
BlobProperties md =
new BlobPropertiesImpl(
currentBlobType,
currentName,
containerUrl.getPath().replace("/", ""),
currentUrl,
currentLastModified,
currentETag,
currentSize,
currentContentType,
currentContentMD5,
currentContentEncoding,
currentContentLanguage,
currentLeaseStatus,
currentMetadata);
blobMetadata.add(md);
currentBlobType = null;
currentName = null;
currentUrl = null;
currentLastModified = null;
currentETag = null;
currentSize = -1;
currentContentType = null;
currentContentEncoding = null;
currentContentLanguage = null;
currentContentMD5 = null;
currentLeaseStatus = null;
currentMetadata = Maps.newHashMap();
} else if (qName.equals("Url")) {
currentUrl = HttpUtils.createUri(currentText.toString().trim());
} else if (qName.equals("Last-Modified")) {
currentLastModified = dateParser.rfc822DateParse(currentText.toString().trim());
} else if (qName.equals("Etag")) {
currentETag = currentText.toString().trim();
} else if (qName.equals("Name")) {
if (inBlob) currentName = currentText.toString().trim();
else if (inBlobPrefix) blobPrefixes.add(currentText.toString().trim());
} else if (qName.equals("Content-Length")) {
currentSize = Long.parseLong(currentText.toString().trim());
} else if (qName.equals("Content-MD5")) {
if (!currentText.toString().trim().equals(""))
currentContentMD5 = CryptoStreams.base64(currentText.toString().trim());
} else if (qName.equals("Content-Type")) {
currentContentType = currentText.toString().trim();
} else if (qName.equals("Content-Encoding")) {
currentContentEncoding = currentText.toString().trim();
if (currentContentEncoding.equals("")) currentContentEncoding = null;
} else if (qName.equals("Content-Language")) {
currentContentLanguage = currentText.toString().trim();
if (currentContentLanguage.equals("")) currentContentLanguage = null;
}
currentText = new StringBuilder();
}