/**
* Check whether the content type returned by the server is acceptable to the endpoint and data
* provider
*/
protected boolean isValidContentType(String contentType, Endpoint endpoint) {
if (endpoint != null && endpoint.getContentTypes().size() > 0) {
ContentType parsed = parseContentType(contentType);
for (ContentType valid : endpoint.getContentTypes()) {
if (valid.matches(parsed) || valid.matchesWildcard(parsed)) {
return true;
}
}
return false;
} else {
// TODO: should probably be removed, since it is not used
for (String type : listMimeTypes()) {
if (type.split(";")[0].equalsIgnoreCase(contentType)) return true;
}
return false;
}
}
@Override
public List<String> handleResponse(HttpResponse response)
throws ClientProtocolException, IOException {
ArrayList<String> requestUrls = new ArrayList<String>();
if (response.getStatusLine().getStatusCode() >= 200
&& response.getStatusLine().getStatusCode() < 400) {
final HttpEntity entity = response.getEntity();
if (entity == null)
throw new IOException("no content returned by Linked Data resource " + resource);
if (!isValidContentType(entity.getContentType().getValue().split(";")[0], endpoint)) {
// FIXME: here was get.abort()
throw new IOException(
"invalid content returned by Linked Data resource "
+ resource
+ ": "
+ entity.getContentType().getValue());
}
this.httpStatus = response.getStatusLine().getStatusCode();
if (entity != null) {
String parseContentType = "application/rdf+xml";
if (endpoint != null && "SPARQL".equals(endpoint.getType())) {
parseContentType = "application/sparql-results+xml";
} else if (entity.getContentType() != null) {
parseContentType = entity.getContentType().getValue().split(";")[0];
}
InputStream in = entity.getContent();
try {
List<String> urls = parseResponse(resource, requestUrl, triples, in, parseContentType);
requestUrls.addAll(urls);
if (expiresDate == null) {
Header expires = response.getFirstHeader("Expires");
if (expires != null) {
try {
expiresDate = DateUtils.parseDate(expires.getValue());
} catch (DateParseException e) {
log.debug("error parsing Expires: header");
}
}
}
} catch (DataRetrievalException e) {
// FIXME: get.abort();
throw new IOException(e);
} finally {
in.close();
}
}
EntityUtils.consume(entity);
} else if (response.getStatusLine().getStatusCode() == 500
|| response.getStatusLine().getStatusCode() == 503
|| response.getStatusLine().getStatusCode() == 504) {
this.httpStatus = response.getStatusLine().getStatusCode();
Header retry = response.getFirstHeader("Retry-After");
if (retry != null) {
try {
int duration = Integer.parseInt(retry.getValue());
expiresDate = new Date(System.currentTimeMillis() + duration * 1000);
} catch (NumberFormatException ex) {
log.debug("error parsing Retry-After: header");
}
} else {
expiresDate = new Date(System.currentTimeMillis() + RETRY_AFTER * 1000);
}
} else {
log.error("the HTTP request failed (status: {})", response.getStatusLine());
throw new ClientProtocolException(
"the HTTP request failed (status: " + response.getStatusLine() + ")");
}
return requestUrls;
}
/**
* Retrieve the data for a resource using the given http client and endpoint definition. The
* service is supposed to manage the connection handling itself. See {@link AbstractHttpProvider}
* for a generic implementation of this method.
*
* @param resource the resource to be retrieved
* @param endpoint the endpoint definition
* @return a completely specified client response, including expiry information and the set of
* triples
*/
@Override
public ClientResponse retrieveResource(String resource, LDClientService client, Endpoint endpoint)
throws DataRetrievalException {
try {
String contentType;
if (endpoint != null && endpoint.getContentTypes().size() > 0) {
contentType =
CollectionUtils.fold(
endpoint.getContentTypes(),
new CollectionUtils.StringSerializer<ContentType>() {
@Override
public String serialize(ContentType contentType) {
return contentType.toString("q");
}
},
",");
} else {
contentType = CollectionUtils.fold(Arrays.asList(listMimeTypes()), ",");
}
long defaultExpires = client.getClientConfiguration().getDefaultExpiry();
if (endpoint != null && endpoint.getDefaultExpiry() != null) {
defaultExpires = endpoint.getDefaultExpiry();
}
final ResponseHandler handler = new ResponseHandler(resource, endpoint);
// a queue for queuing the request URLs needed to build the query response
Queue<String> requestUrls = new LinkedList<String>();
requestUrls.addAll(buildRequestUrl(resource, endpoint));
Set<String> visited = new HashSet<String>();
String requestUrl = requestUrls.poll();
while (requestUrl != null) {
if (!visited.contains(requestUrl)) {
HttpGet get = new HttpGet(requestUrl);
try {
get.setHeader("Accept", contentType);
get.setHeader("Accept-Language", "*"); // PoolParty compatibility
log.info(
"retrieving resource data for {} from '{}' endpoint, request URI is <{}>",
new Object[] {resource, getName(), get.getURI().toASCIIString()});
handler.requestUrl = requestUrl;
List<String> additionalRequestUrls = client.getClient().execute(get, handler);
requestUrls.addAll(additionalRequestUrls);
visited.add(requestUrl);
} finally {
get.releaseConnection();
}
}
requestUrl = requestUrls.poll();
}
Date expiresDate = handler.expiresDate;
if (expiresDate == null) {
expiresDate = new Date(System.currentTimeMillis() + defaultExpires * 1000);
}
long min_expires =
System.currentTimeMillis() + client.getClientConfiguration().getMinimumExpiry() * 1000;
if (expiresDate.getTime() < min_expires) {
log.info(
"expiry time returned by request lower than minimum expiration time; using minimum time instead");
expiresDate = new Date(min_expires);
}
if (log.isInfoEnabled()) {
RepositoryConnection con = handler.triples.getConnection();
log.info(
"retrieved {} triples for resource {}; expiry date: {}",
new Object[] {con.size(), resource, expiresDate});
con.close();
}
ClientResponse result = new ClientResponse(handler.httpStatus, handler.triples);
result.setExpires(expiresDate);
return result;
} catch (RepositoryException e) {
log.error("error while initialising Sesame repository; classpath problem?", e);
throw new DataRetrievalException(
"error while initialising Sesame repository; classpath problem?", e);
} catch (ClientProtocolException e) {
log.error(
"HTTP client error while trying to retrieve resource {}: {}", resource, e.getMessage());
throw new DataRetrievalException(
"I/O error while trying to retrieve resource " + resource, e);
} catch (IOException e) {
log.error("I/O error while trying to retrieve resource {}: {}", resource, e.getMessage());
throw new DataRetrievalException(
"I/O error while trying to retrieve resource " + resource, e);
} catch (RuntimeException ex) {
log.error(
"Unknown error while trying to retrieve resource {}: {}", resource, ex.getMessage());
throw new DataRetrievalException(
"Unknown error while trying to retrieve resource " + resource, ex);
}
}
