public BaseClientResult<ResultType> fetch(Config<ResultType> config)
throws IOException, ParseException, InterruptedException {
PartialBaseClientResult<ResultType> partial_result = partialFetch(config);
try {
return completeFetch(partial_result);
} finally {
closeQuietly(partial_result.getConnection());
}
}
private void setMoreResults(URLConnection conn, PartialBaseClientResult<ResultType> result) {
String more = conn.getHeaderField(X_MORE_RESULTS);
if (more == null) result.setHasMoreResultsHeader(false);
else {
result.setHasMoreResultsHeader(true);
if ("true".equals(more)) result.setHasMoreResults(true);
else result.setHasMoreResults(false);
}
}
/**
* Fetch the API with the given FeedConfig
*
* @throws IOException if there's an error with network transport.
* @throws ParseException if there's a problem parsing the resulting XML.
*/
public BaseClientResult<ResultType> completeFetch(
PartialBaseClientResult<ResultType> partial_result)
throws IOException, ParseException, InterruptedException {
Config<ResultType> config = partial_result.getConfig();
int request_limit = partial_result.getRequestLimit();
String resource = partial_result.getLastRequestURL();
boolean has_results_header = partial_result.getHasMoreResultsHeader();
boolean has_more_results = partial_result.getHasMoreResults();
String next_request = partial_result.getNextRequestURL();
BaseClientResult<ResultType> result = new BaseClientResult<ResultType>(config);
result.setLastRequestURL(resource);
result.setRequestLimit(request_limit);
result.setHasMoreResultsHeadder(has_results_header);
result.setHasMoreResults(has_more_results);
result.setNextRequestURL(next_request);
try {
long before = System.currentTimeMillis();
long call_before = System.currentTimeMillis();
URLConnection conn = partial_result.getConnection();
// TODO: clean up the naming here. getLocalInputStream actually
// reads everything into a byte array in memory.
InputStream localInputStream = getLocalInputStream(conn.getInputStream(), result);
result.setLocalInputStream(localInputStream);
if (GZIP_ENCODING.equals(conn.getContentEncoding())) result.setIsCompressed(true);
InputStream is = result.getInputStream();
long call_after = System.currentTimeMillis();
result.setCallDuration(call_after - call_before);
if (!config.getDisableParse()) {
if (config.getFormat() == Format.PROTOSTREAM)
result.setResults(protobufParse(doProtoStreamFetch(localInputStream, config), config));
else if (config.getFormat() == Format.PROTOBUF)
result.setResults(protobufParse(doProtobufFetch(localInputStream, config), config));
else {
Document doc = doXmlFetch(is, config);
if (doc != null) {
result.setResults(xmlParse(doc, config));
}
}
}
long after = System.currentTimeMillis();
result.setParseDuration(after - before);
} catch (Exception e) {
throw new ParseException(e, "Unable to handle request: " + resource);
}
if (!result.getHasMoreResultsHeadder())
result.setHasMoreResults(result.getResults().size() == request_limit);
return result;
}
/**
* Fetch the API with the given FeedConfig
*
* @throws IOException if there's an error with network transport.
* @throws ParseException if there's a problem parsing the resulting XML.
*/
private PartialBaseClientResult<ResultType> startFetch(
Config<ResultType> config, int request_limit) throws IOException, InterruptedException {
PartialBaseClientResult<ResultType> result = new PartialBaseClientResult<ResultType>(config);
if (config.getVendor() == null) throw new RuntimeException("Vendor not specified");
String resource = config.getNextRequestURL();
// enforce max limit so that we don't generate runtime exceptions.
if (request_limit > config.getMaxLimit()) request_limit = config.getMaxLimit();
if (resource == null) {
resource = config.getFirstRequestURL();
// if the API has NEVER been used before then generate the first
// request URL from the config parameters.
if (resource == null) resource = config.generateFirstRequestURL(request_limit);
}
// apply the request_limit to the current URL. This needs to be done so
// that we can change the limit at runtime. When I originally designed
// the client I didn't want to support introspecting and mutating the URL
// on the client but with the optimial limit performance optimization
// this is impossible.
resource = setParam(resource, "limit", request_limit);
// add a connection number to the vendor code
resource = addConnectionNumber(resource);
// store the last requested URL so we can expose this to the caller for
// debug purposes.
result.setLastRequestURL(resource);
result.setRequestLimit(request_limit);
URLConnection conn = getConnection(resource);
/*
* If this is an http connection and there is an error code,
* return throw an error message containing the response
* message.
*/
if (conn instanceof HttpURLConnection) {
HttpURLConnection httpConn = (HttpURLConnection) conn;
int responseCode = httpConn.getResponseCode();
if (responseCode >= 400) {
StringBuilder message = new StringBuilder("");
InputStream errorStream = httpConn.getErrorStream();
if (errorStream == null)
throw new IOException(String.format("Response code %d received", responseCode));
BufferedReader reader =
new BufferedReader(new InputStreamReader(new GZIPInputStream(errorStream)));
String line;
while ((line = reader.readLine()) != null) message.append(line);
throw new IOException(message.toString());
}
}
result.setConnection(conn);
setMoreResults(conn, result);
result.setNextRequestURL(conn.getHeaderField("X-Next-Request-URL"));
return result;
}
