/** * Makes an HTTP Head request to retrieve the last modified date of the given URL. If the file:// * protocol is specified, then the lastTimestamp of the file is returned. * * @param url the URL to retrieve the timestamp from * @return an epoch timestamp * @throws DownloadFailedException is thrown if an exception occurs making the HTTP request */ public static long getLastModified(URL url) throws DownloadFailedException { long timestamp = 0; // TODO add the FTP protocol? if ("file".equalsIgnoreCase(url.getProtocol())) { File lastModifiedFile; try { lastModifiedFile = new File(url.toURI()); } catch (URISyntaxException ex) { final String msg = format("Unable to locate '%s'", url.toString()); throw new DownloadFailedException(msg); } timestamp = lastModifiedFile.lastModified(); } else { final String httpMethod = determineHttpMethod(); HttpURLConnection conn = null; try { conn = URLConnectionFactory.createHttpURLConnection(url); conn.setRequestMethod(httpMethod); conn.connect(); final int t = conn.getResponseCode(); if (t >= 200 && t < 300) { timestamp = conn.getLastModified(); } else { throw new DownloadFailedException( format("%s request returned a non-200 status code", httpMethod)); } } catch (URLConnectionFailureException ex) { throw new DownloadFailedException( format("Error creating URL Connection for HTTP %s request.", httpMethod), ex); } catch (IOException ex) { analyzeException(ex); try { // retry if (!Settings.getBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP)) { Settings.setBoolean(Settings.KEYS.DOWNLOADER_QUICK_QUERY_TIMESTAMP, true); return getLastModified(url); } } catch (InvalidSettingException ex1) { LOGGER.debug("invalid setting?", ex); } throw new DownloadFailedException(format("Error making HTTP %s request.", httpMethod), ex); } finally { if (conn != null) { try { conn.disconnect(); } finally { conn = null; } } } } return timestamp; }
/** * Retrieves a file from a given URL and saves it to the outputPath. * * @param url the URL of the file to download * @param outputPath the path to the save the file to * @param useProxy whether to use the configured proxy when downloading files * @throws DownloadFailedException is thrown if there is an error downloading the file */ public static void fetchFile(URL url, File outputPath, boolean useProxy) throws DownloadFailedException { if ("file".equalsIgnoreCase(url.getProtocol())) { File file; try { file = new File(url.toURI()); } catch (URISyntaxException ex) { final String msg = format("Download failed, unable to locate '%s'", url.toString()); throw new DownloadFailedException(msg); } if (file.exists()) { try { org.apache.commons.io.FileUtils.copyFile(file, outputPath); } catch (IOException ex) { final String msg = format( "Download failed, unable to copy '%s' to '%s'", url.toString(), outputPath.getAbsolutePath()); throw new DownloadFailedException(msg); } } else { final String msg = format("Download failed, file ('%s') does not exist", url.toString()); throw new DownloadFailedException(msg); } } else { HttpURLConnection conn = null; try { LOGGER.debug("Attempting download of {}", url.toString()); conn = URLConnectionFactory.createHttpURLConnection(url, useProxy); conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); conn.connect(); int status = conn.getResponseCode(); int redirectCount = 0; while ((status == HttpURLConnection.HTTP_MOVED_TEMP || status == HttpURLConnection.HTTP_MOVED_PERM || status == HttpURLConnection.HTTP_SEE_OTHER) && MAX_REDIRECT_ATTEMPTS > redirectCount++) { final String location = conn.getHeaderField("Location"); try { conn.disconnect(); } finally { conn = null; } LOGGER.debug("Download is being redirected from {} to {}", url.toString(), location); conn = URLConnectionFactory.createHttpURLConnection(new URL(location), useProxy); conn.setRequestProperty("Accept-Encoding", "gzip, deflate"); conn.connect(); status = conn.getResponseCode(); } if (status != 200) { try { conn.disconnect(); } finally { conn = null; } final String msg = format( "Error downloading file %s; received response code %s.", url.toString(), status); throw new DownloadFailedException(msg); } } catch (IOException ex) { try { if (conn != null) { conn.disconnect(); } } finally { conn = null; } final String msg = format("Error downloading file %s; unable to connect.", url.toString()); throw new DownloadFailedException(msg, ex); } final String encoding = conn.getContentEncoding(); BufferedOutputStream writer = null; InputStream reader = null; try { if (encoding != null && "gzip".equalsIgnoreCase(encoding)) { reader = new GZIPInputStream(conn.getInputStream()); } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) { reader = new InflaterInputStream(conn.getInputStream()); } else { reader = conn.getInputStream(); } writer = new BufferedOutputStream(new FileOutputStream(outputPath)); final byte[] buffer = new byte[4096]; int bytesRead; while ((bytesRead = reader.read(buffer)) > 0) { writer.write(buffer, 0, bytesRead); } LOGGER.debug("Download of {} complete", url.toString()); } catch (IOException ex) { analyzeException(ex); final String msg = format( "Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n", url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding); throw new DownloadFailedException(msg, ex); } catch (Throwable ex) { final String msg = format( "Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n", url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding); throw new DownloadFailedException(msg, ex); } finally { if (writer != null) { try { writer.close(); } catch (IOException ex) { LOGGER.trace("Error closing the writer in Downloader.", ex); } } if (reader != null) { try { reader.close(); } catch (IOException ex) { LOGGER.trace("Error closing the reader in Downloader.", ex); } } try { conn.disconnect(); } finally { conn = null; } } } }