public void run() { int sleepInterval = runInterval; boolean setInitial = false; while (true) { try { // get desired index state: Object desired[] = getDesiredMD5s(); // get current index state: Object current[] = getCurrentMD5s(); // work to do? HashMap<String, Object> desiredMap = new HashMap<String, Object>(); ArrayList<String> extra = new ArrayList<String>(); for (int i = 0; i < desired.length; i++) { desiredMap.put((String) desired[i], null); } for (int i = 0; i < current.length; i++) { if (desiredMap.containsKey(current[i])) { desiredMap.remove(current[i]); } else { extra.add((String) current[i]); } } Set<String> needed = desiredMap.keySet(); if ((needed.size() + extra.size()) > 0) { // whoops -- we're off: index.setState(DynamicCDXIndex.STATE_SYNCHING); // first remove extras: removeFiles(extra.toArray()); // now get needed: downloadFiles(needed.toArray()); index.setCDXFiles(desired); index.setState(DynamicCDXIndex.STATE_SYNCHED); } else if (!setInitial && current.length > 0) { index.setCDXFiles(desired); index.setState(DynamicCDXIndex.STATE_SYNCHED); } sleep(sleepInterval); } catch (InterruptedException e) { e.printStackTrace(); return; } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } } }
private void removeFiles(Object toBeRemoved[]) throws IOException { for (int i = 0; i < toBeRemoved.length; i++) { File toDelete = index.dataFileForMD5((String) toBeRemoved[i]); if (!toDelete.delete()) { throw new IOException("Failed to remove " + toDelete.getAbsolutePath()); } } }
private void downloadFiles(Object toBeDownloaded[]) throws IOException { for (int i = 0; i < toBeDownloaded.length; i++) { String neededMD5 = (String) toBeDownloaded[i]; File target = index.dataFileForMD5(neededMD5); File tmpTarget = new File(target.getAbsolutePath() + ".TMP"); Object locs[] = md5File.getLocationsForMD5(neededMD5); boolean gotFile = false; for (int j = 0; j < locs.length; j++) { String loc = (String) locs[j]; URL u = new URL(loc); try { if (loc.endsWith(".gz")) { downloader.downloadGZ(u, tmpTarget); } else { downloader.download(u, tmpTarget); } String gotMD5 = downloader.getLastDigest(); if (gotMD5.equals(neededMD5)) { gotFile = true; tmpTarget.renameTo(target); break; } else { tmpTarget.delete(); LOGGER.warning( "Bad file contents. Location(" + loc + ") should have MD5(" + neededMD5 + ") but has MD5(" + gotMD5 + ")"); } } catch (IOException e) { e.printStackTrace(); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } } if (!gotFile) { throw new IOException("Unable to get MD5 " + neededMD5); } } }
private Object[] getCurrentMD5s() { return index.getLocalMD5s(); }