public static List<String> getUrls(ConfigBean config) { List<String> inputData = new ArrayList<String>(); String ftl = config.getBaseDir() + File.separator + config.getUrlFile(); if (!FileUtil.fileToList(null, ftl, inputData)) { throw new IllegalStateException("Couldn't load file " + ftl); } for (int i = 0; i < inputData.size(); i++) { String ln = inputData.get(i); if (ln.matches("^\\s*$")) { inputData.remove(i); logger.warn("Ignoring empty URL from line " + i + " on file " + ftl); /*throw new IllegalArgumentException("The url at line " + i + " of " + ftl + " is invalid (empty)");*/ } } return inputData; }
public static void main(String[] args) { ConfigBean config = new ConfigBean(); CacheBean cacheB = new CacheBean(); String cacheDir = args[1]; String baseDir = args[2]; String urlFile = args[3]; boolean refetch = false; if (args.length == 5 && args[4].equalsIgnoreCase("refetch")) { refetch = true; logger.debug("Forcing refetch"); } else { logger.debug("Not forcing refetch"); } config.initialize(null, cacheDir, baseDir, urlFile, -1); cacheB.initialize(config.getCacheDir(), true); UrlsBean ub = new UrlsBean(); ub.initialize(config, cacheB); ub.startBGFetcher(refetch); }