public static List<String> getUrls(ConfigBean config) {
List<String> inputData = new ArrayList<String>();
String ftl = config.getBaseDir() + File.separator + config.getUrlFile();
if (!FileUtil.fileToList(null, ftl, inputData)) {
throw new IllegalStateException("Couldn't load file " + ftl);
}
for (int i = 0; i < inputData.size(); i++) {
String ln = inputData.get(i);
if (ln.matches("^\\s*$")) {
inputData.remove(i);
logger.warn("Ignoring empty URL from line " + i + " on file " + ftl);
/*throw new IllegalArgumentException("The url at line " + i
+ " of " + ftl + " is invalid (empty)");*/
}
}
return inputData;
}
public static void main(String[] args) {
ConfigBean config = new ConfigBean();
CacheBean cacheB = new CacheBean();
String cacheDir = args[1];
String baseDir = args[2];
String urlFile = args[3];
boolean refetch = false;
if (args.length == 5 && args[4].equalsIgnoreCase("refetch")) {
refetch = true;
logger.debug("Forcing refetch");
} else {
logger.debug("Not forcing refetch");
}
config.initialize(null, cacheDir, baseDir, urlFile, -1);
cacheB.initialize(config.getCacheDir(), true);
UrlsBean ub = new UrlsBean();
ub.initialize(config, cacheB);
ub.startBGFetcher(refetch);
}
