private void requestNextLinks(LevelLink levelLink) throws InterruptedException { int currentLevel = levelLink.getLevel() + 1; String ch = ""; for (int i = 0; i < currentLevel; i++) { ch = ch + "--"; } if (currentLevel >= maxDeep) { return; } randomPause(); Elements links = getLinks(levelLink.getLink()); if (links == null) { return; } LOG.log( Level.DEBUG, " " + ch + " " + levelLink.getLink() + " ||| Thread: " + Thread.currentThread().getName() + " ||| level: " + currentLevel + " newLinks=" + links.size()); // get all ref on a page for (Element element : links) { URL url = null; try { url = new URL(element.attr("abs:href")); } catch (MalformedURLException e) { LOG.log(Level.INFO, "Exception: ", e); } String newURI = Utils.toFullForm(url, false); if (newURI == null) { continue; } putLink(new LevelLink(newURI, currentLevel)); } LevelLink nextLevelLink = readNextLink(); if (nextLevelLink != null) { requestNextLinks(nextLevelLink); } }