/**
* Handles POST requests for /client.html
*
* @param remoteResource the object with form data about entry point.
* @return name of JSP page which will be shown.
*/
@RequestMapping(value = "/client.html", method = RequestMethod.POST)
public String doClient(RemoteResource remoteResource) {
remoteResource.sendRequest();
return "client";
}
/**
* Discovers and crawls in currentResourceNode for aditional resources.
*
* @param currentResourceNode resource which will be crawled.
* @param apiEntry Informations about api entrypoint.
* @param discoveredResources Map which holds informations about already crawled resources.
* @param toVisit queue used to store discovered resources for aditional crawling.
*/
private void doCrawle(
ResourceNode currentResourceNode,
ApiEntry apiEntry,
Map<String, ResourceNode> discoveredResources,
Queue<ResourceNode> toVisit) {
if (discoveredResources.containsKey(currentResourceNode.getCurrentResource().getUrl())) {
currentResourceNode.setCurrentResource(
(RemoteResource)
discoveredResources
.get(currentResourceNode.getCurrentResource().getUrl())
.getCurrentResource());
currentResourceNode.setCurrentResourceOptions(
(RemoteResource)
discoveredResources
.get(currentResourceNode.getCurrentResource().getUrl())
.getCurrentResourceOptions());
return;
}
currentResourceNode.sendOptions();
currentResourceNode.sendRequest();
LinkExtrator links = new LinkExtrator();
if (currentResourceNode.getCurrentResource().getResponseBody() == null) {
return;
}
List<String> urls =
UrlWorker.getUrls(
currentResourceNode.getCurrentResource().getUrl(),
links.grabHTMLLinks(currentResourceNode.getCurrentResource().getResponseBody()));
List<String> validUrls = new ArrayList<String>();
for (String url : urls) {
if (url.startsWith(apiEntry.getBaseUrl())) {
validUrls.add(url);
}
}
if (validUrls.size() > 0) {
RemoteResource nextResource;
int maxResourcesToLoad = apiEntry.getMaxSiblings();
for (String url : validUrls) {
try {
nextResource = (RemoteResource) currentResourceNode.getCurrentResource().clone();
nextResource.deletePreviousResponse();
if (apiEntry.getUrl().contains("?") && !url.contains("?")) {
String append = currentResourceNode.getCurrentResource().getUrl();
append =
currentResourceNode
.getCurrentResource()
.getUrl()
.substring(currentResourceNode.getCurrentResource().getUrl().lastIndexOf('?'));
nextResource.setUrl(url + append);
} else {
nextResource.setUrl(url);
}
ResourceNode nextResourceNode = new ResourceNode(nextResource);
currentResourceNode.getDescendants().add(nextResourceNode);
if (--maxResourcesToLoad < 0) {
// No more crawling for this resource's childs.
} else {
toVisit.add(nextResourceNode);
}
} catch (CloneNotSupportedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
discoveredResources.put(currentResourceNode.getCurrentResource().getUrl(), currentResourceNode);
}
