public String getItem() throws Exception {
logger.finest("Item request initiated.");
String response = null;
if (!status.init) {
if (!status.load(statusFile)) {
status.init = true;
} // else the init was loaded from the serialized status
}
// clear the sitemap and reset if we are at the end of portal
// if the last listing couldn't be loaded, the status was not reset
if (!status.siteMap.isEmpty()) {
if (status.siteMap.size() == status.siteMapIndex) {
status.reset().save(statusFile);
}
}
// load the sitemap if needed
if (status.siteMap.isEmpty()) {
try {
logger.finest("Loading sitemap: " + siteMapUrl);
loadPage(siteMapUrl, siteMapParser);
if (status.siteMap.isEmpty()) {
response = "Failed to load the sitemap - Empty";
logger.severe(response);
return error + response;
} else {
status.save(statusFile);
int size = status.siteMap.size();
logger.info("Sitemap loaded successfuly: " + size + " items");
for (int i = 0; i < size; i++) {
SiteMapLocation sml = status.siteMap.get(i);
logger.finest("Sitemap item #" + i + " : [" + sml.name + "] " + sml.url);
}
}
} catch (Exception e) {
response = "Failed to load the sitemap: " + e.getMessage();
logger.severe(response);
return error + response;
}
}
// load the list if needed
if (status.list.isEmpty() && !status.siteMap.isEmpty()) {
String coordinates =
"[" + status.siteMapIndex + ", " + status.page + ", " + status.pagePosition + "]";
URL listUrl = modifyUrl(status.siteMap.get(status.siteMapIndex).url);
logger.finest("Loading list: " + listUrl + " " + coordinates);
try {
loadPage(listUrl, listParser);
} catch (Exception e) {
response =
"Failed to load the list: ["
+ status.siteMap.get(status.siteMapIndex).url
+ "] "
+ e.getMessage();
logger.severe(response);
return error + response;
} finally {
// if after loading the page list is still empty even after second attempt, we should
// probably try next category, we are out of range of the pager
// this can happen when loading an older status and the page structure changed in the
// meantime
if (status.list.isEmpty()) {
response = "Failed to load list - Empty: " + listUrl + " " + coordinates;
logger.warning(response);
if (firstEmptyList) {
// this is a first empty list, give it one more chance and go to the next page
status.nextPage().save(statusFile);
firstEmptyList = false;
} else {
// this is the second time in a row we've received an empty list, go to the next
// category this time
status.nextCategory().save(statusFile);
firstEmptyList = true;
}
return error + response;
}
}
}
// load the listing
if (!status.list.isEmpty()) {
boolean isError = false;
try {
String coordinates =
"[" + status.siteMapIndex + ", " + status.page + ", " + status.pagePosition + "]";
URL listingUrl = status.list.get(status.pagePosition);
logger.finest("Loading listing: " + listingUrl + " " + coordinates);
loadPage(listingUrl, listingParser);
if (currentListing == null) {
response = "Listing null: " + status.list.get(status.pagePosition);
logger.severe(response);
isError = true;
} else {
// this is where it comes when everything went right
response = currentListing.toString();
}
} catch (Exception e) {
response = "Failed to load the listing: " + status.list.get(status.pagePosition);
logger.severe(response);
isError = true;
} finally {
if (status.pagePosition == status.list.size() - 1) {
if (status.nextPageAvailable) {
status.nextPage().save(statusFile);
status.list.clear();
} else {
status.nextCategory().save(statusFile);
status.list.clear();
if (status.siteMap.size() == status.siteMapIndex) {
status.reset().save(statusFile);
}
}
} else {
status.pagePosition++;
}
}
return (isError) ? error + response : response;
} else {
response = "List empty, can not load listing.";
logger.severe(response);
return error + response;
}
}
