public ArrayList<Listing> getAllListing() throws SQLException {
start();
rs = stmt.executeQuery("SELECT * FROM listingInfo");
ArrayList<Listing> rtrn = new ArrayList<Listing>();
while (rs.next()) {
Listing temp = new Listing();
temp.setUserName(rs.getString(2));
temp.setRacquetID(rs.getInt(3));
temp.setPrice(rs.getDouble(4));
temp.setNewOrOld(rs.getString(5));
temp.setDateListed(rs.getDate(6));
temp.setDateSold(rs.getDate(7));
// temp.setSellerRating(rs.getInt(8));
temp.setDescription(rs.getString(9));
temp.setPicURL(rs.getString(10));
rtrn.add(temp);
}
close();
return rtrn;
}
public static void main(String[] args) throws SQLException, ClassNotFoundException {
Connection conn =
DriverManager.getConnection("jdbc:mysql://localhost:3306/racqual", "root", "99643333");
Statement stmt = conn.createStatement();
ResultSet rs = stmt.executeQuery("SELECT * FROM listingInfo");
ArrayList<Listing> rtrn = new ArrayList<Listing>();
while (rs.next()) {
Listing temp = new Listing();
temp.setUserName(rs.getString(2));
temp.setRacquetID(rs.getInt(3));
temp.setPrice(rs.getDouble(4));
temp.setNewOrOld(rs.getString(5));
temp.setDateListed(rs.getDate(6));
temp.setDateSold(rs.getDate(7));
// temp.setSellerRating(rs.getInt(8));
temp.setDescription(rs.getString(9));
temp.setPicURL(rs.getString(10));
rtrn.add(temp);
}
rs.close();
stmt.close();
conn.close();
}
public String getItem() throws Exception {
logger.finest("Item request initiated.");
String response = null;
if (!status.init) {
if (!status.load(statusFile)) {
status.init = true;
} // else the init was loaded from the serialized status
}
// clear the sitemap and reset if we are at the end of portal
// if the last listing couldn't be loaded, the status was not reset
if (!status.siteMap.isEmpty()) {
if (status.siteMap.size() == status.siteMapIndex) {
status.reset().save(statusFile);
}
}
// load the sitemap if needed
if (status.siteMap.isEmpty()) {
try {
logger.finest("Loading sitemap: " + siteMapUrl);
loadPage(siteMapUrl, siteMapParser);
if (status.siteMap.isEmpty()) {
response = "Failed to load the sitemap - Empty";
logger.severe(response);
return error + response;
} else {
status.save(statusFile);
int size = status.siteMap.size();
logger.info("Sitemap loaded successfuly: " + size + " items");
for (int i = 0; i < size; i++) {
SiteMapLocation sml = status.siteMap.get(i);
logger.finest("Sitemap item #" + i + " : [" + sml.name + "] " + sml.url);
}
}
} catch (Exception e) {
response = "Failed to load the sitemap: " + e.getMessage();
logger.severe(response);
return error + response;
}
}
// load the list if needed
if (status.list.isEmpty() && !status.siteMap.isEmpty()) {
String coordinates =
"[" + status.siteMapIndex + ", " + status.page + ", " + status.pagePosition + "]";
URL listUrl = modifyUrl(status.siteMap.get(status.siteMapIndex).url);
logger.finest("Loading list: " + listUrl + " " + coordinates);
try {
loadPage(listUrl, listParser);
} catch (Exception e) {
response =
"Failed to load the list: ["
+ status.siteMap.get(status.siteMapIndex).url
+ "] "
+ e.getMessage();
logger.severe(response);
return error + response;
} finally {
// if after loading the page list is still empty even after second attempt, we should
// probably try next category, we are out of range of the pager
// this can happen when loading an older status and the page structure changed in the
// meantime
if (status.list.isEmpty()) {
response = "Failed to load list - Empty: " + listUrl + " " + coordinates;
logger.warning(response);
if (firstEmptyList) {
// this is a first empty list, give it one more chance and go to the next page
status.nextPage().save(statusFile);
firstEmptyList = false;
} else {
// this is the second time in a row we've received an empty list, go to the next
// category this time
status.nextCategory().save(statusFile);
firstEmptyList = true;
}
return error + response;
}
}
}
// load the listing
if (!status.list.isEmpty()) {
boolean isError = false;
try {
String coordinates =
"[" + status.siteMapIndex + ", " + status.page + ", " + status.pagePosition + "]";
URL listingUrl = status.list.get(status.pagePosition);
logger.finest("Loading listing: " + listingUrl + " " + coordinates);
loadPage(listingUrl, listingParser);
if (currentListing == null) {
response = "Listing null: " + status.list.get(status.pagePosition);
logger.severe(response);
isError = true;
} else {
// this is where it comes when everything went right
response = currentListing.toString();
}
} catch (Exception e) {
response = "Failed to load the listing: " + status.list.get(status.pagePosition);
logger.severe(response);
isError = true;
} finally {
if (status.pagePosition == status.list.size() - 1) {
if (status.nextPageAvailable) {
status.nextPage().save(statusFile);
status.list.clear();
} else {
status.nextCategory().save(statusFile);
status.list.clear();
if (status.siteMap.size() == status.siteMapIndex) {
status.reset().save(statusFile);
}
}
} else {
status.pagePosition++;
}
}
return (isError) ? error + response : response;
} else {
response = "List empty, can not load listing.";
logger.severe(response);
return error + response;
}
}
protected void loadPage(URL url, Parser parser) {
// toto je prasarna jak kreten
StackTraceElement[] stackTraceElements = Thread.currentThread().getStackTrace();
// if testListing called this, perform no check
if (!stackTraceElements[2].getMethodName().equals("testListing")) {
if (parser.equals(listingParser)) {
if (listingExists(url)) {
currentListing = new Listing();
currentListing.setDuplicate(true);
// avoid the request
return;
}
}
}
CloseableHttpResponse response = null;
int attempts = 0;
int retry_ = (retry == 0) ? 1 : retry;
boolean responseOk = false;
while (attempts < retry_ && !responseOk) {
if (attempts > 0) {
// consume the entity, close the previous response -> release the resources (connection,
// ...)
if (response != null) {
try {
// this will consume the entity and release the resources automatically, e.g. connection
EntityUtils.consume(response.getEntity());
} catch (IOException e) {
logger.severe("Failed to consume the entity: " + e.getMessage());
}
try {
response.close();
} catch (IOException e) {
logger.severe("Failed to close HTTP response: " + e.getMessage());
}
}
try {
logger.finest("Going to sleep for " + retryAfter + "ms " + url);
Thread.sleep(retryAfter);
} catch (InterruptedException e) {
// ignore
logger.severe("Failed to sleep for " + retryAfter + "ms " + url);
}
logger.info(
"Retrying to get page after "
+ retryAfter
+ "ms : Attempt "
+ new Integer(attempts + 1)
+ "/"
+ retry
+ " "
+ url);
}
try {
response = LecturaCrawlerSuite.getResponse(url, useProxy);
} catch (Exception e) {
logger.severe("Failed to retrieve response: " + e.getMessage());
}
responseOk =
(response != null && response.getStatusLine().getStatusCode() == HttpStatus.SC_OK);
attempts++;
}
try {
HttpEntity entity = response.getEntity();
// if the status code is not OK, report a problem
if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
Header[] headers = response.getAllHeaders();
String headersStr = response.getStatusLine().toString();
for (int i = 0; i < headers.length; i++) {
headersStr += " | " + headers[i];
}
logger.warning("Status code not OK: [" + url.toString() + "] " + headersStr);
logger.warning(EntityUtils.toString(entity));
}
if (entity != null) {
Header[] headers = response.getHeaders("Content-Type");
Charset charset = Charset.forName("UTF-8");
String regexCharset = ".*?charset=(.*)$";
for (Header header : headers) {
String val = header.getValue();
if (val.matches(regexCharset)) {
try {
charset = Charset.forName(val.replaceAll(regexCharset, "$1"));
break;
} catch (IllegalCharsetNameException e) {
// ignore, it is already set to utf-8
}
}
}
parser.parse(EntityUtils.toString(entity, charset));
}
} catch (Exception e) {
logger.severe("Failed to load page: [" + url + "] " + e.getMessage());
} finally {
try {
// this will consume the entity and release the resources automatically, e.g. connection
EntityUtils.consume(response.getEntity());
} catch (IOException e) {
logger.severe("Failed to consume the entity: " + e.getMessage());
}
try {
response.close();
} catch (IOException e) {
logger.severe("Failed to close HTTP response: " + e.getMessage());
}
}
}