Exemplo n.º 1
0
 public ArrayList<Listing> getAllListing() throws SQLException {
   start();
   rs = stmt.executeQuery("SELECT * FROM listingInfo");
   ArrayList<Listing> rtrn = new ArrayList<Listing>();
   while (rs.next()) {
     Listing temp = new Listing();
     temp.setUserName(rs.getString(2));
     temp.setRacquetID(rs.getInt(3));
     temp.setPrice(rs.getDouble(4));
     temp.setNewOrOld(rs.getString(5));
     temp.setDateListed(rs.getDate(6));
     temp.setDateSold(rs.getDate(7));
     // temp.setSellerRating(rs.getInt(8));
     temp.setDescription(rs.getString(9));
     temp.setPicURL(rs.getString(10));
     rtrn.add(temp);
   }
   close();
   return rtrn;
 }
Exemplo n.º 2
0
 public static void main(String[] args) throws SQLException, ClassNotFoundException {
   Connection conn =
       DriverManager.getConnection("jdbc:mysql://localhost:3306/racqual", "root", "99643333");
   Statement stmt = conn.createStatement();
   ResultSet rs = stmt.executeQuery("SELECT * FROM listingInfo");
   ArrayList<Listing> rtrn = new ArrayList<Listing>();
   while (rs.next()) {
     Listing temp = new Listing();
     temp.setUserName(rs.getString(2));
     temp.setRacquetID(rs.getInt(3));
     temp.setPrice(rs.getDouble(4));
     temp.setNewOrOld(rs.getString(5));
     temp.setDateListed(rs.getDate(6));
     temp.setDateSold(rs.getDate(7));
     // temp.setSellerRating(rs.getInt(8));
     temp.setDescription(rs.getString(9));
     temp.setPicURL(rs.getString(10));
     rtrn.add(temp);
   }
   rs.close();
   stmt.close();
   conn.close();
 }
Exemplo n.º 3
0
  public String getItem() throws Exception {
    logger.finest("Item request initiated.");

    String response = null;

    if (!status.init) {
      if (!status.load(statusFile)) {
        status.init = true;
      } // else the init was loaded from the serialized status
    }

    // clear the sitemap and reset if we are at the end of portal
    // if the last listing couldn't be loaded, the status was not reset
    if (!status.siteMap.isEmpty()) {
      if (status.siteMap.size() == status.siteMapIndex) {
        status.reset().save(statusFile);
      }
    }

    // load the sitemap if needed
    if (status.siteMap.isEmpty()) {
      try {
        logger.finest("Loading sitemap: " + siteMapUrl);
        loadPage(siteMapUrl, siteMapParser);

        if (status.siteMap.isEmpty()) {
          response = "Failed to load the sitemap - Empty";
          logger.severe(response);
          return error + response;
        } else {
          status.save(statusFile);
          int size = status.siteMap.size();
          logger.info("Sitemap loaded successfuly: " + size + " items");

          for (int i = 0; i < size; i++) {
            SiteMapLocation sml = status.siteMap.get(i);
            logger.finest("Sitemap item #" + i + " : [" + sml.name + "] " + sml.url);
          }
        }

      } catch (Exception e) {
        response = "Failed to load the sitemap: " + e.getMessage();
        logger.severe(response);
        return error + response;
      }
    }

    // load the list if needed
    if (status.list.isEmpty() && !status.siteMap.isEmpty()) {
      String coordinates =
          "[" + status.siteMapIndex + ", " + status.page + ", " + status.pagePosition + "]";
      URL listUrl = modifyUrl(status.siteMap.get(status.siteMapIndex).url);
      logger.finest("Loading list: " + listUrl + " " + coordinates);

      try {
        loadPage(listUrl, listParser);
      } catch (Exception e) {
        response =
            "Failed to load the list: ["
                + status.siteMap.get(status.siteMapIndex).url
                + "] "
                + e.getMessage();
        logger.severe(response);
        return error + response;
      } finally {
        // if after loading the page list is still empty even after second attempt, we should
        // probably try next category, we are out of range of the pager
        // this can happen when loading an older status and the page structure changed in the
        // meantime
        if (status.list.isEmpty()) {
          response = "Failed to load list - Empty: " + listUrl + " " + coordinates;
          logger.warning(response);
          if (firstEmptyList) {
            // this is a first empty list, give it one more chance and go to the next page
            status.nextPage().save(statusFile);
            firstEmptyList = false;
          } else {
            // this is the second time in a row we've received an empty list, go to the next
            // category this time
            status.nextCategory().save(statusFile);
            firstEmptyList = true;
          }
          return error + response;
        }
      }
    }

    // load the listing
    if (!status.list.isEmpty()) {
      boolean isError = false;
      try {
        String coordinates =
            "[" + status.siteMapIndex + ", " + status.page + ", " + status.pagePosition + "]";
        URL listingUrl = status.list.get(status.pagePosition);
        logger.finest("Loading listing: " + listingUrl + " " + coordinates);
        loadPage(listingUrl, listingParser);
        if (currentListing == null) {
          response = "Listing null: " + status.list.get(status.pagePosition);
          logger.severe(response);
          isError = true;
        } else {
          // this is where it comes when everything went right
          response = currentListing.toString();
        }
      } catch (Exception e) {
        response = "Failed to load the listing: " + status.list.get(status.pagePosition);
        logger.severe(response);
        isError = true;
      } finally {
        if (status.pagePosition == status.list.size() - 1) {
          if (status.nextPageAvailable) {
            status.nextPage().save(statusFile);
            status.list.clear();
          } else {
            status.nextCategory().save(statusFile);
            status.list.clear();
            if (status.siteMap.size() == status.siteMapIndex) {
              status.reset().save(statusFile);
            }
          }
        } else {
          status.pagePosition++;
        }
      }

      return (isError) ? error + response : response;
    } else {
      response = "List empty, can not load listing.";
      logger.severe(response);
      return error + response;
    }
  }
Exemplo n.º 4
0
  protected void loadPage(URL url, Parser parser) {
    // toto je prasarna jak kreten
    StackTraceElement[] stackTraceElements = Thread.currentThread().getStackTrace();
    // if testListing called this, perform no check
    if (!stackTraceElements[2].getMethodName().equals("testListing")) {
      if (parser.equals(listingParser)) {
        if (listingExists(url)) {
          currentListing = new Listing();
          currentListing.setDuplicate(true);
          // avoid the request
          return;
        }
      }
    }

    CloseableHttpResponse response = null;
    int attempts = 0;
    int retry_ = (retry == 0) ? 1 : retry;
    boolean responseOk = false;
    while (attempts < retry_ && !responseOk) {

      if (attempts > 0) {

        // consume the entity, close the previous response -> release the resources (connection,
        // ...)
        if (response != null) {
          try {
            // this will consume the entity and release the resources automatically, e.g. connection
            EntityUtils.consume(response.getEntity());
          } catch (IOException e) {
            logger.severe("Failed to consume the entity: " + e.getMessage());
          }

          try {
            response.close();
          } catch (IOException e) {
            logger.severe("Failed to close HTTP response: " + e.getMessage());
          }
        }

        try {
          logger.finest("Going to sleep for " + retryAfter + "ms " + url);
          Thread.sleep(retryAfter);
        } catch (InterruptedException e) {
          // ignore
          logger.severe("Failed to sleep for " + retryAfter + "ms " + url);
        }
        logger.info(
            "Retrying to get page after "
                + retryAfter
                + "ms : Attempt "
                + new Integer(attempts + 1)
                + "/"
                + retry
                + " "
                + url);
      }

      try {
        response = LecturaCrawlerSuite.getResponse(url, useProxy);
      } catch (Exception e) {
        logger.severe("Failed to retrieve response: " + e.getMessage());
      }

      responseOk =
          (response != null && response.getStatusLine().getStatusCode() == HttpStatus.SC_OK);
      attempts++;
    }

    try {
      HttpEntity entity = response.getEntity();
      // if the status code is not OK, report a problem
      if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) {
        Header[] headers = response.getAllHeaders();
        String headersStr = response.getStatusLine().toString();
        for (int i = 0; i < headers.length; i++) {
          headersStr += " | " + headers[i];
        }

        logger.warning("Status code not OK: [" + url.toString() + "] " + headersStr);
        logger.warning(EntityUtils.toString(entity));
      }

      if (entity != null) {
        Header[] headers = response.getHeaders("Content-Type");
        Charset charset = Charset.forName("UTF-8");
        String regexCharset = ".*?charset=(.*)$";
        for (Header header : headers) {
          String val = header.getValue();
          if (val.matches(regexCharset)) {
            try {
              charset = Charset.forName(val.replaceAll(regexCharset, "$1"));
              break;
            } catch (IllegalCharsetNameException e) {
              // ignore, it is already set to utf-8
            }
          }
        }
        parser.parse(EntityUtils.toString(entity, charset));
      }

    } catch (Exception e) {
      logger.severe("Failed to load page: [" + url + "] " + e.getMessage());
    } finally {
      try {
        // this will consume the entity and release the resources automatically, e.g. connection
        EntityUtils.consume(response.getEntity());
      } catch (IOException e) {
        logger.severe("Failed to consume the entity: " + e.getMessage());
      }

      try {
        response.close();
      } catch (IOException e) {
        logger.severe("Failed to close HTTP response: " + e.getMessage());
      }
    }
  }