/**
  * @param webPageEntity
  * @param category
  * @return
  */
 private String[] getNormalizedCategories(WebPageEntity webPageEntity, String category) {
   if (mapping.containsKey(category)) {
     return mapping.get(category).split(",");
   }
   LOGGER.warn("Unknown category: {} url {}", webPageEntity.getCategory(), webPageEntity.getUrl());
   return new String[] {"misc"};
 }
  private Observable<WebPageEntity> parseDocument(DownloadResult downloadResult) {
    Set<WebPageEntity> result = new HashSet<>(1);

    Document document = downloadResult.getDocument();
    if (document != null) {
      Elements elements = document.select(".InfoArea a[title]");
      if (!elements.isEmpty()) {
        for (Element element : elements) {
          WebPageEntity webPageEntity =
              new WebPageEntity(
                  downloadResult.getSourcePage(),
                  "",
                  "productPage",
                  element.attr("abs:href"),
                  downloadResult.getSourcePage().getCategory());
          LOGGER.info("productPageUrl={}", webPageEntity.getUrl());
          result.add(webPageEntity);
        }
      } else {
        WebPageEntity webPageEntity =
            new WebPageEntity(
                downloadResult.getSourcePage(),
                "",
                "productPage",
                downloadResult.getSourcePage().getUrl(),
                downloadResult.getSourcePage().getCategory());
        LOGGER.info("productPageUrl={}", webPageEntity.getUrl());
        result.add(webPageEntity);
      }
    }
    return Observable.from(result);
  }
 /**
  * @param webPageEntity
  * @return
  */
 private String[] getNormalizedCategories(WebPageEntity webPageEntity) {
   String s = mapping.get(webPageEntity.getCategory());
   if (null != s) {
     return s.split(",");
   }
   LOGGER.warn("Unknown category: {} url {}", webPageEntity.getCategory(), webPageEntity.getUrl());
   return new String[] {"misc"};
 }
  /**
   * @param webPageEntity
   * @return
   * @throws Exception
   */
  @Override
  public Observable<ProductEntity> parse(WebPageEntity webPageEntity) {
    HashSet<ProductEntity> result = new HashSet<>();
    try {
      ProductEntity product;
      String productName = null;
      String url = null;
      String regularPrice = null;
      String specialPrice = null;
      String productImage = null;
      String description = null;
      Map<String, String> attr = new HashMap<>();
      String[] category = null;

      url = webPageEntity.getUrl();

      Document document = Jsoup.parse(webPageEntity.getContent(), webPageEntity.getUrl());
      productName = document.select(".naitem").text();
      LOGGER.info("Parsing {}, page={}", productName, webPageEntity.getUrl());

      if (!document.select(".outofstock").isEmpty()) {
        return Observable.empty();
      }

      productImage = document.select(".itemImgDiv img.itemDetailImg").attr("abs:src");
      String priceText = document.select(".itemDetailPrice").text().replace("\\xEF\\xBF\\xBD", " ");
      Matcher matcher = pricePattern.matcher(priceText);
      if (matcher.find()) {
        regularPrice = matcher.group().replace(",", "");
      }
      description = document.select(".itemDescription").text();
      category = getNormalizedCategories(webPageEntity);

      product =
          new ProductEntity(
              productName,
              url,
              regularPrice,
              specialPrice,
              productImage,
              description,
              attr,
              category);
      result.add(product);
    } catch (Exception e) {
      LOGGER.error("Failed to parse: {}", webPageEntity, e);
    }
    return Observable.from(result).doOnNext(e -> parseResultCounter.inc());
  }
 @Override
 public Observable<WebPageEntity> parse(WebPageEntity parent) {
   return client
       .get(parent.getUrl(), new DocumentCompletionHandler(parent))
       .flatMap(this::parseDocument)
       .doOnNext(e -> this.parseResultCounter.inc());
 }
 @Override
 public Observable<WebPageEntity> parse(WebPageEntity webPage) {
   LOGGER.trace("Processing productPage {}", webPage.getUrl());
   return PageDownloader.download(client, webPage, "productPageRaw")
       .filter(data -> null != data)
       .doOnNext(e -> this.parseResultCounter.inc());
 }
  @Override
  public Observable<ProductEntity> parse(WebPageEntity webPageEntity) {
    HashSet<ProductEntity> result = new HashSet<>();

    try {
      ProductEntity product;
      String productName = null;
      String url = null;
      String regularPrice = null;
      String specialPrice = null;
      String productImage = null;
      String description = null;
      Map<String, String> attr = new HashMap<>();
      String[] category = null;

      url = webPageEntity.getUrl();
      Document document = Jsoup.parse(webPageEntity.getContent(), webPageEntity.getUrl());

      productName = document.select(".product_title").text();
      LOGGER.info("Parsing {}, page={}", productName, webPageEntity.getUrl());

      productImage = document.select(".product .wp-post-image").attr("abs:src");
      regularPrice = document.select("meta[itemprop=price]").attr("content");
      description = document.select("#tab-description").text().replace("Product Description", "");
      category = webPageEntity.getCategory().split(",");

      product =
          new ProductEntity(
              productName,
              url,
              regularPrice,
              specialPrice,
              productImage,
              description,
              attr,
              category);
      result.add(product);
    } catch (Exception e) {
      LOGGER.error("Failed to parse: {}", webPageEntity, e);
    }
    return Observable.from(result).doOnNext(e -> parseResultCounter.inc());
  }
  private Observable<WebPageEntity> parseDocument(DownloadResult downloadResult) {
    Set<WebPageEntity> result = new HashSet<>(1);

    Document document = downloadResult.getDocument();
    if (document != null) {
      Elements elements = document.select(".SideCategoryListFlyout a");
      for (Element element : elements) {
        WebPageEntity webPageEntity =
            new WebPageEntity(
                downloadResult.getSourcePage(),
                "",
                "productList",
                element.attr("abs:href"),
                element.text());
        LOGGER.info("Product page listing={}", webPageEntity.getUrl());
        result.add(webPageEntity);
      }
    }
    return Observable.from(result);
  }
 /**
  * @param price
  * @return
  */
 private static String parsePrice(WebPageEntity webPageEntity, String price) {
   Matcher matcher = pricePattern.matcher(price);
   if (matcher.find()) {
     try {
       return matcher.group(1).replace(",", "");
       //                return
       // NumberFormat.getInstance(Locale.US).parse(matcher.group(1)).toString();
     } catch (Exception ignored) {
       return Double.valueOf(matcher.group(1)).toString();
     }
   } else {
     LOGGER.error("failed to parse price {}, page {}", price, webPageEntity.getUrl());
     return price;
   }
 }
  @Override
  public Observable<ProductEntity> parse(WebPageEntity webPageEntity) {
    HashSet<ProductEntity> result = new HashSet<>();

    try {
      ProductEntity product;
      String productName = null;
      String url = null;
      String regularPrice = null;
      String specialPrice = null;
      String productImage = null;
      String description = null;
      Map<String, String> attr = new HashMap<>();
      String[] category = null;

      url = webPageEntity.getUrl();

      Document document = Jsoup.parse(webPageEntity.getContent(), webPageEntity.getUrl());
      if (!document.select(".saleImage").isEmpty()) {
        return Observable.empty();
      }

      productName = document.select("div.innercontentDiv > div > div > h2").text();
      if (productName.isEmpty()) {
        return Observable.empty();
      }

      LOGGER.info("Parsing {}, page={}", productName, webPageEntity.getUrl());

      String manufacturer =
          document.select(".product-details__title .product__manufacturer").text();
      if (!manufacturer.isEmpty()) {
        attr.put("manufacturer", manufacturer);
      }
      productImage = document.select("div.imgLiquidNoFill a").attr("abs:src");
      if (productImage.isEmpty()) {
        productImage = document.select(".es-carousel img").attr("abs:src");
      }

      regularPrice =
          parsePrice(
              webPageEntity,
              document.select("#desPrice > li:nth-child(1) > span.pricetag.show").text());
      specialPrice = document.select("#desPrice > li:nth-child(2) > span.pricetag.show").text();
      if (!specialPrice.isEmpty()) {
        specialPrice = parsePrice(webPageEntity, specialPrice);
      }
      description = document.select("#TabbedPanels1 > div > div:nth-child(1)").text();

      Iterator<Element> labels =
          document.select("table.productTbl > tbody > tr > td:nth-child(1)").iterator();
      Iterator<Element> values =
          document.select("table.productTbl > tbody > tr > td:nth-child(2)").iterator();

      while (labels.hasNext()) {
        String specName =
            CaseFormat.LOWER_UNDERSCORE.to(
                CaseFormat.LOWER_CAMEL,
                labels.next().text().replace(' ', '_').replace(":", "").trim());
        String specValue = values.next().text();
        if (specName.contains("department")) {
          category = getNormalizedCategories(webPageEntity, specValue);
        } else {
          attr.put(specName, specValue);
        }
      }
      if (category == null) {
        LOGGER.warn("Category not found");
      }

      product =
          new ProductEntity(
              productName,
              url,
              regularPrice,
              specialPrice,
              productImage,
              description,
              attr,
              category);
      result.add(product);
    } catch (Exception e) {
      LOGGER.error("Failed to parse: {}", webPageEntity, e);
    }
    return Observable.from(result).doOnNext(e -> parseResultCounter.inc());
  }
 @Override
 public Observable<WebPageEntity> parse(WebPageEntity webPageEntity) {
   Observable<DownloadResult> pages =
       client.get(webPageEntity.getUrl(), new DocumentCompletionHandler(webPageEntity));
   return pages.flatMap(this::parseDocument).doOnNext(e -> this.parseResultCounter.inc());
 }