Exemplo n.º 1
0
  @Override
  public void run() {
    if ((GBDOptions.secureMode() && proxy.isLocal()) || !proxy.isAvailable()) return;

    final ProcessStatus psSigs =
        new ProcessStatus(ExecutionContext.bookInfo.getPagesInfo().getPages().size());

    ExecutorService sigPool = Executors.newCachedThreadPool();
    sigPool.submit(
        () ->
            ExecutionContext.bookInfo
                .getPagesInfo()
                .getPages()
                .parallelStream()
                .forEach(
                    page -> {
                      psSigs.inc();
                      getSig(page);
                    }));

    sigPool.shutdown();
    try {
      sigPool.awaitTermination(100, TimeUnit.MINUTES);
    } catch (InterruptedException e) {
    }

    psSigs.finish();
  }
Exemplo n.º 2
0
  private boolean getSig(PageInfo page) {
    if (!proxy.isAvailable()) return false;

    if (page.dataProcessed.get()
        || page.getSig() != null
        || page.sigChecked.get()
        || page.loadingStarted.get()) return false;

    boolean sigFound = false;
    String rqUrl =
        ExecutionContext.baseUrl
            + ImageExtractor.PAGES_REQUEST_TEMPLATE.replace(
                ImageExtractor.RQ_PG_PLACEHOLDER, page.getPid());

    try {
      HttpResponse resp = getContent(rqUrl, proxy, true);
      if (resp == null) {
        logger.info(String.format(SIG_ERROR_TEMPLATE, rqUrl, proxy.toString()));
        return false;
      }

      String respStr = IOUtils.toString(resp.getContent());
      PagesInfo framePages = Mapper.objectMapper.readValue(respStr, PagesInfo.class);

      PageInfo[] pages = framePages.getPagesArray();
      for (PageInfo framePage : pages)
        if (framePage.getOrder() >= page.getOrder() && framePage.getSrc() != null) {
          PageInfo _page =
              ExecutionContext.bookInfo.getPagesInfo().getPageByPid(framePage.getPid());

          if (_page.dataProcessed.get() || _page.getSig() != null || _page.sigChecked.get())
            continue;

          String _frameSrc = framePage.getSrc();
          if (_frameSrc != null) _page.setSrc(_frameSrc);

          if (_page.getSig() != null) {
            if (_page.getPid().equals(page.getPid())) sigFound = true;
            _page.sigChecked.set(true);

            proxy.promoteProxy();

            // Если есть возможность - пытаемся грузить страницу сразу
            Pools.imgExecutor.execute(new PageImgProcessor(_page, proxy));
          }

          if (_page.getSrc() != null && _page.getSig() == null)
            logger.finest(String.format(SIG_WRONG_FORMAT, _page.getSrc()));
        }
    } catch (JsonParseException
        | JsonMappingException
        | SocketTimeoutException
        | SocketException
        | NoHttpResponseException ce) {
      if (!proxy.isLocal()) {
        proxy.registerFailure();
        logger.info(String.format("Proxy %s failed!", proxy.toString()));
      }

      ce.printStackTrace();
    } catch (Exception ex) {
      ex.printStackTrace();
    }

    return sigFound;
  }