@Test public void testGetUpdateArticle() { String updateUrl = "http://tailicaiop.fero.com.cn/spiderapi/article/source"; String SID = TLCSpiderPropertiesUtil.getResource("tlc.spider.article.sid"); String TOKEN = TLCSpiderPropertiesUtil.getResource("tlc.spider.article.token"); Map<String, String> param = new HashMap(); param.put(TLCSpiderConstants.SPIDER_PARAM_SID, SID); param.put(TLCSpiderConstants.SPIDER_PARAM_TOKEN, TOKEN); String response = TLCSpiderRequest.post(updateUrl, param); System.out.println(response); }
@Test public void testHttpsProxy() { String url = TLCSpiderPropertiesUtil.getResource("tlc.spider.p2p.rdnsyherjz.url.list"); Map<String, String> param = new HashMap(); param.put("PageIndex", TLCSpiderConstants.SPIDER_PARAM_PAGE_ONE); param.put("PageSize", "10"); param.put("targetAction", "CmbFinancingSearch"); param.put("Interest", ""); param.put("Duration", ""); param.put("ProjectStatus", ""); param.put("ProjectAmount", ""); String countContent = TLCSpiderRequest.postViaProxy(url, param, TLCSpiderRequest.ProxyType.HTTPS); // String countContent = TLCSpiderRequest.post(url, param); String countStr = TLCSpiderJsonUtil.getString(countContent, "Data"); String totalCount = TLCSpiderJsonUtil.getString(countStr, "TotalCount"); System.out.println(totalCount); }
// 尧都农商银行E融九州抓取 public class RDNSYHERJZJob extends TLCSpiderJob { // detail: https://e.ydnsh.com/home/detail?FinancingId=15b43001-555c-4dda-845c-31b62879fbe7 private static final String URL_PRODUCT_LIST = TLCSpiderPropertiesUtil.getResource("tlc.spider.p2p.rdnsyherjz.url.list"); private static final String SID = TLCSpiderPropertiesUtil.getResource("tlc.spider.p2p.rdnsyherjz.sid"); private static final String TOKEN = TLCSpiderPropertiesUtil.getResource("tlc.spider.p2p.rdnsyherjz.token"); private static final String JOB_TITLE = TLCSpiderPropertiesUtil.getResource("tlc.spider.p2p.rdnsyherjz.title"); private static final String PAGE_NAME = "PageIndex"; private static final String PAGE_SIZE = "10"; @Override public Map<String, String> constructSystemInteractiveParam() { Map<String, String> param = new HashMap(); param.put( TLCSpiderConstants.SPIDER_PARAM_STATUS_NAME, TLCSpiderConstants.SPIDER_PARAM_STATUS_SUCCESS_CODE); param.put(TLCSpiderConstants.SPIDER_PARAM_SID, SID); param.put(TLCSpiderConstants.SPIDER_PARAM_TOKEN, TOKEN); param.put(TLCSpiderConstants.SPIDER_CONST_JOB_TITLE, JOB_TITLE); param.put(TLCSpiderConstants.SPIDER_PARAM_PAGE_NAME, PAGE_NAME); return param; } @Override public Map<String, String> constructSpiderFetchParam() { Map<String, String> param = new HashMap(); param.put(PAGE_NAME, TLCSpiderConstants.SPIDER_PARAM_PAGE_ONE); param.put("PageSize", PAGE_SIZE); param.put("targetAction", "CmbFinancingSearch"); param.put("Interest", ""); param.put("Duration", ""); param.put("ProjectStatus", ""); param.put("ProjectAmount", ""); return param; } @Override public int getTotalPage(Map<String, String> param) { String countContent = TLCSpiderRequest.postViaProxy(URL_PRODUCT_LIST, param, TLCSpiderRequest.ProxyType.HTTPS); String countStr = TLCSpiderJsonUtil.getString(countContent, "Data"); String totalCount = TLCSpiderJsonUtil.getString(countStr, "TotalCount"); int pageSize = Integer.parseInt(PAGE_SIZE); int totalPage = Integer.parseInt(totalCount) % pageSize == 0 ? Integer.parseInt(totalCount) / pageSize : (Integer.parseInt(totalCount) / pageSize + 1); return totalPage; } @Override public List<TransObject> getSpiderDataList(Map<String, String> param) { String productContent = TLCSpiderRequest.postViaProxy(URL_PRODUCT_LIST, param, TLCSpiderRequest.ProxyType.HTTPS); String productJsonStr = TLCSpiderJsonUtil.getString(productContent, "Data"); List<RDNSYHERJZ> productList = TLCSpiderJsonUtil.json2Array(productJsonStr, "ResultList", RDNSYHERJZ.class, "YMInterest"); List<TransObject> transObjectList = new ArrayList(); for (RDNSYHERJZ product : productList) { TransObject transObject = convertToTransObject(product); transObjectList.add(transObject); } return transObjectList; } private TransObject convertToTransObject(RDNSYHERJZ product) { TransObject transObject = new TransObject(); transObject.setFinancingId(product.getFinancingId()); transObject.setProjectCode(product.getProjectCode()); transObject.setProjectName(product.getProjectName()); transObject.setBindUserId(product.getBindUserId()); transObject.setBindUserName(product.getBindUserName()); transObject.setBindCompanyId(product.getBindCompanyId()); transObject.setBindCompanyName(product.getBindCompanyName()); transObject.setAmount(product.getAmount()); transObject.setPartsCount(product.getPartsCount()); transObject.setBankInterest(product.getBankInterest()); transObject.setInvestmentInterest(product.getInvestmentInterest()); transObject.setDuration(product.getDuration()); transObject.setRepayType(product.getRepayType()); transObject.setValueBegin( TLCSpiderDateFormatUtil.formatDateTime("MM/dd/yyyy HH:mm:ss", product.getValueBegin())); transObject.setRepayBegin( TLCSpiderDateFormatUtil.formatDateTime("MM/dd/yyyy HH:mm:ss", product.getRepayBegin())); transObject.setProjectBeginTime( TLCSpiderDateFormatUtil.formatDateTime( "MM/dd/yyyy HH:mm:ss", product.getProjectBeginTime())); transObject.setReadyBeginTime( TLCSpiderDateFormatUtil.formatDateTime("MM/dd/yyyy HH:mm:ss", product.getReadyBeginTime())); transObject.setProjectStatus(product.getProjectStatus()); transObject.setJmBeginTime( TLCSpiderDateFormatUtil.formatDateTime("MM/dd/yyyy HH:mm:ss", product.getjMBeginTime())); transObject.setCreateTime( TLCSpiderDateFormatUtil.formatDateTime("MM/dd/yyyy HH:mm:ss", product.getCreateTime())); transObject.setIsShow(product.getIsShow()); transObject.setProjectType(product.getProjectType()); transObject.setIsExclusivePublic(product.getIsExclusivePublic()); transObject.setMinInvestPartsCount(product.getMinInvestPartsCount()); transObject.setExclusiveCode(product.getExclusiveCode()); transObject.setRealProgress(product.getRealProgress()); transObject.setProgress(product.getProgress()); transObject.setFinanceApplyStatus(product.getFinanceApplyStatus()); transObject.setHotStatus(product.getHotStatus()); // TODO 未处理属性 YMInterest: 0.5 return transObject; } }