ybf

ybf


1.2中国银行汇率抓取代码参考

/**

  • 中国银行网汇率抓取USD */ @Service @Slf4j public class BankChinaExchangePrice implements CoinPriceFetchService {

    private static final String BASE_URL="https://srh.bankofchina.com/search/whpj/search_cn.jsp";; private static final String USD_CNY="美元"; private static final String appName="中国银行";

    private final BaseExchangeRateServiceImpl baseExchangeRateService; private final ExchangeRateCheckService exchangeRateService;

    @Autowired private MerchantsBankExchangePrice merchantsBankExchangePrice;

    @Value("${exchange.rate.discard.time:2000}") private long exchangeRateDiscardTime;

    @Autowired private Environment environment;

    @Autowired public BankChinaExchangePrice(BaseExchangeRateServiceImpl baseExchangeRateService , ExchangeRateCheckService exchangeRateService ) { this.baseExchangeRateService = baseExchangeRateService; this.exchangeRateService = exchangeRateService; }

    @Override public void fetchPriceJsonAndSave() { fetchExchange(); }

    private static String buildParams() { StringBuilder url = new StringBuilder(BASE_URL+"?"); url.append("pjname=").append(USD_CNY); return url.toString(); }

    public List<BaseExchangeRateHistoryDTO> getBankChinaExchangeList() { List<BaseExchangeRateHistoryDTO> exchangeRates=Lists.newArrayList(); exchangeRates.addAll(this.sendRequest(USD_CNY)); return exchangeRates; }

    public void fetchExchange() { List<BaseExchangeRateHistoryDTO> exchangeRates=Lists.newArrayList(); exchangeRates.addAll(this.sendRequest(USD_CNY));

    long thresholdCount = 0L;
    log.info(&quot;【中国银行】查询汇率{},开始批量入库:{}&quot;,exchangeRates.size()&gt;0?&quot;成功&quot;:&quot;失败&quot;, JSON.toJSONString(exchangeRates));
    //批量保存中国银行汇率结果集
    if(!exchangeRates.isEmpty()){
        //如果此次抓取时间超过阀值,则放弃此次入库
        exchangeRates.stream().filter(e-&gt;e.getDuration()&lt;=exchangeRateDiscardTime).forEach(exchangeRateHistoryDTO-&gt;{
            baseExchangeRateService.mergeBaseExchangeRate(exchangeRateHistoryDTO, true);
        });
        thresholdCount = exchangeRates.stream().filter(e -&gt; e.getDuration() &gt; exchangeRateDiscardTime).count();
    }else{
        log.warn(&quot;【中国银行】查询汇率失败,修改通道状态为失败&quot;);
        baseExchangeRateService.lambdaUpdate().in(BaseExchangeRate::getApiInfoId,
                ApiInfoEnum.BANKCHINA_CNY_USD_FIRST_BUY_PRICE.getId(), ApiInfoEnum.BANKCHINA_CNY_USD_FIRST_SELL_PRICE.getId())
                .set(BaseExchangeRate::getStatus,Integer.parseInt(BaseRateEnum.NO.getCode()))
                .set(BaseExchangeRate::getUpdatedAt,new Date())
                .update();
    }
    
    List&lt;BaseExchangeRateHistoryDTO&gt; cmbExchangeRates = merchantsBankExchangePrice.fetchExchangeRate();
    log.info(&quot;【招商银行】查询汇率{},开始批量入库:{}&quot;,cmbExchangeRates.size()&gt;0?&quot;成功&quot;:&quot;失败&quot;,JSON.toJSONString(cmbExchangeRates));
    //批量保存招商银行汇率结果集
    if(!cmbExchangeRates.isEmpty()){
        //如果此次抓取时间超过阀值,则放弃此次入库
        cmbExchangeRates.stream().filter(e-&gt;e.getDuration()&lt;=exchangeRateDiscardTime).forEach(exchangeRateHistoryDTO-&gt;{
            baseExchangeRateService.mergeBaseExchangeRate(exchangeRateHistoryDTO, true);
        });
    }else{
        log.warn(&quot;【招商银行】查询汇率失败,修改通道状态为失败&quot;);
        baseExchangeRateService.lambdaUpdate().in(BaseExchangeRate::getApiInfoId,
                ApiInfoEnum.CMBCHINA_CNY_USD_FIRST_BUY_PRICE.getId(), ApiInfoEnum.CMBCHINA_CNY_USD_FIRST_SELL_PRICE.getId())
                .set(BaseExchangeRate::getStatus,Integer.parseInt(BaseRateEnum.NO.getCode()))
                .set(BaseExchangeRate::getUpdatedAt,new Date())
                .update();
    }
    
    //批量保存系统基础汇率,如果thresholdCount&gt;0说明此次中国银行抓取延迟时间不满足(延迟要小于等于{exchangeRateDiscardTime}秒)条件,则使用招商银行汇率作为基础汇率
    BaseExchangeRateHistoryDTO baseExchangeRateHistoryDTO = new BaseExchangeRateHistoryDTO();
    if(!exchangeRates.isEmpty()&amp;&amp;thresholdCount==0){
        log.info(&quot;【中国银行】基础汇率入库,中国银行数据:{},招商银行数据:{}&quot;,JSON.toJSONString(exchangeRates),JSON.toJSONString(cmbExchangeRates));
        exchangeRates.forEach(e-&gt;{
            BeanUtil.copyProperties(e,baseExchangeRateHistoryDTO);
            if(e.getDirection().equals(BUY_DIRECTION)){
                baseExchangeRateHistoryDTO.setApiInfo(ApiInfoEnum.DEFAULT_CNY_USD_FIRST_BUY_PRICE);
            }else{
                baseExchangeRateHistoryDTO.setApiInfo(ApiInfoEnum.DEFAULT_CNY_USD_FIRST_SELL_PRICE);
            }
            baseExchangeRateHistoryDTO.setApiInfoDes(&quot;中行&quot;);
            baseExchangeRateHistoryDTO.setApiSource(RateSourceEnum.SYSTEM);
            baseExchangeRateHistoryDTO.setDuration(0L);
            baseExchangeRateService.mergeBaseExchangeRate(baseExchangeRateHistoryDTO,false);
        });
        return;
    }else if(!cmbExchangeRates.isEmpty()){
        //获取此次抓取时间是否超过阈值
        thresholdCount = cmbExchangeRates.stream().filter(e -&gt; e.getDuration() &gt; exchangeRateDiscardTime).count();
        if(thresholdCount==0){
            log.info(&quot;【招商银行】基础汇率入库,中国银行数据:{},招商银行数据:{}&quot;,JSON.toJSONString(exchangeRates),JSON.toJSONString(cmbExchangeRates));
            cmbExchangeRates.forEach(e-&gt;{
                BeanUtil.copyProperties(e,baseExchangeRateHistoryDTO);
                if(e.getDirection().equals(BUY_DIRECTION)){
                    baseExchangeRateHistoryDTO.setApiInfo(ApiInfoEnum.DEFAULT_CNY_USD_FIRST_BUY_PRICE);
                }else{
                    baseExchangeRateHistoryDTO.setApiInfo(ApiInfoEnum.DEFAULT_CNY_USD_FIRST_SELL_PRICE);
                }
                baseExchangeRateHistoryDTO.setApiInfoDes(&quot;招行&quot;);
                baseExchangeRateHistoryDTO.setApiSource(RateSourceEnum.SYSTEM);
                baseExchangeRateHistoryDTO.setDuration(0L);
                baseExchangeRateService.mergeBaseExchangeRate(baseExchangeRateHistoryDTO,false);
            });
            return;
        }
    }
    //如果本次汇率都未入库,则修改基础汇率状态为不可用
    baseExchangeRateService.lambdaUpdate().eq(BaseExchangeRate::getApiInfoId,ApiInfoEnum.DEFAULT_CNY_USD_FIRST_BUY_PRICE.getId())
            .set(BaseExchangeRate::getStatus,Integer.parseInt(BaseRateEnum.NO.getCode()))
            .set(BaseExchangeRate::getUpdatedAt,new Date())
            .update();
    baseExchangeRateService.lambdaUpdate().eq(BaseExchangeRate::getApiInfoId,ApiInfoEnum.DEFAULT_CNY_USD_FIRST_SELL_PRICE.getId())
            .set(BaseExchangeRate::getStatus,Integer.parseInt(BaseRateEnum.NO.getCode()))
            .set(BaseExchangeRate::getUpdatedAt,new Date())
            .set(BaseExchangeRate::getApiName,String.format(ApiInfoEnum.DEFAULT_CNY_USD_FIRST_SELL_PRICE.getDescription(),&quot;未知&quot;))
            .update();
    log.warn(&quot;【USD-CNY汇率】基础汇率创建失败,中国银行:{} \n招商银行:{}&quot;,JSON.toJSONString(exchangeRates),JSON.toJSONString(cmbExchangeRates));

    }

    private List<BaseExchangeRateHistoryDTO> sendRequest(String symbol){ List<BaseExchangeRateHistoryDTO> list= Lists.newArrayList(); String url=buildParams(); Long start = System.currentTimeMillis(); CloseableHttpClient httpClient = HttpClients.createDefault(); CloseableHttpResponse response = null; HttpPost httpPost =setHttpHead(new HttpPost(url)); String html = ""; try { response = httpClient.execute(httpPost); //判断响应状态为200,进行处理 if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { html = EntityUtils.toString(response.getEntity(), "utf-8"); } log.info("【中国银行】汇率抓取,请求地址:{},响应结果:{},耗时:{}ms",url,html,System.currentTimeMillis()-start); } catch (Exception e) { log.error("【中国银行】汇率抓取异常:{}",e.getMessage(),e); html=e.getMessage(); } finally { org.apache.http.client.utils.HttpClientUtils.closeQuietly(response); org.apache.http.client.utils.HttpClientUtils.closeQuietly(httpClient); } if(StringUtils.isNotBlank(html)){ list=assembleObjByHtml(html,symbol,list,start); } if(list.isEmpty()){ list.add(newInstance(null,html,BUY_DIRECTION, ApiInfoEnum.BANKCHINA_CNY_USD_FIRST_BUY_PRICE, RateSourceEnum.BANKOFCHINA,exchangeRateDiscardTime+1)); list.add(newInstance(null,html,SELL_DIRECTION,ApiInfoEnum.BANKCHINA_CNY_USD_FIRST_SELL_PRICE, RateSourceEnum.BANKOFCHINA,exchangeRateDiscardTime+1)); } return list;

    }

    private static BaseExchangeRateHistoryDTO newInstance(BigDecimal price,String responseString, Integer tradeType,ApiInfoEnum apiInfo,RateSourceEnum apiSource,Long delayTime ){ BaseExchangeRateHistoryDTO baseExchangeRateHistoryDTO = new BaseExchangeRateHistoryDTO(); baseExchangeRateHistoryDTO.setCoinType(AssetType.USDT); baseExchangeRateHistoryDTO.setPriceTrading(RateTradingEnum.CNY_USD); baseExchangeRateHistoryDTO.setPriceUnit(AssetType.CNY); baseExchangeRateHistoryDTO.setDuration(delayTime); baseExchangeRateHistoryDTO.setResponse(responseString); baseExchangeRateHistoryDTO.setPrice(price); baseExchangeRateHistoryDTO.setDirection(tradeType); baseExchangeRateHistoryDTO.setApiInfo(apiInfo); baseExchangeRateHistoryDTO.setApiSource(apiSource); log.info("bankChina baseExchangeRateHistoryDTO: {}", JacksonUtils.toJson(baseExchangeRateHistoryDTO)); return baseExchangeRateHistoryDTO; }

    /**

    • 根据取得的网页,解析html中的内容 先不做业务逻辑,全部查询
    • @param html 要解析的html
    • @return */ private static List<BaseExchangeRateHistoryDTO> assembleObjByHtml(String html,String symbol,List<BaseExchangeRateHistoryDTO> baseExchangeRateHistoryDTOList,Long start) { Document document = Jsoup.parse(html); Elements tables = document.getElementsByTag("table"); int tableIndex = -1; for (int i = 0; i < tables.size(); i++) { Element element = tables.get(i); String text = element.text(); if (text.indexOf("货币名称") > -1) { tableIndex = i; break; } } if (tableIndex > -1) { Element table = tables.get(tableIndex); Elements trs = table.select("tr"); for (int i = 1; i < trs.size(); ++i) { Element tr = trs.get(i); log.info("tr is {}",tr); Elements tds = tr.select("td"); if(tds!=null && tds.size()==7){ Long delayTime = System.currentTimeMillis() - start; BigDecimal baseNum=new BigDecimal(100); //买入汇率 BigDecimal sellRate=StringUtils.isEmpty(tds.get(1).text()) ? BigDecimal.ZERO:new BigDecimal(tds.get(1).text()); //卖出汇率 BigDecimal buyRate=StringUtils.isEmpty(tds.get(3).text()) ? BigDecimal.ZERO:new BigDecimal(tds.get(3).text()); AssetType assetType=symbol.equals(USD_CNY) ? AssetType.USD : AssetType.CNY;

              //现汇买入价
              baseExchangeRateHistoryDTOList.add(newInstance(buyRate.divide(baseNum,6,RoundingMode.DOWN),tr.html(),
                      TradeDirectionEnum.BUY_FROM_USER.getCode(),ApiInfoEnum.BANKCHINA_CNY_USD_FIRST_BUY_PRICE,RateSourceEnum.BANKOFCHINA,delayTime));
              //现汇卖出价
              baseExchangeRateHistoryDTOList.add(newInstance(sellRate.divide(baseNum,6, RoundingMode.DOWN),tr.html(),
                      TradeDirectionEnum.SELL_TO_USER.getCode(),ApiInfoEnum.BANKCHINA_CNY_USD_FIRST_SELL_PRICE,RateSourceEnum.BANKOFCHINA,delayTime));
              log.info(&quot;newInstance Done&quot;);
              break;
          }
      }

      } return baseExchangeRateHistoryDTOList; }

    HttpPost setHttpHead(HttpPost httpPost){ httpPost.addHeader("Content-Type", "application/x-www-form-urlencoded;charset=utf-8"); httpPost.setHeader("Accept", "Accept: text/plain, /"); httpPost.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3724.8 Safari/537.36"); httpPost.addHeader("x-amazon-user-agent", "AmazonJavascriptScratchpad/1.0 (Language=Javascript)"); httpPost.addHeader("X-Requested-With", "XMLHttpRequest"); return httpPost; } }

页面列表

ITEM_HTML