代理IP访问目标网络

说明:

本方法使用“快代理” + Redis 实现代理IP功能

一. 引入 Maven 依赖

<!-- 代理IP网站 -->
<dependency>
    <groupId>com.github.kuaidaili</groupId>
    <artifactId>kdl</artifactId>
    <version>0.1.1</version>
</dependency>

二. 工具类

package com.moji.utils;

import kdl.Auth;
import kdl.Client;
import lombok.extern.slf4j.Slf4j;

import java.util.*;

/**
 * @author :zhangbocong
 * @version :V1.0
 * @program :test-email.json
 * @date : 2021/8/26
 * @description :
 */
@Slf4j
public class AgentIpUtils {

    /**
     * 获取代理ip
     *
     * @param number 获取数量
     * @return java.util.List<java.lang.String> IP集合
     */
    public static List<String> getIpByNumber(Integer number) {
        Client client = getClient();

        /* 2.获取IP集合 */
        List<String> ips = new ArrayList<>();
        try {
            // 2.1 获取ip
            String[] dps_proxies = client.get_dps(number);
            ips = Arrays.asList(dps_proxies);
        } catch (Exception e) {
            log.error("获取代理IP错误:" + e);
        }
        return ips;
    }

    /**
     * 校验IP有效性
     * 
     * @param ips 需要校验的
     * @return java.util.Map<java.lang.String, java.lang.Boolean>
     */
    public static Map<String, Boolean> checkIdValid(List<String> ips) {
        Client client = getClient();
        Map<String, Boolean> ipMap = new HashMap<>(ips.size());
        try {
            String[] objects = new String[ips.size()];
            for (int i = 0; i < ips.size(); i++) {
                objects[i] = ips.get(i);
            }
            ipMap = client.check_dps_valid(objects, "hmacsha1");
        } catch (Exception e) {
            log.error("校验代理IP错误:" + e);
        }
        return ipMap;
    }

    /**
     * 获取客户端
     *
     * @return kdl.Client
     */
    public static Client getClient() {
        /* 1.设置授权参数 */
        Auth auth = new Auth("953012222256898", "2vsge6vzd84mbn5wy6j7zr9e94sjr2sf");
        // 1.1 获取授权
        Client client = null;
        try {
            client = new Client(auth);
        } catch (Exception e) {
            log.error("获取客户端错误:" + e);
        }
        return client;
    }
}

三. 实现业务

package com.moji.service.douyin;

import com.moji.utils.AgentIpUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * @author :zhangbocong
 * @version :V1.0
 * @program :test-email.json
 * @date : 2021/8/26
 * @description :
 */
@Slf4j
@Service
public class AgentIpServiceImpl {
    public static String AGENT_IP = "agent_ip";


    @Resource
    private RedisTemplate<String, String> redisTemplate;

    /**
     * 根据数量获取代理ip
     */
    public String getIpList() {
        // index:索引(下标)
        Long size = redisTemplate.boundListOps(AGENT_IP).size();
        if (size <= 0) {
            List<String> ipByNumber = AgentIpUtils.getIpByNumber(1);
            log.info("获取新ip:" + ipByNumber.get(0));
            // 存入reids
            addIpByRedis(ipByNumber);
            return ipByNumber.get(0);
        }
        List<String> ips = redisTemplate.boundListOps(AGENT_IP).range(0, size);

        // 校验IP是否合理
        Map<String, Boolean> stringBooleanMap = AgentIpUtils.checkIdValid(ips);
        // 排除失效IP
        stringBooleanMap.entrySet().removeIf(next -> !next.getValue());
        // 获取所有有效的IP
        List<String> ipList = new ArrayList<>(stringBooleanMap.keySet());
        if (ipList.size() <= 0) {
            // 补充数量
            List<String> ipByNumber = AgentIpUtils.getIpByNumber(1);
            log.info("校验失败,获取新ip:" + ipByNumber.get(0));
            // 整合所有有效IP
            ipList.addAll(ipByNumber);
            // 存入reids
            addIpByRedis(ipByNumber);
        }
        // 返回可用IP
        return ipList.get(0);
    }

    /**
     * 将ip添加到redis
     *
     * @param ips ip集合
     */
    public void addIpByRedis(List<String> ips) {
        for (String ip : ips) {
            // 左侧新增,返回当前数组长度
            redisTemplate.boundListOps(AGENT_IP).leftPush(ip);
        }
    }
}