干货 I 隧道代理搭建教程

"""ProxyManager.py~~~~~~~~~~~~~~~~~~~~~简易代理池管理工具，直接从URL中读取所有最新的代理，并写入Redis。"""import yamlimport timeimport jsonimport redisimport datetimeimport requests  class ProxyManager:    def __init__(self):        self.config = self.read_config()        self.redis_config = self.config['redis']        self.client = redis.Redis(host=self.redis_config['host'],                                  password=self.redis_config['password'],                                  port=self.redis_config['port'])        self.instance_dict = {}     def read_config(self):        with open('config.yaml') as f:            config = yaml.safe_load(f.read())            return config     def read_ip(self):        resp = requests.get(self.config['proxy']).text        if '{' in resp:            return []        proxy_list = resp.split()        return proxy_list     def delete_ip(self, live_ips, pool_ips):        ip_to_removed = set(pool_ips) - set(live_ips)        if ip_to_removed:            print('ip to be removed:', ip_to_removed)            self.client.hdel(self.redis_config['key'], *list(ip_to_removed))     def add_new_ips(self, live_ips, pool_ips):        ip_to_add = set(live_ips) - set(pool_ips)        if ip_to_add:            print('ip to add:', ip_to_add)            ips = {}            for ip in ip_to_add:                ips[ip] = json.dumps({'private_ip': ip,                                      'ts': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})            self.client.hset(self.redis_config['key'], mapping=ips)     def run(self):        while True:            live_ips = self.read_ip()            pool_ips = [x.decode() for x in self.client.hgetall(self.redis_config['key'])]            self.delete_ip(live_ips, pool_ips)            self.add_new_ips(live_ips, pool_ips)            time.sleep(40)  if __name__ == '__main__':    manager = ProxyManager()    manager.run()

# python 3.6+ import requests url = "百度一下，你就知道" ip, port = "39.137.95.73", "8080" proxies = {"http": f"http://{ip}:{port}"} #空白位置为测试HTTP代理和HTTP代理使用端口 headers = {"User-Agent": "Mozilla/5.0"} #响应头 res = requests.get(url, proxies=proxies, headers=headers) #发起请求 print(res.status_code) #返回响应码

worker_processes  16;        #nginx worker 数量error_log /usr/local/openresty/nginx/logs/error.log;   #指定错误日志文件路径events {    worker_connections 1024;}  stream {    ## TCP 代理日志格式定义    log_format tcp_proxy '$remote_addr [$time_local] '                         '$protocol $status $bytes_sent $bytes_received '                         '$session_time "$upstream_addr" '                         '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';    ## TCP 代理日志配置    access_log /usr/local/openresty/nginx/logs/access.log tcp_proxy;    open_log_file_cache off;     ## TCP 代理配置    upstream backend{        server 127.0.0.2:1101;# 爱写啥写啥  反正下面的代码也给你改了        balancer_by_lua_block {            -- 初始化balancer            local balancer = require "ngx.balancer"            local host = ""            local port = 0            host = ngx.ctx.proxy_host            port = ngx.ctx.proxy_port            -- 设置 balancer            local ok, err = balancer.set_current_peer(host, port)            if not ok then                ngx.log(ngx.ERR, "failed to set the peer: ", err)            end        }    }      server {        preread_by_lua_block{             local redis = require("resty.redis")            --创建实例            local redis_instance = redis:new()            --设置超时（毫秒）            redis_instance:set_timeout(3000)            --建立连接，请在这里配置Redis 的 IP 地址、端口号、密码和用到的 Key            local rhost = "123.45.67.89"            local rport = 6739            local rpass = "abcdefg"            local rkey = "proxy:pool"            local ok, err = redis_instance:connect(rhost, rport)            ngx.log(ngx.ERR, "1111111 ", ok, " ", err)             -- 如果没有密码，移除下面这一行            local res, err = redis_instance:auth(rpass)            local res, err = redis_instance:hkeys(rkey)            if not res then                ngx.log(ngx.ERR,"res num error : ", err)                return redis_instance:close()            end            math.randomseed(tostring(ngx.now()):reverse():sub(1, 6))            local proxy = res[math.random(#res)]            local colon_index = string.find(proxy, ":")            local proxy_ip = string.sub(proxy, 1, colon_index - 1)            local proxy_port = string.sub(proxy, colon_index + 1)            ngx.log(ngx.ERR,"redis data = ", proxy_ip, ":", proxy_port);            ngx.ctx.proxy_host = proxy_ip            ngx.ctx.proxy_port = proxy_port            redis_instance:close()        }        #  下面是本机的端口，也就是爬虫固定写死的端口       listen 0.0.0.0:9976; #监听本机地址和端口，当使用keeplived的情况下使用keeplived VIP       proxy_connect_timeout 3s;       proxy_timeout 10s;       proxy_pass backend; #这里填写对端的地址    }}

from openresty/openresty:centos copy nginx_redis.conf /usr/local/openresty/nginx/conf/nginx.conf

docker build --network host -t tunnel_proxy:0.01 .docker run --name tunnel_proxy --network host -it tunnel_proxy:0.01

import requestsimport time proxies = {'http': 'http://13.88.220.207:9976'}for _ in range(10):    resp = requests.get('http://httpbin.org/ip', proxies=proxies).text    print(resp)    time.sleep(1)