worker_processes  16;        #nginx worker 数量
error_log /usr/local/openresty/nginx/logs/error.log;   #指定错误日志文件路径 这里要根据你自己的路径来设置
events {worker_connections 1024;
}
stream {## TCP 代理日志格式定义log_format tcp_proxy '$remote_addr [$time_local] ''$protocol $status $bytes_sent $bytes_received ''$session_time "$upstream_addr" ''"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';## TCP 代理日志配置access_log /usr/local/openresty/nginx/logs/access.log tcp_proxy; #这里要根据你自己的路径来设置open_log_file_cache off;## TCP 代理配置upstream backend{server 127.0.0.2:1101;# 这里不用改balancer_by_lua_block {-- 初始化balancerlocal balancer = require "ngx.balancer"local host = ""local port = 0host = ngx.ctx.proxy_hostport = ngx.ctx.proxy_port-- 设置 balancerlocal ok, err = balancer.set_current_peer(host, port)if not ok thenngx.log(ngx.ERR, "failed to set the peer: ", err)end}}server {preread_by_lua_block{local redis = require("resty.redis")--创建实例local redis_instance = redis:new()--设置超时（毫秒）redis_instance:set_timeout(3000)--建立连接，请在这里配置Redis 的 IP 地址、端口号、密码和用到的 Key#redis需要改一下，没有密码就把下面说的那行删掉就行了local rhost = "123.45.67.89"local rport = 6739local rpass = "abcdefg"local rkey = "proxy:pool"local ok, err = redis_instance:connect(rhost, rport)ngx.log(ngx.ERR, "1111111 ", ok, " ", err)-- 如果没有密码，移除下面这一行local res, err = redis_instance:auth(rpass)local res, err = redis_instance:hkeys(rkey)if not res thenngx.log(ngx.ERR,"res num error : ", err)return redis_instance:close()endmath.randomseed(tostring(ngx.now()):reverse():sub(1, 6))local proxy = res[math.random(#res)]local colon_index = string.find(proxy, ":")local proxy_ip = string.sub(proxy, 1, colon_index - 1)local proxy_port = string.sub(proxy, colon_index + 1)ngx.log(ngx.ERR,"redis data = ", proxy_ip, ":", proxy_port);ngx.ctx.proxy_host = proxy_ipngx.ctx.proxy_port = proxy_portredis_instance:close()}#  下面是本机的端口，也就是爬虫固定写死的端口#端口可以改一下，我是改成了5位数listen 0.0.0.0:9976; #监听本机地址和端口，当使用keeplived的情况下使用keeplived VIPproxy_connect_timeout 3s;proxy_timeout 10s;proxy_pass backend; #这里填写对端的地址}
}

"""
ProxyManager.py
~~~~~~~~~~~~~~~~~~~~~
简易代理池管理工具，直接从URL中读取所有
最新的代理，并写入Redis。
"""
import yaml
import time
import json
import redis
import datetime
import requests
class ProxyManager:def __init__(self):self.config = self.read_config()self.redis_config = self.config['redis']self.client = redis.Redis(host=self.redis_config['host'],password=self.redis_config['password'],port=self.redis_config['port'])self.instance_dict = {}def read_config(self):with open('config.yaml') as f:config = yaml.safe_load(f.read())return configdef read_ip(self):resp = requests.get(self.config['proxy']).textif '{' in resp:return []proxy_list = resp.split()return proxy_listdef delete_ip(self, live_ips, pool_ips):ip_to_removed = set(pool_ips) - set(live_ips)if ip_to_removed:print('ip to be removed:', ip_to_removed)self.client.hdel(self.redis_config['key'], *list(ip_to_removed))def add_new_ips(self, live_ips, pool_ips):ip_to_add = set(live_ips) - set(pool_ips)if ip_to_add:print('ip to add:', ip_to_add)ips = {}for ip in ip_to_add:ips[ip] = json.dumps({'private_ip': ip,'ts': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')})self.client.hset(self.redis_config['key'], mapping=ips)def run(self):while True:live_ips = self.read_ip()pool_ips = [x.decode() for x in self.client.hgetall(self.redis_config['key'])]self.delete_ip(live_ips, pool_ips)self.add_new_ips(live_ips, pool_ips)time.sleep(40)
if __name__ == '__main__':manager = ProxyManager()manager.run()

redis:host: redis IP地址port: 6379password: 密码，没有随便写点啥或者删掉就行了key: 'proxy:pool'
proxy:'这里写去ip代理的url'