0.2.0
This commit is contained in:
504
spider/main.py
504
spider/main.py
@@ -1,9 +1,6 @@
|
||||
from math import log
|
||||
import random
|
||||
from re import S
|
||||
import time
|
||||
from datetime import datetime
|
||||
from tkinter import N
|
||||
from DrissionPage import Chromium
|
||||
from loguru import logger
|
||||
from work import get_random_canada_info
|
||||
@@ -11,7 +8,10 @@ from mail_ import mail_
|
||||
from bit_browser import bit_browser
|
||||
from api import api
|
||||
from proxys import proxy_list
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
|
||||
|
||||
class Auto:
|
||||
@@ -523,60 +523,148 @@ class Auto:
|
||||
jc += 1
|
||||
|
||||
|
||||
# 取对应城市的代理
|
||||
def get_proxy(city: str):
|
||||
if city == "Calgary":
|
||||
return "us.novproxy.io:1000:ozua8623-region-CA-st-Alberta-city-Calgary:6wdcv4gq".split(':')
|
||||
elif city == 'Edmonton':
|
||||
return 'us.novproxy.io:1000:ozua8623-region-CA-st-Alberta-city-Edmonton:6wdcv4gq'.split(':')
|
||||
elif city == 'Vancouver':
|
||||
return 'us.novproxy.io:1000:ozua8623-region-CA-st-British Columbia-city-Vancouver:6wdcv4gq'.split(':')
|
||||
elif city == 'Halifax':
|
||||
return 'us.novproxy.io:1000:ozua8623-region-CA-st-Nova Scotia-city-Halifax:6wdcv4gq'.split(':')
|
||||
elif city == 'Toronto':
|
||||
return 'us.novproxy.io:1000:ozua8623-region-CA-st-Ontario-city-Toronto:6wdcv4gq'.split(':')
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def get_random_proxy() -> list[str] | None:
|
||||
def parse_proxy(proxy: str) -> tuple[str, int, str, str] | None:
|
||||
"""
|
||||
随机选择一个代理配置（按指纹浏览器数量随机取 IP）
|
||||
解析代理字符串为四元组 `(host, port, user, pwd)`
|
||||
|
||||
参数:
|
||||
proxy: 形如 `host:port:user:pwd`
|
||||
|
||||
返回值:
|
||||
list[str] | None: 代理参数列表 `[host, port, user, pwd]`；无可用代理返回 None
|
||||
(host, port, user, pwd) 或 None（格式错误）
|
||||
"""
|
||||
proxy_list = [
|
||||
"us.novproxy.io:1000:zhiyu111-region-CA:zhiyu111",
|
||||
"us.novproxy.io:1000:zhiyu222-region-US:zhiyu222",
|
||||
"us.novproxy.io:1000:zhiyu333-region-CA:zhiyu333",
|
||||
"us.novproxy.io:1000:zhiyu444-region-US:zhiyu444",
|
||||
]
|
||||
try:
|
||||
return random.choice(proxy_list).split(':')
|
||||
host, port, user, pwd = proxy.split(":", 3)
|
||||
return host, int(port), user, pwd
|
||||
except Exception:
|
||||
logger.error(f"代理格式错误: {proxy}")
|
||||
return None
|
||||
|
||||
|
||||
def get_all_proxies() -> list[list[str]]:
|
||||
def create_fingerprint_browser(proxy: str) -> tuple[str, str] | None:
|
||||
"""
|
||||
返回固定代理列表（与提供的代理一一对应）
|
||||
创建指纹浏览器并打开窗口，返回 `(browser_id, debugger_http)`
|
||||
|
||||
参数:
|
||||
proxy: 代理字符串
|
||||
|
||||
返回值:
|
||||
list[list[str]]: 每个元素为 `[host, port, user, pwd]`
|
||||
(browser_id, http) 或 None（失败）
|
||||
"""
|
||||
proxy_list = [
|
||||
"us.novproxy.io:1000:zhiyu111-region-CA:zhiyu111",
|
||||
"us.novproxy.io:1000:zhiyu222-region-US:zhiyu222",
|
||||
"us.novproxy.io:1000:zhiyu333-region-CA:zhiyu333",
|
||||
"us.novproxy.io:1000:zhiyu444-region-US:zhiyu444",
|
||||
]
|
||||
return [p.split(":") for p in proxy_list]
|
||||
info = parse_proxy(proxy)
|
||||
if info is None:
|
||||
return None
|
||||
host, port, user, pwd = info
|
||||
try:
|
||||
browser_id = bit_browser.bit_browser_create(
|
||||
remark=f"{user}",
|
||||
proxy_type="socks5",
|
||||
host=host,
|
||||
port=str(port),
|
||||
proxy_user=user,
|
||||
proxy_pwd=pwd,
|
||||
)
|
||||
if not browser_id:
|
||||
return None
|
||||
logger.info(f"创建指纹浏览器成功: {browser_id}")
|
||||
time.sleep(0.1)
|
||||
http = bit_browser.bit_browser_open(browser_id)
|
||||
if not http:
|
||||
return None
|
||||
logger.info(f"打开指纹浏览器成功: {browser_id}")
|
||||
return browser_id, http
|
||||
except Exception as e:
|
||||
logger.error(f"创建指纹浏览器失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def is_quiet_time() -> bool:
|
||||
def close_and_delete_browser(browser_id: str) -> None:
|
||||
"""
|
||||
判断当前是否处于禁跑时段（18:30~20:00）
|
||||
关闭并删除指定指纹浏览器
|
||||
|
||||
参数:
|
||||
browser_id: 指纹浏览器ID
|
||||
"""
|
||||
try:
|
||||
bit_browser.bit_browser_close(browser_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"关闭浏览器失败或已关闭: {browser_id} - {e}")
|
||||
time.sleep(0.1)
|
||||
try:
|
||||
bit_browser.bit_browser_delete(browser_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"删除浏览器失败或已删除: {browser_id} - {e}")
|
||||
|
||||
|
||||
def run_task_with_proxy(proxy: str, stop_event: threading.Event) -> None:
|
||||
"""
|
||||
使用代理创建指纹浏览器、执行自动化，并在结束后清理
|
||||
|
||||
参数:
|
||||
proxy: 代理字符串
|
||||
"""
|
||||
browser_id: str | None = None
|
||||
try:
|
||||
created = create_fingerprint_browser(proxy)
|
||||
if not created:
|
||||
return
|
||||
browser_id, http = created
|
||||
if stop_event.is_set():
|
||||
return
|
||||
auto = Auto(http=http)
|
||||
auto.open_url('https://veritaconnect.ca/canadianbreadsettlement/en-us')
|
||||
if stop_event.is_set():
|
||||
return
|
||||
if not auto.wait_home():
|
||||
return
|
||||
if stop_event.is_set():
|
||||
return
|
||||
if not auto.click_continue():
|
||||
return
|
||||
if stop_event.is_set():
|
||||
return
|
||||
auto.fill_questionnaire()
|
||||
except Exception as e:
|
||||
logger.error(f"执行任务异常: {e}")
|
||||
finally:
|
||||
if browser_id:
|
||||
try:
|
||||
close_and_delete_browser(browser_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def proxy_loop(proxy: str, stop_event: threading.Event) -> None:
|
||||
"""
|
||||
为单个代理保持持续运行：任务结束后立即重建并再次执行
|
||||
|
||||
参数:
|
||||
proxy: 代理字符串
|
||||
stop_event: 停止事件，用于外部触发退出循环
|
||||
"""
|
||||
while not stop_event.is_set():
|
||||
try:
|
||||
if is_forbidden_time():
|
||||
cleanup_all_browsers()
|
||||
secs = seconds_until(20, 0)
|
||||
if stop_event.wait(timeout=secs):
|
||||
break
|
||||
continue
|
||||
run_task_with_proxy(proxy, stop_event)
|
||||
except Exception as e:
|
||||
logger.error(f"代理循环异常: {proxy} - {e}")
|
||||
if stop_event.is_set():
|
||||
break
|
||||
if stop_event.wait(timeout=0.1):
|
||||
break
|
||||
|
||||
|
||||
def is_forbidden_time() -> bool:
|
||||
"""
|
||||
判断当前是否处于禁跑时段（每日 18:30 ~ 20:00，本地时间）
|
||||
|
||||
返回值:
|
||||
bool: True 表示处于禁跑时段
|
||||
"""
|
||||
now = datetime.now()
|
||||
start = now.replace(hour=18, minute=30, second=0, microsecond=0)
|
||||
@@ -584,260 +672,132 @@ def is_quiet_time() -> bool:
|
||||
return start <= now < end
|
||||
|
||||
|
||||
def sleep_until_quiet_end():
|
||||
def seconds_until(hour: int, minute: int) -> float:
|
||||
"""
|
||||
在禁跑时段内休眠至 20:00
|
||||
计算到今天指定时间点的剩余秒数
|
||||
|
||||
参数:
|
||||
hour: 目标小时（24小时制）
|
||||
minute: 目标分钟
|
||||
|
||||
返回值:
|
||||
float: 剩余秒数，若目标时间已过则为 0
|
||||
"""
|
||||
now = datetime.now()
|
||||
end = now.replace(hour=20, minute=0, second=0, microsecond=0)
|
||||
if now < end:
|
||||
seconds = (end - now).total_seconds()
|
||||
logger.info(f"当前处于禁跑时段，休眠至 20:00（{int(seconds)} 秒）")
|
||||
time.sleep(seconds)
|
||||
target = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||
if target <= now:
|
||||
return 0.0
|
||||
return (target - now).total_seconds()
|
||||
|
||||
|
||||
"""指纹浏览器操作"""
|
||||
# 创建指纹浏览器
|
||||
|
||||
|
||||
def create_fingerprint_browser(city: str | None = None):
|
||||
def count_fingerprint_browsers() -> int:
|
||||
"""
|
||||
创建指纹浏览器并执行一次流程（支持随机 IP 与指定城市）
|
||||
统计当前指纹浏览器数量
|
||||
|
||||
参数:
|
||||
city (str | None): 指定城市使用其对应代理；None 则使用随机代理并随机选择城市
|
||||
返回值:
|
||||
int: 当前总数量
|
||||
"""
|
||||
browser_id = None
|
||||
try:
|
||||
if is_quiet_time():
|
||||
logger.info("处于禁跑时段（18:30~20:00），跳过本次运行")
|
||||
return
|
||||
if city is not None:
|
||||
proxy = get_proxy(city)
|
||||
if proxy is None:
|
||||
logger.error(f"{city} 未配置对应代理，结束该线程")
|
||||
return
|
||||
remark = city
|
||||
else:
|
||||
proxy = get_random_proxy()
|
||||
if proxy is None:
|
||||
logger.error("未获取到随机代理，结束该线程")
|
||||
return
|
||||
remark = "random-ip"
|
||||
logger.info("准备创建指纹浏览器")
|
||||
browser_id = bit_browser.bit_browser_create(
|
||||
remark=remark,
|
||||
host=proxy[0],
|
||||
port=proxy[1],
|
||||
proxy_user=proxy[2],
|
||||
proxy_pwd=proxy[3],
|
||||
proxy_type='socks5'
|
||||
)
|
||||
logger.debug(browser_id)
|
||||
# 打开指纹浏览器
|
||||
http = bit_browser.bit_browser_open(browser_id)
|
||||
logger.debug(http)
|
||||
auto = Auto(http)
|
||||
auto.open_url(
|
||||
"https://veritaconnect.ca/canadianbreadsettlement/en-us/Claimant/UnknownClaimForm")
|
||||
bol = auto.wait_home()
|
||||
if not bol:
|
||||
logger.error("进入首页失败，结束该线程")
|
||||
return
|
||||
|
||||
bol = auto.click_continue()
|
||||
if not bol:
|
||||
logger.error("点击 Continue 失败，结束该线程")
|
||||
return
|
||||
auto.fill_questionnaire()
|
||||
# fill_city = city if city is not None else random.choice(['Calgary', 'Edmonton', 'Vancouver', 'Halifax', 'Toronto'])
|
||||
# auto.fill_questionnaire(fill_city)
|
||||
# time.sleep(5)
|
||||
finally:
|
||||
if browser_id:
|
||||
# 关闭指纹浏览器
|
||||
try:
|
||||
bit_browser.bit_browser_close(browser_id)
|
||||
except Exception as e:
|
||||
logger.error(f"关闭浏览器异常: {e}")
|
||||
# 删除指纹浏览器
|
||||
try:
|
||||
bit_browser.bit_browser_delete(browser_id)
|
||||
except Exception as e:
|
||||
logger.error(f"删除浏览器异常: {e}")
|
||||
res = bit_browser.bit_browser_get(0, 100)
|
||||
data = res.get("data", {}) if isinstance(res, dict) else {}
|
||||
total = data.get("totalNum")
|
||||
lst = data.get("list", [])
|
||||
if isinstance(total, int) and total >= 0:
|
||||
return total
|
||||
return len(lst)
|
||||
except Exception as e:
|
||||
logger.warning(f"统计指纹浏览器数量失败: {e}")
|
||||
return 0
|
||||
|
||||
|
||||
def run_city_forever(city: str):
|
||||
def cleanup_all_browsers() -> None:
|
||||
"""
|
||||
持续循环运行指定城市流程：完成一次即关闭并删除浏览器，然后重新创建继续运行
|
||||
|
||||
参数:
|
||||
city (str): 城市名称
|
||||
关闭并删除所有指纹浏览器
|
||||
"""
|
||||
while True:
|
||||
if is_quiet_time():
|
||||
sleep_until_quiet_end()
|
||||
continue
|
||||
try:
|
||||
create_fingerprint_browser(city)
|
||||
except Exception as e:
|
||||
logger.error(f"{city} 流程异常: {e}")
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def run_all_cities_concurrently(num: int):
|
||||
"""
|
||||
多线程并发运行城市流程（支持随机选择）
|
||||
|
||||
参数:
|
||||
num (int | None): 随机选择并启动的城市数量；None 表示全部
|
||||
"""
|
||||
import threading
|
||||
threads = []
|
||||
for i in range(num):
|
||||
t = threading.Thread(target=run_random_ips_forever,
|
||||
name=f"random-ip-thread-{i}")
|
||||
t.start()
|
||||
threads.append(t)
|
||||
logger.info(f"随机 IP 线程 {i} 已启动")
|
||||
for t in threads:
|
||||
t.join()
|
||||
logger.info("所有随机 IP 流程执行完成")
|
||||
|
||||
|
||||
def run_random_ips_forever():
|
||||
"""
|
||||
持续使用随机 IP 执行流程：每次完成后关闭并删除浏览器再重建
|
||||
"""
|
||||
while True:
|
||||
if is_quiet_time():
|
||||
sleep_until_quiet_end()
|
||||
continue
|
||||
try:
|
||||
create_fingerprint_browser(None)
|
||||
except Exception as e:
|
||||
logger.error(f"随机 IP 流程异常: {e}")
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def run_random_ips_concurrently(num: int):
|
||||
"""
|
||||
根据指纹浏览器数量并发运行流程（随机取 IP）
|
||||
|
||||
参数:
|
||||
num (int): 并发指纹浏览器数量（每个使用随机代理）
|
||||
"""
|
||||
import threading
|
||||
if num <= 0:
|
||||
logger.warning("num 不合法（<=0），不启动任何线程")
|
||||
return
|
||||
threads = []
|
||||
for i in range(num):
|
||||
t = threading.Thread(target=run_random_ips_forever,
|
||||
name=f"random-ip-thread-{i}")
|
||||
t.start()
|
||||
threads.append(t)
|
||||
logger.info(f"随机 IP 线程 {i} 已启动")
|
||||
for t in threads:
|
||||
t.join()
|
||||
logger.info("随机 IP 并发流程执行完成")
|
||||
|
||||
|
||||
def create_fingerprint_browser_with_proxy(proxy: list[str]):
|
||||
"""
|
||||
使用指定代理创建指纹浏览器并执行一次流程（一一对应）
|
||||
|
||||
参数:
|
||||
proxy (list[str]): `[host, port, user, pwd]`
|
||||
"""
|
||||
browser_id = None
|
||||
try:
|
||||
if is_quiet_time():
|
||||
logger.info("处于禁跑时段（18:30~20:00），跳过本次运行")
|
||||
return
|
||||
if not proxy or len(proxy) < 4:
|
||||
logger.error("代理参数不完整，结束该线程")
|
||||
return
|
||||
# 随机等待0.1秒
|
||||
time.sleep(random.uniform(0.1, 1.0))
|
||||
logger.info(f"使用代理 {proxy[2]} 创建浏览器")
|
||||
browser_id = bit_browser.bit_browser_create(
|
||||
remark=f"{proxy[2]}",
|
||||
host=proxy[0],
|
||||
port=proxy[1],
|
||||
proxy_user=proxy[2],
|
||||
proxy_pwd=proxy[3],
|
||||
proxy_type='socks5'
|
||||
)
|
||||
logger.debug(f"创建浏览器 {browser_id}")
|
||||
time.sleep(random.uniform(0.1, 1.0))
|
||||
http = bit_browser.bit_browser_open(browser_id)
|
||||
logger.debug(f"打开浏览器 {browser_id}")
|
||||
auto = Auto(http)
|
||||
auto.open_url(
|
||||
"https://veritaconnect.ca/canadianbreadsettlement/en-us/Claimant/UnknownClaimForm")
|
||||
bol = auto.wait_home()
|
||||
if not bol:
|
||||
logger.error("进入首页失败，结束该线程")
|
||||
return
|
||||
bol = auto.click_continue()
|
||||
if not bol:
|
||||
logger.error("点击 Continue 失败，结束该线程")
|
||||
return
|
||||
auto.fill_questionnaire()
|
||||
finally:
|
||||
if browser_id:
|
||||
try:
|
||||
bit_browser.bit_browser_close(browser_id)
|
||||
except Exception as e:
|
||||
logger.error(f"关闭浏览器异常: {e}")
|
||||
try:
|
||||
bit_browser.bit_browser_delete(browser_id)
|
||||
except Exception as e:
|
||||
logger.error(f"删除浏览器异常: {e}")
|
||||
res = bit_browser.bit_browser_get(0, 100)
|
||||
data = res.get("data", {}) if isinstance(res, dict) else {}
|
||||
lst = data.get("list", [])
|
||||
ids = [i.get("id") for i in lst if i.get("id")]
|
||||
for bid in ids:
|
||||
close_and_delete_browser(bid)
|
||||
except Exception as e:
|
||||
logger.warning(f"清理所有指纹浏览器失败: {e}")
|
||||
|
||||
|
||||
def run_proxies_forever(proxy: list[str]):
|
||||
def monitor_browsers_and_restart(limit: int, stop_event: threading.Event, restart_event: threading.Event) -> None:
|
||||
"""
|
||||
持续使用指定代理执行流程：完成后关闭并删除浏览器再重建
|
||||
每 30 秒检测指纹浏览器数量，超过 `limit` 则触发重启事件并清理所有浏览器
|
||||
|
||||
参数:
|
||||
proxy (list[str]): `[host, port, user, pwd]`
|
||||
limit: 允许的最大浏览器数量（通常为代理数量）
|
||||
restart_event: 触发重启的事件
|
||||
"""
|
||||
while not stop_event.is_set():
|
||||
time.sleep(30)
|
||||
count = count_fingerprint_browsers()
|
||||
if count > limit:
|
||||
logger.warning(f"指纹浏览器数量 {count} 超过限制 {limit}，执行重启")
|
||||
restart_event.set()
|
||||
stop_event.set()
|
||||
cleanup_all_browsers()
|
||||
break
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
多线程并发管理：按代理数量并发创建指纹浏览器并执行任务；每 30 秒监控数量，超限则重启。
|
||||
"""
|
||||
proxies = list(proxy_list)
|
||||
while True:
|
||||
if is_quiet_time():
|
||||
sleep_until_quiet_end()
|
||||
stop_event = threading.Event()
|
||||
restart_event = threading.Event()
|
||||
|
||||
if is_forbidden_time():
|
||||
cleanup_all_browsers()
|
||||
secs = seconds_until(20, 0)
|
||||
logger.info(f"处于禁跑时段，休眠至20:00，剩余 {int(secs)} 秒")
|
||||
time.sleep(secs)
|
||||
continue
|
||||
try:
|
||||
create_fingerprint_browser_with_proxy(proxy)
|
||||
except Exception as e:
|
||||
logger.error(f"固定代理流程异常: {e}")
|
||||
time.sleep(2)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=len(proxies)) as executor:
|
||||
futures_map = {executor.submit(proxy_loop, p, stop_event): p for p in proxies}
|
||||
|
||||
monitor_thread = threading.Thread(
|
||||
target=monitor_browsers_and_restart,
|
||||
args=(len(proxies), stop_event, restart_event),
|
||||
daemon=True,
|
||||
)
|
||||
monitor_thread.start()
|
||||
|
||||
while True:
|
||||
if restart_event.is_set():
|
||||
stop_event.set()
|
||||
try:
|
||||
executor.shutdown(wait=False)
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
# 进入禁跑时段时，立即停止并清理浏览器
|
||||
if is_forbidden_time():
|
||||
logger.info("进入禁跑时段，停止当前批次并清理指纹浏览器")
|
||||
stop_event.set()
|
||||
try:
|
||||
executor.shutdown(wait=False)
|
||||
except Exception:
|
||||
pass
|
||||
cleanup_all_browsers()
|
||||
break
|
||||
time.sleep(0.2)
|
||||
|
||||
try:
|
||||
monitor_thread.join(timeout=5)
|
||||
except Exception:
|
||||
pass
|
||||
continue
|
||||
|
||||
|
||||
def run_all_proxies_concurrently():
|
||||
"""
|
||||
按固定代理列表一一创建并发浏览器
|
||||
"""
|
||||
import threading
|
||||
# proxies = get_all_proxies()
|
||||
proxies = [p.split(":") for p in proxy_list]
|
||||
if not proxies:
|
||||
logger.warning("未找到可用代理，结束执行")
|
||||
return
|
||||
threads = []
|
||||
for i, proxy in enumerate(proxies):
|
||||
t = threading.Thread(target=run_proxies_forever,
|
||||
args=(proxy,), name=f"proxy-thread-{i}")
|
||||
t.start()
|
||||
threads.append(t)
|
||||
logger.info(f"固定代理线程 {i} 已启动: {proxy[0]}:{proxy[1]} @ {proxy[2]}")
|
||||
for t in threads:
|
||||
t.join()
|
||||
logger.info("固定代理并发流程执行完成")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# auto = Auto()
|
||||
# auto.get_random_food('a')
|
||||
run_all_proxies_concurrently()
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user