fix(lightnovel): 尝试修复轻小说推荐功能,增加代理池

This commit is contained in:
ClovertaTheTrilobita 2025-02-15 18:11:32 +08:00
parent 511cd92c3a
commit 2c9935482d
2 changed files with 28 additions and 6 deletions

View file

@ -2,7 +2,9 @@ from os import getcwd
import requests import requests
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from src.configs.api_config import wenku8_username, wenku8_password from src.configs.api_config import wenku8_username, wenku8_password, proxy_api
# 登录页面的URL # 登录页面的URL
login_url = 'https://www.wenku8.net/login.php?jumpurl=http%3A%2F%2Fwww.wenku8.net%2Findex.php' login_url = 'https://www.wenku8.net/login.php?jumpurl=http%3A%2F%2Fwww.wenku8.net%2Findex.php'
@ -11,7 +13,7 @@ index_url = 'https://www.wenku8.net/index.php'
headers = { headers = {
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2', 'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Connection': 'close', 'Connection': 'close',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.41', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.41'
} }
# 登录表单数据 # 登录表单数据
@ -22,12 +24,26 @@ login_data = {
'action': 'login' 'action': 'login'
} }
def get_proxy(headers):
#proxy_url可通过多米HTTP代理网站购买后生成代理api链接每次请求api链接都是新的ip
proxy_url = proxy_api
aaa=requests.get(proxy_url, headers=headers).text
proxy_host = aaa.splitlines()[0]
print('代理IP为'+proxy_host)
#proxy_host='117.35.254.105:22001'
#proxy_host='192.168.0.134:1080'
proxy = {
'http': 'http://'+proxy_host,
'https': 'http://'+proxy_host
}
return proxy
async def login(): async def login():
# 发送登录请求 # 发送登录请求
with requests.Session() as session: with requests.Session() as session:
proxy = get_proxy(headers)
# 注意这里使用了Session对象来保持会话状态 # 注意这里使用了Session对象来保持会话状态
login_response = session.post(login_url, data=login_data, headers=headers) login_response = session.post(login_url, data=login_data, headers=headers, proxies=proxy)
# 检查登录是否成功(根据实际需求调整) # 检查登录是否成功(根据实际需求调整)
if login_response.status_code == 200: if login_response.status_code == 200:
@ -54,8 +70,8 @@ async def get_books():
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.41', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.41',
'Cookie': cookie 'Cookie': cookie
} }
proxy = get_proxy(headers1)
response = requests.get(index_url, headers=headers1) response = requests.get(index_url, headers=headers1, proxies=proxy)
print(response) print(response)
html = response.content.decode('gbk') html = response.content.decode('gbk')
soup = BeautifulSoup(html, 'html.parser') soup = BeautifulSoup(html, 'html.parser')
@ -80,6 +96,7 @@ async def get_books():
<!-- <link rel="stylesheet" href="main.css">--> <!-- <link rel="stylesheet" href="main.css">-->
</head> </head>
""" """
# print(orders[7].text)
with open(getcwd() + "/src/clover_lightnovel/output1.html", 'w', encoding='utf-8') as file: with open(getcwd() + "/src/clover_lightnovel/output1.html", 'w', encoding='utf-8') as file:
file.write(head + str(orders[7]).replace('(<a href="https://www.wenku8.net/zt/sugoi/2025.php"', '').replace( file.write(head + str(orders[7]).replace('(<a href="https://www.wenku8.net/zt/sugoi/2025.php"', '').replace(
'target="_blank">查看 这本轻小说真厉害2025 TOP榜单</a>)', '') + str(orders[8]) + str(orders[9])) 'target="_blank">查看 这本轻小说真厉害2025 TOP榜单</a>)', '') + str(orders[8]) + str(orders[9]))

View file

@ -27,4 +27,9 @@ deepseek_key= "<KEY>"
Wenku8账号 Wenku8账号
""" """
wenku8_username = "<user_name>" wenku8_username = "<user_name>"
wenku8_password = "<passwd>" wenku8_password = "<passwd>"
"""
多米HTTP代理api
"""
proxy_api = "<KEY>"