Files
2026-09-20 21:04:34 +08:00

484 lines
19 KiB
Python

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
OctoWow (octowow.st/db) 任务奖励数据抓取脚本 —— pfExtend 插件数据生成器
用法:
python scrape.py catalog 阶段A: 抓分类菜单 + 分类列表页 -> catalog.json
(任务ID/名称/等级/阵营/经验/固定奖励物品/自选奖励物品)
python scrape.py build 汇总 catalog.json -> rewards_data.lua (pfExtend 数据文件)
python scrape.py all catalog + build (完整流程, 约110次请求)
python scrape.py rewards [--limit N] [--ids 1,2,3]
(可选) 阶段B: 逐任务抓详情页补充满级折算金钱(mml)
-> cache/quests/<id>.json (断点续传)
说明: 分类列表页本身已包含经验(xp)与物品奖励(itemrewards/itemchoices),
且 xp 与详情页 Gains 展示值一致, 因此阶段B不是必须的;
详情页 Quick Facts 的 RewXP 字段与展示值存在偏差, 不采用。
工作目录结构:
questGaindb/
scrape.py 本脚本
cookies.txt BlazingFast 通行 Cookie (7天有效, 过期自动重新挑战)
catalog.json 阶段A产物: 全部任务目录 {id: {name,level,reqlevel,side,xp,category,...}}
cache/
categories/ 每个分类列表页解析结果 <parent>_<child>.json
quests/ 每个任务页解析结果 <id>.json
html/ 任务页原始HTML <id>.html (便于离线重解析)
rewards_data.lua 最终产物: pfExtend 任务奖励数据文件
"""
import argparse
import json
import os
import random
import re
import subprocess
import sys
import time
BASE = "https://octowow.st/db/"
SITE = "https://octowow.st/" # 挑战相关资源在站点根路径
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0 Safari/537.36")
WORKDIR = os.path.dirname(os.path.abspath(__file__))
COOKIE_FILE = os.path.join(WORKDIR, "cookies.txt")
CATALOG_FILE = os.path.join(WORKDIR, "catalog.json")
LUA_FILE = os.path.join(WORKDIR, "rewards_data.lua")
DIR_CATEGORIES = os.path.join(WORKDIR, "cache", "categories")
DIR_QUESTS = os.path.join(WORKDIR, "cache", "quests")
DIR_HTML = os.path.join(WORKDIR, "cache", "html")
MIN_INTERVAL = 1.0 # 请求最小间隔(秒)
JITTER = 0.5 # 随机抖动上限(秒)
MAX_RETRIES = 5 # 单请求最大重试次数
# ---------------------------------------------------------------- 站点会话
def _find_curl():
for cand in ("curl", "curl.exe"):
try:
subprocess.run([cand, "--version"], capture_output=True, check=True)
return cand
except (OSError, subprocess.CalledProcessError):
continue
sys.exit("未找到 curl。请安装 curl 或在含 curl 的环境中运行 "
"(Windows 10+ 自带 curl.exe, Git Bash 自带 curl)")
CURL = _find_curl()
class Resp:
def __init__(self, status, text):
self.status_code = status
self.text = text
class OctoSession:
"""基于 curl 子进程的 HTTP 会话。
为什么不用 requests: BlazingFast 防护按 TLS 指纹识别客户端,
Python requests 的每次请求都会被重新挑战(实测), 而 curl 通过挑战后
服务端会按 IP+Cookie 放行。因此全部流量经 curl 收发。
Cookie 以 Netscape 格式存于 cookies.txt, 7 天有效。
"""
def __init__(self):
self.last_request = 0.0
def has_clearance(self):
if not os.path.exists(COOKIE_FILE):
return False
with open(COOKIE_FILE, encoding="utf-8", errors="replace") as f:
return "__bf_clearance_v2" in f.read()
def _curl(self, args):
return subprocess.run(
[CURL, "-s", "--compressed", "--max-time", "40",
"-b", COOKIE_FILE, "-c", COOKIE_FILE, "-A", UA] + args,
capture_output=True)
def _curl_body(self, args):
p = self._curl(args)
if p.returncode != 0:
raise RuntimeError("curl 退出码 %d: %s"
% (p.returncode, p.stderr.decode("utf-8", "replace")[:200]))
return p.stdout.decode("utf-8", "replace")
@staticmethod
def _solve_expr(expr):
"""计算挑战算术表达式, 已观测格式: a+b*c (乘法优先)"""
m = re.fullmatch(r"\s*(\d+)\s*\+\s*(\d+)\s*\*\s*(\d+)\s*", expr)
if m:
a, b, c = (int(x) for x in m.groups())
return a + b * c
m = re.fullmatch(r"\s*(\d+)\s*\*\s*(\d+)\s*\+\s*(\d+)\s*", expr)
if m:
a, b, c = (int(x) for x in m.groups())
return a * b + c
raise ValueError("未知挑战表达式: %r" % expr)
def solve_challenge(self, html, referer):
chal = re.search(r'name="bf_challenge" value="([0-9a-f]{64})"', html).group(1)
bfu = re.search(r'name="bfu" value="([^"]*)"', html).group(1)
expr = re.search(r'bf-v2-answer"\)\.value=([^;]+)', html).group(1)
answer = self._solve_expr(expr)
# 服务端要求先拉取挑战脚本资源(站点根路径), 否则 409 challenge_asset_missing
self._curl(["-e", referer, SITE + "bf.jquery.max.js?bf_challenge=" + chal,
"-o", os.devnull])
time.sleep(6) # 官方页面本身延迟5.1秒提交, 必须模仿
self._curl(["-e", referer, "-X", "POST", SITE + "blzgfst-shark/",
"--data-urlencode", "bf_challenge=" + chal,
"--data-urlencode", "bfu=" + bfu,
"--data-urlencode", "blazing_answer=" + str(answer),
"-o", os.devnull])
if not self.has_clearance():
raise RuntimeError("挑战失败, 未获得通行 Cookie")
print("[challenge] 已通过 BlazingFast 挑战")
def get(self, url):
"""限速 + 挑战重试的 GET, 返回 Resp"""
for attempt in range(MAX_RETRIES):
wait = MIN_INTERVAL + random.random() * JITTER - (time.time() - self.last_request)
if wait > 0:
time.sleep(wait)
self.last_request = time.time()
try:
out = self._curl_body(["-L", "-w", "\n%{http_code}", url])
body, _, code = out.rpartition("\n")
status = int(code)
except (RuntimeError, ValueError) as e:
print("[net] %s (第%d次)" % (e, attempt + 1))
time.sleep(min(120, 5 * 2 ** attempt))
continue
if 'name="bf_challenge"' in body:
self.solve_challenge(body, url)
continue
if status == 200:
return Resp(200, body)
print("[http] %s -> %d (第%d次)" % (url, status, attempt + 1))
time.sleep(min(120, 5 * 2 ** attempt))
raise RuntimeError("请求失败(重试%d次): %s" % (MAX_RETRIES, url))
# ---------------------------------------------------------------- 阶段A: 目录
def parse_menu(js_text):
"""解析 locale_enus.js 中的 mn_quests 分类树
返回 [(parent_id, parent_name, [(child_id, child_name), ...] or None), ...]"""
i = js_text.find("var mn_quests")
if i < 0:
raise ValueError("未找到 mn_quests")
start = js_text.index("[", i)
depth, end = 0, start
for end in range(start, len(js_text)):
if js_text[end] == "[":
depth += 1
elif js_text[end] == "]":
depth -= 1
if depth == 0:
break
body = js_text[start + 1:end] # 根数组内部
# 按深度0处的逗号切分顶层条目 (嵌套数组内的逗号不受影响)
entries, depth, last = [], 0, 0
for idx, c in enumerate(body):
if c == "[":
depth += 1
elif c == "]":
depth -= 1
elif c == "," and depth == 0:
entries.append(body[last:idx])
last = idx + 1
entries.append(body[last:])
head_re = re.compile(r'\[\s*(-?\d+)\s*,\s*"([^"]+)"')
pair_re = re.compile(r'\[\s*(-?\d+)\s*,\s*"([^"]+)"\s*\]')
menu = []
for e in entries:
e = e.strip()
if not e:
continue
h = head_re.search(e)
if not h:
continue
children = [(int(a), b) for a, b in pair_re.findall(e[h.end():])]
menu.append((int(h.group(1)), h.group(2), children if children else None))
return menu
ENTRY_RE = re.compile(r"\{id:\s*'(\d+)',([^{}]*)\}")
FIELD_RES = {
"name": re.compile(r"name:\s*'((?:[^'\\]|\\.)*)'"),
"level": re.compile(r"level:\s*'(\d+)'"),
"reqlevel": re.compile(r"reqlevel:(\d+)"),
"side": re.compile(r"side:\s*'(\d+)'"),
"xp": re.compile(r"xp:(-?\d+)"),
"category": re.compile(r"category:(-?\d+)"),
"category2": re.compile(r"category2:(-?\d+)"),
"type": re.compile(r"type:(-?\d+)"),
}
ITEMS_RE = re.compile(r"item(rewards|choices):(\[\[[0-9,\[\]]*\]\])")
PAIR_RE = re.compile(r"\[(\d+),(\d+)\]")
def parse_quest_list(html):
"""解析分类列表页内嵌 Listview 数据 (含可选的 itemrewards/itemchoices 字段)"""
out = []
for m in ENTRY_RE.finditer(html):
body = m.group(2)
q = {"id": int(m.group(1))}
for key, pat in FIELD_RES.items():
mm = pat.search(body)
if not mm:
continue
v = mm.group(1)
if key == "name":
q[key] = v.replace("\\'", "'").replace("\\\\", "\\").strip()
else:
q[key] = int(v)
for kind, arr in ITEMS_RE.findall(body):
items = [[int(a), int(b)] for a, b in PAIR_RE.findall(arr)]
if items:
q["reward" if kind == "rewards" else "choice"] = items
out.append(q)
return out
def cmd_catalog(sess, force=False):
os.makedirs(DIR_CATEGORIES, exist_ok=True)
r = sess.get(BASE + "templates/wowhead/js/locale_enus.js")
menu = parse_menu(r.text)
jobs = [] # (cache_name, url, desc)
for pid, pname, children in menu:
if children is None:
jobs.append(("%d" % pid, "%s?quests=%d" % (BASE, pid),
"%d %s" % (pid, pname)))
else:
for cid, cname in children:
jobs.append(("%d_%d" % (pid, cid), "%s?quests=%d.%d" % (BASE, pid, cid),
"%d.%d %s/%s" % (pid, cid, pname, cname)))
catalog = {}
done = skipped = 0
for idx, (cache_name, url, desc) in enumerate(jobs, 1):
cache_path = os.path.join(DIR_CATEGORIES, cache_name + ".json")
if os.path.exists(cache_path) and not force:
quests = json.load(open(cache_path, encoding="utf-8"))
skipped += 1
else:
r = sess.get(url)
quests = parse_quest_list(r.text)
json.dump(quests, open(cache_path, "w", encoding="utf-8"),
ensure_ascii=False, indent=1)
print("[catalog] (%d/%d) %s -> %d 个任务" % (idx, len(jobs), desc, len(quests)))
done += 1
for q in quests:
catalog.setdefault(q["id"], q)
json.dump(catalog, open(CATALOG_FILE, "w", encoding="utf-8"),
ensure_ascii=False, indent=1, sort_keys=True)
print("[catalog] 完成: 新抓 %d 个分类, 复用缓存 %d 个, 目录共 %d 个任务"
% (done, skipped, len(catalog)))
return catalog
# ---------------------------------------------------------------- 阶段B: 奖励
REWARD_SECTION_RE = re.compile(r"<h3>Reward</h3>(.*?)(?=<h3>|<h2>)", re.S)
PAGEINFO_RE = re.compile(r"g_pageInfo\s*=\s*\{type:\s*(\d+),\s*typeId:\s*(\d+)")
ITEM_LINK_RE = re.compile(r"\?item=(\d+)")
MONEY_TEXT_RE = re.compile(r"also receive[^0-9]*([\d,]+)\s*(Gold|Silver|Copper)", re.I)
def _extract_items(segment):
return [int(x) for x in ITEM_LINK_RE.findall(segment)]
def parse_quest_page(html, quest_id):
"""解析任务详情页 -> dict (验证失败返回 None)"""
m = PAGEINFO_RE.search(html)
if not m or int(m.group(2)) != quest_id:
return None
info = {"id": quest_id}
m = re.search(r"RewXP:\s*(\d+)", html)
if m:
info["xp"] = int(m.group(1))
m = re.search(r"RewMoneyMaxLevel:\s*(\d+)", html)
if m:
info["mml"] = int(m.group(1))
sec = REWARD_SECTION_RE.search(html)
if sec:
body = sec.group(1)
# 标记点: 固定奖励 / 自选奖励 / 法术奖励
marks = []
for label, pat in (("reward", r"You will receive"),
("choice", r"choose one of these awards"),
("spell", r"The following spell")):
mm = re.search(pat, body)
if mm:
marks.append((mm.start(), label))
marks.sort()
for i, (pos, label) in enumerate(marks):
if label == "spell":
continue
end = marks[i + 1][0] if i + 1 < len(marks) else len(body)
items = _extract_items(body[pos:end])
if items:
info[label] = items
return info
def cmd_rewards(sess, limit=None, ids=None):
os.makedirs(DIR_QUESTS, exist_ok=True)
os.makedirs(DIR_HTML, exist_ok=True)
if ids:
queue = ids
else:
if not os.path.exists(CATALOG_FILE):
sys.exit("缺少 catalog.json, 请先运行: python scrape.py catalog")
catalog = json.load(open(CATALOG_FILE, encoding="utf-8"))
queue = sorted(int(qid) for qid in catalog.keys())
todo = [qid for qid in queue
if not os.path.exists(os.path.join(DIR_QUESTS, "%d.json" % qid))]
if limit:
todo = todo[:limit]
print("[rewards] 待抓取 %d 个任务 (已完成 %d 个)" % (len(todo), len(queue) - len(todo)))
consecutive_fail = 0
for idx, qid in enumerate(todo, 1):
url = "%s?quest=%d" % (BASE, qid)
try:
r = sess.get(url)
except RuntimeError as e:
print("[rewards] %s" % e)
consecutive_fail += 1
if consecutive_fail >= 5:
sys.exit("连续失败5次, 中止 (可重新运行续传)")
continue
info = parse_quest_page(r.text, qid)
if info is None:
info = {"id": qid, "missing": True}
print("[rewards] (%d/%d) 任务 %d 页面无效, 标记 missing" % (idx, len(todo), qid))
json.dump(info, open(os.path.join(DIR_QUESTS, "%d.json" % qid), "w",
encoding="utf-8"), ensure_ascii=False)
with open(os.path.join(DIR_HTML, "%d.html" % qid), "w", encoding="utf-8") as f:
f.write(r.text)
consecutive_fail = 0
if idx % 25 == 0 or idx == len(todo):
print("[rewards] 进度 %d/%d (最近: %d)" % (idx, len(todo), qid))
print("[rewards] 本轮完成")
# ---------------------------------------------------------------- 汇总输出
def _fmt_items(items):
"""[[id,count],...] -> Lua 片段; count为1时写裸ID以压缩体积"""
parts = []
for iid, count in items:
parts.append(str(iid) if count == 1 else "{ %d, %d }" % (iid, count))
return "{ %s }" % ", ".join(parts)
def cmd_build():
if not os.path.exists(CATALOG_FILE):
sys.exit("缺少 catalog.json, 请先运行: python scrape.py catalog")
catalog = json.load(open(CATALOG_FILE, encoding="utf-8"))
entries = {}
stats = {"total": 0, "with_reward": 0, "with_choice": 0, "with_xp": 0,
"with_mml": 0}
for qid_str, q in catalog.items():
qid = int(qid_str)
stats["total"] += 1
entry = {}
if q.get("xp"):
entry["xp"] = q["xp"]
stats["with_xp"] += 1
if q.get("reward"):
entry["reward"] = q["reward"]
stats["with_reward"] += 1
if q.get("choice"):
entry["choice"] = q["choice"]
stats["with_choice"] += 1
# 可选: 阶段B详情页缓存中的满级折算金钱
page_path = os.path.join(DIR_QUESTS, "%d.json" % qid)
if os.path.exists(page_path):
page = json.load(open(page_path, encoding="utf-8"))
if page.get("mml"):
entry["mml"] = page["mml"]
stats["with_mml"] += 1
if entry:
entries[qid] = entry
lines = [
"-- pfExtend 任务奖励数据 (物品奖励/经验)",
"-- 数据来源: https://octowow.st/db/ (分类列表页批量抓取)",
"-- 生成时间: %s" % time.strftime("%Y-%m-%d %H:%M:%S"),
"-- 任务数: %d (含固定奖励 %d, 含自选奖励 %d, 含经验 %d)"
% (len(entries), stats["with_reward"], stats["with_choice"], stats["with_xp"]),
"-- 物品格式: 单数量写裸ID, 多数量写 { ID, 数量 }",
"PfExtend_QuestRewards = {",
]
for qid in sorted(entries):
e = entries[qid]
parts = []
if "xp" in e:
parts.append("xp = %d" % e["xp"])
if "reward" in e:
parts.append("reward = %s" % _fmt_items(e["reward"]))
if "choice" in e:
parts.append("choice = %s" % _fmt_items(e["choice"]))
if "mml" in e:
parts.append("mml = %d" % e["mml"])
lines.append(" [%d] = { %s }," % (qid, ", ".join(parts)))
lines.append("}")
with open(LUA_FILE, "w", encoding="utf-8") as f:
f.write("\n".join(lines) + "\n")
print("[build] %s 已生成: %d 条数据, %.1f KB" %
(LUA_FILE, len(entries), os.path.getsize(LUA_FILE) / 1024))
print("[build] 统计: 目录 %d, 含固定奖励 %d, 含自选奖励 %d, 含经验 %d, 含满级金钱 %d"
% (stats["total"], stats["with_reward"], stats["with_choice"],
stats["with_xp"], stats["with_mml"]))
# 同步到插件目录 (pfExtend 通过 init/modules.xml 加载)
deploy = os.path.join(WORKDIR, "..", "modules", "QuestHelper", "rewards_data.lua")
try:
import shutil
shutil.copyfile(LUA_FILE, deploy)
print("[build] 已同步到插件: %s" % os.path.abspath(deploy))
except OSError as e:
print("[build] 同步到插件目录失败: %s" % e)
# ---------------------------------------------------------------- 入口
def main():
ap = argparse.ArgumentParser(description="OctoWow 任务奖励抓取")
ap.add_argument("cmd", choices=["catalog", "rewards", "build", "all"])
ap.add_argument("--limit", type=int, default=None, help="阶段B本轮最多抓取数(调试用)")
ap.add_argument("--ids", type=str, default=None, help="只抓指定任务ID, 逗号分隔")
ap.add_argument("--force", action="store_true", help="忽略缓存强制重抓(阶段A)")
args = ap.parse_args()
sess = OctoSession()
if args.cmd in ("catalog", "all"):
cmd_catalog(sess, force=args.force)
if args.cmd == "rewards":
ids = [int(x) for x in args.ids.split(",")] if args.ids else None
cmd_rewards(sess, limit=args.limit, ids=ids)
if args.cmd in ("build", "all"):
cmd_build()
if __name__ == "__main__":
main()