Mistral 网络抓取：读取隐藏在 HTML 属性中的数据

pip install "mistralai==2.7.2" "openai==2.48.0" requests

export MISTRAL_API_KEY="your_mistral_key"
export SCRAPELESS_API_KEY="sk_your_scrapeless_key"

# fetch_laptops.py — 一次POST返回渲染的笔记本电脑目录页面
import os

import requests

resp = requests.post(
    "https://api.scrapeless.com/api/v2/unlocker/request",
    headers={
        "Content-Type": "application/json",
        "x-api-token": os.environ["SCRAPELESS_API_KEY"],
    },
    json={
        "actor": "unlocker.webunlocker",
        "input": {
            "url": "https://webscraper.io/test-sites/e-commerce/static/computers/laptops",
            "method": "GET",
            "js_render": True,
        },
    },
    timeout=120,
)
resp.raise_for_status()
html = resp.json().get("data", "")
print(f"抓取了 {len(html):,} 个字符")
print("产品卡片:", html.count('class="card thumbnail"'))

with open("page.html", "w", encoding="utf-8") as f:
    f.write(html)

# extract_mistral.py — 原生 Mistral 提取（需要 MISTRAL_API_KEY）
import json
import os

from mistralai.client import Mistral

client = Mistral(api_key=os.environ["MISTRAL_API_KEY"])

page_html = open("page.html", encoding="utf-8").read()

response = client.chat.complete(
    model="ministral-3b-2512",
    temperature=0,
    max_tokens=2000,
    response_format={"type": "json_object"},
    messages=[
        {
            "role": "system",
            "content": '提取每一款笔记本电脑。仅用 JSON 回复： '
            '{"laptops":[{"title":str,"price_usd":number,"rating":int,"review_count":int}]}. '
            "title 是链接的 title 属性中的完整产品名称，而不是截断的可见文本。 "
            "rating 为 1-5，从 data-rating 属性中读取。",
        },
        {"role": "user", "content": page_html},
    ],
)

data = json.loads(response.choices[0].message.content)
print(f"提取了 {len(data['laptops'])} 款笔记本电脑")

# extract_openrouter.py — 通过 OpenRouter 执行相同的提取
import json
import os

import requests
from openai import OpenAI

resp = requests.post(
    "https://api.scrapeless.com/api/v2/unlocker/request",
    headers={
        "Content-Type": "application/json",
        "x-api-token": os.environ["SCRAPELESS_API_KEY"],
    },
    json={
        "actor": "unlocker.webunlocker",
        "input": {
            "url": "https://webscraper.io/test-sites/e-commerce/static/computers/laptops",
            "method": "GET",
            "js_render": True,
        },
    },
    timeout=120,
)
resp.raise_for_status()
page_html = resp.json().get("data", "")

client = OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"])

completion = client.chat.completions.create(
    model="mistralai/ministral-3b-2512",
    temperature=0,
    max_tokens=2000,
    response_format={"type": "json_object"},
    messages=[
        {
            "role": "system",
            "content": '提取每一款笔记本电脑。仅用 JSON 回复： '
            '{"laptops":[{"title":str,"price_usd":number,"rating":int,"review_count":int}]}. '
            "title 是链接的 title 属性中的完整产品名称，而不是截断的可见文本。 "
            "rating 为 1-5，从 data-rating 属性中读取。",
        },
        {"role": "user", "content": page_html},
    ],
)

data = json.loads(completion.choices[0].message.content)
print(f"提取了 {len(data['laptops'])} 款笔记本电脑")
for row in data["laptops"]:
    print(json.dumps(row, ensure_ascii=False))

提取了 6 款笔记本电脑
{"title": "Packard 255 G2", "price_usd": 416.99, "rating": 2, "review_count": 2}
{"title": "Aspire E1-510", "price_usd": 306.99, "rating": 3, "review_count": 2}
{"title": "ThinkPad T540p", "price_usd": 1178.99, "rating": 1, "review_count": 2}
{"title": "ProBook", "price_usd": 739.99, "rating": 4, "review_count": 8}
{"title": "ThinkPad X240", "price_usd": 1311.99, "rating": 3, "review_count": 12}
{"title": "Aspire E1-572G", "price_usd": 581.99, "rating": 1, "review_count": 2}