Playwright + Scrapeless Scraping Browser：捕获并重放隐藏的 GraphQL API

pip install playwright

export SCRAPELESS_API_KEY="your_scrapeless_api_key"

import os
from urllib.parse import urlencode

API_KEY = os.environ["SCRAPELESS_API_KEY"]

def scraping_browser_url(proxy_country="US", session_ttl=120):
    params = urlencode({
        "token": API_KEY,
        "sessionTTL": session_ttl,
        "proxyCountry": proxy_country,
    })
    return f"wss://browser.scrapeless.com/api/v2/browser?{params}"

import json
import os
from urllib.parse import urlencode

from playwright.sync_api import sync_playwright

API_KEY = os.environ["SCRAPELESS_API_KEY"]

QUERY = (
    "query GetCharacters($page: Int, $name: String) { "
    "characters(page: $page, filter: { name: $name }) { "
    "info { count pages } "
    "results { id name status species } } }"
)
VARIABLES = '{"page": 1, "name": "rick"}'

def scraping_browser_url(proxy_country="US", session_ttl=120):
    params = urlencode({
        "token": API_KEY,
        "sessionTTL": session_ttl,
        "proxyCountry": proxy_country,
    })
    return f"wss://browser.scrapeless.com/api/v2/browser?{params}"

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(scraping_browser_url())
    page = browser.new_page()
    page.goto("https://rickandmortyapi.com/graphql", wait_until="domcontentloaded")

    query_editor = page.locator(".graphiql-query-editor .CodeMirror").first
    query_editor.click()
    page.keyboard.press("Control+A")
    page.keyboard.insert_text(QUERY)

    page.locator("button:has-text('Variables')").first.click()
    variables_editor = page.locator(".graphiql-editor-tool .CodeMirror").first
    variables_editor.click()
    page.keyboard.press("Control+A")
    page.keyboard.insert_text(VARIABLES)

    with page.expect_response(lambda r: "graphcdn.app" in r.url and r.request.method == "POST") as run:
        page.locator("button.graphiql-execute-button").click()

    resp = run.value
    sent = json.loads(resp.request.post_data)
    data = resp.json()["data"]["characters"]

    print("POST target:", resp.request.url)
    print("operationName:", sent["operationName"])
    print("variables sent:", sent["variables"])
    print("info:", data["info"])
    print("first result:", data["results"][0])
    print("result count in this page:", len(data["results"]))

    browser.close()

POST target: https://rickandmortyapi.graphcdn.app/
operationName: GetCharacters
variables sent: {'page': 1, 'name': 'rick'}
info: {'count': 107, 'pages': 6}
first result: {'id': '1', 'name': 'Rick Sanchez', 'status': 'Alive', 'species': 'Human'}
result count in this page: 20

import json
import os
from urllib.parse import urlencode

from playwright.sync_api import sync_playwright

API_KEY = os.environ["SCRAPELESS_API_KEY"]

QUERY = (
    "query GetCharacters($page: Int, $name: String) { "
    "characters(page: $page, filter: { name: $name }) { "
    "info { count pages } "
    "results { id name status species } } }"
)
VARIABLES = '{"page": 2, "name": "rick"}'

captured = {}

def scraping_browser_url(proxy_country="US", session_ttl=120):
    params = urlencode({
        "token": API_KEY,
        "sessionTTL": session_ttl,
        "proxyCountry": proxy_country,
    })
    return f"wss://browser.scrapeless.com/api/v2/browser?{params}"

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(scraping_browser_url())
    page = browser.new_page()
    cdp = page.context.new_cdp_session(page)
    cdp.send("Network.enable")

    def on_request(event):
        # The playground also fires a schema-introspection POST to this same
        # URL on load. Matching on operationName in the body -- not the URL
        # -- is what separates it from the query this script triggers.
        request = event["request"]
        if "graphcdn.app" in request["url"] and "GetCharacters" in request.get("postData", ""):
            captured[event["requestId"]] = None

    def on_finished(event):
        request_id = event["requestId"]
        if request_id in captured and captured[request_id] is None:
            body = cdp.send("Network.getResponseBody", {"requestId": request_id})
            captured[request_id] = json.loads(body["body"])

    cdp.on("Network.requestWillBeSent", on_request)
    cdp.on("Network.loadingFinished", on_finished)

    page.goto("https://rickandmortyapi.com/graphql", wait_until="domcontentloaded")

    query_editor = page.locator(".graphiql-query-editor .CodeMirror").first
    query_editor.click()
    page.keyboard.press("Control+A")
    page.keyboard.insert_text(QUERY)

    page.locator("button:has-text('Variables')").first.click()
    variables_editor = page.locator(".graphiql-editor-tool .CodeMirror").first
    variables_editor.click()
    page.keyboard.press("Control+A")
    page.keyboard.insert_text(VARIABLES)

    page.locator("button.graphiql-execute-button").click()
    for _ in range(30):
        if captured and all(v is not None for v in captured.values()):
            break
        page.wait_for_timeout(300)

    data = next(iter(captured.values()))["data"]["characters"]
    print("requests matched by body content:", len(captured))
    print("info:", data["info"])
    print("first result:", data["results"][0])

    browser.close()

requests matched by body content: 1
info: {'count': 107, 'pages': 6}
first result: {'id': '218', 'name': 'Mechanical Rick', 'status': 'unknown', 'species': 'Robot'}

import json
import urllib.error
import urllib.request

ENDPOINT = "https://rickandmortyapi.graphcdn.app/"

QUERY = (
    "query GetCharacters($page: Int, $name: String) { "
    "characters(page: $page, filter: { name: $name }) { "
    "info { count pages } "
    "results { id name status species } } }"
)

payload = json.dumps({
    "query": QUERY,
    "variables": {"page": 1, "name": "rick"},
    "operationName": "GetCharacters",
}).encode("utf-8")

req = urllib.request.Request(
    ENDPOINT,
    data=payload,
    # A default urllib request declares "Python-urllib/x.y" as its User-Agent
    # and the gateway's edge rejects that outright -- see "When the Browser
    # Stays in the Loop" below for what's actually being checked.
    headers={
        "Content-Type": "application/json",
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
        ),
    },
    method="POST",
)

with urllib.request.urlopen(req, timeout=10) as resp:
    if resp.status != 200:
        raise urllib.error.HTTPError(ENDPOINT, resp.status, "unexpected status", resp.headers, None)
    body = json.loads(resp.read())

data = body["data"]["characters"]
print("status: 200, no browser process involved")
print("info:", data["info"])
print("first result:", data["results"][0])
print("result count in this page:", len(data["results"]))

status: 200, no browser process involved
info: {'count': 107, 'pages': 6}
first result: {'id': '1', 'name': 'Rick Sanchez', 'status': 'Alive', 'species': 'Human'}
result count in this page: 20