PlaywrightとScrapelessを使ってlocalStorageをスクレイピングする方法

python -m pip install "playwright==1.59.0"

import os
from urllib.parse import urlencode
from playwright.sync_api import sync_playwright

params = urlencode({
    "token": os.environ["SCRAPELESS_API_KEY"],
    "sessionTTL": 120,
    "proxyCountry": "US",
})
cdp_url = f"wss://browser.scrapeless.com/api/v2/browser?{params}"

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(cdp_url)
    context = browser.contexts[0]
    page = context.pages[0]
    # ここでコンテキストとページを使ってストレージステップを実行します。
    browser.close()

TARGET_URL = "https://mdn.github.io/dom-examples/web-storage/"

page.goto(TARGET_URL, wait_until="domcontentloaded")
stored = page.evaluate(
    "Object.fromEntries(Object.entries(localStorage))"
)
print("初期ストレージ:", stored)
print("セッションストレージ:", page.evaluate("sessionStorage.length"))

初期ストレージ: {'font': 'sans-serif', 'image': 'images/firefoxos.png', 'bgcolor': 'FF0000'}
セッションストレージ: 0

observer_page = context.new_page()
observer_page.goto(TARGET_URL, wait_until="domcontentloaded")
observer_page.evaluate("""
    window.capturedStorageEvents = [];
    addEventListener('storage', event => {
        capturedStorageEvents.push({
            key: event.key,
            oldValue: event.oldValue,
            newValue: event.newValue,
        });
    });
""")

page.evaluate("""
    bgcolor.value = '00FF00';
    bgcolor.dispatchEvent(new Event('change', {bubbles: true}));
""")
page.select_option("#font", "serif")
page.select_option("#image", "images/crocodile.png")
page.wait_for_timeout(500)

print("更新されたストレージ:", page.evaluate(
    "Object.fromEntries(Object.entries(localStorage))"
))
print("イベント:", observer_page.evaluate("capturedStorageEvents"))

更新されたストレージ: {'font': 'serif', 'image': 'images/crocodile.png', 'bgcolor': '00FF00'}
イベント: [{'key': 'bgcolor', 'oldValue': 'FF0000', 'newValue': '00FF00'}, {'key': 'font', 'oldValue': 'sans-serif', 'newValue': 'serif'}, {'key': 'image', 'oldValue': 'images/firefoxos.png', 'newValue': 'images/crocodile.png'}]

page.reload(wait_until="domcontentloaded")
reloaded = page.evaluate(
    "Object.fromEntries(Object.entries(localStorage))"
)
controls = {
    "bgcolor": page.locator("#bgcolor").input_value(),
    "font": page.locator("#font").input_value(),
    "image": page.locator("#image").input_value(),
}

print("リロードされたストレージ:", reloaded)
print("コントロールの値:", controls)

リロードされたストレージ: {'font': 'serif', 'image': 'images/crocodile.png', 'bgcolor': '00FF00'}
コントロールの値: {'bgcolor': '00FF00', 'font': 'serif', 'image': 'images/crocodile.png'}

from playwright.sync_api import sync_playwright

TARGET_URL = "https://mdn.github.io/dom-examples/web-storage/"

with sync_playwright() as p:
    browser = p.chromium.launch(
        executable_path="/usr/bin/google-chrome",
        headless=True,
    )
    context = browser.new_context()
    page = context.new_page()
    observer_page = context.new_page()

    observer_page.goto(TARGET_URL, wait_until="domcontentloaded")
    observer_page.evaluate("""
        window.capturedStorageEvents = [];
        addEventListener('storage', event => {
            capturedStorageEvents.push({
                key: event.key,
                oldValue: event.oldValue,
                newValue: event.newValue,
            });
        });
    """)
    page.goto(TARGET_URL, wait_until="domcontentloaded")

    initial = page.evaluate(
        "Object.fromEntries(Object.entries(localStorage))"
    )
    page.evaluate("""
        bgcolor.value = '00FF00';
        bgcolor.dispatchEvent(new Event('change', {bubbles: true}));
    """)
    page.select_option("#font", "serif")
    page.select_option("#image", "images/crocodile.png")
    page.wait_for_timeout(500)

    updated = page.evaluate(
        "Object.fromEntries(Object.entries(localStorage))"
    )
    events = observer_page.evaluate("capturedStorageEvents")
    page.reload(wait_until="domcontentloaded")
    reloaded = page.evaluate(
        "Object.fromEntries(Object.entries(localStorage))"
    )
    controls = {
        "bgcolor": page.locator("#bgcolor").input_value(),
        "font": page.locator("#font").input_value(),
        "image": page.locator("#image").input_value(),
    }

    assert len(events) == 3
    assert updated == reloaded == controls
    assert page.evaluate("sessionStorage.length") == 0

    print("タイトル:", page.title())
    print("初期ストレージ:", initial)
    print("更新されたストレージ:", updated)
    print("イベントキー:", [event["key"] for event in events])
    print("イベント数:", len(events))
    print("リロードされたストレージ:", reloaded)
    print("コントロールの値:", controls)
    print("セッションストレージのエントリ:", page.evaluate("sessionStorage.length"))
    browser.close()