MutationObserverとPlaywrightを使って動的DOMの変化をキャプチャする

python -m pip install "playwright==1.59.0"

import os
from urllib.parse import urlencode
from playwright.sync_api import sync_playwright

params = urlencode({
    "token": os.environ["SCRAPELESS_API_KEY"],
    "sessionTTL": 120,
    "proxyCountry": "US",
})
cdp_url = f"wss://browser.scrapeless.com/api/v2/browser?{params}"

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(cdp_url)
    page = browser.contexts[0].pages[0]
    # 下のオブザーバーをインストールして使用します。
    browser.close()

TARGET_URL = "https://www.selenium.dev/selenium/web/dynamic.html"

page.goto(TARGET_URL, wait_until="domcontentloaded")
print("box before:", page.locator("#box0").count())
print("input visible before:", page.locator("#revealed").is_visible())

box before: 0
input visible before: False

page.evaluate("""
    window.capturedMutations = [];
    window.scrapeObserver = new MutationObserver(records => {
        for (const record of records) {
            for (const node of record.addedNodes) {
                if (node.nodeType === Node.ELEMENT_NODE) {
                    capturedMutations.push({
                        type: 'added',
                        id: node.id,
                        display: getComputedStyle(node).display,
                    });
                }
            }
            if (record.type === 'attributes') {
                capturedMutations.push({
                    type: 'attribute',
                    id: record.target.id,
                    name: record.attributeName,
                    display: getComputedStyle(record.target).display,
                });
            }
        }
    });
    scrapeObserver.observe(document.body, {
        childList: true,
        subtree: true,
        attributes: true,
        attributeFilter: ['style'],
    });
""")

page.click("#adder")
page.click("#reveal")
page.wait_for_selector("#box0")
page.locator("#revealed").wait_for(state="visible")

changes = page.evaluate("""() => {
    scrapeObserver.disconnect();
    return capturedMutations;
}""")

print("changes:", changes)
print("change count:", len(changes))
print("box matches:", page.locator("#box0").count())
print("input visible:", page.locator("#revealed").is_visible())

changes: [{'type': 'added', 'id': 'box0', 'display': 'block'}, {'type': 'attribute', 'id': 'revealed', 'name': 'style', 'display': 'inline-block'}]
change count: 2
box matches: 1
input visible: True

from playwright.sync_api import sync_playwright

TARGET_URL = "https://www.selenium.dev/selenium/web/dynamic.html"

with sync_playwright() as p:
    browser = p.chromium.launch(
        executable_path="/usr/bin/google-chrome",
        headless=True,
    )
    page = browser.new_page()
    page.goto(TARGET_URL, wait_until="domcontentloaded")

    box_before = page.locator("#box0").count()
    input_before = page.locator("#revealed").is_visible()

    page.evaluate("""
        window.capturedMutations = [];
        window.scrapeObserver = new MutationObserver(records => {
            for (const record of records) {
                for (const node of record.addedNodes) {
                    if (node.nodeType === Node.ELEMENT_NODE) {
                        capturedMutations.push({
                            type: 'added',
                            id: node.id,
                            display: getComputedStyle(node).display,
                        });
                    }
                }
                if (record.type === 'attributes') {
                    capturedMutations.push({
                        type: 'attribute',
                        id: record.target.id,
                        name: record.attributeName,
                        display: getComputedStyle(record.target).display,
                    });
                }
            }
        });
        scrapeObserver.observe(document.body, {
            childList: true,
            subtree: true,
            attributes: true,
            attributeFilter: ['style'],
        });
    """)

    page.click("#adder")
    page.click("#reveal")
    page.wait_for_selector("#box0")
    page.locator("#revealed").wait_for(state="visible")

    changes = page.evaluate("""() => {
        scrapeObserver.disconnect();
        return capturedMutations;
    }""")
    box_after = page.locator("#box0").count()
    input_after = page.locator("#revealed").is_visible()

    assert box_before == 0 and input_before is False
    assert changes == [
        {"type": "added", "id": "box0", "display": "block"},
{
    "type": "attribute",
    "id": "revealed",
    "name": "style",
    "display": "inline-block"
}
）

箱が 1 のとき、入力が True であることを確認します。

```python
print("箱の前:", box_before)
print("入力の表示の前:", input_before)
print("変更:", changes)
print("変更の数:", len(changes))
print("箱の後:", box_after)
print("入力の表示の後:", input_after)
browser.close()

{
  "before": {
    "box_matches": 0,
    "input_visible": false
  },
  "mutations": [
    {
      "type": "added",
      "id": "box0",
      "display": "block"
    },
    {
      "type": "attribute",
      "id": "revealed",
      "name": "style",
      "display": "inline-block"
    }
  ],
  "after": {
    "box_matches": 1,
    "input_visible": true
  }
}