PlaywrightとScrapelessを使用してIndexedDBデータを抽出する方法

python -m pip install "playwright==1.59.0"

import os
from urllib.parse import urlencode
from playwright.sync_api import sync_playwright

params = urlencode({
    "token": os.environ["SCRAPELESS_API_KEY"],
    "sessionTTL": 120,
    "proxyCountry": "US",
})
cdp_url = f"wss://browser.scrapeless.com/api/v2/browser?{params}"

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(cdp_url)
    page = browser.contexts[0].pages[0]
    # 以下のIndexedDBクエリをナビゲートして評価します。
    browser.close()

TARGET_URL = (
    "https://mdn.github.io/dom-examples/"
    "indexeddb-examples/idbindex/"
)

page.goto(TARGET_URL, wait_until="domcontentloaded")
page.wait_for_selector("tbody tr")
databases = page.evaluate("indexedDB.databases()")
print("databases:", databases)

databases: [{'name': 'contactsList', 'version': 1}]

schema = page.evaluate("""async () => {
    const opened = await new Promise((resolve, reject) => {
        const request = indexedDB.open('contactsList');
        request.onsuccess = () => resolve(request.result);
        request.onerror = () => reject(request.error);
    });
    const storeName = opened.objectStoreNames[0];
    const transaction = opened.transaction(storeName, 'readonly');
    const store = transaction.objectStore(storeName);
    const result = {
        storeName,
        indexes: Array.from(store.indexNames),
    };
    opened.close();
    return result;
}""")

print("オブジェクトストア:", schema["storeName"])
print("インデックス:", schema["indexes"])

オブジェクトストア: contactsList
インデックス: ['age', 'company', 'eMail', 'fName', 'jTitle', 'lName', 'phone']

records = page.evaluate("""async () => {
    const database = await new Promise((resolve, reject) => {
        const request = indexedDB.open('contactsList');
        request.onsuccess = () => resolve(request.result);
        request.onerror = () => reject(request.error);
    });
    const transaction = database.transaction('contactsList', 'readonly');
    const store = transaction.objectStore('contactsList');
    const rows = await new Promise((resolve, reject) => {
        const request = store.getAll();
        request.onsuccess = () => resolve(request.result);
        request.onerror = () => reject(request.error);
    });
    database.close();
    return rows;
}""")

print("レコード数:", len(records))
print("最初:", records[0])
print("最後:", records[-1])

table_rows = page.locator("tbody tr").count()
arkham_rows = [row for row in records if row["company"] == "Arkham"]
ages = [row["age"] for row in records]

print("テーブル行数:", table_rows)
print("データベース行数:", len(records))
print("Arkham 行数:", len(arkham_rows))
print("年齢範囲:", min(ages), max(ages))

テーブル行数: 10
データベース行数: 10
Arkham 行数: 2
年齢範囲: 24 55

from playwright.sync_api import sync_playwright

TARGET_URL = (
    "https://mdn.github.io/dom-examples/"
    "indexeddb-examples/idbindex/"
)

QUERY = """async () => {
    const databases = await indexedDB.databases();
    const database = await new Promise((resolve, reject) => {
        const request = indexedDB.open('contactsList');
        request.onsuccess = () => resolve(request.result);
        request.onerror = () => reject(request.error);
    });
    const storeName = database.objectStoreNames[0];
    const transaction = database.transaction(storeName, 'readonly');
    const store = transaction.objectStore(storeName);
    const indexes = Array.from(store.indexNames);
    const rows = await new Promise((resolve, reject) => {
        const request = store.getAll();
        request.onsuccess = () => resolve(request.result);
        request.onerror = () => reject(request.error);
    });
    database.close();
    return {databases, storeName, indexes, rows};
}"""

with sync_playwright() as p:
    browser = p.chromium.launch(
        executable_path="/usr/bin/google-chrome",
        headless=True,
    )
    page = browser.new_page()
    page.goto(TARGET_URL, wait_until="domcontentloaded")
    page.wait_for_selector("tbody tr")

    result = page.evaluate(QUERY)
    rows = result["rows"]
    table_rows = page.locator("tbody tr").count()

arkham_rows = [row for row in rows if row["company"] == "Arkham"]
ages = [row["age"] for row in rows]

assert result["databases"] == [{"name": "contactsList", "version": 1}]
assert result["storeName"] == "contactsList"
assert len(result["indexes"]) == 7
assert len(rows) == table_rows == 10
assert len(arkham_rows) == 2

print("タイトル:", page.title())
print("データベース:", result["databases"])
print("オブジェクトストア:", result["storeName"])
print("インデックス:", result["indexes"])
print("データベースの行:", len(rows))
print("テーブルの行:", table_rows)
print("最初の連絡先:", rows[0]["fName"], rows[0]["lName"])
print("最後の連絡先:", rows[-1]["fName"], rows[-1]["lName"])
print("アーカムの行:", len(arkham_rows))
print("年齢範囲:", min(ages), max(ages))
browser.close()

{
  "database": {
    "name": "contactsList",
    "version": 1
  },
  "object_store": "contactsList",
  "index_count": 7,
  "database_rows": 10,
  "rendered_rows": 10,
  "matching_company_rows": 2,
  "age_range": {
    "minimum": 24,
    "maximum": 55
  }
}

**Q: IndexedDBはブラウザセッションを超えて持続しますか？**

ブラウザのプロファイルが保持されている限り持続しますが、オリジンまたはブラウザがそれをクリアすると消えます。一時的なコンテキストは、コンテキストが閉じると消える可能性があります。

**Q: IndexedDBデータの抽出は常に許可されていますか？**

いいえ。認可されたセッションと必要な公開データのみを検査し、保持されるフィールドを最小限に抑え、適用可能な場合はサイトの利用規約と<a href="https://datatracker.ietf.org/doc/html/rfc9309" rel="nofollow"><strong>ロボットポリシー</strong></a>に従い、機密用途については法的助言を受けてください。