如何使用Python抓取动态网站：综合指南

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from webdriver_manager.chrome import ChromeDriverManager

def scrape_with_selenium(url):
    service = Service(ChromeDriverManager().install())
    driver = webdriver.Chrome(service=service)
    driver.get(url)
    try:
        # 等待元素出现
        WebDriverWait(driver, 10).until(
            EC.presence_of_element_located((By.ID, "some_dynamic_element"))
        )
        content = driver.find_element(By.ID, "some_dynamic_element").text
        print(f"内容: {content}")
    finally:
        driver.quit()

# 示例用法：
# scrape_with_selenium("https://example.com/dynamic-page")

from playwright.sync_api import sync_playwright

def scrape_with_playwright(url):
    with sync_playwright() as p:
        browser = p.chromium.launch()
        page = browser.new_page()
        page.goto(url)
        # 等待内容加载，例如，通过等待特定选择器
        page.wait_for_selector("#dynamic_content_id")
        content = page.inner_text("#dynamic_content_id")
        print(f"内容: {content}")
        browser.close()

# 示例用法：
# scrape_with_playwright("https://example.com/another-dynamic-page")

from requests_html import HTMLSession

def scrape_with_requests_html(url):
    session = HTMLSession()
    r = session.get(url)
    # 在页面上渲染 JavaScript
    r.html.render(sleep=1, scrolldown=True)
    
    # 找到 JavaScript 渲染后的元素
    title = r.html.find("title", first=True).text
    print(f"标题：{title}")
    
    session.close()

# 示例用法：
# scrape_with_requests_html("https://example.com/js-rendered-page")

import requests
import json

def scrape_with_api_interception(api_url, headers=None, params=None):
    response = requests.get(api_url, headers=headers, params=params)
    response.raise_for_status() # 对 HTTP 错误引发异常
    data = response.json() # 假设为 JSON 响应
    print(json.dumps(data, indent=2))

# 示例用法（替换为实际 API URL 和参数）：
# api_endpoint = "https://api.example.com/products?page=1"
# custom_headers = {"User-Agent": "Mozilla/5.0"}
# scrape_with_api_interception(api_endpoint, headers=custom_headers)

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
from bs4 import BeautifulSoup
import time

def scrape_with_bs_and_selenium(url):
    service = Service(ChromeDriverManager().install())
    options = webdriver.ChromeOptions()
    options.add_argument("--headless") # 在无头模式下运行
    driver = webdriver.Chrome(service=service, options=options)
    driver.get(url)
    time.sleep(5) # 给时间让 JavaScript 执行
    
    html_content = driver.page_source
    driver.quit()
    
    soup = BeautifulSoup(html_content, "html.parser")
    # 示例：查找所有链接
    links = [a.get("href") for a in soup.find_all("a", href=True)]
    print(f"找到的链接：{links[:5]}...") # 打印前 5 个链接

# 示例用法：
# 使用bs和selenium进行抓取("https://example.com/dynamic-content")

import asyncio
from pyppeteer import launch

async def scrape_with_pyppeteer(url):
    browser = await launch(headless=True)
    page = await browser.newPage()
    await page.goto(url)
    await page.waitForSelector("#content_area") # 等待特定元素
    content = await page.evaluate("document.querySelector(\"#content_area\").innerText")
    print(f"内容: {content}")
    await browser.close()

# 示例用法：
# asyncio.get_event_loop().run_until_complete(scrape_with_pyppeteer("https://example.com/async-dynamic-page"))

import requests
import time
from random import uniform

def get_random_user_agent():
    user_agents = [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/109.0.0.0 Safari/537.36",
        # 添加更多用户代理
    ]
    return random.choice(user_agents)

def make_request_with_anti_bot_measures(url, proxies=None):
    headers = {"User-Agent": get_random_user_agent()}
    try:
        response = requests.get(url, headers=headers, proxies=proxies)
        response.raise_for_status()
        time.sleep(uniform(2, 5)) # 随机延迟
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"请求失败: {e}")
        return None

# 示例用法（需要代理设置）：
# proxies = {"http": "http://user:pass@proxy.example.com:8080"}
# content = make_request_with_anti_bot_measures("https://example.com/protected-page", proxies=proxies)

import requests
from bs4 import BeautifulSoup

def check_dynamic_content(url, expected_element_id):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")
    
    if soup.find(id=expected_element_id):
        print(f"在初始 HTML 中找到 ID 为 '{expected_element_id}' 的元素。页面可能是静态的或内容同步加载。")
        return True
    else:
        print(f"在初始 HTML 中未找到 ID 为 '{expected_element_id}' 的元素。页面可能是动态的，需要 JavaScript 渲染。")
        return False

# 示例用法：
# is_dynamic = check_dynamic_content("https://example.com/some-page", "main-content")
# if not is_dynamic:
#     # 继续使用无头浏览器或 API 拦截
#     pass

import requests

def scrape_with_api(api_endpoint, target_url, api_key):
    payload = {
        "url": target_url,
        "api_key": api_key,
        "render_js": True, # 指示 API 渲染 JavaScript
        # 添加其他参数，如代理设置、国家等。
    }
    response = requests.post(api_endpoint, json=payload)
    response.raise_for_status()
    return response.json() # 或 response.text 如果返回 HTML

# 示例用法（替换为实际的 API 端点和密钥）：
# scraping_api_url = "https://api.scraping-service.com/scrape"
# my_api_key = "YOUR_API_KEY"
# data = scrape_with_api(scraping_api_url, "https://example.com/dynamic-site", my_api_key)
# print(data)

import requests

def scrape_with_splash(url, splash_url="http://localhost:8050/render.html"):
    params = {
        "url": url,
        "wait": 0.5, # 等待 0.5 秒以便 JavaScript 执行
        "timeout": 90,
        "render_all": 1 # 渲染所有内容，包括屏幕外的内容
    }
    response = requests.get(splash_url, params=params)
    response.raise_for_status()
    return response.text

# 示例用法（假设 Splash 正在 localhost:8050 运行）：
# html_content = scrape_with_splash("https://example.com/dynamic-site-with-splash")
# if html_content:
#     print("成功使用 Splash 进行抓取！")