如何使用不被检测的ChromeDriver进行网页抓取

mkdir undetected-chromedriver-scraper
cd undetected-chromedriver-scraper
python -m venv env

# 在 Linux 或 macOS 上
source env/bin/activate

# 在 Windows 上
env\Scripts\activate

pip install undetected_chromedriver

import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
import json

# 初始化 Chrome 实例
driver = uc.Chrome()

# 连接到目标页面
driver.get("https://scrapeless.com")

# 抓取逻辑……

# 关闭浏览器
driver.quit()

import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
import json
import time

# 创建一个 Chrome 网页驱动程序实例
driver = uc.Chrome()

# 连接到 Apple 网站
driver.get("https://www.apple.com/fr/")

# 给页面一些时间完全加载
time.sleep(3)

# 用于存储产品信息的字典
apple_products = {}

try:
    # 查找产品部分（使用提供的 HTML 中的类）
    product_sections = driver.find_elements(By.CSS_SELECTOR, ".homepage-section.collection-module .unit-wrapper")
    
    for i, section in enumerate(product_sections):
        try:
            # 提取产品名称（标题）
            headline = section.find_element(By.CSS_SELECTOR, ".headline, .logo-image").get_attribute("textContent").strip()
            
            # 提取说明（副标题）
            subhead_element = section.find_element(By.CSS_SELECTOR, ".subhead")
            subhead = subhead_element.text
            
            # 获取链接（如果可用）
            link = ""
            try:
                link_element = section.find_element(By.CSS_SELECTOR, ".unit-link")
                link = link_element.get_attribute("href")
            except:
                pass
            
            apple_products[f"product_{i+1}"] = {
                "name": headline,
                "description": subhead,
                "link": link
            }
        except Exception as e:
            print(f"Error processing section {i+1}: {e}")
    
    # 将抓取的数据导出到 JSON
    with open("apple_products.json", "w", encoding="utf-8") as json_file:
        json.dump(apple_products, json_file, indent=4, ensure_ascii=False)
    
    print(f"Successfully scraped {len(apple_products)} Apple products")
    
except Exception as e:
    print(f"Error during scraping: {e}")

finally:
    # 关闭浏览器并释放其资源
    driver.quit()

python scraper.py

import undetected_chromedriver as uc

# 指定目标版本的 Chrome
driver = uc.Chrome(version_main=105)

import undetected_chromedriver as uc

with uc.Chrome() as driver:
    driver.get("https://example.com")
    # 代码的其余部分……

driver = uc.Chrome(headless=True)

const {chromium} = require('playwright-core');

// 使用您的令牌的 Scrapeless 连接 URL
const connectionURL = 'wss://browser.scrapeless.com/browser?token=YOUR_TOKEN_HERE&session_ttl=180&proxy_country=ANY';

(async () => {
  // 连接到远程 Scrapeless 浏览器
  const browser = await chromium.connectOverCDP(connectionURL);
  
  try {
    // 创建一个新页面
    const page = await browser.newPage();
    
    // 导航到 Apple 的网站
    console.log('Navigating to Apple website...');
    await page.goto('https://www.apple.com/fr/', {
      waitUntil: 'domcontentloaded',
      timeout: 60000
    });
    
    console.log('Page loaded successfully');
    
    // 等待产品部分可用
    await page.waitForSelector('.homepage-section.collection-module', { timeout: 10000 });
    
    // 获取首页的特色产品
    const products = await page.evaluate(() => {
      const results = [];
      
      // 获取所有产品部分
      const productSections = document.querySelectorAll('.homepage-section.collection-module .unit-wrapper');
      
      productSections.forEach((section, index) => {
        try {
          // 获取产品名称 - 可能在 .headline 或 .logo-image 中
          const headlineEl = section.querySelector('.headline') || section.querySelector('.logo-image');
          const headline = headlineEl ? headlineEl.textContent.trim() : 'Unknown Product';
          
          // 获取产品说明
          const subheadEl = section.querySelector('.subhead');
          const subhead = subheadEl ? subheadEl.textContent.trim() : '';
          
          // 获取产品链接
          const linkEl = section.querySelector('.unit-link');
          const link = linkEl ? linkEl.getAttribute('href') : '';
          
          results.push({
            name: headline,
            description: subhead,
            link: link
          });
        } catch (err) {
          console.error(`Error processing section ${index}: ${err.message}`);
        }
      });
      
      return results;
    });
    
    // 显示结果
    console.log('Found Apple products:');
    console.log(JSON.stringify(products, null, 2));
    console.log(`Total products found: ${products.length}`);
    
  } catch (error) {
    console.error('An error occurred:', error);
  } finally {
    // 关闭浏览器
    await browser.close();
    console.log('Browser closed');
  }
})();

Undetected ChromeDriver 源代码的简化摘录

def _patch_driver_executable():
    """
    修补 ChromeDriver 二进制文件以删除自动化的蛛丝马迹
    """
    linect = 0
    replacement = os.urandom(32).hex()
    with io.open(self.executable_path, "r+b") as fh:
        for line in iter(lambda: fh.readline(), b""):
            if b"cdc_" in line.lower():
                fh.seek(-len(line), 1)
                newline = re.sub(
                    b"cdc_.{22}", b"cdc_" + replacement.encode(), line
                )
                fh.write(newline)
                linect += 1
    return linect