XPath与CSS选择器：在网页抓取中使用哪个

from parsel import Selector
import requests

url = "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
response = requests.get(url, timeout=30)
response.raise_for_status()
sel = Selector(text=response.content.decode("utf-8"))

pairs = [
    ("article.product_pod h3 a::attr(href)", "//article[@class='product_pod']/h3/a/@href"),
    ("p.star-rating::attr(class)",           "//p[contains(@class,'star-rating')]/@class"),
    ("div.image_container img::attr(alt)",   "//div[@class='image_container']//img/@alt"),
]
for css_query, xpath_query in pairs:
    css_hits = sel.css(css_query).getall()
    xpath_hits = sel.xpath(xpath_query).getall()
    print(len(css_hits), len(xpath_hits), css_hits == xpath_hits)

axes = [
    ("parent::",            "//p[@class='price_color']/parent::div/@class"),
    ("ancestor::",          "//h3/ancestor::article/@class"),
    ("preceding-sibling::", "//div[@class='product_price']/preceding-sibling::h3/a/@title"),
    ("following-sibling::", "//h3/following-sibling::div[@class='product_price']//p[@class='price_color']/text()"),
]
for label, query in axes:
    hits = sel.xpath(query).getall()
    print(f"{label:20s} {len(hits):2d} hits   {hits[0]!r}")

parent::             20 hits   'product_price'
ancestor::           20 hits   'product_pod'
preceding-sibling::  20 hits   'Sharp Objects'
following-sibling::  20 hits   '£47.82'

print(len(sel.css("a:contains('Sharp')").getall()))       # 1
print(len(sel.xpath("//a[contains(., 'Sharp')]").getall()))  # 1

import os
from urllib.parse import urlencode
from playwright.sync_api import sync_playwright

JS = """() => {
  const out = {};
  try { out.css_class = document.querySelectorAll('article.product_pod h3 a').length; }
  catch (e) { out.css_class = 'ERR ' + e.name; }
  try { out.css_contains = document.querySelectorAll("a:contains('Sharp')").length; }
  catch (e) { out.css_contains = 'ERR ' + e.name + ': ' + e.message.slice(0, 60); }
  const xp = (q) => document.evaluate(q, document, null,
    XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null).snapshotLength;
  out.xpath_class    = xp("//article[@class='product_pod']/h3/a");
  out.xpath_contains = xp("//a[contains(., 'Sharp')]");
  out.xpath_parent   = xp("//p[@class='price_color']/parent::div");
  out.xpath_ancestor = xp("//h3/ancestor::article");
  return out;
}"""

url = "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
endpoint = "wss://browser.scrapeless.com/api/v2/browser?" + urlencode({
    "token": os.environ["SCRAPELESS_API_KEY"], "sessionTTL": 300, "proxyCountry": "US",
})
with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(endpoint)
    page = browser.new_page()
    page.goto(url, wait_until="domcontentloaded")
    for key, value in page.evaluate(JS).items():
        print(f"{key:16s} {value}")
    browser.close()

css_class        20
css_contains     ERR SyntaxError: Failed to execute 'querySelectorAll' on 'Document': 'a:conta
xpath_class      20
xpath_contains   1
xpath_parent     20
xpath_ancestor   20

import time

N = 2000
start = time.perf_counter()
for _ in range(N):
    sel.css("article.product_pod h3 a::attr(title)").getall()
css_ms = (time.perf_counter() - start) * 1000

start = time.perf_counter()
for _ in range(N):
    sel.xpath("//article[@class='product_pod']/h3/a/@title").getall()
xpath_ms = (time.perf_counter() - start) * 1000

print(f"CSS   {css_ms:8.1f} ms  ({css_ms / N * 1000:6.1f} us/call)")
print(f"XPath {xpath_ms:8.1f} ms  ({xpath_ms / N * 1000:6.1f} us/call)")

CSS      487.1 ms  ( 243.5 us/call)
XPath    309.1 ms  ( 154.5 us/call)

from cssselect import GenericTranslator
print(GenericTranslator().css_to_xpath("article.product_pod h3 a"))

descendant-or-self::article[@class and contains(concat(' ', normalize-space(@class), ' '), ' product_pod ')]/descendant-or-self::*/h3/descendant-or-self::*/a

import requests
from parsel import Selector

url = "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
response = requests.get(url, timeout=30)
response.raise_for_status()

print(response.headers.get("content-type"))
print(response.encoding)
print(response.apparent_encoding)
print(Selector(text=response.text).css("p.price_color::text").get())
print(Selector(text=response.content.decode("utf-8")).css("p.price_color::text").get())

text/html
ISO-8859-1
utf-8
'£47.82'
'£47.82'

import os
from urllib.parse import urlencode
from playwright.sync_api import sync_playwright

url = "https://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
cdp_endpoint = "wss://browser.scrapeless.com/api/v2/browser?" + urlencode({
    "token": os.environ["SCRAPELESS_API_KEY"],
    "sessionTTL": 300,
    "proxyCountry": "US",
})

with sync_playwright() as p:
    browser = p.chromium.connect_over_cdp(cdp_endpoint)
    page = browser.new_page()
    page.goto(url, wait_until="domcontentloaded")
    titles = page.eval_on_selector_all(
        "article.product_pod h3 a", "els => els.map(e => e.title)"
    )
    print(len(titles), titles[0])
    browser.close()

20 Sharp Objects