如何用Python抓取Google产品评论结果

import json
import requests

class Payload:
    def __init__(self, actor, input_data):
        self.actor = actor
        self.input = input_data

def send_request():
    host = "api.scrapeless.com"
    url = f"https://{host}/api/v1/scraper/request"
    token = "your_token"

    headers = {
        "x-api-token": token
    }

    input_data = {
        "engine": "google_product",
        "product_id": "4172129135583325756",
        "gl": "us",
        "hl": "en",
    }

    payload = Payload("scraper.google.product", input_data)

    json_payload = json.dumps(payload.__dict__)

    response = requests.post(url, headers=headers, data=json_payload)

    if response.status_code != 200:
        print("Error:", response.status_code, response.text)
        return

    print("body", response.text)


if __name__ == "__main__":
    send_request()

body {"product_results":{"product_id":"4172129135583325756","title":"Apple iPhone 12 Pro - 128 GB - Silver - AT\u0026T","price":["$314.84","$298.00","$325.33"],"reviews":14303,"rating":4.4,"extensions":["Smartphone","Dual SIM","5G","With Wireless Charging","With Fast Charging","AT\u0026T","Dual Lens","iOS","GSM","CDMA"],"description":"5G goes Pro. A14 Bionic rockets past every other smartphone chip. The iPhone 12 Pro features a 6.1” Super Retina XDR display, LiDAR scanner for ultrafast and accurate depth maps of whatever space you're in and MagSafe wireless charging. The Pro camera system takes low-light ...More5G goes Pro. A14 Bionic rockets past every other smartphone chip. The iPhone 12 Pro features a 6.1” Super Retina XDR display, LiDAR scanner for ultrafast and accurate depth maps of whatever space you're in and MagSafe wireless charging. The Pro camera system takes low-light photography to the next level with Night mode available in both the Wide and Ultra Wide cameras, so it’s better than ever at capturing incredible low-light shots. And Ceramic Shield delivers four times better drop performance.Less","media":[{"type":"image","link":"https://encrypted-tbn0.gstatic.com/shopping?q=tbn:ANd9GcStw-jdTZtGmdXcVKCqweq6wxzU5tpRTTbl6stPV97GpGVR6XY\u0026usqp=CAY"},{"type":"image","link":"https://encrypted-tbn1.gstatic.com/shopping?q=tbn:ANd9GcR0wJ1fsUOPAGDMtjdtx1zsd5ZWUXwnNe70fmZszERkEihkYCKnZoGJ3Y4lqSQTyR4soiTVWFVzllzYTHJBTRXegTR7Pj83RA\u0026usqp=CAY"},{"type":"image","link":"https://encrypted-tbn1.gstatic.com/shopping?q=tbn:ANd9GcR4L66Gss9O5HSL00NLxaHu0pl5huMUojbC9tO9FKCRpCQObUqdHWsSPYZJ4lU8eETn-MlJx4Hni_oc_l5mxIs_l-Z2htBiaA\u0026usqp=CAY"},{"type":"image","link":"https://encrypted-tbn0.gstatic.com/shopping?q=tbn:ANd9GcQLv5xOi-9b-Mka7jfFnQzlXkTrEAsjPAzumbUB2D6Ddgl3FHGZOQXAUGQAv6WkUeZsbsdvKA2NRF1-h8EOBSQPLmuPMLPQ2Q\u0026usqp=CAY"}],"sizes":{"128 GB":{"link":"https://www.google.com/shopping/product/4172129135583325756?gl=us\u0026hl=en\u0026sourceid=chrome\u0026ie=UTF-8","product_id":"4172129135583325756"},"256 GB":{"link":"https://www.google.com/shopping/product/1700752269234454309?gl=us\u0026hl=en\u0026sourceid=chrome\u0026ie=UTF-8\u0026prds=opd:11579479524734831751,rsk:PC_14243855303706753583\u0026sa=X\u0026ved=0ahUKEwjxhNXT2ZeMAxVcK7kGHUh1ErMQlIUHCEQoAQ","product_id":"1700752269234454309"},"512 GB":{"link":"https://www.google.com/shopping/product/14752474427020499512?gl=us\u0026hl=en\u0026sourceid=chrome\u0026ie=UTF-8\u0026prds=opd:11579479524734831751,rsk:PC_14243855303706753583\u0026sa=X\u0026ved=0ahUKEwjxhNXT2ZeMAxVcK7kGHUh1ErMQlIUHCEUoAg","product_id":"14752474427020499512"}},"highlight":["5G transforms iPhone with accelerated wireless speeds and better performance on congested networks","A14 Bionic: generations ahead of any other smartphone chip","Night mode comes to both the Wide and Ultra Wide cameras, and it's better than ever at capturing incredible low-light shots",

pip install selenium beautifulsoup4 pandas

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time
service = Service('path/to/chromedriver')  # 更新为正确的路径
options = webdriver.ChromeOptions()
options.add_argument('--headless')  # 在无头模式下运行
driver = webdriver.Chrome(service=service, options=options)
driver.get('https://www.google.com/shopping/product/1234567890/reviews')  # 示例 URL

from bs4 import BeautifulSoup
def extract_reviews(driver):
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    reviews = []

    for review in soup.find_all('div', class_='sh-dgr__content'):
        rating = review.find('div', class_='sh-dgr__rating')
        text = review.find('div', class_='sh-dgr__review-text')

        if rating and text:
            reviews.append({
                'rating': rating.text.strip(),
                'review': text.text.strip()
            })
    return reviews
data = extract_reviews(driver)
print(data)

def scrape_multiple_pages(driver):
    all_reviews = []
    while True:
        all_reviews.extend(extract_reviews(driver))

        try:
            next_button = driver.find_element(By.XPATH, '//a[@aria-label="Next page"]')
            next_button.click()
            time.sleep(2)  # 等待新页面加载
        except:
            break
    return all_reviews
data = scrape_multiple_pages(driver)
print(f'Total Reviews Scraped: {len(data)}')

import pandas as pd

df = pd.DataFrame(data)
df.to_csv('google_reviews.csv', index=False)
print('Reviews saved to google_reviews.csv')