Cách dùng Python để Scrape thông tin người bán sản phẩm trực tuyến trên Google

import json
import requests

class Payload:
    def __init__(self, actor, input_data):
        self.actor = actor
        self.input = input_data

def send_request():
    host = "api.scrapeless.com"
    url = f"https://{host}/api/v1/scraper/request"
    token = "your_token"

    headers = {
        "x-api-token": token
    }

    input_data = {
        "engine": "google_product",
        "product_id": "4172129135583325756",
        "gl": "us",
        "hl": "en",
    }

    payload = Payload("scraper.google.product", input_data)
    json_payload = json.dumps(payload.__dict__)

    response = requests.post(url, headers=headers, data=json_payload)

    if response.status_code != 200:
        print("Error:", response.status_code, response.text)
        return

    print("body", response.text)

if __name__ == "__main__":
    send_request()

$ pip install requests beautifulsoup4

def google_product():
     headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jpeg,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Language": "en-US,en;q=0.9",
        "Cache-Control": "no-cache",
        "Pragma": "no-cache",
        "Priority": "u=0, i",
        "Sec-Ch-Ua": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
    }
    response = requests.get('https://www.google.com/shopping/product/4172129135583325756?gl=us&hl=en&prds=pid:4172129135583325756,freeship:1&sourceid=chrome&ie=UTF-8', headers=headers)

online_sellers = []
soup = BeautifulSoup(response.content, 'html.parser')
for i, row in enumerate(soup.find_all("tr", {"jscontroller": "d5bMlb"})):
    name = row.find("a").get_text(strip=True)
    payment_methods = row.find_all("div")[1].get_text(strip=True)
    link = row.find("a")['href']
    details_and_offers_text = row.find_all("td")[1].get_text(strip=True)
    base_price = row.find("span", class_="g9WBQb fObmGc").get_text(strip=True)
    badge = row.find("span", class_="XhDkmd").get_text(strip=True) if row.find("span", class_="XhDkmd") else ""
    link = f"https://www.google.com/{link}"
    parse = urlparse(link)
    direct_link = parse_qs(parse.query).get("q", [""])[0]
    online_sellers.append({
        'name' : name,
        'payment_methods' : payment_methods,
        'link' : link,
        'direct_link' : direct_link,
        'details_and_offers' : [{"text": details_and_offers_text}],
        'base_price' : base_price,
        'badge' : badge
        })

import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, parse_qs


class AdditionalPrice:
    def __init__(self, shipping, tax):
        self.shipping = shipping
        self.tax = tax


class OnlineSeller:
    def __init__(self, position, name, payment_methods, link, direct_link, details_and_offers, base_price,
                 additional_price, badge, total_price):
        self.position = position
        self.name = name
        self.payment_methods = payment_methods
        self.link = link
        self.direct_link = direct_link
        self.details_and_offers = details_and_offers
        self.base_price = base_price
        self.additional_price = additional_price
        self.badge = badge
        self.total_price = total_price


def sellers_results(url):
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/jpeg,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Language": "en-US,en;q=0.9",
        "Cache-Control": "no-cache",
        "Pragma": "no-cache",
        "Priority": "u=0, i",
        "Sec-Ch-Ua": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
        "Sec-Ch-Ua-Mobile": "?0",
        "Sec-Ch-Ua-Platform": '"Windows"',
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
    }

    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')
    online_sellers = []

    for i, row in enumerate(soup.find_all("tr", {"jscontroller": "d5bMlb"})):
        name = row.find("a").get_text(strip=True)
        payment_methods = row.find_all("div")[1].get_text(strip=True)
        link = row.find("a")['href']
        details_and_offers_text = row.find_all("td")[1].get_text(strip=True)
        base_price = row.find("span", class_="g9WBQb fObmGc").get_text(strip=True)
        badge = row.find("span", class_="XhDkmd").get_text(strip=True) if row.find("span", class_="XhDkmd") else ""
        link = f"https://www.google.com/{link}"
        parse = urlparse(link)
        direct_link = parse_qs(parse.query).get("q", [""])[0]
        online_sellers.append({
            'name' : name,
            'payment_methods' : payment_methods,
            'link' : link,
            'direct_link' : direct_link,
            'details_and_offers' : [{"text": details_and_offers_text}],
            'base_price' : base_price,
            'badge' : badge
            })
    return online_sellers


url = 'https://www.google.com/shopping/product/4172129135583325756?gl=us&hl=en&prds=pid:4172129135583325756,freeship:1&sourceid=chrome&ie=UTF-8'
sellers = sellers_results(url)
for seller in sellers:
    print(seller)

pip install selenium
pip install webdriver_manager

options = webdriver.ChromeOptions()
options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")

driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

driver.get(url)
time.sleep(5)  #wait page

online_sellers = []

rows = driver.find_elements(By.CSS_SELECTOR, "tr[jscontroller='d5bMlb']")
for i, row in enumerate(rows):
    name = row.find_element(By.TAG_NAME, "a").text.strip()
    payment_methods = row.find_elements(By.TAG_NAME, "div")[1].text.strip()
    link = row.find_element(By.TAG_NAME, "a").get_attribute('href')
    details_and_offers_text = row.find_elements(By.TAG_NAME, "td")[1].text.strip()
    base_price = row.find_element(By.CSS_SELECTOR, "span.g9WBQb.fObmGc").text.strip()
    badge = row.find_element(By.CSS_SELECTOR, "span.XhDkmd").text.strip() if row.find_elements(By.CSS_SELECTOR, "span.XhDkmd") else ""

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
import time
from urllib.parse import urlparse, parse_qs

class AdditionalPrice:
    def __init__(self, shipping, tax):
        self.shipping = shipping
        self.tax = tax

class OnlineSeller:
    def __init__(self, position, name, payment_methods, link, direct_link, details_and_offers, base_price, additional_price, badge, total_price):
        self.position = position
        self.name = name
        self.payment_methods = payment_methods
        self.link = link
        self.direct_link = direct_link
        self.details_and_offers = details_and_offers
        self.base_price = base_price
        self.additional_price = additional_price
        self.badge = badge
        self.total_price = total_price

def sellers_results(url):
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    options.add_argument("--no-sandbox")
    options.add_argument("--disable-dev-shm-usage")

    driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options)

    try:
        driver.get(url)
        time.sleep(5)

        online_sellers = []

        rows = driver.find_elements(By.CSS_SELECTOR, "tr[jscontroller='d5bMlb']")
        for i, row in enumerate(rows):
            name = row.find_element(By.TAG_NAME, "a").text.strip()
            payment_methods = row.find_elements(By.TAG_NAME, "div")[1].text.strip()
            link = row.find_element(By.TAG_NAME, "a").get_attribute('href')
            details_and_offers_text = row.find_elements(By.TAG_NAME, "td")[1].text.strip()
            base_price = row.find_element(By.CSS_SELECTOR, "span.g9WBQb.fObmGc").text.strip()
            badge = row.find_element(By.CSS_SELECTOR, "span.XhDkmd").text.strip() if row.find_elements(By.CSS_SELECTOR, "span.XhDkmd") else ""

            parse = urlparse(link)
            direct_link = parse_qs(parse.query).get("q", [""])[0]

            online_seller = {
                'name': name,
                'payment_methods': payment_methods,
                'link': link,
                'direct_link': direct_link,
                'details_and_offers': [{"text": details_and_offers_text}],
                'base_price': base_price,
                'badge': badge
            }
            online_sellers.append(online_seller)

        return online_sellers

    finally:
        driver.quit() 

url = 'https://www.google.com/shopping/product/4172129135583325756?gl=us&hl=en&prds=pid:4172129135583325756,freeship:1&sourceid=chrome&ie=UTF-8'
sellers = sellers_results(url)
for seller in sellers:
    print(seller)