如何用Python爬取Google学术？

python --version

python -m venv google_scholar_env

google_scholar_env\Scripts\activate

source google_scholar_env/bin/activate

pip install requests
pip install beautifulsoup4

def scrape_scholar_title(listing):
    title_element = listing.select_one('h3.gs_rt a')
    return title_element .text.strip()

 def scrape_scholar_publication_info(listing):
    publication_info_element = listing.select_one('div.gs_a')
    return publication_info_element .text.strip()

def scrape_scholar_snippet(listing):
    snippet_element = listing.select_one('div.gs_rs')
    return snippet_element  .text.strip()

# Import necessary libraries
import time

import requests
from bs4 import BeautifulSoup
import json

# Function to scrape listing elements from google_scholar
def scrape_listings(soup):
    return soup.select('div.gs_r.gs_or.gs_scl')

# Function to scrape title from google_scholar
def scrape_scholar_title(listing):
    title_element = listing.select_one('h3.gs_rt > a')
    print(title_element.text)
    return title_element.text.strip()

# Function to scrape publication_info from google_scholar
def scrape_scholar_publication_info(listing):
    publication_info_element = listing.select_one('div.gs_a')
    print(publication_info_element.text)
    return publication_info_element.text.strip()

# Function to scrape snippet from google_scholar
def scrape_scholar_snippet(listing):
    snippet_element = listing.select_one('div.gs_rs')
    print(snippet_element.text)
    return snippet_element.text.strip()

# Main function
def main():
    # Make a request to google_scholar URL and parse HTML
    url = 'https://scholar.google.com/scholar?hl=en&q=biology'
    response = requests.get(url, verify=False)
    time.sleep(2)
    soup = BeautifulSoup(response.text, 'html.parser')

    # Scrape scholar listings
    listings = scrape_listings(soup)
    print(listings)

    # Iterate through each listing and extract scholar information
    scholar_data = []
    for listing in listings:
        title = scrape_scholar_title(listing)
        publication_info = scrape_scholar_publication_info(listing)
        snippet = scrape_scholar_snippet(listing)

        # Store scholar information in a dictionary
        scholar_info = {
            'title': title,
            'publication_info': publication_info,
            'snippet': snippet
        }

        scholar_data.append(scholar_info)

    # Save results to a JSON file
    with open('google_scholar_data.json', 'w') as json_file:
        json.dump(scholar_data, json_file, indent=4)

if __name__ == "__main__":
    main()

[
    {
        "title": "A new biology for a new century",
        "publication_info": "CR Woese\u00a0- Microbiology and molecular biology reviews, 2004 - Am Soc Microbiol",
        "snippet": "\u2026 molecular biology's lead \u2026 in biology that 20th century biology, molecular biology, could not \nhandle and, so, avoided. The former course, though highly productive, is certain to turn biology \u2026"
    },
    {
        "title": "Biology",
        "publication_info": "PH Raven, GB Johnson, KA Mason - 2011 - thuvienso.hoasen.edu.vn",
        "snippet": "\u2026 25.1 Overview of Evolutionary Developmental Biology 492 25.2 One or Two Gene \nMutations, New Form 495 25.3 Same Gene, New Function 496 25.4 Different Genes\u00a0\u2026"
    },
    {
        "title": "General biology",
        "publication_info": "R Fayer\u00a0- Cryptosporidium and cryptosporidiosis, 2007 - taylorfrancis.com",
        "snippet": "Some species of Cryptosporidium infect many host species, whereas others appear restricted \nto groups such as rodents or ruminants, and others are known to infect only one host \u2026"
    },
    {
        "title": "The molecular biology of coronaviruses",
        "publication_info": "PS Masters\u00a0- Advances in virus research, 2006 - Elsevier",
        "snippet": "Coronaviruses are large, enveloped RNA viruses of both medical and veterinary importance. \nInterest in this viral family has intensified in the past few years as a result of the \u2026"
    },
    {
        "title": "Biology",
        "publication_info": "SS Mader - 2010 - thuvienso.hoasen.edu.vn",
        "snippet": "\u2026 Comparative Animal Biology 576 \u2026 1.1 How to Define Life 2 1.3 Evolution, the Unifying \nConcept of Biology 6 1.3 How the Biosphere Is Organized 9 1.4 The Process of Science 11\u00a0\u2026"
    },
    {
        "title": "Sealice on salmonids: their biology and control",
        "publication_info": "AW Pike, SL Wadsworth\u00a0- Advances in parasitology, 1999 - Elsevier",
        "snippet": "\u2026 This review examines the voluminous literature on the biology and control of sealice and \nbrings together ideas for developing our knowledge of these organisms. Research on the \u2026"
    },
    {
        "title": "Biology data book",
        "publication_info": "PL Altman, DS Dittmer - 1972 - bionumbers.hms.harvard.edu",
        "snippet": "Embryos were raised at constant temperature in circulating nalis\" u 10% smaller. For \nadditional information on salmowater, from three hours after fertilization. Age= time from nids, \u2026"
    },
    {
        "title": "The biology of Pseudocalanus",
        "publication_info": "CJ Corkett, IA McLaren\u00a0- Advances in marine biology, 1979 - Elsevier",
        "snippet": "Publisher Summary Pseudocalanus is typical of most crustaceans in that after hatching at \nan early stage of development it adds successively new segments and appendages. \u2026"
    },
    {
        "title": "Introduction to a submolecular biology",
        "publication_info": "A Szent-Gyorgyi - 2012 - books.google.com",
        "snippet": "\u2026 Biology is the science of the improbable and I think it is on principle that the body works \nonly with reactions which are statistically improbable. If metabolism were built of a series of \u2026"
    },
    {
        "title": "The biology of mycorrhiza.",
        "publication_info": "JL Harley - 1959 - cabidigitallibrary.org",
        "snippet": "Since Dr. Rayner published her book on mycorrhiza in 1927 there has not been a comprehensive \naccount of this subject, although the need for a critical re-appraisal of the extensive \u2026"
    }
]

import json
import requests


class Payload:
    def __init__(self, actor, input_data):
        self.actor = actor
        self.input = input_data


def send_request():
    host = "api.scrapeless.com"
    url = f"https://{host}/api/v1/scraper/request"
    token = your_token ## replace with your API Token

    headers = {
        "x-api-token": token
    }

    input_data = {
        "engine": "google_scholar",
        "q": "biology",
    }

    payload = Payload("scraper.google.scholar", input_data)

    json_payload = json.dumps(payload.__dict__)

    response = requests.post(url, headers=headers, data=json_payload)

    if response.status_code != 200:
        print("Error:", response.status_code, response.text)
        return

    print("body", response.text)


if __name__ == "__main__":
    send_request()