PythonでGoogle Scholarをスクレイピングする方法

python --version

python -m venv google_scholar_env

google_scholar_env\Scripts\activate

source google_scholar_env/bin/activate

pip install requests
pip install beautifulsoup4

def scrape_scholar_title(listing):
    title_element = listing.select_one('h3.gs_rt a')
    return title_element .text.strip()

 def scrape_scholar_publication_info(listing):
    publication_info_element = listing.select_one('div.gs_a')
    return publication_info_element .text.strip()

def scrape_scholar_snippet(listing):
    snippet_element = listing.select_one('div.gs_rs')
    return snippet_element  .text.strip()

# 必要なライブラリのインポート
import time

import requests
from bs4 import BeautifulSoup
import json

# google_scholarからリスト要素をスクレイピングする関数
def scrape_listings(soup):
    return soup.select('div.gs_r.gs_or.gs_scl')

# google_scholarからタイトルをスクレイピングする関数
def scrape_scholar_title(listing):
    title_element = listing.select_one('h3.gs_rt > a')
    print(title_element.text)
    return title_element.text.strip()

# google_scholarからpublication_infoをスクレイピングする関数
def scrape_scholar_publication_info(listing):
    publication_info_element = listing.select_one('div.gs_a')
    print(publication_info_element.text)
    return publication_info_element.text.strip()

# google_scholarからスニペットをスクレイピングする関数
def scrape_scholar_snippet(listing):
    snippet_element = listing.select_one('div.gs_rs')
    print(snippet_element.text)
    return snippet_element.text.strip()

# メイン関数
def main():
    # google_scholar URLへのリクエストを行い、HTMLを解析します
    url = 'https://scholar.google.com/scholar?hl=en&q=biology'
    response = requests.get(url, verify=False)
    time.sleep(2)
    soup = BeautifulSoup(response.text, 'html.parser')

    # scholarリストのスクレイピング
    listings = scrape_listings(soup)
    print(listings)

    # 各リストを反復処理し、scholar情報を抽出します
    scholar_data = []
    for listing in listings:
        title = scrape_scholar_title(listing)
        publication_info = scrape_scholar_publication_info(listing)
        snippet = scrape_scholar_snippet(listing)

        # scholar情報を辞書に格納します
        scholar_info = {
            'title': title,
            'publication_info': publication_info,
            'snippet': snippet
        }

        scholar_data.append(scholar_info)

    # 結果をJSONファイルに保存します
    with open('google_scholar_data.json', 'w') as json_file:
        json.dump(scholar_data, json_file, indent=4)

if __name__ == "__main__":
    main()

[
    {
        "title": "A new biology for a new century",
        "publication_info": "CR Woese\u00a0- Microbiology and molecular biology reviews, 2004 - Am Soc Microbiol",
        "snippet": "\u2026 molecular biology's lead \u2026 in biology that 20th century biology, molecular biology, could not \nhandle and, so, avoided. The former course, though highly productive, is certain to turn biology \u2026"
    },
    {
        "title": "Biology",
        "publication_info": "PH Raven, GB Johnson, KA Mason - 2011 - thuvienso.hoasen.edu.vn",
        "snippet": "\u2026 25.1 Overview of Evolutionary Developmental Biology 492 25.2 One or Two Gene \nMutations, New Form 495 25.3 Same Gene, New Function 496 25.4 Different Genes\u00a0\u2026"
    },
    {
        "title": "General biology",
        "publication_info": "R Fayer\u00a0- Cryptosporidium and cryptosporidiosis, 2007 - taylorfrancis.com",
        "snippet": "Some species of Cryptosporidium infect many host species, whereas others appear restricted \nto groups such as rodents or ruminants, and others are known to infect only one host \u2026"
    },
    {
        "title": "The molecular biology of coronaviruses",
        "publication_info": "PS Masters\u00a0- Advances in virus research, 2006 - Elsevier",
        "snippet": "Coronaviruses are large, enveloped RNA viruses of both medical and veterinary importance. \nInterest in this viral family has intensified in the past few years as a result of the \u2026"
    },
    {
        "title": "Biology",
        "publication_info": "SS Mader - 2010 - thuvienso.hoasen.edu.vn",
        "snippet": "\u2026 Comparative Animal Biology 576 \u2026 1.1 How to Define Life 2 1.3 Evolution, the Unifying \nConcept of Biology 6 1.3 How the Biosphere Is Organized 9 1.4 The Process of Science 11\u00a0\u2026"
    },
    {
        "title": "Sealice on salmonids: their biology and control",
        "publication_info": "AW Pike, SL Wadsworth\u00a0- Advances in parasitology, 1999 - Elsevier",
        "snippet": "\u2026 This review examines the voluminous literature on the biology and control of sealice and \nbrings together ideas for developing our knowledge of these organisms. Research on the \u2026"
    },
    {
        "title": "Biology data book",
        "publication_info": "PL Altman, DS Dittmer - 1972 - bionumbers.hms.harvard.edu",
        "snippet": "Embryos were raised at constant temperature in circulating nalis\" u 10% smaller. For \nadditional information on salmowater, from three hours after fertilization. Age= time from nids, \u2026"
    },
    {
        "title": "The biology of Pseudocalanus",
        "publication_info": "CJ Corkett, IA McLaren\u00a0- Advances in marine biology, 1979 - Elsevier",
        "snippet": "Publisher Summary Pseudocalanus is typical of most crustaceans in that after hatching at \nan early stage of development it adds successively new segments and appendages. \u2026"
    },
    {
        "title": "Introduction to a submolecular biology",
        "publication_info": "A Szent-Gyorgyi - 2012 - books.google.com",
        "snippet": "\u2026 Biology is the science of the improbable and I think it is on principle that the body works \nonly with reactions which are statistically improbable. If metabolism were built of a series of \u2026"
    },
    {
        "title": "The biology of mycorrhiza.",
        "publication_info": "JL Harley - 1959 - cabidigitallibrary.org",
        "snippet": "Since Dr. Rayner published her book on mycorrhiza in 1927 there has not been a comprehensive \naccount of this subject, although the need for a critical re-appraisal of the extensive \u2026"
    }
]

import json
import requests


class Payload:
    def __init__(self, actor, input_data):
        self.actor = actor
        self.input = input_data


def send_request():
    host = "api.scrapeless.com"
    url = f"https://{host}/api/v1/scraper/request"
    token = your_token ## APIトークンに置き換えます

    headers = {
        "x-api-token": token
    }

    input_data = {
        "engine": "google_scholar",
        "q": "biology",
    }

    payload = Payload("scraper.google.scholar", input_data)

    json_payload = json.dumps(payload.__dict__)

    response = requests.post(url, headers=headers, data=json_payload)

    if response.status_code != 200:
        print("Error:", response.status_code, response.text)
        return

    print("body", response.text)


if __name__ == "__main__":
    send_request()