如何抓取谷歌学术公开搜索结果

import requests
from bs4 import BeautifulSoup

url = "https://scholar.google.com/scholar?q=machine+learning"
headers = {"User-Agent": "Mozilla/5.0"}
response = requests.get(url, headers=headers)

soup = BeautifulSoup(response.text, "html.parser")
results = soup.find_all("div", class_="gs_r")

for result in results:
    title = result.find("h3").text if result.find("h3") else "No Title"
    print(title)

import http.client
import json

conn = http.client.HTTPSConnection("api.scrapeless.com")
payload = json.dumps({
   "actor": "scraper.google.scholar",
   "input": {
      "engine": "google_scholar",
      "q": "biology"
   }
})
headers = {
   'Content-Type': 'application/json'
}
conn.request("POST", "/api/v1/scraper/request", payload, headers)
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))

import requests
import json
# 设置API请求URL
url = "https://api.scrapeless.com/api/v1/scraper/request"
# 定义请求的有效负载
payload = json.dumps({
    "actor": "scraper.google.scholar",
    "input": {
        "engine": "google_scholar",
        "q": "machine learning",        # 搜索查询
        "cites": "KNJ0p4CbwgoJ",        # 可选：查找引用此论文的文章
        "as_ylo": 2015,                 # 可选：过滤自此年份的结果（起始年份）
        "as_yhi": 2023,                 # 可选：过滤截止到此年份的结果（结束年份）
        "hl": "en",                     # 可选：语言设置（默认为英语）
        "num": 10                       # 可选：结果数量（默认为10）
    }
})
# 设置请求头
headers = {
    'Content-Type': 'application/json'
}
# 发送请求
response = requests.request("POST", url, headers=headers, data=payload)
# 打印响应
print(response.text)

{
    "search_information": {
        "total_results": 5000000,
        "time_taken_displayed": 0.05,
        "query_displayed": "machine learning"
    },
    "organic_result": [
        {
            "position": 1,
            "title": "A survey on machine learning methods",
            "result_id": "KNJ0p4CbwgoJ",
            "link": "https://example.com/article1",
            "snippet": "This article provides a comprehensive survey of machine learning methods, including supervised and unsupervised learning.",
            "publication_info": {
                "summary": "Author1, Author2 - Journal of Machine Learning, 2020"
            }
        },
        {
            "position": 2,
            "title": "Deep learning in artificial intelligence",
            "result_id": "KNJ0p4CbwgoK",
            "link": "https://example.com/article2",
            "snippet": "This paper discusses the applications of deep learning in various fields, including computer vision and natural language processing.",
            "publication_info": {
                "summary": "Author3, Author4 - AI Journal, 2021"
            }
        }
    ]
}

import http.client
import json

conn = http.client.HTTPSConnection("api.scrapeless.com")

# 定义请求有效负载，指定爬虫引擎和作者ID
payload = json.dumps({
   "actor": "scraper.google.scholar",
   "input": {
      "engine": "google_scholar_author",
      "author_id": "LSsXyncAAAAJ"  # 示例作者ID
   }
})

# 设置请求头
headers = {
   'Content-Type': 'application/json'
}

# 发出POST请求
conn.request("POST", "/api/v1/scraper/request", payload, headers)

# 获取响应结果
res = conn.getresponse()

# 读取并输出数据
data = res.read()
print(data.decode("utf-8"))

{
  "name": "John Doe",
  "institution": "Harvard University",
  "research_areas": [
    {
      "title": "Epigenetics",
      "link": "https://scholar.google.com/citations?view_op=search_authors&hl=en&mauthors=label:epigenetics"
    },
    {
      "title": "Gene Regulation",
      "link": "https://scholar.google.com/citations?view_op=search_authors&hl=en&mauthors=label:gene_regulation"
    },
    {
      "title": "Genomics",
      "link": "https://scholar.google.com/citations?view_op=search_authors&hl=en&mauthors=label:genomics"
    },
    {
      "title": "Transcription Factors",
      "link": "https://scholar.google.com/citations?view_op=search_authors&hl=en&mauthors=label:transcription_factors"
    }
...

import http.client
import json

conn = http.client.HTTPSConnection("api.scrapeless.com")
payload = json.dumps({
   "actor": "scraper.google.scholar",
   "input": {
      "engine": "google_scholar_cite",
      "q": "s1QWFy06YAYJ"
   }
})
headers = {
   'Content-Type': 'application/json'
}
conn.request("POST", "/api/v1/scraper/request", payload, headers)
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))

{
    "citations": [
        {
            "title": "MLA",
            "snippet": "Masters, Paul S. \"The molecular biology of coronaviruses.\" Advances in virus research 66 (2006): 193-292."
        },
        {
            "title": "APA",
            "snippet": "Masters, P. S. (2006). The molecular biology of coronaviruses. Advances in virus research, 66, 193-292."
        },
        {
            "title": "Chicago",
            "snippet": "Masters, Paul S. \"The molecular biology of coronaviruses.\" Advances in virus research 66 (2006): 193-292."
        },
        {
            "title": "Harvard",
            "snippet": "Masters, P.S., 2006. The molecular biology of coronaviruses. Advances in virus research, 66, pp.193-292."
        },
        {
            "title": "Vancouver",
...
    ]
}

"citations": [
    {
        "title": "MLA",
        "snippet": "Masters, Paul S. \"The molecular biology of coronaviruses.\" Advances in virus research 66 (2006): 193-292."
    },
    {
        "title": "APA",
        "snippet": "Masters, P. S. (2006). The molecular biology of coronaviruses. Advances in virus research, 66, 193-292."
    }
]

"links": [
    {
        "name": "BibTeX",
        "link": "https://scholar.googleusercontent.com/scholar.bib?q=info:s1QWFy06YAYJ..."
    },
    {
        "name": "EndNote",
        "link": "https://scholar.googleusercontent.com/scholar.enw?q=info:s1QWFy06YAYJ..."
    }
]

import http.client
import json

conn = http.client.HTTPSConnection("api.scrapeless.com")
payload = json.dumps({
   "actor": "scraper.google.scholar",
   "input": {
      "engine": "google_scholar_author",
      "author_id": "LSsXyncAAAAJ"
   }
})
headers = {
   'Content-Type': 'application/json'
}
conn.request("POST", "/api/v1/scraper/request", payload, headers)
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))

{
    "pagination": {
        "next": "https://scholar.google.com//citations?view_op=search_authors&hl=en&mauthors=Mike&after_author=pnnfAUQM__8J&astart=10",
        "next_page_token": "pnnfAUQM__8J"
    },
    "profiles": [

{
  "name": "Mike Robb",
  "link": "https://scholar.google.com//citations?hl=en&user=kq0NYnMAAAAJ",
  "author_id": "kq0NYnMAAAAJ",
  "affiliations": "帝国理工学院化学系",
  "email": "已验证的帝国理工学院邮箱",
  "cited_by": 230346,
  "interests": [
    {
      "title": "计算化学",
      "link": "https://scholar.google.com//citations?hl=en&view_op=search_authors&mauthors=label:computational_chemistry"
    },
    {
      "title": "理论化学",
      "link": "https://scholar.google.com//citations?hl=en&view_op=search_authors&mauthors=label:theoretical_chemistry"
    },
    {
      "title": "圆锥交叉",
      "link": "https://scholar.google.com//citations?hl=en&view_op=search_authors&mauthors=label:conical_intersections"
    },
    {
      "title": "非绝热动力学",
      "link": "https://scholar.google.com//citations?hl=en&view_op=search_authors&mauthors=label:non_adiabatic_dynamics"
    }
  ],
  "thumbnail": "https://scholar.google.com//citations?hl=en&user=kq0NYnMAAAAJ"
},
{
  "name": "Mike A. Nalls",
  "link": "https://scholar.google.com//citations?hl=en&user=ZjfgPLMAAAAJ",
  "author_id": "ZjfgPLMAAAAJ",
  "affiliations": "Data Tecnica International创始人/顾问 + NIH中心数据科学主管……",
  "email": "已验证的nih.gov邮箱",
  "cited_by": 175760,
  "interests": [
    {
      "title": "统计遗传学",
      "link": "https://scholar.google.com//citations?hl=en&view_op=search_authors&mauthors=label:statistical_genetics"
    },
    {