Pythonを使ってGoogleマップをスクレイピングする方法

import http.client
import json

conn = http.client.HTTPSConnection("api.scrapeless.com")
payload = json.dumps({
   "actor": "scraper.google.maps",
   "input": {
      "engine": "google_maps",
      "q": "coffee",
      "type": "search",
      "ll": "@40.7455096,-74.0083012,14z",
      "hl": "en",
      "gl": "us"
   }
})
headers = {
   'Content-Type': 'application/json'
}
conn.request("POST", "/api/v1/scraper/request", payload, headers)
res = conn.getresponse()
data = res.read()
print(data.decode("utf-8"))

var myHeaders = new Headers();
myHeaders.append("Content-Type", "application/json");

var raw = JSON.stringify({
   "actor": "scraper.google.maps",
   "input": {
      "engine": "google_maps",
      "q": "coffee",
      "type": "search",
      "ll": "@40.7455096,-74.0083012,14z",
      "hl": "en",
      "gl": "us"
   }
});

var requestOptions = {
   method: 'POST',
   headers: myHeaders,
   body: raw,
   redirect: 'follow'
};

fetch("https://api.scrapeless.com/api/v1/scraper/request", requestOptions)
   .then(response => response.text())
   .then(result => console.log(result))
   .catch(error => console.log('error', error));

python --version

python -m venv google_flights_env

google_flights_env\Scripts\activate

source google_flights_env/bin/activate

pip install requests
pip install beautifulsoup4

pip install selenium

def scrape_title(listing):
    maps_element = listing.select_one('div.NrDZNb div.qBF1Pd.fonHeadlineSmall')
    return maps_element.text.strip()

def scrape_rating(listing):
    maps_element = listing.select_one('span.MW4etd[aria-hidden="true"]')
    if maps_element is None:
        return None
    else:
        return maps_element.text.strip()

def scrape_image_links(listing):
    maps_image_link = listing.select_one('div.SpFAAb').select_one("img")
    return maps_image_link['src']

# 必要なライブラリのインポート
import time

from bs4 import BeautifulSoup
import json
from selenium import webdriver

driver = webdriver.Chrome()

# Google Mapsからリスト要素をスクレイピングする関数
def scrape_listings(soup):
    return soup.select('div.Nv2PK.THOPZb.CpccDe')

# google_mapsからタイトルをスクレイピングする関数
def scrape_title(listing):
    maps_element = listing.select_one('div.NrDZNb').select_one('div.qBF1Pd.fontHeadlineSmall')
    return maps_element.text.strip()

# google_mapsから評価をスクレイピングする関数
def scrape_rating(listing):
    maps_element = listing.select_one('span.MW4etd[aria-hidden="true"]')
    if maps_element is None:
        return None
    else:
        return maps_element.text.strip()

def scrape_image_links(listing):
    maps_image_link = listing.select_one('div.SpFAAb').select_one("img")
    return maps_image_link['src']

# メイン関数
def main():
    # Google MapsのURLにリクエストを行い、HTMLを解析する
    url = 'https://www.google.com/maps/search/coffee/@47.4641284,-122.3855982,11z?entry=ttu&g_ep=EgoyMDI1MDIwOS4wIKXMDSoASAFQAw=='
    driver.get(url)
    time.sleep(2)
    page_source = driver.page_source
    soup = BeautifulSoup(page_source, 'html.parser')

    # マップリストのスクレイピング
    listings = scrape_listings(soup)

    # 各リストを反復処理し、マップ情報を抽出する
    maps_data = []
    for listing in listings:
        title = scrape_title(listing)
        rating = scrape_rating(listing)
        image_link = scrape_image_links(listing)

        # マップ情報を辞書に格納する
        maps_info = {
            'title': title,
            'rating': rating,
            'image_link': image_link
        }

        maps_data.append(maps_info)

    # 結果をJSONファイルに保存する
    with open('google_maps_data.json', 'w') as json_file:
        json.dump(maps_data, json_file, indent=4)

if __name__ == "__main__":
    main()

[
    {
        "title": "Lucky Mugs",
        "rating": null,
        "image_link": "https://streetviewpixels-pa.googleapis.com/v1/thumbnail?panoid=36eAZcAEe5VpSUaFnPcOVA&cb_client=search.gws-prod.gps&w=80&h=92&yaw=139.65103&pitch=0&thumbfov=100"
    },
    {
        "title": "Retro Coffee",
        "rating": "4.4",
        "image_link": "https://lh5.googleusercontent.com/p/AF1QipPxJNNq_ddJeY4metdP0-Yv1gfAb8hwqtdjEk3R=w80-h106-k-no"
    },
    {
        "title": "Caffe Migliore.",
        "rating": "4.4",
        "image_link": "https://lh5.googleusercontent.com/p/AF1QipN5LhTxOs7TEicnyDcaeIwp5iOVD46aQ-vMpco=w80-h106-k-no"
    },
    {
        "title": "Local Coffee Spot",
        "rating": "4.2",
        "image_link": "https://lh5.googleusercontent.com/p/AF1QipO9UVq4zJ3G-xw62mFyQEj8jTdlehq7eLlZ4RyL=w163-h92-k-no"
    },
    {
        "title": "BigFoot Java",
        "rating": "4.1",
        "image_link": "https://lh3.googleusercontent.com/gps-proxy/ALd4DhGHbsVNZu27z_GxIxq3jTyZbJI5MUY_rpPTL7yZQX0Mbx4BEbKTwMTz6vJ16y1u1qVA97qxlMtjejRYMjEk7Riqx2dLl1CeDEcGfIWKWmIF5E3qcna_9faDEBRwLvhoiBcJywVx_TZ7jtM1tKUmynpAZdn-vvYju1pEGKDA34jUT38S8qip3Drk=w92-h92-k-no"
    },
    {
        "title": "Caffe Ladro - Downtown on Pine Street",
        "rating": "4.3",
        "image_link": "https://lh5.googleusercontent.com/p/AF1QipO84r3OrCAb6IjtMYbki4pAMpuWf8KoK5_ods89=w92-h92-k-no"
    },
    {
        "title": "Coffee Tree",
        "rating": "4.4",
        "image_link": "https://lh5.googleusercontent.com/p/AF1QipPYbEDGEskTGDCcpBHxF-nQqrFhR2yFi5cBt6Lu=w122-h92-k-no"
    }
]

def scrape_via(listing):
    maps_element = listing.select_one('h1.VuCHmb.fontHeadlineSmall')
    return maps_element.text.strip()

def scrape_time(listing):
    maps_element = listing.select_one('div.Fk3sm.fontHeadlineSmall.delay-light')
    return maps_element.text.strip()

def scrape_distance(listing):
    maps_element = listing.select_one('div.ivN21e.tUEI83.fontBodyMedium').select_one('div')
    return maps_element.text.strip()

# 必要なライブラリのインポート
import time

from bs4 import BeautifulSoup
import json
from selenium import webdriver

driver = webdriver.Chrome()

# Google Mapsからリスト要素をスクレイピングする関数
def scrape_listings(soup):
    return soup.select('div.UgZKXd')

# google_mapsから経由をスクレイピングする関数
def scrape_via(listing):
    maps_element = listing.select_one('h1.VuCHmb.fontHeadlineSmall')
    return maps_element.text.strip()

# google_mapsから時間をスクレイピングする関数
def scrape_time(listing):
    maps_element = listing.select_one('div.Fk3sm.fontHeadlineSmall.delay-light')
    return maps_element.text.strip()

# google_mapsから距離をスクレイピングする関数
def scrape_distance(listing):
    maps_element = listing.select_one('div.ivN21e.tUEI8e.fontBodyMedium').select_one('div')
    return maps_element.text.strip()

# メイン関数
def main():
    # google mapsのURLにリクエストを行い、HTMLを解析する
    url = 'https://www.google.com/maps/dir/Austin-Bergstrom+International+Airport/5540+N+Lamar+Blvd,+Austin,+TX+78756/@30.2603068,-97.7871692,12z/data=!3m1!4b1!4m13!4m12!1m5!1m1!1s0x8644b13b8b4aff45:0x1ca7fca8c9dc2768!2m2!1d-97.6710889!2d30.194085!1m5!1m1!1s0x8644cba140fad1fb:0x2db903443245739c!2m2!1d-97.7286733!2d30.3247493?hl=en&entry=ttu&g_ep=EgoyMDI1MDIwOS4wIKXMDSoASAFQAw%3D%3D'
    driver.get(url)
    time.sleep(2)
    page_source = driver.page_source
    soup = BeautifulSoup(page_source, 'html.parser')

    # マップリストのスクレイピング
    listings = scrape_listings(soup)

    # 各リストを反復処理し、マップ情報を抽出する
    maps_data = []
    for listing in listings:
        via = scrape_via(listing)
        time_consuming = scrape_time(listing)
        distance = scrape_distance(listing)

        # マップ情報を辞書に格納する
        maps_info = {
            'via': via,
            'time': time_consuming,
            'distance': distance
        }

        maps_data.append(maps_info)

    # 結果をJSONファイルに保存する
    with open('google_maps_dir_data.json', 'w') as json_file:
        json.dump(maps_data, json_file, indent=4)

if __name__ == "__main__":
    main()

[
    {
        "via": "via 183 Toll",
        "time": "17 min",
        "distance": "13.4 miles"
    },
    {
        "via": "via State Hwy 71 W and I-35 N",
        "time": "19 min",
        "distance": "15.5 miles"
    },
    {
        "via": "via S Hwy 183 and 183 Toll",
        "time": "17 min",
        "distance": "13.6 miles"
    }
]