GEO向けのChatGPTでバルククエリを実行する方法：Scrapeless Cloud Browserを使用した自動化ソリューション

npm install puppeteer-core @scrapeless-ai/sdk node-fetch

import puppeteer, { Browser, Page } from 'puppeteer-core';
import { Scrapeless, PuppeteerLaunchOptions } from '@scrapeless-ai/sdk';

const scrapeless = new Scrapeless();

async function connectBrowser(): Promise<Browser> {
  const options: PuppeteerLaunchOptions = {
    session_name: 'Batch ChatGPT',
    session_ttl: 600,
    fingerprint: {
      platform: 'macOS',
      localization: { timezone: 'America/New_York' },
      args: { '--window-size': '1920,1080' },
    },
  };

  const { browserWSEndpoint } = await scrapeless.browser.createSession(options);
  const browser = await puppeteer.connect({ browserWSEndpoint, defaultViewport: null });
  return browser;
}

async function queryChatGPT(browser: Browser, prompt: string): Promise<string> {
  const page = await browser.newPage();
  await page.goto('https://chatgpt.com/');

  const inputSelector = '[placeholder="Ask anything"]';
  await page.waitForSelector(inputSelector, { visible: true });
  await page.type(inputSelector, prompt);
  await page.keyboard.press('Enter');

  await page.waitForSelector('[data-message-author-role="assistant"]');
  const result = await page.evaluate(() => {
    const messages = document.querySelectorAll('[data-message-author-role="assistant"]');
    return messages[messages.length - 1]?.textContent || '';
  });

  await page.close();
  return result;
}

let gptAnswer: string;
gptAnswer = await waitForChatGPTResponse(page);

let gptImageCards: ChatgptResponse['image_cards'] = [];
// セレクタを使用して画像を抽出し、{ url, position }を構築

const gptRecommendProducts: ChatgptResponse['products'] = [];
// セレクタを使用して製品リンク、タイトル、画像を抽出

let gptCitations: ChatgptResponse['citations'] = [];
// 脚注ボタンを使用して引用リンク、アイコン、タイトル、説明を抽出

let gptLinksAttached: ChatgptResponse['links_attached'] = [];
// マークダウンリンクセレクタを使用してリンクとそのテキストを抽出

const body = await page.evaluate(() => document.body.innerHTML);

const cleanBody = body
  .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
  .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
  .replace(/\s+/g, ' ')
  .trim();

2. **関数の戻り値**

```ts
return createResponse(JSON.stringify(payload), payload.url);

import puppeteer, { Browser, Page, Target } from 'puppeteer-core';
import fetch from 'node-fetch';
import { PuppeteerLaunchOptions, Scrapeless } from '@scrapeless-ai/sdk';
import { Logger } from '@nestjs/common';

export interface BaseInput {
  task_id: string;
  proxy_url: string;
  timeout: number;
}

export interface BaseOutput {
  url: string;
  data: number[];
  collection?: string;
  dataType?: string;
}

export interface QueryChatgptRequest extends BaseInput {
  prompt: string;
  webhook?: string;
  session_name?: string;
  web_search?: boolean;
  session_recording?: boolean;
  answer_type?: 'text' | 'html' | 'raw';
}

export interface ChatgptResponse {
  prompt: string;
  task_id?: string;
  duration?: number;
  answer?: string;
  url: string;
  success: boolean;
  country_code: string;
  error_reason?: string;
  links_attached?: Partial<{ position: number; text: string; url: string }>[];
  citations?: Partial<{ url: string; icon: string; title: string; description: string }>[];
  products?: Partial<{ url: string; title: string; image_urls: (string | null)[] }>[];
  image_cards?: Partial<{ position: number; url: string }>[];
}

interface StartChatParams extends QueryChatgptRequest {
  page: Page;
  browser: Browser;
}

export class ChatgptService {
  logger = new Logger(this.constructor.name);
  scrapeless = new Scrapeless();

  private timeoutMultiplier = 2;
  private defaultTimeout = 3 * 60 * 1000;
  private internalErrorSymbol = '[InternalError]:';

  async solver(input: QueryChatgptRequest, checkTimeout: () => boolean): Promise<BaseOutput> {
    const { session_name, task_id, webhook, session_recording, proxy_url } = input;

    let browser: Browser;

    const startTime = performance.now();
    const successful = false;

    const getTotalDuration = () => {
      const endTime = performance.now();
      const totalDuration = ((endTime - startTime) / 1000).toFixed(2);
      return totalDuration;
    };

    const handleChatResponse = (data: Partial<ChatgptResponse>) => {
      const payload = { ...data, task_id, duration: getTotalDuration() };
      return payload;
    };

    const createResponse = (data: string, url = 'https://chatgpt.com'): BaseOutput => {
      return {
        url: url,
        data: Array.from(Buffer.from(data)),
        dataType: 'json',
      };
    };

    try {
      browser = await this.connectToBrowser(
        {
          session_name,
          session_ttl: 600,
          session_recording,
          proxy_url,
          fingerprint: {
            platform: 'macOS',
            localization: {
              timezone: 'America/New_York',
            },
            args: {
              '--window-size': '1920,1080',
            },
          },
        },
        checkTimeout,
      );

      const page = await browser.newPage();

      await this.fakePageDate(page);

      const chatParams: StartChatParams = { ...input, page, browser };
      const results = await this.startChat(chatParams);
      const payload = handleChatResponse(results);
      this.pushToMessage(payload, webhook);
      return createResponse(JSON.stringify(payload), payload.url);
    } catch (error) {
      if (error.success) {
        const payload = handleChatResponse(error);
        this.pushToMessage(payload, webhook);
        return createResponse(JSON.stringify(payload), error.url);
      }
      if (error.error_reason) {
        const errorMessage = error.error_reason;
        const payload = handleChatResponse(error);
        this.pushToMessage(payload, webhook);
        this.logger.warn(`処理に失敗しました: ${errorMessage}`);
        throw { message: !errorMessage.includes(this.internalErrorSymbol) ? errorMessage : '' };
      }
      const errorMessage = error.message || '不明なエラー';
      const payload = handleChatResponse({
        success: false,
        error_reason: errorMessage,
      });
      this.pushToMessage(payload, webhook);
      this.logger.warn(`処理に失敗しました: ${errorMessage}`);
      throw error;
    } finally {
      const totalDuration = getTotalDuration();
      this.logger.log(
        `処理は${successful ? '成功' : '完了'}しました | 合計時間: ${totalDuration}秒`,
      );
    }
  }

if (!input.prompt) {
  this.logger.error(`プロンプトは必須です`);
  throw new Error('プロンプトは必須です');
}

return {
  ...input,
  timeout: input.timeout || this.defaultTimeout,
  web_search: input.web_search ?? true,
  session_name: input.session_name || 'Chatgptの回答',
  session_recording: input.session_recording || false,
  answer_type: input.answer_type || 'テキスト',
};

  this.logger.debug((action = 'ブラウザに接続中'));

  const proxy_country = /-country_([A-Z]{2,3})/.exec(proxy_url)?.[1] || 'ANY';
  const query = new URLSearchParams({ q: prompt });
  if (web_search) {
    query.set('hints', 'search');
  }
  const baseUrl = 'https://chatgpt.com';
  const _url = `${baseUrl}/?${query.toString()}`;

  function waitForChatGPTResponse(page: Page): Promise<string> {
    return new Promise((resolve, reject) => {
      let retryCount = 0;
      const CHECK_INTERVAL = 500; // 500msごとにチェック

      const checkResponse = async () => {
        try {
          if (isAborted) {
            resolve('タイムアウト');
            return;
          }
          const currentContent = await page.evaluate(() => {
            const assistantMessage = '[data-message-author-role="assistant"]';
            const $assistantMessages = document.querySelectorAll(assistantMessage);
            const lastMessage = $assistantMessages[$assistantMessages.length - 1];
            if (!lastMessage) return null;

            // コピーボタンが一つ以上現れた場合、gptの回答が完了したことを意味します
            const $answerCopyButtons = document.querySelectorAll('button[data-testid="copy-turn-action-button"]');
            if ($answerCopyButtons.length > 1) {
              return lastMessage.textContent || '内容なし';
            } else {
              return null;
            }
          });

          // コンテンツが変更されておらず、空でない場合
          if (currentContent) {
            retryCount++;
            // コンテンツが安定していて空でない場合、応答は完了とみなされます
            if (retryCount >= 3) {
              // 1.5秒間コンテンツが安定している
              resolve(currentContent);
              return;
            }
          }

          // チェックを続ける
          setTimeout(checkResponse, CHECK_INTERVAL);
        } catch (error) {
          reject(error);
        }
      };

      // チェックを開始する
      checkResponse();
    });
  }

  async function receiveChatGPTStream() {
    const client = await page.createCDPSession();
    await client.send('Fetch.enable', {
      patterns: [
        {
          urlPattern: '*conversation',
          requestStage: 'Response',
        },
      ],
    });

    client.on('Fetch.requestPaused', async (event) => {
      const { requestId, request, responseHeaders } = event;
      const isSSE = responseHeaders?.some(
        (h) => h.name?.toLowerCase() === 'content-type' && h.value?.includes('text/event-stream'),
      );
      if (request.url.includes('/conversation') && isSSE) {
        try {
          const { body, base64Encoded } = await client.send('Fetch.getResponseBody', { requestId });
          rawResponse = base64Encoded ? Buffer.from(body, 'base64').toString('utf-8') : body;
        } catch (err) {
          console.warn('ストリーム応答の取得に失敗しました', err.message);
        }
      }
      await client.send('Fetch.continueRequest', { requestId });
    });
  }

  function throwError(errorReason: string) {
    const error: ChatgptResponse = {
      prompt,
      success: false,
      country_code: proxy_country,
      error_reason: errorReason,
      url: _url,
    };
    reject(error);
  }

  const timeoutId = setTimeout(() => {
    isAborted = true;
    throwError(`チャットは${timeout}ms後にタイムアウトしました`);
  }, timeout);

  try {
    this.logger.debug((action = 'GPTストリームデータをキャプチャするためにCDPを登録中 (raw_response)'));
    await receiveChatGPTStream();
    this.logger.debug((action = 'chatgpt.comに移動中'));

    // URL変更リスナーを追加
    page.on('framenavigated', async (frame) => {
      if (frame !== page.mainFrame()) return;
      const url = frame.url();
      if (!url.startsWith('https://auth.openai.com')) return;
      isAborted = true;
      throwError(`<<${_url}>>からOpenAIログインページにリダイレクトされました - ${action}`);
      return;
    });

    if (isAborted) return;
    await this.wait(50, 150);
    this.logger.debug((action = '入力が存在することを確認'));
    const inputs = ['#prompt-textarea', '[placeholder="Ask anything"]'];
    try {
      await Promise.race(
        inputs.map(async (input) => {
          await page.waitForSelector(input, {
            timeout: 20_000 * this.timeoutMultiplier,
            visible: true,
          });
          return input;
        }),
      );
    } catch {
      throwError('現在の地域が利用できないか、ログインページにリダイレクトされました');
      return;
    }

    if (isAborted) return;
    await this.wait(150, 250);
    this.logger.debug((action = 'GPTの応答を待機'));
    let gptAnswer: string;
    try {
      gptAnswer = await waitForChatGPTResponse(page);
      this.logger.debug((action = 'GPTの応答を受信しました'));
    } catch (error: any) {
      this.logger.error(`応答の取得に失敗しました: ${error.message}`);
      throwError(`chatgptの応答取得に失敗しました`);
      return;
    }

    if (isAborted) return;
    await this.wait(150, 250);
    this.logger.debug((action = 'chatgptの画像カードを取得'));
    const imageCardsSelector = 'div.no-scrollbar:has(button img) img';
    const imageCardsLightBoxSelector = 'div[data-testid="modal-image-gen-lightbox"] ol li img';
    const imageCardsLightBoxCloseSelector = 'div[data-testid="modal-image-gen-lightbox"] button';
    let gptImageCards: ChatgptResponse['image_cards'] = [];
    try {
      const firstImageCard = await page.$(imageCardsSelector);
      if (firstImageCard) {
        firstImageCard.click();
        await page.waitForSelector(imageCardsLightBoxSelector);
        gptImageCards = await page.$$eval(imageCardsLightBoxSelector, (elements) => {
          return elements.map((element, index) => {
            const url = element.getAttribute('src') || '';
            return { url, position: index + 1 };
          });
        });
        await page.waitForSelector(imageCardsLightBoxCloseSelector);
        await page.click(imageCardsLightBoxCloseSelector);
      } else {
        this.logger.debug((action = '画像カードが見つかりませんでした'));
      }
    } catch (error: any) {
      this.logger.debug((action = `chatgptの画像カードを取得: ${error.toString()}`));
    }

    if (isAborted) return;
    await this.wait(300, 450);
    this.logger.debug((action = 'chatgptの推奨商品を取得'));
    const closeButtonSelector = `button[data-testid="close-button"]`;
    const recommendProductsSelector = 'div.markdown div.relative > div.flex.flex-row:has(img):not(a) > div img';
    const recommendProductDetailsSelector = `section[screen-anchor="top"] div[slot="content"]`;
    const detailLinkSelector = `${recommendProductDetailsSelector} span a`;
    const gptRecommendProducts: ChatgptResponse['products'] = [];
    try {
      const recommendProducts = await page.$$(recommendProductsSelector);
      if (recommendProducts.length) {
        let lastUrl = '';
        for (const [index] of recommendProducts.entries()) {
          // 外部リンクジャンプが発生する可能性があります
          let newPage: Page = null as unknown as Page;
          const targetCreatedHandler = async (target: Target) => {
            this.logger.debug((action = `chatgptの推奨商品を取得: ${target.type()}`));
            try {
              if (target.type() === 'page') {
                const pageTarget = await target.page();
                const opener = await target.opener();
                if (opener && (opener as any)?._targetId === (page.target() as any)?._targetId) {
                  newPage = pageTarget as Page;
                }
              }
            } catch (e) {}
          };
          browser.once('targetcreated', targetCreatedHandler);

          // 推奨商品をクリック
          await page.evaluate(
            (selector, index) => {
              const currentProduct = document.querySelectorAll(selector)?.[index];

(currentProduct as HTMLElement)?.click();
                },
                recommendProductsSelector,
                index,
              );

              await this.wait(750, 950);
              browser.off('targetcreated', targetCreatedHandler);

              if (newPage) {
                const url = newPage.url();
                const title = await newPage.title();
                gptRecommendProducts.push({ url, title, image_urls: [] });
                await newPage.close();
                continue;
              }

              await page.waitForSelector(detailLinkSelector, { timeout: 20_000 * this.timeoutMultiplier });

              // 詳細が変わるのを待つ
              let maxRetry = 30;
              while (maxRetry-- > 0) {
                const currentUrl = await page.$eval(detailLinkSelector, (el) => el.getAttribute('href') || '');
                if (currentUrl && currentUrl !== lastUrl) {
                  lastUrl = currentUrl;
                  break;
                }
                await this.wait(200, 300);
              }

              const info = await page.$eval(
                recommendProductDetailsSelector,
                (element, currentUrl) => {
                  const title = element.querySelector('div.text-xl')?.textContent || '';
                  const image_urls = Array.from(element.querySelectorAll('.no-scrollbar img')).map((img) =>
                    img.getAttribute('src'),
                  );
                  return { url: currentUrl, title, image_urls };
                },
                lastUrl,
              );

              gptRecommendProducts.push(info);
            }
            await page.click(closeButtonSelector);
          } else {
            this.logger.debug((action = 'おすすめの製品は見つかりませんでした'));
          }
        } catch (error: any) {
          this.logger.debug((action = `ChatGPTのおすすめ製品を取得: ${error.toString()}`));
        }

        if (isAborted) return;
        await this.wait(500, 1000);
        this.logger.debug((action = 'ChatGPTの引用を取得'));
        const citationsEntranceSelector = `button.group\\/footnote`;
        const citationsContentLinkSelector = `section[screen-anchor="top"] div[slot="content"] a`;
        let gptCitations: ChatgptResponse['citations'] = [];
        await page.bringToFront();

        try {
          const citationsButton = await page.waitForSelector(citationsEntranceSelector, {
            timeout: 3_000,
          });
          if (citationsButton) {
            await citationsButton.click();
            const citationsContent = await page.waitForSelector(citationsContentLinkSelector, {
              timeout: 20_000 * this.timeoutMultiplier,
            });
            if (citationsContent) {
              gptCitations = await page.$$eval(citationsContentLinkSelector, (elements) => {
                return elements.map((element) => {
                  const url = element.href || '';
                  const icon = element.querySelector('img')?.getAttribute?.('src');
                  const title = element.querySelector('div:nth-child(2)')?.textContent || '';
                  const description = element.querySelector('div:nth-child(3)')?.textContent || '';
                  return { url, icon, title, description };
                });
              });
              await page.click(closeButtonSelector);
            }
          } else {
            this.logger.debug((action = '引用は見つかりませんでした'));
          }
        } catch (error: any) {
          this.logger.debug((action = `ChatGPTの引用を取得: ${error.toString()}`));
        }

        // 場合によっては、予期しないクリックを防ぐためにページに固定位置の要素を追加する必要があります
        if (isAborted) return;
        this.logger.debug((action = '予期しないクリックを避けるために固定要素を追加'));
        await page.evaluate(() => {
          const element = document.createElement('div');
          element.style.position = 'fixed';
          element.style.top = '0';
          element.style.left = '0';
          element.style.width = '100%';
          element.style.height = '100%';
          element.style.zIndex = '1000';
          document.body.appendChild(element);
        });

        if (isAborted) return;
        await this.wait(150, 250);
        this.logger.debug((action = 'ChatGPTの添付リンクを取得'));
        const markdownLinksSelector = 'div.markdown a';
        let gptLinksAttached: ChatgptResponse['links_attached'] = [];
        try {
          gptLinksAttached = await page.$$eval(markdownLinksSelector, (elements) => {
            return elements.map((element, index) => {
              const url = element.href || '';
              const title = element.textContent || '';
              return { url, title, position: index + 1 };
            });
          });
        } catch (error: any) {

    this.logger.debug((action = 'ボディを取得中'));
    const body = await page.evaluate(() => document.body.innerHTML);
    const cleanBody = body
      .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // スクリプトタグを削除
      .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // スタイルタグを削除
      .replace(/<svg[^>]*>[\s\S]*?<\/svg>/gi, '') // SVGタグを削除
      .replace(/<img[^>]*\/?>/gi, '') // imgタグを削除
      .replace(/style="[^"]*"/gi, '') // すべてのスタイル属性を削除
      .replace(/class="[^"]*"/gi, '') // すべてのクラス属性を削除
      .replace(/<!--[\s\S]*?-->/g, '') // コメントを削除
      // 関連する置換をマッピング
      .replace(/<span>·<\/span>/g, '') // ·を含むspanタグを削除
      .replace(/<a href="https:\/\/www\.google\.com\/maps\/[^"]*"[^>]*>[\s\S]*?<\/a>/g, '') // すべてのGoogleマップリンクを削除
      .replace(/<a href="tel:+[^"]*"[^>]*>[\s\S]*?<\/a>/g, '') // すべての電話リンクを削除
      .replace(/\s+/g, ' ') // スペースを正規化
      .trim();

    this.logger.debug((action = 'エラー応答を確認中'));
    const hasError = [
      '応答の生成中にエラーが発生しました。',
      'あなたのデバイスから異常な活動が検出されました。',
      'エラーが発生しました。リクエストしたエンジンが存在しないか、リクエスト処理中に別の問題が発生しました。',
    ].some((message) => cleanBody.includes(message));
    if (hasError) {
      throwError(`ChatGPTは現在利用できません`);
      return;
    }

    this.logger.log((action = 'チャット成功'));

    const answerMap: Record<QueryChatgptRequest['answer_type'], string> = {
      html: cleanBody,
      raw: rawResponse,
      text: gptAnswer,
    };
    const answerResponse = answerMap[answer_type] ?? answerMap.text;

    resolve({
      prompt,
      success: true,
      answer: answerResponse,
      country_code: proxy_country,
      citations: gptCitations,
      links_attached: gptLinksAttached,
      image_cards: gptImageCards,
      products: gptRecommendProducts,
      url: _url,
    });
  } catch (error: any) {
    if (!isAborted) {
      throwError(this.internalErrorSymbol + (error.message || String(error)));
    }
  } finally {
    clearTimeout(timeoutId);
    try {
      await page.close();
      await browser.close();
    } catch {}
  }
});

  class FakeDate extends OriginalDate {
    constructor(...args: Parameters<typeof Date>) {
      super();
      if (args.length === 0) {
        return new OriginalDate(fixedDate);
      }
      return new OriginalDate(...args);
    }

    static now() {
      return fixedDate.getTime();
    }

    static parse(str: string) {
      return OriginalDate.parse(str);
    }

    static UTC(...args: Parameters<typeof Date.UTC>) {