【Selenium】初めてのスクレイピング

[Selenium] -> [ChromeDriver] -> [Google Chrome]

driver = Chrome('path', options=options)
driver.get('url')

import csv
import pandas as pd
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.common.by import By


options = ChromeOptions()
options.add_argument('--headless')

print('Connectiong to Remote Browser...')
driver = Chrome('chromedriverがあるパス', options=options)

# 検索結果
driver.get('対象のurl')

# 一覧ページ取得
detail_links = driver.find_elements(By.XPATH,'//div[@class="productImg-wrap"]')
print(len(detail_links))

# 詳細ページリスト化
detail_results = []
for element in detail_links:
    aTag    = element.find_element_by_tag_name("a")
    url     = aTag.get_attribute("href")
    detail_results.append(url)

# 空のDataFrameを用意
cols = ['brand_name', 'url']
df = pd.DataFrame(index=[], columns=cols)

# 結果をリスト化
final_results = []
for url in detail_results:
    driver.get(url)
    article_links = driver.find_elements(By.XPATH,'//div[@class="attributes-list"]')
    print(article_links)

    # データフレームを作成
    record = pd.Series([article_links[0].text, url], index=df.columns)
    df = df.append(record, ignore_index=True)
    df.to_csv("output.csv")

# ブラウザを終了
driver.quit()