실전 프로젝트 - 5일차

데이터 크롤링 코드(python)

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import csv

url = 'http://www.yes24.com/Main/default.aspx'
driver = webdriver.Chrome()
driver.get(url)
time.sleep(1)

# 국내 도서 클릭
driver.find_element('xpath', '//*[@id="bigbanner_domestic_area"]/h3/a').click()

#대분류 클릭(중고등)
driver.find_element('xpath', '//*[@id="mCateLi"]/li[26]/a/em').click()

#소분류 클릭 (고등학교)
driver.find_element('xpath', '//*[@id="cateSubListWrap"]/dl[2]/dt/a').click()

#문제집
driver.find_element('xpath', '//*[@id="cateSubListWrap"]/dl[1]/dt/a').click()

#csv 파일 생성
f = open(r"C:\Users\sjhj8\Desktop\crawlingPrac\중고등_고등_문제집.csv", 'w', encoding='CP949', newline='')
csvWriter = csv.writer(f)


#페이지
for i in range(1, 1346):
    driver.get(('http://www.yes24.com/24/Category/Display/001001013003003?PageNumber={0}').format(i))

    lis = driver.find_element(By.ID, 'category_layout').find_elements(By.CSS_SELECTOR, '#category_layout > ul > li')

    for li in lis:
        print('start')

        try:
            title = li.find_element(By.CSS_SELECTOR, 'div.goods_name > a:nth-child(2)').text
            price = li.find_element(By.CLASS_NAME, 'yes_b').text
            image = li.find_element(By.CSS_SELECTOR, 'div > p > span > span > a > img').get_attribute('src')
            author = li.find_element(By.CSS_SELECTOR,
                                     'div > div.goods_info > div.goods_pubGrp > span.goods_auth').text
            publish = li.find_element(By.CSS_SELECTOR, 'div > div.goods_info > div.goods_pubGrp > span.goods_pub').text
            since = li.find_element(By.CSS_SELECTOR, 'div > div.goods_info > div.goods_pubGrp > span.goods_date').text
            star = li.find_element(By.CSS_SELECTOR,
                                   'div > div.goods_info > div.goods_rating > span.gd_rating > em').text
            link = li.find_element(By.CSS_SELECTOR, 'div.goods_name > a:nth-child(2)').get_attribute('href')

        except Exception:
            print('-----------exception-----------')
            pass
        print(title, price, image, author, publish, since, star, link)
        csvWriter.writerow([title, price, image, author, publish, since, star, link])
driver.close()

정상 작동, 데이터 정상 수록

 

개선할 점 : 각 분류마다 코드를 수정해주어야함(xPath) => 상당한 노동력이 들어 갈 것으로 예상, 조금 더 간편하게 할 수 있게 개선 필요