[ABC 220911 - 17일차] 실습 (혼자 해봄)

유튜브 댓글 크롤링 및 워드클라우드 실습

라이브러리 설치

!pip install selenium
!apt-get update
!apt install chromium-chromedriver
!cp /usr/lib/chromium-browser/chromedriver /usr/bin

한글 깨짐 방지 코드

import matplotlib as mpl
import matplotlib.pyplot as plt
 
%config InlineBackend.figure_format = 'retina'
 
!apt -qq -y install fonts-nanum
 
import matplotlib.font_manager as fm
fontpath = '/usr/share/fonts/truetype/nanum/NanumBarunGothic.ttf'
font = fm.FontProperties(fname=fontpath, size=9)
plt.rc('font', family='NanumBarunGothic') 
mpl.font_manager._rebuild()

라이브러리 임포트

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from bs4 import BeautifulSoup
import time

import pandas as pd

import warnings
warnings.filterwarnings('ignore')

유튜브 댓글 크롤링

# https://youtu.be/Za3OW0TjdQw

# 웹 드라이버 셋팅
options = webdriver.ChromeOptions()
options.add_argument('--headless') # Head-less 설정
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

# 웹 드라이버
driver = webdriver.Chrome('chromedriver', options=options)

driver.get('https://youtu.be/Za3OW0TjdQw')
driver.implicitly_wait(3)

time.sleep(1.5)

driver.execute_script('window.scrollTo(0,800)')
time.sleep(3)

# 댓글 수집을위한 스크롤 내리기

# 최초 접속 시 스크롤 높이 초기화
last_height = driver.execute_script('return document.documentElement.scrollHeight')

while True :
  driver.execute_script('window.scrollTo(0, document.documentElement.scrollHeight);')
  time.sleep(1.5)

  new_height = driver.execute_script('return document.documentElement.scrollHeight')

  if new_height == last_height :
    break
  
  last_height = new_height
  time.sleep(1.5)

  try :
    driver.find_element_by_css_seletor('#dismiss-button > a').click
    time.sleep(1.5)

  except :
    pass

# 댓글 크롤링
html_source = driver.page_source
soup = BeautifulSoup(html_source, 'html.parser')

id_list = soup.select('div#header-author > h3 > #author-text > span')
comment_list = soup.select('yt-formatted-string#content-text')

id_final = []
comment_final = []

for i in range(len(comment_list)) :
  temp_id = id_list[i].text
  temp_id = temp_id.replace('\n', '').replace('\t', '').replace(' ', '').strip()
  id_final.append(temp_id) # 댓글 작성자

  temp_comment = comment_list[i].text
  temp_comment = temp_comment.replace('\n', '').replace('\t', '').replace('\r', '').strip()
  comment_final.append(temp_comment) # 댓글 내용

# DataFrame 만들기(list -> dic -> dataframe)

youtube_dic = {'아이디':id_final, '댓글내용':comment_final}
youtube_pd = pd.DataFrame(youtube_dic)

youtube_pd.head()

youtube_pd.info()

크롤링 데이터 저장

youtube_pd.to_csv('유튜브댓글_크롤링_실습_20220911.csv', encoding='utf-8-sig', index=False)

워드 클라우드 시각화

df = pd.read_csv('/content/유튜브댓글_크롤링_실습_20220911.csv')
df.info()

df.head(20)

데이터 전처리

text=''.join(li for li in df['댓글내용'].astype(str))
text

워드 클라우드 시각화

import matplotlib.pyplot as plt
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

plt.subplots(figsize=(25, 15))
wordcloud = WordCloud(background_color='white', width=1000, height=700, font_path=fontpath).generate(text)
plt.axis('off')
plt.imshow(wordcloud, interpolation='bilinear')
plt.show()