requests 抓取分页评论数据 使用pandas写入csv

import requests
from lxml import etree
import pandas as pd
import numpy as np

def comment_gain(url):
    headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36'}
    r = requests.get (url ,headers=headers )
    t = r.text
    s = etree.HTML ( t )
    x = (s.xpath ( '//*[@id="comments"]/ul[1]/li/div[2]/p/span/text()' )) 
    print(x)
    df = pd.DataFrame ( x )
    return df
if __name__ == '__main__':
    url = 'https://book.douban.com/subject/34857216/comments/hot?p={}'
    for page in range ( 1, 4 ):
       df = comment_gain(url.format(page))
       # path : 你要保存文件的地址
       df.to_csv ( r'path/duanping6.csv', encoding="utf_8_sig",mode='a' )

最后

以上就是现实外套最近收集整理的关于requests 抓取分页评论数据 使用pandas写入csv的全部内容，更多相关requests内容请搜索靠谱客的其他文章。