python使用bs4库爬取网站数据存储到excel

80 阅读 0 评论 53 点赞

我是靠谱客的博主背后毛巾，这篇文章主要介绍python使用bs4库爬取网站数据存储到excel，现在分享给大家，希望可以做个参考。

复制代码

from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas as pd
import re
import pprint
class Crawler(object):
def __init__(self, url):
self.url = url
def parser_page(self):
html = urlopen(self.url)
soup = BeautifulSoup(html, "html.parser")
tables = soup.find_all('div', class_='col middle-column-home')
print(tables)
for table in tables:
# 此时的table是这样的<h4>【学习 HTML】</h4>
# 方法一用正则去掉标签<h4>
# name_regx = re.compile(r">(.*)<")
# name = name_regx.findall(str(table))[0]
# 方法二用bs4自带的直接取name.text
name_infos = table.find_all('h4')
name_list = [i.text for i in name_infos]
describe_infos = table.find_all('strong')
info_list = [i.text for i in describe_infos]
table_dict = dict(zip(name_list, info_list))
# pprint.pprint(table_dict)
return table_dict
def write_excel(self):
data = self.parser_page()
with pd.ExcelWriter("test_data.xlsx") as wt:
df = pd.DataFrame(data, index=['1'])
df.to_excel(wt, sheet_name="lable", index=False)
if __name__ == '__main__':
cs = Crawler("url")
cs.parser_page()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
from urllib.request import urlopen
from bs4 import BeautifulSoup
import pandas as pd
import re
import pprint
class Crawler(object):
def __init__(self, url):
self.url = url
def parser_page(self):
html = urlopen(self.url)
soup = BeautifulSoup(html, "html.parser")
tables = soup.find_all('div', class_='col middle-column-home')
print(tables)
for table in tables:
# 此时的table是这样的<h4>【学习 HTML】</h4>
# 方法一用正则去掉标签<h4>
# name_regx = re.compile(r">(.*)<")
# name = name_regx.findall(str(table))[0]
# 方法二用bs4自带的直接取name.text
name_infos = table.find_all('h4')
name_list = [i.text for i in name_infos]
describe_infos = table.find_all('strong')
info_list = [i.text for i in describe_infos]
table_dict = dict(zip(name_list, info_list))
# pprint.pprint(table_dict)
return table_dict
def write_excel(self):
data = self.parser_page()
with pd.ExcelWriter("test_data.xlsx") as wt:
df = pd.DataFrame(data, index=['1'])
df.to_excel(wt, sheet_name="lable", index=False)
if __name__ == '__main__':
cs = Crawler("url")
cs.parser_page()