使用BeautifulSoup实现的图片爬虫

81 阅读 0 评论 54 点赞

我是靠谱客的博主紧张歌曲，这篇文章主要介绍使用BeautifulSoup实现的图片爬虫，现在分享给大家，希望可以做个参考。

主要用了BeautifulSoup获取图片链接，然后使用urllib.retrieve()这个函数把图片下载到对应的文件夹

复制代码

from
bs4 import BeautifulSoup
from urllib.request import urlopen
import urllib
import re
import os
def get_web_bsobj(target_url):
req = urllib.request.Request(target_url)
h = urllib.request.urlopen(req)
bsobj = BeautifulSoup(h, "html5lib")
print(bsobj.title)
return
bsobj
def get_file_bsobj():
h = urlopen("file:c:/mzt.html")
bsobj = BeautifulSoup(h,'html5lib')
return bsobj
def download_img(target_url,file_path):
bsobj = get_web_bsobj(target_url)
print(str(bsobj.title.text)+"解析就绪")
content = bsobj.findAll('li',{"id":re.compile("comment-[0-9]+")},recursive=True);
my_img_url=[]
for i in content:
temp = i.find_all('a',{'class':'view_img_link'})
for j in temp:
print(j['href'])
my_img_url.append(j['href'])
print("###########")
print("^^^^^^^^^^^the next^^^^^^^^^^^^^")
j=1
for i in my_img_url:
print(i)
fn = file_path+str(j)+".jpg"
urllib.request.urlretrieve(i,fn)
j=j+1
print("saved %s" %j)
def run():
print("""
#########################################
#
煎蛋网妹子图片下载器
#####
#########################################
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
运行之前请先在D盘下面新建mzt文件夹 摸摸蛋
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
""")
start_page_num = int(input("输入页码:"))
total_page_num = int(input("输入要获取的页数"))
s_url = "http://jandan.net/ooxx/page-"
file_path = "d:/mzt/"
for i in range(total_page_num):
temp_dir = file_path + str(start_page_num) + "/"
print(temp_dir)
os.mkdir(temp_dir)
url = s_url + str(start_page_num)
start_page_num += 1
print(url)
download_img(url, temp_dir)
print("##################" + url + "下载完毕" + "################")
if __name__ == '__main__':
run()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
from
bs4 import BeautifulSoup
from urllib.request import urlopen
import urllib
import re
import os
def get_web_bsobj(target_url):
req = urllib.request.Request(target_url)
h = urllib.request.urlopen(req)
bsobj = BeautifulSoup(h, "html5lib")
print(bsobj.title)
return
bsobj
def get_file_bsobj():
h = urlopen("file:c:/mzt.html")
bsobj = BeautifulSoup(h,'html5lib')
return bsobj
def download_img(target_url,file_path):
bsobj = get_web_bsobj(target_url)
print(str(bsobj.title.text)+"解析就绪")
content = bsobj.findAll('li',{"id":re.compile("comment-[0-9]+")},recursive=True);
my_img_url=[]
for i in content:
temp = i.find_all('a',{'class':'view_img_link'})
for j in temp:
print(j['href'])
my_img_url.append(j['href'])
print("###########")
print("^^^^^^^^^^^the next^^^^^^^^^^^^^")
j=1
for i in my_img_url:
print(i)
fn = file_path+str(j)+".jpg"
urllib.request.urlretrieve(i,fn)
j=j+1
print("saved %s" %j)
def run():
print("""
#########################################
#
煎蛋网妹子图片下载器
#####
#########################################
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
运行之前请先在D盘下面新建mzt文件夹 摸摸蛋
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
""")
start_page_num = int(input("输入页码:"))
total_page_num = int(input("输入要获取的页数"))
s_url = "http://jandan.net/ooxx/page-"
file_path = "d:/mzt/"
for i in range(total_page_num):
temp_dir = file_path + str(start_page_num) + "/"
print(temp_dir)
os.mkdir(temp_dir)
url = s_url + str(start_page_num)
start_page_num += 1
print(url)
download_img(url, temp_dir)
print("##################" + url + "下载完毕" + "################")
if __name__ == '__main__':
run()