python 爬虫爬取图片 BeautifulSoup

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#coding=utf-8
import requests
from bs4 import BeautifulSoup
import os
from multiprocessing import Pool
# http请求头
Hostreferer = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
'Referer': 'http://www.mzitu.com'
}
Picreferer = {
'User-Agent': 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
'Referer': 'http://i.meizitu.net'
}
# 此请求头破解盗链
all_url = 'http://www.mzitu.com'
# 保存地址
path = 'D:/mzitu/'
def Download(href,title):
if(os.path.exists(path+title.strip().replace('?','')) and len(os.listdir(path+title.strip().replace('?',''))) >= int(pic_max)):
print('已完毕，跳过'+title)
return 1
html = requests.get(href,headers = Hostreferer)
soup = BeautifulSoup(html.text,'html.parser')
pic_max = soup.find_all('span')
pic_max = pic_max[10].text
# 最大页数
print("开始扒取：" + title)
os.makedirs(path+title.strip().replace('?',''))
os.chdir(path + title.strip().replace('?',''))
for num in range(1,int(pic_max)+1):
pic = href+'/'+str(num)
#print(pic)
html = requests.get(pic,headers = Hostreferer)
mess = BeautifulSoup(html.text,"html.parser")
pic_url = mess.find('img',alt = title)
html = requests.get(pic_url['src'],headers = Picreferer)
file_name = pic_url['src'].split(r'/')[-1]
f = open(file_name,'wb')
f.write(html.content)
f.close()
print('完成'+title)
if __name__=='__main__':
start_html = requests.get(all_url, headers=Hostreferer)
# 找寻最大页数
soup = BeautifulSoup(start_html.text, "html.parser")
page = soup.find_all('a', class_='page-numbers')
max_page = page[-2].text
same_url = 'http://www.mzitu.com/page/'
pool = Pool(15)
for n in range(1, int(max_page) + 1):
ul = same_url + str(n)
start_html = requests.get(ul, headers=Hostreferer)
soup = BeautifulSoup(start_html.text, "html.parser")
all_a = soup.find('div', class_='postlist').find_all('a', target='_blank')
for a in all_a:
title = a.get_text()
# 提取文本
if (title != ''):
href = a['href']
pool.apply_async(Download, args=(href, title))
pool.close()
pool.join()
print('所有图片已下完')