一个Python 爬虫程序

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# -*- coding:utf-8 -*-
'''
version:Python 2.6
standard libs: urllib
author:Dead_morning
system: cetos 6.5
'''
import re
import urllib

def get_content(html_page):
'''html downladd'''
    html = urllib.urlopen(html_page)
    content = html.read()
    html.close()
    return content

def get_images(info):
'''html parser'''
    regex = r'href="//wx(.+?.(?:gif|jpg|jpeg|png))" ' # download original picture
    #使用正则表达式为了下载原图，这里可使用 soupbeautiful 模块替代正则表达式
    pat = re.compile(regex)
    image_code = map(lambda x: 'http://wx'+ x , re.findall(pat,info))
    return image_code

def Download_image():
''' image download'''
    for image_url in get_images(info):
        print image_url
        image_name = image_url.split('/')[-1]
        # 给文件命名 
        urllib.urlretrieve(image_url,image_name)

def html_pages():
''' URl list'''
#因为煎蛋网的网址比较有规律，所以就用了一个简单的List替代了从网页里解析
    b = []
    for a in range (1 ,95):
        url= 'http://jandan.net/ooxx/page-%s#comments' %a
        b.append(url)
    return b

if __name__ == '__main__':   
    for html_page in html_pages():
        info = get_content(html_page)
        print Download_image()