技校新闻爬虫食用方法抓取思路分析正文一、学校新闻抓取二、党群工作新闻抓取三、技能培训新闻抓取四、教学科研新闻抓取五、 拼接数据六、保存数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

import requests
from bs4 import BeautifulSoup
import re

url = '''http://www.gdnjsxy.com/Home/Article/lists/category/3.html'''

def get_html(url):
    headers = {
   'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36'}

    response = requests.get(url=url,headers=headers)

    result_str = response.content.decode("utf-8")

    soup = BeautifulSoup(result_str,'lxml')
    return soup

soup = get_html(url)
print(soup.prettify())

1
2
3
print(soup.find_all("div",attrs={
   "class":re.compile("f-fl")}))

1
2
3
print(soup.find_all("a",attrs={
   "class":"atitle"}))

1
2
3
4
5
6
# 列表页中每篇文章标题
title_texts = [i.get_text() for i in soup.find_all("a",attrs={
   "class":"atitle"})]

title_texts

1
2
3
4
5
6
7
8
9
# 每篇文章地址
urls_texts = [i.get_text() for i in soup.find_all("a",attrs={
   "class":"atitle"})]

links = ["http://www.gdnjsxy.com{}".format(i.attrs["href"] )
         for i in  soup.find_all("a",attrs={
   "class":"atitle"})]
links

1
2
3
4
5
6
#构建学校新闻列表页地址
pagesize = 16
urls = ['http://www.gdnjsxy.com/home/article/lists/category/3/p/{}.html/'.format(i) for i in range(1,pagesize+1)]
for i in range(0,16):
    print(urls[i])

1
2
3
4
5
6
7
8

import requests
from bs4 import BeautifulSoup
import re
# 获取页面    
def get_html(url):
    headers = {