爬取100多个城市的历史天气

import pandas as pd
import numpy as np
import re
import os
import json
#import js2py
import requests
from bs4 import BeautifulSoup
from urllib.parse import urlencode
import time   #用于时间的停顿
import random 

接着取出想要爬取的城市名

file = pd.read_csv(r'C:Users信息明细表.csv').loc[:,['service_date','city','service_result']]
city_name = list(file.groupby('city').city.count().index)
city_name

output:
[‘三亚市’,
‘三门峡市’,
‘上海市’,
‘上饶市’,
‘东莞市’,
‘东营市’,
…]
由于发现2345天气的城市 不带“市”，所以去掉

city=[]
for i in city_name:
    r = re.sub('市', '', i) 
    city.append(r)

找出城市对应的ID

def get_areaid(city):
    url = 'http://tianqi.2345.com/tqpcimg/tianqiimg/theme4/js/citySelectData2.js'
    r = requests.get(url)
    print(r.encoding, r.apparent_encoding)
    r.encoding = r.apparent_encoding
    if r.status_code == 200:  # 防止url请求
        h = r.text.encode('gbk').decode('gbk')
    areaid = re.findall('{}-(d*)'.format(city), h)[-1]
    print('get areaid：',city,areaid)
    return  areaid

对雨量进行标记

def rain2num(a):
    '''雨量标记'''
    if '暴雨' in a:
        return 4
    elif '大雨' in a:
        return 3
    elif '中雨' in a:
        return 2
    elif ('小雨' in a) or ('阵雨' in a):
        return 1
    else:
        return 0

爬取天气

def get_weather(cityid):
        # 生成请求头
    headers = {
        'Accept': '*/*',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,zh;q=0.9',
        'Connection': 'keep-alive',
        'Cookie': 'widget_dz_id=54511; widget_dz_cityValues=,; timeerror=1; defaultCityID=54511; defaultCityName=%u5317%u4EAC; Hm_lvt_a3f2879f6b3620a363bec646b7a8bcdd=1516245199; Hm_lpvt_a3f2879f6b3620a363bec646b7a8bcdd=1516245199; addFavorite=clicked',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3236.0 Safari/537.36'
    }

    # 生成所有需要抓取的链接
    urls = []
    for year in range(2021, 2022):  #遍历2011年至2017年
        for month in range(11, 12):  #遍历1月份至12

            #用于2011年~2016年的链接与2017年的链接存在一点点差异，需要分支处理
                if month < 10:
                    urls.append('http://tianqi.2345.com/t/wea_history/js/%s0%s/%d_%s0%s.js' % (year, month, int(cityid), year, month))
                else:
                    urls.append('http://tianqi.2345.com/t/wea_history/js/%s%s/%d_%s%s.js' % (year, month, int(cityid), year, month))
    #         if year <= 2016:
    #             urls.append('http://tianqi.2345.com/t/wea_history/js/58362_%s%s.js' % (year, month))
    #         else:
    #             #2017年的月份中，1~9月前面都有数字0，需要分支处理
    #             if month < 10:
    #                 urls.append('http://tianqi.2345.com/t/wea_history/js/%s0%s/58362_%s0%s.js' % (year, month, year, month))
    #             else:
    #                 urls.append('http://tianqi.2345.com/t/wea_history/js/%s%s/58362_%s%s.js' % (year, month, year, month))

    # 循环并通过正则匹配获取相关数据
    info = []  #构建空列表，用于所有天气数据的存储
    for url in urls:
        seconds = random.randint(3, 6)  #每次循环，都随机生成一个3~6之间的整数
        response = requests.get(url, headers=headers).text  #发送url链接的请求，并返回响应数据
        ymd = re.findall("ymd:'(.*?)',", response)  #正则表达式获取日期数据
        high = re.findall("bWendu:'(.*?)℃',", response)  #正则表达式获取最高气温数据
        low = re.findall("yWendu:'(.*?)℃',", response)  #正则表达式获取最低气温数据
        tianqi = re.findall("tianqi:'(.*?)',", response)  #正则表达式获取天气状况数据
        fengxiang = re.findall("fengxiang:'(.*?)',", response)  #正则表达式获取风向数据
        fengli = re.findall(",fengli:'(.*?)'", response)  #正则表达式获取风力数据
        aqi = re.findall("aqi:'(.*?)',", response)  #正则表达式获取空气质量指标数据
        aqiInfo = re.findall("aqiInfo:'(.*?)',", response)  #正则表达式获取空气质量说明数据
        aqiLevel = re.findall(",aqiLevel:'(.*?)'", response)  #正则表达式获取空气质量水平数据

        # 由于2011~2015没有空气质量相关的数据，故需要分开处理
        if len(aqi) == 0:
            aqi = None
            aqiInfo = None
            aqiLevel = None
            info.append(pd.DataFrame(
                {'ymd': ymd, 'high': high, 'low': low, 'tianqi': tianqi, 'fengxiang': fengxiang, 'fengli': fengli,
                 'aqi': aqi, 'aqiInfo': aqiInfo, 'aqiLevel': aqiLevel}))
        else:
            info.append(pd.DataFrame(
                {'ymd': ymd, 'high': high, 'low': low, 'tianqi': tianqi, 'fengxiang': fengxiang, 'fengli': fengli,
                 'aqi': aqi, 'aqiInfo': aqiInfo, 'aqiLevel': aqiLevel}))
#         print(info)
        time.sleep(seconds)  #每循环一次，都随机停顿几秒

    # 将存储的所有的天气数据进行合并，生成数据表格
    weather = pd.concat(info)
    weather['ymd'] = weather['ymd'].apply(lambda x: pd.to_datetime(x))
    weather['雨量'] = weather['tianqi'].apply(rain2num)
    return weather

for i in range(54, len(city_name)):
    try:
        cityid = get_areaid(city[i])
        weather = get_weather(cityid)
    except Exception as e:
        pass
    continue

最后

以上就是火星上石头为你收集整理的爬取100多个城市的历史天气的全部内容，希望文章能够帮你解决爬取100多个城市的历史天气所遇到的程序开发问题。

如果觉得靠谱客网站的内容还不错，欢迎将靠谱客网站推荐给程序员好友。