通过requests读取web数据，使用pandas 完成csv读写

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import requests
import json
import pandas as pd
import csv
import numpy as np
# # # 拷贝一份原文件，并添加新字段
####
# 读取1.csv
data1 = pd.read_csv(r".data1.csv")
# 增加新列
data1['DZDM'] = ''
data1['DZQC'] = ''
data1['parent_address_code'] = ''
data1['address_level'] = ''
data1['SCORES'] = ''
# 写入2.csv
写入header，不写index
data1.to_csv(r".data2.csv", encoding='utf-8', header=True, index=False)
print("已经把文件拷贝到另外一个csv！")
# # #
读取csv文件 # # #
# 读取2.csv
data2 = pd.read_csv(r".data2.csv")
# 把文件转换为str
# data2 = data2.astype(str)
# data = pd.read_table(r".data2.csv", sep=',')
# 获取文件的总行数,默认跳过第一行
rows = len(data2)
# 获取文件的总行数的另外一种方法
# rows = data.shape[0]
# 获取文件的总列数
colls = data2.shape[1]
print("新文件的总行数(不包括字段行)是：")
print(rows)
print("文件的总列数是：")
print(colls)
# 读取列名
# mycolls = pd.DataFrame(columns=data2.columns.values)
mycolls = data2.columns.values
print(mycolls)
# # #
写csv # # # #
url = 'http://ip/-svc/adress/query'
# 循环读取每一行数据
for i in range(rows):
# 读取第i行数据
# 使用loc 读取出来的是？
# rr = data.loc[i:i]
# print(rr['SFZ'].values)
# 使用iloc 整数索引,r是一个series
r = data2.iloc[i]
# 对DZ字段进行规格化
addr = r['DZ']
# 把addr 转换成字符串
addr = ''.join(addr)
# 去除头尾的空格
addr = addr.strip()
# print(r['DZ'])
# 如果addr 不为空
if len(addr) > 0 :
params = json.dumps({'access_key': 'token', 'address': addr, 'from': 0, 'size': 1})
headers = {'content-type': 'application/json'}
rr = requests.post(url, data=params, headers=headers)
res = rr.json()
if len(res['data']) > 0 :
# 记录处理到第几行
print('匹配到了这一行：')
print(i+2)
print(addr)
data2.loc[i, ['DZDM']] = res['data'][0]['address_code']
data2.loc[i, ['DZQC']] = res['data'][0]['address']
data2.loc[i, ['parent_address_code']] = res['data'][0]['parent_address_code']
data2.loc[i, ['address_level']] = res['data'][0]['address_level']
data2.loc[i, ['SCORES']] = res['data'][0]['_score']
data2.to_csv(r".data3.csv", encoding='UTF-8', header=True, index=False)