Python + ElasticSearch：有了这个超级武器，你也可以报名参加诗词大会了！

1
2
pip install elasticsearch

1
2
3
4
5
>>> from elasticsearch import Elasticsearch
>>> es = Elasticsearch()
>>> es.info()
{'name': 'LAPTOP-8507OGEN', 'cluster_name': 'elasticsearch', 'cluster_uuid': 'OwrXmbSwTk6LB-q9lFDV0w', 'version': {'number': '7.5.2', 'build_flavor': 'default', 'build_type': 'zip', 'build_hash': '8bec50e1e0ad29dad5653712cf3bb580cd1afcdf', 'build_date': '2020-01-15T12:11:52.313576Z', 'build_snapshot': False, 'lucene_version': '8.3.0', 'minimum_wire_compatibility_version': '6.8.0', 'minimum_index_compatibility_version': '6.0.0-beta1'}, 'tagline': 'You Know, for Search'}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
>>> from elasticsearch import Elasticsearch, client
>>> es = Elasticsearch()
>>> ic = client.IndicesClient(es)
>>> doc = {
        "mappings": {
            "properties": {
                "title": { # 题目 
                    "type":  "keyword" 
                },
                "epigraph": { # 词牌名 
                    "type":  "keyword" 
                },
                "dynasty": { # 朝代  
                    "type":  "keyword" 
                },
                "author": { # 作者 
                    "type":  "keyword" 
                },
                "content": { # 内容 
                    "type":  "text" 
                }
            }
        }
    }
>>> ic.create(index='poetry', body=doc)
{'acknowledged': True, 'shards_acknowledged': True, 'index': 'poetry'}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
>>> from elasticsearch import Elasticsearch
>>> es = Elasticsearch()
>>> doc = {
    'title':title,
    'dynasty':dynasty, 
    'author':author,
    'content':content
}
>>> ret = es.index(index='poetry', body=doc)
>>> print(json.dumps(ret, indent=4, separators=(',', ': '), ensure_ascii=False))
{
    "_index": "test",
    "_type": "test",
    "_id": "bp3tNnABnXfYifgM_GRI",
    "_version": 1,
    "result": "created",
    "_shards": {
        "total": 2,
        "successful": 1,
        "failed": 0
    },
    "_seq_no": 0,
    "_primary_term": 1
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
>>> ret = es.get(index='poetry', id='bp3tNnABnXfYifgM_GRI')
>>> print(json.dumps(ret, indent=4, separators=(',', ': '), ensure_ascii=False))
{
    "_index": "test",
    "_type": "test",
    "_id": "bp3tNnABnXfYifgM_GRI",
    "_version": 1,
    "_seq_no": 0,
    "_primary_term": 1,
    "found": true,
    "_source": {
        "title": "登鹳雀楼",
        "dynasty": "唐代",
        "author": "王之涣",
        "content": "白日依山尽，黄河入海流。欲穷千里目，更上一层楼。"
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
#!/usr/bin/env python
# coding:utf-8

import lxml
import requests
from bs4 import BeautifulSoup
from elasticsearch import Elasticsearch, client

def create_index():
    es = Elasticsearch()
    ic = client.IndicesClient(es)

    # 判断索引是否存在
    if not ic.exists(index="poetry"):
        # 创建索引
        doc = {
            "mappings": {
                "properties": {
                    "title": {
                        "type":  "keyword" 
                    },
                    "epigraph": {
                        "type":  "keyword" 
                    },
                    "dynasty": {
                        "type":  "keyword" 
                    },
                    "author": {
                        "type":  "keyword" 
                    },
                    "content": {
                        "type":  "text" 
                    }
                }
            }
        }

        ic.create(index='poetry', body=doc)

def get_poetry(list_url):
    es = Elasticsearch()

    # 取得列表页面
    html = requests.get(list_url).text
    soup = BeautifulSoup(html, "lxml")
    typecont = soup.find_all(attrs={"class":"typecont"})

    # 遍历列表
    for div in typecont:
        for ch in div.children:
            if ch.name == 'span':
                # 取得诗词内容
                print('get:', ch.a.text, ch.a.attrs['href'])
                html = requests.get('https://so.gushiwen.org' + ch.a.attrs['href']).text
                soup = BeautifulSoup(html, "lxml")
                cont = soup.select('.main3 .left .sons .cont')[0]

                # 标题
                title = cont.h1.text

                # 词牌
                epigraph = ""
                if '·' in title:
                    epigraph = title[:title.index('·')]

                al = cont.p.select('a')

                # 朝代
                dynasty = al[0].text

                # 作者
                author = al[1].text

                # 内容
                content = cont.select('.contson')[0].text.strip()

                # 索引数据
                doc = {
                    "title": title,
                    "epigraph": epigraph,
                    "dynasty": dynasty,
                    "author": author,
                    "content": content
                }
                # ret = es.index(index='poetry', doc_type='poetry', body=doc)
                ret = es.index(index='poetry', body=doc)
                print(ret)
def main():
    create_index()
    get_poetry('https://so.gushiwen.org/gushi/tangshi.aspx')
    get_poetry('https://so.gushiwen.org/gushi/songsan.aspx')

if __name__ == '__main__':
    main()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
>>> from elasticsearch import Elasticsearch
>>> es = Elasticsearch()
>>> ret = es.search(index='poetry')
>>> ret.keys() # 查询结果是一个字典，可以逐一查看其数据
dict_keys(['took', 'timed_out', '_shards', 'hits'])
>>> ret['hits']['total']['value'] # 这是查询到的文档总数
613
>>> for item in ret['hits']['hits']: # 这里是部分查询到的文档（ElasticSearch默认只返回前10条）
	print(item['_source']['title'], '--', item['_source']['author'])
	
早发白帝城 / 白帝下江陵 -- 李白
夜上受降城闻笛 -- 李益
贾生 -- 李商隐
隋宫 -- 李商隐
瑶池 -- 李商隐
芙蓉楼送辛渐 -- 王昌龄
闺怨 -- 王昌龄
春宫曲 -- 王昌龄
九月九日忆山东兄弟 -- 王维
凉州词 -- 王翰

1
2
3
4
5
6
7
8
9
10
11
12
13
14
>>> ret = es.search(index='poetry', body={'size':2})
>>> ret['hits']['total']['value'] # 查询总数任然是613
613
>>> for item in ret['hits']['hits']:  # 返回2个文档
	print(item['_source']['title'], '--', item['_source']['author'])
	print(item['_source']['content'])
	print()
	
早发白帝城 / 白帝下江陵 -- 李白
朝辞白帝彩云间，千里江陵一日还。两岸猿声啼不住，轻舟已过万重山。

夜上受降城闻笛 -- 李益
回乐烽前沙似雪，受降城外月如霜。(回乐烽 一作：回乐峰）不知何处吹芦管，一夜征人尽望乡。

1
2
3
4
5
>>> condition = {"query":{"match":{"author":"李白"}},"size":0}
>>> ret = es.search(index='poetry', body=condition)
>>> ret['hits']['total']['value']
37

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
>>> ret = es.search(index='poetry', body={'size':0, 'aggs': {'authors':{"terms": { "field": "author"}}}})
>>> for item in ret['aggregations']['authors']['buckets']:
	print(item['key'], item['doc_count'])
	
杜甫 39
李白 37
王维 29
李商隐 24
苏轼 16
辛弃疾 16
孟浩然 15
李清照 14
周邦彦 12
韦应物 12

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
>>> ret = es.search(index='poetry', body={'size':0, 'aggs': {'epigraphs':{"terms": { "field": "epigraph",'size':11 }}}})
>>> for item in ret['aggregations']['epigraphs']['buckets']:
	print(item['key'], item['doc_count'])
	
 273
清平乐 11
浣溪沙 9
蝶恋花 9
鹧鸪天 9
卜算子 8
西江月 8
菩萨蛮 7
贺新郎 7
长相思 7
临江仙 6

1
2
3
>>> es.index(index='poetry', doc_type='_mapping', body={"properties": {"content": {"type": "text","fielddata": True}}})
{'acknowledged': True}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
>>> ret = es.search(index='poetry', body={'size':0, 'aggs': {'content':{"terms": { "field": "content",'size':20 }}}})
>>> 
>>> for item in ret['aggregations']['content']['buckets']:
	print(item['key'], item['doc_count'])
	
一 288
人 287
不 280
风 265
山 217
无 208
月 205
花 205
天 185
来 184
春 182
时 178
云 177
日 171
上 168
何 166
水 166
夜 163
有 155
雨 146

1
2
3
4
5
6
7
8
>>> ret = es.search(index='poetry', body={"query":{"match":{"content":"江"}}, "highlight":{"fields":{"content":{}}}})
>>> ret['hits']['total']['value']
138
>>> for item in ret['hits']['hits']:
	print(item['_source']['title'], item['_source']['author'])
	print(item['highlight']['content'])
	print()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
>>> condition = {
    "query" : {
        "match" : {
            "content" : {
		"query": "江 水",
		"operator" : "and"
	    }
	    
        }
    },
    "highlight": {
        "fields" : {
            "content" : {}
        }
    }
}
>>> ret = es.search(index='poetry', body=condition)
>>> ret['hits']['total']['value']
52
>>> for item in ret['hits']['hits']:
	print(item['_source']['title'], item['_source']['author'])
	print(item['highlight']['content'])
	print()