1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
数据情况：
不重复单词(彩票开奖记录)的个数: 988
开奖期数: 4656期
行数: 4657
平均每行单词数: 1.0

开奖记录从 0 到 10:
202
243
580
306
598
900
761
262
891
623

1
2
3
4
5
6
7
8
9
10
11
12
import numpy as np
from collections import Counter

def create_lookup_tables():
    """
    Create lookup tables for vocabulary
    :param text: The text of tv scripts split into words
    :return: A tuple of dicts (vocab_to_int, int_to_vocab)
    """
    vocab_to_int = {str(ii).zfill(3) : ii for ii in range(1000)}
    int_to_vocab = {ii : str(ii).zfill(3) for ii in range(1000)}
    return vocab_to_int, int_to_vocab

1
2
3
4
5
6
7
8
9
def get_inputs():
    """
    Create TF Placeholders for input, targets, and learning rate.
    :return: Tuple (input, targets, learning rate)
    """
    inputs = tf.placeholder(tf.int32, [None, None], name="input")
    targets = tf.placeholder(tf.int32, [None, None], name="targets")
    LearningRate = tf.placeholder(tf.float32)
    return inputs, targets, LearningRate

1
2
3
4
5
6
7
8
9
10
11
12
def get_init_cell(batch_size, rnn_size):
    """
    Create an RNN Cell and initialize it.
    :param batch_size: Size of batches
    :param rnn_size: Size of RNNs
    :return: Tuple (cell, initialize state)
    """
    lstm_cell = tf.contrib.rnn.BasicLSTMCell(rnn_size)#num_units=embed_dim
    cell = tf.contrib.rnn.MultiRNNCell([lstm_cell] * 2)
    InitialState = cell.zero_state(batch_size, tf.float32)
    InitialState = tf.identity(InitialState, name="initial_state")
    return cell, InitialState

1
2
3
4
5
6
7
8
9
10
11
def get_embed(input_data, vocab_size, embed_dim):
    """
    Create embedding for <input_data>.
    :param input_data: TF placeholder for text input.
    :param vocab_size: Number of words in vocabulary.
    :param embed_dim: Number of embedding dimensions
    :return: Tuple (Embedded input, embed_matrix)
    """
    embed_matrix = tf.Variable(tf.random_uniform([vocab_size, embed_dim], -1, 1))
    embed_layer = tf.nn.embedding_lookup(embed_matrix, input_data)
    return embed_layer, embed_matrix

1
2
3
4
5
6
7
8
9
10
def build_rnn(cell, inputs):
    """
    Create a RNN using a RNN Cell
    :param cell: RNN Cell
    :param inputs: Input text data
    :return: Tuple (Outputs, Final State)
    """
    Outputs, final_State = tf.nn.dynamic_rnn(cell, inputs, dtype=tf.float32)
    final_State = tf.identity(final_State, name="final_state")
    return Outputs, final_State

1
2
3
4
5
6
7
8
9
10
11
12
13
14
def build_nn(cell, rnn_size, input_data, vocab_size, embed_dim):
    """
    Build part of the neural network
    :param cell: RNN cell
    :param rnn_size: Size of rnns
    :param input_data: Input data
    :param vocab_size: Vocabulary size
    :param embed_dim: Number of embedding dimensions
    :return: Tuple (Logits, FinalState, embed_matrix)
    """
    embed_layer, embed_matrix = get_embed(input_data, vocab_size, embed_dim)
    Outputs, final_State = build_rnn(cell, embed_layer)
    logits = tf.layers.dense(Outputs, vocab_size)
    return logits, final_State, embed_matrix

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Number of Epochs
num_epochs = 25
# Batch Size
batch_size = 32
# RNN Size
rnn_size = 1000
# Embedding Dimension Size
embed_dim = 1000
# Sequence Length
seq_length = 1
# Learning Rate
learning_rate = 0.01
# Show stats for every n number of batches
show_every_n_batches = 10

save_dir = './save'

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
from tensorflow.contrib import seq2seq

tf.reset_default_graph()
train_graph = tf.Graph()
with train_graph.as_default():
    vocab_size = len(int_to_vocab)
    input_text, targets, lr = get_inputs()
    input_data_shape = tf.shape(input_text)
    cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)
    logits, final_state, embed_matrix = build_nn(cell, rnn_size, input_text, vocab_size, embed_dim)

    # Probabilities for generating words
    probs = tf.nn.softmax(logits, name='probs')

    # Loss function
    cost = seq2seq.sequence_loss(
        logits,
        targets,
        tf.ones([input_data_shape[0], input_data_shape[1]]))

    # We use the cosine distance:
    norm = tf.sqrt(tf.reduce_sum(tf.square(embed_matrix), 1, keep_dims=True))
    normalized_embedding = embed_matrix / norm

    probs_embeddings = tf.nn.embedding_lookup(normalized_embedding, tf.squeeze(tf.argmax(probs, 2)))#np.squeeze(probs.argmax(2))
    probs_similarity = tf.matmul(probs_embeddings, tf.transpose(normalized_embedding))

    y_embeddings = tf.nn.embedding_lookup(normalized_embedding, tf.squeeze(targets))
    y_similarity = tf.matmul(y_embeddings, tf.transpose(normalized_embedding))

    similar_loss = tf.reduce_mean(tf.abs(y_similarity - probs_similarity))
    total_loss = cost + similar_loss

    # Optimizer
    optimizer = tf.train.AdamOptimizer(lr)

    # Gradient Clipping
    gradients = optimizer.compute_gradients(total_loss)
    capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]  #clip_by_norm
    train_op = optimizer.apply_gradients(capped_gradients)

    # Accuracy
    correct_pred = tf.equal(tf.argmax(probs, 2), tf.cast(targets, tf.int64))
    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name='accuracy')

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import seaborn as sns
import matplotlib.pyplot as plt

batches = get_batches(int_text[:-(batch_size+1)], batch_size, seq_length)
test_batches = get_batches(int_text[-(batch_size+1):], batch_size, seq_length)
top_k = 10
#预测结果的Top K准确率
topk_acc_list = []
topk_acc = 0
#与预测结果距离最近的Top K准确率
sim_topk_acc_list = []
sim_topk_acc = 0

#表示k值是一个范围，不像Top K是最开始的K个
range_k = 5
#以每次训练得出的距离中位数为中心，以范围K为半径的准确率，使用预测结果向量
floating_median_idx = 0
floating_median_acc_range_k = 0
floating_median_acc_range_k_list = []

#同上，使用的是相似度向量
floating_median_sim_idx = 0
floating_median_sim_acc_range_k = 0
floating_median_sim_acc_range_k_list = []

#保存训练损失和测试损失
losses = {'train':[], 'test':[]}
#保存各类准确率
accuracies = {'accuracy':[], 'topk':[], 'sim_topk':[], 'floating_median_acc_range_k':[], 'floating_median_sim_acc_range_k':[]}

with tf.Session(graph=train_graph) as sess:
    sess.run(tf.global_variables_initializer())
    saver = tf.train.Saver()
    for epoch_i in range(num_epochs):
        state = sess.run(initial_state, {input_text: batches[0][0]})

        #训练的迭代，保存训练损失
        for batch_i, (x, y) in enumerate(batches):
            feed = {
                input_text: x,
                targets: y,
                initial_state: state,
                lr: learning_rate}
            train_loss, state, _ = sess.run([total_loss, final_state, train_op], feed)  #cost
            losses['train'].append(train_loss)

            # Show every <show_every_n_batches> batches
            if (epoch_i * len(batches) + batch_i) % show_every_n_batches == 0:
                print('Epoch {:>3} Batch {:>4}/{}   train_loss = {:.3f}'.format(
                    epoch_i,
                    batch_i,
                    len(batches),
                    train_loss))

        #使用测试数据的迭代
        acc_list = []
        prev_state = sess.run(initial_state, {input_text: np.array([[1]])})
        for batch_i, (x, y) in enumerate(test_batches):
            # Get Prediction
            test_loss, acc, probabilities, prev_state = sess.run(
                [total_loss, accuracy, probs, final_state],
                {input_text: x, 
                 targets: y,
                 initial_state: prev_state})  

            #保存测试损失和准确率
            acc_list.append(acc)
            losses['test'].append(test_loss)
            accuracies['accuracy'].append(acc)

            print('Epoch {:>3} Batch {:>4}/{}   test_loss = {:.3f}'.format(
                epoch_i,
                batch_i,
                len(test_batches),
                test_loss))

            #利用嵌入矩阵和生成的预测计算得到相似度矩阵sim
            valid_embedding = tf.nn.embedding_lookup(normalized_embedding, np.squeeze(probabilities.argmax(2)))
            similarity = tf.matmul(valid_embedding, tf.transpose(normalized_embedding))
            sim = similarity.eval()

            #保存预测结果的Top K准确率和与预测结果距离最近的Top K准确率
            topk_acc = 0
            sim_topk_acc = 0
            for ii in range(len(probabilities)):

                nearest = (-sim[ii, :]).argsort()[0:top_k]
                if y[ii] in nearest:
                    sim_topk_acc += 1

                if y[ii] in (-probabilities[ii]).argsort()[0][0:top_k]:
                    topk_acc += 1

            topk_acc = topk_acc / len(y)
            topk_acc_list.append(topk_acc)
            accuracies['topk'].append(topk_acc)

            sim_topk_acc = sim_topk_acc / len(y)
            sim_topk_acc_list.append(sim_topk_acc)
            accuracies['sim_topk'].append(sim_topk_acc)

            #计算真实值在预测值中的距离数据
            realInSim_distance_list = []
            realInPredict_distance_list = []
            for ii in range(len(probabilities)):
                sim_nearest = (-sim[ii, :]).argsort()
                idx = list(sim_nearest).index(y[ii])
                realInSim_distance_list.append(idx)

                nearest = (-probabilities[ii]).argsort()[0]
                idx = list(nearest).index(y[ii])
                realInPredict_distance_list.append(idx)

            print('真实值在预测值中的距离数据：')
            print('max distance : {}'.format(max(realInPredict_distance_list)))
            print('min distance : {}'.format(min(realInPredict_distance_list)))
            print('平均距离 : {}'.format(np.mean(realInPredict_distance_list)))
            print('距离中位数 : {}'.format(np.median(realInPredict_distance_list)))
            print('距离标准差 : {}'.format(np.std(realInPredict_distance_list)))

            print('真实值在预测值相似向量中的距离数据：')
            print('max distance : {}'.format(max(realInSim_distance_list)))
            print('min distance : {}'.format(min(realInSim_distance_list)))
            print('平均距离 : {}'.format(np.mean(realInSim_distance_list)))
            print('距离中位数 : {}'.format(np.median(realInSim_distance_list)))
            print('距离标准差 : {}'.format(np.std(realInSim_distance_list)))

            #计算以距离中位数为中心，范围K为半径的准确率
            floating_median_sim_idx = int(np.median(realInSim_distance_list))
            floating_median_sim_acc_range_k = 0

            floating_median_idx = int(np.median(realInPredict_distance_list))
            floating_median_acc_range_k = 0
            for ii in range(len(probabilities)):
                nearest_floating_median = (-probabilities[ii]).argsort()[0][floating_median_idx - range_k:floating_median_idx + range_k]
                if y[ii] in nearest_floating_median:
                    floating_median_acc_range_k += 1

                nearest_floating_median_sim = (-sim[ii, :]).argsort()[floating_median_sim_idx - range_k:floating_median_sim_idx + range_k]
                if y[ii] in nearest_floating_median_sim:
                    floating_median_sim_acc_range_k += 1

            floating_median_acc_range_k = floating_median_acc_range_k / len(y)
            floating_median_acc_range_k_list.append(floating_median_acc_range_k)
            accuracies['floating_median_acc_range_k'].append(floating_median_acc_range_k)

            floating_median_sim_acc_range_k = floating_median_sim_acc_range_k / len(y)
            floating_median_sim_acc_range_k_list.append(floating_median_sim_acc_range_k)
            accuracies['floating_median_sim_acc_range_k'].append(floating_median_sim_acc_range_k)

        print('Epoch {:>3} floating median sim range k accuracy {} '.format(epoch_i, np.mean(floating_median_sim_acc_range_k_list)))#:.3f
        print('Epoch {:>3} floating median range k accuracy {} '.format(epoch_i, np.mean(floating_median_acc_range_k_list)))#:.3f
        print('Epoch {:>3} similar top k accuracy {} '.format(epoch_i, np.mean(sim_topk_acc_list)))#:.3f
        print('Epoch {:>3} top k accuracy {} '.format(epoch_i, np.mean(topk_acc_list)))#:.3f
        print('Epoch {:>3} accuracy {} '.format(epoch_i, np.mean(acc_list)))#:.3f

    # Save Model
    saver.save(sess, save_dir) 
    print('Model Trained and Saved')
    embed_mat = sess.run(normalized_embedding)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
真实值在预测值中的距离数据：
max distance : 998
min distance : 9
平均距离 : 534.59375
距离中位数 : 489.0
距离标准差 : 299.3345598338713
真实值在预测值相似向量中的距离数据：
max distance : 988
min distance : 1
平均距离 : 505.28125
距离中位数 : 443.5
距离标准差 : 297.11700498025607
Epoch  24 floating median sim range k accuracy 0.0325 
Epoch  24 floating median range k accuracy 0.01125 
Epoch  24 similar top k accuracy 0.0275 
Epoch  24 top k accuracy 0.025 
Epoch  24 accuracy 0.0 

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
def pick_word(probabilities, sim, int_to_vocab, top_n = 5, pred_mode = 'sim'):

    vocab_size = len(int_to_vocab)

    if pred_mode == 'sim':
        p = np.squeeze(sim)
        p[np.argsort(p)[:-top_n]] = 0
        p = p / np.sum(p)
        c = np.random.choice(vocab_size, 1, p=p)[0]
        return int_to_vocab[c]
    elif pred_mode == 'median':
        p = np.squeeze(sim)
        p[np.argsort(p)[:floating_median_sim_idx - top_n]] = 0
        p[np.argsort(p)[floating_median_sim_idx + top_n:]] = 0
        p = np.abs(p) / np.sum(np.abs(p))
        c = np.random.choice(vocab_size, 1, p=p)[0]
        return int_to_vocab[c]
    elif pred_mode == 'topk':
        p = np.squeeze(probabilities)
        p[np.argsort(p)[:-top_n]] = 0
        p = p / np.sum(p)
        c = np.random.choice(vocab_size, 1, p=p)[0]
        return int_to_vocab[c]
    elif pred_mode == 'max':
        return int_to_vocab[probabilities.argmax()]

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
gen_length = 17
prime_word = '202'

loaded_graph = tf.Graph()  #loaded_graph
with tf.Session(graph=train_graph) as sess:
    # Load saved model
    loader = tf.train.import_meta_graph(load_dir + '.meta')
    loader.restore(sess, load_dir)

    # Get Tensors from loaded model
    input_text, initial_state, final_state, probs = get_tensors(train_graph)  #loaded_graph

    # Sentences generation setup
    gen_sentences = [prime_word]
    prev_state = sess.run(initial_state, {input_text: np.array([[1]])})

    # Generate sentences
    for n in range(gen_length):
        # Dynamic Input
        dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]
        dyn_seq_length = len(dyn_input[0])

        # Get Prediction
        probabilities, prev_state = sess.run(
            [probs, final_state],
            {input_text: dyn_input, initial_state: prev_state})

        valid_embedding = tf.nn.embedding_lookup(normalized_embedding, probabilities.argmax())
        valid_embedding = tf.reshape(valid_embedding, (1, len(int_to_vocab)))
        similarity = tf.matmul(valid_embedding, tf.transpose(normalized_embedding))
        sim = similarity.eval()

        pred_word = pick_word(probabilities[dyn_seq_length-1], sim, int_to_vocab, 5, 'median')

        gen_sentences.append(pred_word)

    cp_script = ' '.join(gen_sentences)
    cp_script = cp_script.replace('n ', 'n')
    cp_script = cp_script.replace('( ', '(')

    print(cp_script)

1
2
3
4
5
Epoch 24 floating median sim range k accuracy 0.0325
Epoch 24 floating median range k accuracy 0.01125
Epoch 24 similar top k accuracy 0.0275
Epoch 24 top k accuracy 0.025
Epoch 24 accuracy 0.0

1
2
3
4
5
6
7
8
int_sentences = [int(words) for words in gen_sentences]
int_sentences = int_sentences[1:]

val_data = [[103],[883],[939],[36],[435],[173],[572],[828],[509],[723],[145],[621],[535],[385],[98],[321],[427]]
plt.plot(int_sentences, label='History')
plt.plot(val_tests, label='val_data')
plt.legend()
_ = plt.ylim()