-
Notifications
You must be signed in to change notification settings - Fork 0
/
data_load.py
74 lines (63 loc) · 2.3 KB
/
data_load.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import codecs
import numpy as np
def load_vocab():
vocab = ['<eos>'] #<sos>句子开始符号,<eos>句子结束符号
file = codecs.open('data/vocab', 'r', 'utf-8')
for line in file:
word = line.split()[0]
vocab.append(word)
word2idx = {word: idx for idx, word in enumerate(vocab)} # 词和词频:索引
#idx2word = {idx: word for idx, word in enumerate(vocab)} # 索引:词和词频
fileout = codecs.open('data/word2idx', 'w', 'utf-8')
for key in word2idx:
fileout.write(key + '\t'+ str(word2idx[key])+'\n')
return word2idx
def load_train():
word2idx = load_vocab()
x_list=[]
file = codecs.open('data/ptb.train.txt', 'r', 'utf-8')
for line in file:
wordlist = (line.strip() + ' '+ '<eos>').split()
for word in wordlist:
x = word2idx.get(word, 2)#2是<UNK>索引
x_list.append(x)
return x_list
def load_test():
word2idx = load_vocab()
x_list=[]
file = codecs.open('data/ptb.test.txt', 'r', 'utf-8')
for line in file:
wordlist = (line.strip() + ' '+ '<eos>').split()
for word in wordlist:
x = word2idx.get(word, 2)#2是<UNK>索引
x_list.append(x)
return x_list
def load_valid():
word2idx = load_vocab()
x_list=[]
file = codecs.open('data/ptb.valid.txt', 'r', 'utf-8')
for line in file:
wordlist = (line.strip() + ' '+ '<eos>').split()
for word in wordlist:
x = word2idx.get(word, 2)#2是<UNK>索引
x_list.append(x)
return x_list
def data_iterator(raw_data, batch_size, num_steps):
data_len = len(raw_data)
batch_len = data_len // batch_size
data = []
for i in range(batch_size):
x = raw_data[batch_len * i:batch_len * (i + 1)]
data.append(x)
epoch_size = (batch_len - 1) // num_steps
if epoch_size == 0:
raise ValueError("epoch_size == 0, decrease batch_size or num_steps")
for i in range(epoch_size):
xs = list()
ys = list()
for j in range(batch_size):
x = data[j][i * num_steps:(i + 1) * num_steps]
y = data[j][i * num_steps + 1:(i + 1) * num_steps + 1]
xs.append(x)
ys.append(y)
yield (xs, ys)