-
Notifications
You must be signed in to change notification settings - Fork 48
/
Copy pathBatch_gen.py
177 lines (163 loc) · 6.02 KB
/
Batch_gen.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
import multiprocessing
import numpy as np
_Dataset = None
_batch_size = None
_num_negatives = None
_num_items = None
_user_input = None
_item_input = None
_labels = None
_index = None
_num_batch = None
_batch_length = None
def shuffle(dataset, batch_choice, num_negatives): #negative sampling and shuffle the data
global _Dataset
global _batch_size
global _num_negatives
global _num_items
global _user_input
global _item_input
global _labels
global _index
global _num_batch
global _batch_length
_Dataset = dataset
_num_negatives = num_negatives
if batch_choice == 'user':
_num_items, _user_input, _item_input, _labels, _batch_length = _get_train_data_user()
_num_batch = len(_batch_length)
return _preprocess(_get_train_batch_user)
else:
batch_choices = batch_choice.split(":")
if batch_choices[0] == 'fixed':
_batch_size = int(batch_choices[1])
_num_items, _user_input, _item_input, _labels = _get_train_data_fixed()
iterations = len(_user_input)
_index = np.arange(iterations)
_num_batch = iterations / _batch_size
return _preprocess(_get_train_batch_fixed)
else:
print "invalid batch size !"
def batch_gen(batches, i):
return [(batches[r])[i] for r in range(4)]
def _preprocess(get_train_batch): #generate the masked batch list
user_input_list, num_idx_list, item_input_list, labels_list = [], [], [], []
cpu_count = multiprocessing.cpu_count()
if cpu_count == 1:
for i in range(_num_batch):
ui, ni, ii, l = get_train_batch(i)
user_input_list.append(ui)
num_idx_list.append(ni)
item_input_list.append(ii)
labels_list.append(l)
else:
pool = multiprocessing.Pool(cpu_count)
res = pool.map(get_train_batch, range(_num_batch))
pool.close()
pool.join()
user_input_list = [r[0] for r in res]
num_idx_list = [r[1] for r in res]
item_input_list = [r[2] for r in res]
labels_list = [r[3] for r in res]
return (user_input_list, num_idx_list, item_input_list, labels_list)
def _get_train_data_user():
user_input, item_input, labels, batch_length = [],[],[],[]
train = _Dataset.trainMatrix
trainList = _Dataset.trainList
num_items = train.shape[1]
num_users = train.shape[0]
for u in range(num_users):
if u == 0:
batch_length.append((1+_num_negatives) * len(trainList[u]))
else:
batch_length.append((1+_num_negatives) * len(trainList[u])+batch_length[u-1])
for i in trainList[u]:
# positive instance
user_input.append(u)
item_input.append(i)
labels.append(1)
# negative instances
for t in xrange(_num_negatives):
j = np.random.randint(num_items)
while j in trainList[u]:
j = np.random.randint(num_items)
user_input.append(u)
item_input.append(j)
labels.append(0)
return num_items, user_input, item_input, labels, batch_length
def _get_train_batch_user(i):
#represent the feature of users via items rated by him/her
user_list, num_list, item_list, labels_list = [],[],[],[]
trainList = _Dataset.trainList
if i == 0:
begin = 0
else:
begin = _batch_length[i-1]
batch_index = range(begin, _batch_length[i])
np.random.shuffle(batch_index)
for idx in batch_index:
user_idx = _user_input[idx]
item_idx = _item_input[idx]
nonzero_row = []
nonzero_row += trainList[user_idx]
num_list.append(_remove_item(_num_items, nonzero_row, item_idx))
user_list.append(nonzero_row)
item_list.append(item_idx)
labels_list.append(_labels[idx])
user_input = np.array(_add_mask(_num_items, user_list, max(num_list)))
num_idx = np.array(num_list)
item_input = np.array(item_list)
labels = np.array(labels_list)
return (user_input, num_idx, item_input, labels)
def _get_train_data_fixed():
user_input, item_input, labels = [],[],[]
train = _Dataset.trainMatrix
num_items = train.shape[1]
for (u, i) in train.keys():
# positive instance
user_items = []
user_input.append(u)
item_input.append(i)
labels.append(1)
# negative instances
for t in xrange(_num_negatives):
j = np.random.randint(num_items)
while train.has_key((u, j)):
j = np.random.randint(num_items)
user_input.append(u)
item_input.append(j)
labels.append(0)
return num_items, user_input, item_input, labels
def _get_train_batch_fixed(i):
#represent the feature of users via items rated by him/her
user_list, num_list, item_list,labels_list = [],[],[],[]
trainList = _Dataset.trainList
begin = i * _batch_size
for idx in range(begin, begin+_batch_size):
user_idx = _user_input[_index[idx]]
item_idx = _item_input[_index[idx]]
nonzero_row = []
nonzero_row += trainList[user_idx]
num_list.append(_remove_item(_num_items, nonzero_row, item_idx))
user_list.append(nonzero_row)
item_list.append(item_idx)
labels_list.append(_labels[_index[idx]])
user_input = np.array(_add_mask(_num_items, user_list, max(num_list)))
num_idx = np.array(num_list)
item_input = np.array(item_list)
labels = np.array(labels_list)
return (user_input, num_idx, item_input, labels)
def _remove_item(feature_mask, users, item):
flag = 0
for i in range(len(users)):
if users[i] == item:
users[i] = users[-1]
users[-1] = feature_mask
flag = 1
break
return len(users) - flag
def _add_mask(feature_mask, features, num_max):
#uniformalize the length of each batch
for i in xrange(len(features)):
features[i] = features[i] + [feature_mask] * (num_max+1 - len(features[i]))
return features