1#!/usr/bin/python3
2
3from __future__ import print_function
4
5from keras.models import Sequential
6from keras.models import Model
7from keras.layers import Input
8from keras.layers import Dense
9from keras.layers import LSTM
10from keras.layers import GRU
11from keras.layers import CuDNNGRU
12from keras.layers import SimpleRNN
13from keras.layers import Dropout
14from keras import losses
15import h5py
16from keras.optimizers import Adam
17
18from keras.constraints import Constraint
19from keras import backend as K
20import numpy as np
21
22import tensorflow as tf
23from keras.backend.tensorflow_backend import set_session
24config = tf.ConfigProto()
25config.gpu_options.per_process_gpu_memory_fraction = 0.44
26set_session(tf.Session(config=config))
27
28def binary_crossentrop2(y_true, y_pred):
29    return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_true, y_pred), axis=-1)
30
31def binary_accuracy2(y_true, y_pred):
32    return K.mean(K.cast(K.equal(y_true, K.round(y_pred)), 'float32') + K.cast(K.equal(y_true, 0.5), 'float32'), axis=-1)
33
34def quant_model(model):
35    weights = model.get_weights()
36    for k in range(len(weights)):
37        weights[k] = np.maximum(-128, np.minimum(127, np.round(128*weights[k])*0.0078125))
38    model.set_weights(weights)
39
40class WeightClip(Constraint):
41    '''Clips the weights incident to each hidden unit to be inside a range
42    '''
43    def __init__(self, c=2):
44        self.c = c
45
46    def __call__(self, p):
47        return K.clip(p, -self.c, self.c)
48
49    def get_config(self):
50        return {'name': self.__class__.__name__,
51            'c': self.c}
52
53reg = 0.000001
54constraint = WeightClip(.998)
55
56print('Build model...')
57
58main_input = Input(shape=(None, 25), name='main_input')
59x = Dense(32, activation='tanh', kernel_constraint=constraint, bias_constraint=constraint)(main_input)
60#x = CuDNNGRU(24, return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x)
61x = GRU(24, recurrent_activation='sigmoid', activation='tanh', return_sequences=True, kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(x)
62x = Dense(2, activation='sigmoid', kernel_constraint=constraint, bias_constraint=constraint)(x)
63model = Model(inputs=main_input, outputs=x)
64
65batch_size = 2048
66
67print('Loading data...')
68with h5py.File('features10b.h5', 'r') as hf:
69    all_data = hf['data'][:]
70print('done.')
71
72window_size = 1500
73
74nb_sequences = len(all_data)//window_size
75print(nb_sequences, ' sequences')
76x_train = all_data[:nb_sequences*window_size, :-2]
77x_train = np.reshape(x_train, (nb_sequences, window_size, 25))
78
79y_train = np.copy(all_data[:nb_sequences*window_size, -2:])
80y_train = np.reshape(y_train, (nb_sequences, window_size, 2))
81
82print("Marking ignores")
83for s in y_train:
84    for e in s:
85        if (e[1] >= 1):
86            break
87        e[0] = 0.5
88
89all_data = 0;
90x_train = x_train.astype('float32')
91y_train = y_train.astype('float32')
92
93print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape)
94
95model.load_weights('newweights10a1b_ep206.hdf5')
96
97#weights = model.get_weights()
98#for k in range(len(weights)):
99#    weights[k] = np.round(128*weights[k])*0.0078125
100#model.set_weights(weights)
101
102# try using different optimizers and different optimizer configs
103model.compile(loss=binary_crossentrop2,
104              optimizer=Adam(0.0001),
105              metrics=[binary_accuracy2])
106
107print('Train...')
108quant_model(model)
109model.fit(x_train, y_train,
110          batch_size=batch_size,
111          epochs=10, validation_data=(x_train, y_train))
112model.save("newweights10a1c_ep10.hdf5")
113
114quant_model(model)
115model.fit(x_train, y_train,
116          batch_size=batch_size,
117          epochs=50, initial_epoch=10)
118model.save("newweights10a1c_ep50.hdf5")
119
120model.compile(loss=binary_crossentrop2,
121              optimizer=Adam(0.0001),
122              metrics=[binary_accuracy2])
123
124quant_model(model)
125model.fit(x_train, y_train,
126          batch_size=batch_size,
127          epochs=100, initial_epoch=50)
128model.save("newweights10a1c_ep100.hdf5")
129
130quant_model(model)
131model.fit(x_train, y_train,
132          batch_size=batch_size,
133          epochs=150, initial_epoch=100)
134model.save("newweights10a1c_ep150.hdf5")
135
136quant_model(model)
137model.fit(x_train, y_train,
138          batch_size=batch_size,
139          epochs=200, initial_epoch=150)
140model.save("newweights10a1c_ep200.hdf5")
141
142quant_model(model)
143model.fit(x_train, y_train,
144          batch_size=batch_size,
145          epochs=201, initial_epoch=200)
146model.save("newweights10a1c_ep201.hdf5")
147
148quant_model(model)
149model.fit(x_train, y_train,
150          batch_size=batch_size,
151          epochs=202, initial_epoch=201, validation_data=(x_train, y_train))
152model.save("newweights10a1c_ep202.hdf5")
153
154quant_model(model)
155model.fit(x_train, y_train,
156          batch_size=batch_size,
157          epochs=203, initial_epoch=202, validation_data=(x_train, y_train))
158model.save("newweights10a1c_ep203.hdf5")
159
160quant_model(model)
161model.fit(x_train, y_train,
162          batch_size=batch_size,
163          epochs=204, initial_epoch=203, validation_data=(x_train, y_train))
164model.save("newweights10a1c_ep204.hdf5")
165
166quant_model(model)
167model.fit(x_train, y_train,
168          batch_size=batch_size,
169          epochs=205, initial_epoch=204, validation_data=(x_train, y_train))
170model.save("newweights10a1c_ep205.hdf5")
171
172quant_model(model)
173model.fit(x_train, y_train,
174          batch_size=batch_size,
175          epochs=206, initial_epoch=205, validation_data=(x_train, y_train))
176model.save("newweights10a1c_ep206.hdf5")
177
178