-
Notifications
You must be signed in to change notification settings - Fork 1
/
adaptive_learning.py
92 lines (83 loc) · 3.92 KB
/
adaptive_learning.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
# -*- encoding:utf-8 -*-
from __future__ import print_function
from keras.callbacks import ModelCheckpoint, EarlyStopping, CSVLogger
from util.data_load import load_data2, load_data3
from util.data_load import make_err_dataset
from util import data_process
from model.model2 import mlp2
from util.data_load import generate_imdb_model2_data
from util.util import cal_err_ratio, cal_err_ratio_only
import numpy as np
from model_use import model_use
from model.model1 import lstm_mul_model
# train model1
def model1(index):
results_flag = True
i = index
if index >= 10:
i = index % 10
model2_file = './modfile/model2file/imdb.mlp.best_model.h5'
result_file = './data/err_data/imdb_'+str(i)+'.data'
data2_path = './data/model2_data/imdb_'+str(i)+'_data.csv'
train_file = "./data/part_data_all/train_" + str(i) + ".txt"
# train model1
monitor = 'val_acc'
filepath = "./modfile/model1file/lstm.best_model_"+str(i)+".h5"
check_pointer = ModelCheckpoint(filepath=filepath, monitor=monitor, verbose=1,
save_best_only=True, save_weights_only=True)
early_stopping = EarlyStopping(patience=3)
csv_logger = CSVLogger('logs/imdb_model2_mlp_' + str(i) + '.log')
Xtrain, Xtest, ytrain, ytest = data_process.get_imdb_part_data(raw_file=train_file)
vocab_size = data_process.get_imdb_vocab_size(train_file)
model = lstm_mul_model(vocab_size=vocab_size)
model.fit(Xtrain, ytrain, batch_size=32, epochs=50, validation_data=(Xtest, ytest), verbose=1, shuffle=True,
callbacks=[check_pointer, early_stopping, csv_logger])
if results_flag:
print('Generate model2 dataset ...')
result_path = './data/model2_data/imdb_' + str(i) + '_data.csv'
model_file = './modfile/model1file/lstm.best_model_'
test_file = './data/part_data_all/test_0.txt'
generate_imdb_model2_data(model_file=model_file, result_path=result_path, test_file=test_file, count=10)
print('Load result ...')
X_test, Y_test = load_data3(data_path=data2_path)
mlp2_model = mlp2(sample_dim=X_test.shape[1], class_count=2)
mlp2_model.load_weights(filepath=model2_file)
results = mlp2_model.predict(X_test)
label = np.argmax(results, axis=1)
y_label = Y_test
make_err_dataset(result_path=result_file, label=label, x_test=X_test, y_test=y_label)
cal_err_ratio(file_name='train', label=label, y_test=y_label)
print('***** End Model1 Train *****')
# train model2
def model2(i):
results_flag = True
data_path = './data/model2_data/imdb_'+str(i)+'_data.csv'
filepath = "./modfile/model2file/imdb.mlp.best_model.h5"
print('***** Start Model2 Train *****')
print('Loading data ...')
x_train, y_train, x_test, y_test = load_data2(data_path=data_path)
print('Training MLP model ...')
monitor = 'val_acc'
check_pointer = ModelCheckpoint(filepath=filepath, monitor=monitor, verbose=1,
save_best_only=True, save_weights_only=True)
early_stopping = EarlyStopping(patience=5)
csv_logger = CSVLogger('logs/imdb_model2_mlp_'+str(i)+'.log')
mlp_model2 = mlp2(sample_dim=x_train.shape[1], class_count=2)
mlp_model2.fit(x_train, y_train, batch_size=128, epochs=100, verbose=1, shuffle=True, validation_data=(x_test, y_test),
callbacks=[check_pointer, early_stopping, csv_logger])
if results_flag:
print('Test Model2 ...')
mlp_model2.load_weights(filepath=filepath)
results = mlp_model2.predict(x_test)
label = np.argmax(results, axis=1)
y_test = np.argmax(y_test, axis=1)
cal_err_ratio_only(label=label, y_test=y_test)
print('***** End Model2 Train *****')
if __name__ == '__main__':
model2(0)
for i in range(1, 11):
print('***** ' + str(i) + ' START! *****')
model1(i)
model2(i)
model_use(i)
print('***** '+str(i)+' FINISHED! *****')