本文介绍哈工大团队提出的一种动态ReLU（Dynamic ReLU）激活函数，即自适应参数化ReLU激活函数，原本是应用在基于一维振动信号的故障诊断，能够让每个样本有自己独特的ReLU参数，在2019年5月3日投稿至IEEE Transactions on Industrial Electronics，2020年1月24日录用，2020年2月13日在IEEE官网公布。
本文在调参记录21的基础上，增加卷积核的个数，也就是增加深度神经网络的宽度，继续尝试深度残差网络 自适应参数化ReLU激活函数在Cifar10数据集上的效果。
自适应参数化ReLU激活函数的原理如下：
自适应参数化ReLU：一种动态ReLU（Dynamic ReLU）激活函数Keras程序：
代码语言：python代码运行次数：0复制#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 14 04:17:45 2020
Implemented using TensorFlow 1.0.1 and Keras 2.2.1
Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht,
Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis,
IEEE Transactions on Industrial Electronics, DOI: 10.1109/TIE.2020.2972458,
Date of Publication: 13 February 2020
@author: Minghang Zhao
"""
from __future__ import print_function
import keras
import numpy as np
from keras.datasets import cifar10
from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Minimum
from keras.layers import AveragePooling2D, Input, GlobalAveragePooling2D, Concatenate, Reshape
from keras.regularizers import l2
from keras import backend as K
from keras.models import Model
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
K.set_learning_phase(1)
# The data, split between train and test sets
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_test = x_test-np.mean(x_train)
x_train = x_train-np.mean(x_train)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)
# Schedule the learning rate, multiply 0.1 every 150 epoches
def scheduler(epoch):
if epoch % 150 == 0 and epoch != 0:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * 0.1)
print("lr changed to {}".format(lr * 0.1))
return K.get_value(model.optimizer.lr)
# An adaptively parametric rectifier linear unit (APReLU)
def aprelu(inputs):
# get the number of channels
channels = inputs.get_shape().as_list()[-1]
# get a zero feature map
zeros_input = keras.layers.subtract([inputs, inputs])
# get a feature map with only positive features
pos_input = Activation('relu')(inputs)
# get a feature map with only negative features
neg_input = Minimum()([inputs,zeros_input])
# define a network to obtain the scaling coefficients
scales_p = GlobalAveragePooling2D()(pos_input)
scales_n = GlobalAveragePooling2D()(neg_input)
scales = Concatenate()([scales_n, scales_p])
scales = Dense(channels//16, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
scales = Activation('relu')(scales)
scales = Dense(channels, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
scales = Activation('sigmoid')(scales)
scales = Reshape((1,1,channels))(scales)
# apply a paramtetric relu
neg_part = keras.layers.multiply([scales, neg_input])
return keras.layers.add([pos_input, neg_part])
# Residual Block
def residual_block(incoming, nb_blocks, out_channels, downsample=False,
downsample_strides=2):
residual = incoming
in_channels = incoming.get_shape().as_list()[-1]
for i in range(nb_blocks):
identity = residual
if not downsample:
downsample_strides = 1
residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
residual = Activation('relu')(residual)
residual = Conv2D(out_channels, 3, strides=(downsample_strides, downsample_strides),
padding='same', kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(residual)
residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
residual = Activation('relu')(residual)
residual = Conv2D(out_channels, 3, padding='same', kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(residual)
residual = aprelu(residual)
# Downsampling
if downsample_strides > 1:
identity = AveragePooling2D(pool_size=(1,1), strides=(2,2))(identity)
# Zero_padding to match channels
if in_channels != out_channels:
zeros_identity = keras.layers.subtract([identity, identity])
identity = keras.layers.concatenate([identity, zeros_identity])
in_channels = out_channels
residual = keras.layers.add([residual, identity])
return residual
# define and train a model
inputs = Input(shape=(32, 32, 3))
net = Conv2D(32, 3, padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(inputs)
net = residual_block(net, 20, 32, downsample=False)
net = residual_block(net, 1, 64, downsample=True)
net = residual_block(net, 19, 64, downsample=False)
net = residual_block(net, 1, 128, downsample=True)
net = residual_block(net, 19,128, downsample=False)
net = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(net)
net = Activation('relu')(net)
net = GlobalAveragePooling2D()(net)
outputs = Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(net)
model = Model(inputs=inputs, outputs=outputs)
sgd = optimizers.SGD(lr=0.1, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
# data augmentation
datagen = ImageDataGenerator(
# randomly rotate images in the range (deg 0 to 180)
rotation_range=30,
# Range for random zoom
zoom_range = 0.2,
# shear angle in counter-clockwise direction in degrees
shear_range = 30,
# randomly flip images
horizontal_flip=True,
# randomly shift images horizontally
width_shift_range=0.125,
# randomly shift images vertically
height_shift_range=0.125)
reduce_lr = LearningRateScheduler(scheduler)
# fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train, y_train, batch_size=100),
validation_data=(x_test, y_test), epochs=500,
verbose=1, callbacks=[reduce_lr], workers=4)
# get results
K.set_learning_phase(0)
DRSN_train_score = model.evaluate(x_train, y_train, batch_size=100, verbose=0)
print('Train loss:', DRSN_train_score[0])
print('Train accuracy:', DRSN_train_score[1])
DRSN_test_score = model.evaluate(x_test, y_test, batch_size=100, verbose=0)
print('Test loss:', DRSN_test_score[0])
print('Test accuracy:', DRSN_test_score[1])实验结果如下：
代码语言：javascript复制Using TensorFlow backend.
x_train shape: (50000, 32, 32, 3)
50000 train samples
10000 test samples
Epoch 1/500
177s 354ms/step - loss: 5.4649 - acc: 0.3787 - val_loss: 4.7056 - val_acc: 0.5364
Epoch 2/500
134s 268ms/step - loss: 4.3931 - acc: 0.5448 - val_loss: 3.8649 - val_acc: 0.6462
Epoch 3/500
134s 267ms/step - loss: 3.6521 - acc: 0.6280 - val_loss: 3.2183 - val_acc: 0.7019
Epoch 4/500
134s 267ms/step - loss: 3.0934 - acc: 0.6764 - val_loss: 2.7007 - val_acc: 0.7536
Epoch 5/500
134s 267ms/step - loss: 2.6582 - acc: 0.7114 - val_loss: 2.3136 - val_acc: 0.7880
Epoch 6/500
134s 267ms/step - loss: 2.3051 - acc: 0.7361 - val_loss: 2.0292 - val_acc: 0.7951
Epoch 7/500
134s 267ms/step - loss: 2.0207 - acc: 0.7557 - val_loss: 1.7687 - val_acc: 0.8134
Epoch 8/500
134s 267ms/step - loss: 1.7859 - acc: 0.7732 - val_loss: 1.5536 - val_acc: 0.8292
Epoch 9/500
134s 267ms/step - loss: 1.6043 - acc: 0.7845 - val_loss: 1.3916 - val_acc: 0.8385
Epoch 10/500
134s 267ms/step - loss: 1.4540 - acc: 0.7942 - val_loss: 1.2741 - val_acc: 0.8426
Epoch 11/500
134s 267ms/step - loss: 1.3272 - acc: 0.8037 - val_loss: 1.1760 - val_acc: 0.8461
Epoch 12/500
133s 267ms/step - loss: 1.2285 - acc: 0.8110 - val_loss: 1.0927 - val_acc: 0.8514
Epoch 13/500
133s 267ms/step - loss: 1.1442 - acc: 0.8174 - val_loss: 0.9914 - val_acc: 0.8639
Epoch 14/500
133s 267ms/step - loss: 1.0733 - acc: 0.8252 - val_loss: 0.9532 - val_acc: 0.8634
Epoch 15/500
133s 267ms/step - loss: 1.0186 - acc: 0.8297 - val_loss: 0.9055 - val_acc: 0.8638
Epoch 16/500
133s 267ms/step - loss: 0.9759 - acc: 0.8329 - val_loss: 0.8713 - val_acc: 0.8664
Epoch 17/500
133s 267ms/step - loss: 0.9408 - acc: 0.8357 - val_loss: 0.8173 - val_acc: 0.8798
Epoch 18/500
133s 267ms/step - loss: 0.9071 - acc: 0.8412 - val_loss: 0.8102 - val_acc: 0.8730
Epoch 19/500
133s 267ms/step - loss: 0.8804 - acc: 0.8437 - val_loss: 0.7703 - val_acc: 0.8819
Epoch 20/500
133s 267ms/step - loss: 0.8613 - acc: 0.8478 - val_loss: 0.7809 - val_acc: 0.8739
Epoch 21/500
133s 267ms/step - loss: 0.8408 - acc: 0.8498 - val_loss: 0.7378 - val_acc: 0.8870
Epoch 22/500
133s 267ms/step - loss: 0.8260 - acc: 0.8515 - val_loss: 0.7501 - val_acc: 0.8812
Epoch 23/500
133s 267ms/step - loss: 0.8093 - acc: 0.8561 - val_loss: 0.7612 - val_acc: 0.8719
Epoch 24/500
134s 267ms/step - loss: 0.8009 - acc: 0.8579 - val_loss: 0.7348 - val_acc: 0.8814
Epoch 25/500
133s 267ms/step - loss: 0.7908 - acc: 0.8585 - val_loss: 0.7542 - val_acc: 0.8741
Epoch 26/500
133s 266ms/step - loss: 0.7787 - acc: 0.8636 - val_loss: 0.7407 - val_acc: 0.8771
Epoch 27/500
133s 266ms/step - loss: 0.7765 - acc: 0.8622 - val_loss: 0.6996 - val_acc: 0.8934
Epoch 28/500
133s 266ms/step - loss: 0.7662 - acc: 0.8658 - val_loss: 0.7118 - val_acc: 0.8855
Epoch 29/500
133s 266ms/step - loss: 0.7623 - acc: 0.8661 - val_loss: 0.7267 - val_acc: 0.8808
Epoch 30/500
133s 266ms/step - loss: 0.7654 - acc: 0.8652 - val_loss: 0.7112 - val_acc: 0.8846
Epoch 31/500
133s 266ms/step - loss: 0.7575 - acc: 0.8675 - val_loss: 0.6885 - val_acc: 0.8944
Epoch 32/500
133s 267ms/step - loss: 0.7513 - acc: 0.8691 - val_loss: 0.6925 - val_acc: 0.8930
Epoch 33/500
133s 267ms/step - loss: 0.7455 - acc: 0.8724 - val_loss: 0.6935 - val_acc: 0.8910
Epoch 34/500
133s 267ms/step - loss: 0.7411 - acc: 0.8722 - val_loss: 0.6856 - val_acc: 0.8938
Epoch 35/500
133s 267ms/step - loss: 0.7418 - acc: 0.8729 - val_loss: 0.7001 - val_acc: 0.8881
Epoch 36/500
133s 267ms/step - loss: 0.7354 - acc: 0.8739 - val_loss: 0.6869 - val_acc: 0.8895
Epoch 37/500
133s 266ms/step - loss: 0.7337 - acc: 0.8767 - val_loss: 0.6840 - val_acc: 0.8962
Epoch 38/500
133s 266ms/step - loss: 0.7360 - acc: 0.8765 - val_loss: 0.6967 - val_acc: 0.8914
Epoch 39/500
133s 267ms/step - loss: 0.7316 - acc: 0.8780 - val_loss: 0.6687 - val_acc: 0.8993
Epoch 40/500
133s 267ms/step - loss: 0.7253 - acc: 0.8811 - val_loss: 0.6886 - val_acc: 0.8949
Epoch 41/500
133s 267ms/step - loss: 0.7240 - acc: 0.8809 - val_loss: 0.7086 - val_acc: 0.8887
Epoch 42/500
133s 267ms/step - loss: 0.7247 - acc: 0.8802 - val_loss: 0.6879 - val_acc: 0.8944
Epoch 43/500
133s 267ms/step - loss: 0.7266 - acc: 0.8794 - val_loss: 0.6762 - val_acc: 0.9009
Epoch 44/500
133s 266ms/step - loss: 0.7206 - acc: 0.8820 - val_loss: 0.7067 - val_acc: 0.8874
Epoch 45/500
133s 266ms/step - loss: 0.7233 - acc: 0.8823 - val_loss: 0.6840 - val_acc: 0.8944
Epoch 46/500
133s 266ms/step - loss: 0.7163 - acc: 0.8839 - val_loss: 0.6924 - val_acc: 0.8926
Epoch 47/500
133s 266ms/step - loss: 0.7189 - acc: 0.8842 - val_loss: 0.6761 - val_acc: 0.8982
Epoch 48/500
133s 266ms/step - loss: 0.7137 - acc: 0.8841 - val_loss: 0.7079 - val_acc: 0.8931
Epoch 49/500
133s 266ms/step - loss: 0.7139 - acc: 0.8851 - val_loss: 0.6882 - val_acc: 0.8954
Epoch 50/500
133s 266ms/step - loss: 0.7129 - acc: 0.8859 - val_loss: 0.6681 - val_acc: 0.9011
Epoch 51/500
133s 266ms/step - loss: 0.7157 - acc: 0.8838 - val_loss: 0.6726 - val_acc: 0.9000
Epoch 52/500
133s 266ms/step - loss: 0.7108 - acc: 0.8858 - val_loss: 0.6720 - val_acc: 0.9002
Epoch 53/500
133s 266ms/step - loss: 0.7137 - acc: 0.8866 - val_loss: 0.6790 - val_acc: 0.8982
Epoch 54/500
133s 266ms/step - loss: 0.7151 - acc: 0.8859 - val_loss: 0.6823 - val_acc: 0.8998
Epoch 55/500
133s 266ms/step - loss: 0.7139 - acc: 0.8870 - val_loss: 0.7120 - val_acc: 0.8894
Epoch 56/500
133s 266ms/step - loss: 0.7093 - acc: 0.8884 - val_loss: 0.6790 - val_acc: 0.9013
Epoch 57/500
133s 266ms/step - loss: 0.7113 - acc: 0.8880 - val_loss: 0.6772 - val_acc: 0.9038
Epoch 58/500
133s 266ms/step - loss: 0.7042 - acc: 0.8908 - val_loss: 0.6758 - val_acc: 0.9042
Epoch 59/500
133s 266ms/step - loss: 0.7107 - acc: 0.8881 - val_loss: 0.6771 - val_acc: 0.9001
Epoch 60/500
133s 266ms/step - loss: 0.7082 - acc: 0.8878 - val_loss: 0.6848 - val_acc: 0.8998
Epoch 61/500
133s 266ms/step - loss: 0.7039 - acc: 0.8920 - val_loss: 0.6842 - val_acc: 0.9002
Epoch 62/500
133s 266ms/step - loss: 0.7049 - acc: 0.8908 - val_loss: 0.6577 - val_acc: 0.9076
Epoch 63/500
133s 265ms/step - loss: 0.7005 - acc: 0.8914 - val_loss: 0.6904 - val_acc: 0.8962
Epoch 64/500
133s 266ms/step - loss: 0.7042 - acc: 0.8916 - val_loss: 0.7025 - val_acc: 0.8910
Epoch 65/500
133s 266ms/step - loss: 0.7037 - acc: 0.8904 - val_loss: 0.6811 - val_acc: 0.9038
Epoch 66/500
133s 266ms/step - loss: 0.7085 - acc: 0.8908 - val_loss: 0.7166 - val_acc: 0.8915
Epoch 67/500
133s 265ms/step - loss: 0.6981 - acc: 0.8939 - val_loss: 0.6934 - val_acc: 0.8978
Epoch 68/500
133s 266ms/step - loss: 0.7087 - acc: 0.8917 - val_loss: 0.6868 - val_acc: 0.9026
Epoch 69/500
133s 266ms/step - loss: 0.6994 - acc: 0.8932 - val_loss: 0.6792 - val_acc: 0.9016
Epoch 70/500
133s 266ms/step - loss: 0.7040 - acc: 0.8931 - val_loss: 0.6695 - val_acc: 0.9042
Epoch 71/500
133s 266ms/step - loss: 0.7022 - acc: 0.8933 - val_loss: 0.6771 - val_acc: 0.9039
Epoch 72/500
133s 266ms/step - loss: 0.6975 - acc: 0.8954 - val_loss: 0.6789 - val_acc: 0.9043
Epoch 73/500
133s 266ms/step - loss: 0.6935 - acc: 0.8953 - val_loss: 0.6664 - val_acc: 0.9070
Epoch 74/500
133s 266ms/step - loss: 0.6956 - acc: 0.8943 - val_loss: 0.6633 - val_acc: 0.9124
Epoch 75/500
133s 266ms/step - loss: 0.6966 - acc: 0.8934 - val_loss: 0.6719 - val_acc: 0.9057
Epoch 76/500
133s 266ms/step - loss: 0.7008 - acc: 0.8942 - val_loss: 0.6872 - val_acc: 0.8993
Epoch 77/500
133s 266ms/step - loss: 0.6923 - acc: 0.8950 - val_loss: 0.6961 - val_acc: 0.9007
Epoch 78/500
133s 266ms/step - loss: 0.6966 - acc: 0.8951 - val_loss: 0.6771 - val_acc: 0.9010
Epoch 79/500
133s 266ms/step - loss: 0.6988 - acc: 0.8952 - val_loss: 0.6752 - val_acc: 0.9046
Epoch 80/500
133s 266ms/step - loss: 0.6946 - acc: 0.8970 - val_loss: 0.6716 - val_acc: 0.9073
Epoch 81/500
133s 266ms/step - loss: 0.6979 - acc: 0.8950 - val_loss: 0.6785 - val_acc: 0.9049
Epoch 82/500
133s 266ms/step - loss: 0.6956 - acc: 0.8968 - val_loss: 0.6916 - val_acc: 0.8987
Epoch 83/500
133s 266ms/step - loss: 0.6946 - acc: 0.8964 - val_loss: 0.6816 - val_acc: 0.9054
Epoch 84/500
133s 266ms/step - loss: 0.6921 - acc: 0.8972 - val_loss: 0.6834 - val_acc: 0.9044
Epoch 85/500
133s 265ms/step - loss: 0.6909 - acc: 0.8983 - val_loss: 0.6983 - val_acc: 0.8966
Epoch 86/500
133s 266ms/step - loss: 0.6991 - acc: 0.8959 - val_loss: 0.6677 - val_acc: 0.9096
Epoch 87/500
133s 266ms/step - loss: 0.6932 - acc: 0.8996 - val_loss: 0.6768 - val_acc: 0.9078
Epoch 88/500
133s 266ms/step - loss: 0.6961 - acc: 0.8974 - val_loss: 0.6895 - val_acc: 0.9016
Epoch 89/500
133s 266ms/step - loss: 0.6919 - acc: 0.9001 - val_loss: 0.6846 - val_acc: 0.9060
Epoch 90/500
133s 267ms/step - loss: 0.6937 - acc: 0.8986 - val_loss: 0.6677 - val_acc: 0.9106
Epoch 91/500
134s 268ms/step - loss: 0.6880 - acc: 0.9007 - val_loss: 0.6800 - val_acc: 0.9038
Epoch 92/500
134s 268ms/step - loss: 0.6910 - acc: 0.8982 - val_loss: 0.6843 - val_acc: 0.9035
Epoch 93/500
134s 268ms/step - loss: 0.6888 - acc: 0.8995 - val_loss: 0.7000 - val_acc: 0.8988
Epoch 94/500
134s 268ms/step - loss: 0.6865 - acc: 0.8998 - val_loss: 0.6852 - val_acc: 0.9047
Epoch 95/500
134s 268ms/step - loss: 0.6970 - acc: 0.8963 - val_loss: 0.7136 - val_acc: 0.8964
Epoch 96/500
134s 268ms/step - loss: 0.6883 - acc: 0.9005 - val_loss: 0.6620 - val_acc: 0.9128
Epoch 97/500
134s 268ms/step - loss: 0.6923 - acc: 0.8986 - val_loss: 0.6725 - val_acc: 0.9088
Epoch 98/500
134s 268ms/step - loss: 0.6889 - acc: 0.9005 - val_loss: 0.6813 - val_acc: 0.9058
Epoch 99/500
134s 268ms/step - loss: 0.6915 - acc: 0.8992 - val_loss: 0.6781 - val_acc: 0.9048
Epoch 100/500
134s 268ms/step - loss: 0.6876 - acc: 0.9011 - val_loss: 0.6740 - val_acc: 0.9062
Epoch 101/500
134s 268ms/step - loss: 0.6886 - acc: 0.9015 - val_loss: 0.6744 - val_acc: 0.9074
Epoch 102/500
134s 268ms/step - loss: 0.6904 - acc: 0.8995 - val_loss: 0.6853 - val_acc: 0.9028
Epoch 103/500
134s 268ms/step - loss: 0.6860 - acc: 0.9018 - val_loss: 0.6714 - val_acc: 0.9111
Epoch 104/500
134s 268ms/step - loss: 0.6921 - acc: 0.8997 - val_loss: 0.6827 - val_acc: 0.9026
Epoch 105/500
134s 268ms/step - loss: 0.6849 - acc: 0.9013 - val_loss: 0.7103 - val_acc: 0.8968
Epoch 106/500
134s 268ms/step - loss: 0.6896 - acc: 0.9010 - val_loss: 0.6898 - val_acc: 0.9017
Epoch 107/500
134s 268ms/step - loss: 0.6874 - acc: 0.9018 - val_loss: 0.6835 - val_acc: 0.9019
Epoch 108/500
134s 268ms/step - loss: 0.6879 - acc: 0.9017 - val_loss: 0.6864 - val_acc: 0.9057
Epoch 109/500
134s 268ms/step - loss: 0.6900 - acc: 0.9024 - val_loss: 0.6853 - val_acc: 0.9032
Epoch 110/500
134s 268ms/step - loss: 0.6811 - acc: 0.9038 - val_loss: 0.6793 - val_acc: 0.9062
Epoch 111/500
134s 268ms/step - loss: 0.6848 - acc: 0.9011 - val_loss: 0.6824 - val_acc: 0.9069
Epoch 112/500
134s 268ms/step - loss: 0.6864 - acc: 0.9025 - val_loss: 0.6786 - val_acc: 0.9075
Epoch 113/500
134s 268ms/step - loss: 0.6831 - acc: 0.9023 - val_loss: 0.6813 - val_acc: 0.9018
Epoch 114/500
134s 268ms/step - loss: 0.6832 - acc: 0.9033 - val_loss: 0.6756 - val_acc: 0.9078
Epoch 115/500
134s 268ms/step - loss: 0.6813 - acc: 0.9049 - val_loss: 0.6847 - val_acc: 0.9030
Epoch 116/500
134s 268ms/step - loss: 0.6899 - acc: 0.8999 - val_loss: 0.6872 - val_acc: 0.9067
Epoch 117/500
134s 268ms/step - loss: 0.6816 - acc: 0.9038 - val_loss: 0.6873 - val_acc: 0.9084
Epoch 118/500
134s 268ms/step - loss: 0.6832 - acc: 0.9025 - val_loss: 0.6646 - val_acc: 0.9142
Epoch 119/500
134s 268ms/step - loss: 0.6754 - acc: 0.9053 - val_loss: 0.6790 - val_acc: 0.9053
Epoch 120/500
134s 268ms/step - loss: 0.6800 - acc: 0.9050 - val_loss: 0.6888 - val_acc: 0.9062
Epoch 121/500
134s 268ms/step - loss: 0.6821 - acc: 0.9043 - val_loss: 0.6804 - val_acc: 0.9076
Epoch 122/500
134s 268ms/step - loss: 0.6821 - acc: 0.9047 - val_loss: 0.6873 - val_acc: 0.9074
Epoch 123/500
134s 268ms/step - loss: 0.6862 - acc: 0.9017 - val_loss: 0.6817 - val_acc: 0.9061
Epoch 124/500
134s 267ms/step - loss: 0.6827 - acc: 0.9034 - val_loss: 0.6852 - val_acc: 0.9070
Epoch 125/500
133s 266ms/step - loss: 0.6801 - acc: 0.9050 - val_loss: 0.6793 - val_acc: 0.9080
Epoch 126/500
133s 266ms/step - loss: 0.6857 - acc: 0.9038 - val_loss: 0.6788 - val_acc: 0.9059
Epoch 127/500
133s 266ms/step - loss: 0.6817 - acc: 0.9042 - val_loss: 0.6804 - val_acc: 0.9065
Epoch 128/500
133s 266ms/step - loss: 0.6851 - acc: 0.9036 - val_loss: 0.7013 - val_acc: 0.9027
Epoch 129/500
133s 266ms/step - loss: 0.6850 - acc: 0.9024 - val_loss: 0.6965 - val_acc: 0.9042
Epoch 130/500
133s 266ms/step - loss: 0.6846 - acc: 0.9050 - val_loss: 0.6797 - val_acc: 0.9104
Epoch 131/500
133s 266ms/step - loss: 0.6814 - acc: 0.9058 - val_loss: 0.6740 - val_acc: 0.9107
Epoch 132/500
133s 266ms/step - loss: 0.6835 - acc: 0.9044 - val_loss: 0.7089 - val_acc: 0.8962
Epoch 133/500
133s 266ms/step - loss: 0.6808 - acc: 0.9066 - val_loss: 0.6767 - val_acc: 0.9105
Epoch 134/500
133s 265ms/step - loss: 0.6847 - acc: 0.9035 - val_loss: 0.6932 - val_acc: 0.9055
Epoch 135/500
133s 266ms/step - loss: 0.6832 - acc: 0.9058 - val_loss: 0.6916 - val_acc: 0.9058
Epoch 136/500
133s 265ms/step - loss: 0.6801 - acc: 0.9041 - val_loss: 0.6851 - val_acc: 0.9073
Epoch 137/500
133s 266ms/step - loss: 0.6809 - acc: 0.9056 - val_loss: 0.6726 - val_acc: 0.9108
Epoch 138/500
133s 266ms/step - loss: 0.6813 - acc: 0.9053 - val_loss: 0.6590 - val_acc: 0.9143
Epoch 139/500
133s 266ms/step - loss: 0.6814 - acc: 0.9057 - val_loss: 0.6746 - val_acc: 0.9085
Epoch 140/500
133s 266ms/step - loss: 0.6804 - acc: 0.9060 - val_loss: 0.6839 - val_acc: 0.9068
Epoch 141/500
133s 266ms/step - loss: 0.6810 - acc: 0.9065 - val_loss: 0.7121 - val_acc: 0.8978
Epoch 142/500
133s 266ms/step - loss: 0.6831 - acc: 0.9054 - val_loss: 0.6893 - val_acc: 0.9067
Epoch 143/500
133s 266ms/step - loss: 0.6785 - acc: 0.9069 - val_loss: 0.6754 - val_acc: 0.9105
Epoch 144/500
133s 266ms/step - loss: 0.6810 - acc: 0.9049 - val_loss: 0.6889 - val_acc: 0.9064
Epoch 145/500
133s 266ms/step - loss: 0.6807 - acc: 0.9074 - val_loss: 0.7067 - val_acc: 0.9023
Epoch 146/500
133s 266ms/step - loss: 0.6845 - acc: 0.9057 - val_loss: 0.6855 - val_acc: 0.9055
Epoch 147/500
133s 267ms/step - loss: 0.6779 - acc: 0.9055 - val_loss: 0.6928 - val_acc: 0.9040
Epoch 148/500
134s 269ms/step - loss: 0.6781 - acc: 0.9069 - val_loss: 0.6760 - val_acc: 0.9086
Epoch 149/500
133s 267ms/step - loss: 0.6834 - acc: 0.9064 - val_loss: 0.6991 - val_acc: 0.9012
Epoch 150/500
135s 270ms/step - loss: 0.6809 - acc: 0.9071 - val_loss: 0.6887 - val_acc: 0.9069
Epoch 151/500
lr changed to 0.010000000149011612
133s 267ms/step - loss: 0.5790 - acc: 0.9415 - val_loss: 0.5901 - val_acc: 0.9381
Epoch 152/500
134s 267ms/step - loss: 0.5211 - acc: 0.9595 - val_loss: 0.5735 - val_acc: 0.9413
Epoch 153/500
133s 267ms/step - loss: 0.4983 - acc: 0.9645 - val_loss: 0.5626 - val_acc: 0.9440
Epoch 154/500
134s 267ms/step - loss: 0.4793 - acc: 0.9686 - val_loss: 0.5532 - val_acc: 0.9434
Epoch 155/500
135s 269ms/step - loss: 0.4689 - acc: 0.9709 - val_loss: 0.5510 - val_acc: 0.9434
Epoch 156/500
133s 267ms/step - loss: 0.4579 - acc: 0.9716 - val_loss: 0.5398 - val_acc: 0.9444
Epoch 157/500
133s 266ms/step - loss: 0.4461 - acc: 0.9739 - val_loss: 0.5347 - val_acc: 0.9459
Epoch 158/500
133s 266ms/step - loss: 0.4325 - acc: 0.9769 - val_loss: 0.5237 - val_acc: 0.9461
Epoch 159/500
133s 266ms/step - loss: 0.4263 - acc: 0.9767 - val_loss: 0.5284 - val_acc: 0.9435
Epoch 160/500
133s 266ms/step - loss: 0.4159 - acc: 0.9773 - val_loss: 0.5137 - val_acc: 0.9458
Epoch 161/500
133s 266ms/step - loss: 0.4084 - acc: 0.9782 - val_loss: 0.5121 - val_acc: 0.9457
Epoch 162/500
133s 266ms/step - loss: 0.4002 - acc: 0.9792 - val_loss: 0.5061 - val_acc: 0.9463
Epoch 163/500
133s 266ms/step - loss: 0.3892 - acc: 0.9812 - val_loss: 0.5056 - val_acc: 0.9454
Epoch 164/500
133s 266ms/step - loss: 0.3828 - acc: 0.9816 - val_loss: 0.5098 - val_acc: 0.9438
Epoch 165/500
133s 266ms/step - loss: 0.3795 - acc: 0.9811 - val_loss: 0.4993 - val_acc: 0.9436
Epoch 166/500
133s 266ms/step - loss: 0.3708 - acc: 0.9829 - val_loss: 0.4963 - val_acc: 0.9439
Epoch 167/500
133s 266ms/step - loss: 0.3640 - acc: 0.9835 - val_loss: 0.4935 - val_acc: 0.9428
Epoch 168/500
133s 266ms/step - loss: 0.3581 - acc: 0.9835 - val_loss: 0.4856 - val_acc: 0.9440
Epoch 169/500
133s 266ms/step - loss: 0.3534 - acc: 0.9836 - val_loss: 0.4830 - val_acc: 0.9441
Epoch 170/500
133s 266ms/step - loss: 0.3478 - acc: 0.9841 - val_loss: 0.4819 - val_acc: 0.9452
Epoch 171/500
133s 266ms/step - loss: 0.3438 - acc: 0.9836 - val_loss: 0.4810 - val_acc: 0.9432
Epoch 172/500
133s 266ms/step - loss: 0.3365 - acc: 0.9847 - val_loss: 0.4694 - val_acc: 0.9430
Epoch 173/500
133s 266ms/step - loss: 0.3307 - acc: 0.9859 - val_loss: 0.4657 - val_acc: 0.9454
Epoch 174/500
133s 266ms/step - loss: 0.3266 - acc: 0.9849 - val_loss: 0.4566 - val_acc: 0.9474
Epoch 175/500
133s 265ms/step - loss: 0.3199 - acc: 0.9860 - val_loss: 0.4570 - val_acc: 0.9442
Epoch 176/500
133s 266ms/step - loss: 0.3156 - acc: 0.9863 - val_loss: 0.4640 - val_acc: 0.9426
Epoch 177/500
133s 266ms/step - loss: 0.3134 - acc: 0.9863 - val_loss: 0.4648 - val_acc: 0.9405
Epoch 178/500
133s 266ms/step - loss: 0.3089 - acc: 0.9856 - val_loss: 0.4527 - val_acc: 0.9450
Epoch 179/500
134s 267ms/step - loss: 0.3021 - acc: 0.9871 - val_loss: 0.4506 - val_acc: 0.9429
Epoch 180/500
133s 266ms/step - loss: 0.2990 - acc: 0.9868 - val_loss: 0.4441 - val_acc: 0.9460
Epoch 181/500
133s 267ms/step - loss: 0.2960 - acc: 0.9870 - val_loss: 0.4532 - val_acc: 0.9413
Epoch 182/500
133s 266ms/step - loss: 0.2916 - acc: 0.9874 - val_loss: 0.4430 - val_acc: 0.9435
Epoch 183/500
133s 266ms/step - loss: 0.2905 - acc: 0.9860 - val_loss: 0.4414 - val_acc: 0.9445
Epoch 184/500
133s 266ms/step - loss: 0.2851 - acc: 0.9875 - val_loss: 0.4303 - val_acc: 0.9460
Epoch 185/500
133s 266ms/step - loss: 0.2804 - acc: 0.9873 - val_loss: 0.4317 - val_acc: 0.9441
Epoch 186/500
133s 266ms/step - loss: 0.2789 - acc: 0.9872 - val_loss: 0.4294 - val_acc: 0.9449
Epoch 187/500
133s 266ms/step - loss: 0.2748 - acc: 0.9874 - val_loss: 0.4248 - val_acc: 0.9428
Epoch 188/500
133s 266ms/step - loss: 0.2700 - acc: 0.9882 - val_loss: 0.4242 - val_acc: 0.9461
Epoch 189/500
133s 266ms/step - loss: 0.2690 - acc: 0.9875 - val_loss: 0.4235 - val_acc: 0.9402
Epoch 190/500
133s 266ms/step - loss: 0.2669 - acc: 0.9868 - val_loss: 0.4353 - val_acc: 0.9404
Epoch 191/500
133s 266ms/step - loss: 0.2636 - acc: 0.9874 - val_loss: 0.4182 - val_acc: 0.9432
Epoch 192/500
133s 266ms/step - loss: 0.2595 - acc: 0.9882 - val_loss: 0.4160 - val_acc: 0.9439
Epoch 193/500
133s 266ms/step - loss: 0.2579 - acc: 0.9879 - val_loss: 0.4147 - val_acc: 0.9445
Epoch 194/500
133s 266ms/step - loss: 0.2556 - acc: 0.9876 - val_loss: 0.4161 - val_acc: 0.9428
Epoch 195/500
133s 266ms/step - loss: 0.2530 - acc: 0.9875 - val_loss: 0.4120 - val_acc: 0.9420
Epoch 196/500
133s 266ms/step - loss: 0.2525 - acc: 0.9873 - val_loss: 0.4121 - val_acc: 0.9420
Epoch 197/500
133s 266ms/step - loss: 0.2492 - acc: 0.9877 - val_loss: 0.4260 - val_acc: 0.9390
Epoch 198/500
133s 266ms/step - loss: 0.2488 - acc: 0.9867 - val_loss: 0.4126 - val_acc: 0.9424
Epoch 199/500
133s 266ms/step - loss: 0.2458 - acc: 0.9869 - val_loss: 0.3974 - val_acc: 0.9442
Epoch 200/500
133s 266ms/step - loss: 0.2418 - acc: 0.9881 - val_loss: 0.3993 - val_acc: 0.9431
Epoch 201/500
133s 266ms/step - loss: 0.2390 - acc: 0.9878 - val_loss: 0.3998 - val_acc: 0.9448
Epoch 202/500
133s 266ms/step - loss: 0.2387 - acc: 0.9871 - val_loss: 0.3926 - val_acc: 0.9451
Epoch 203/500
133s 266ms/step - loss: 0.2354 - acc: 0.9879 - val_loss: 0.4007 - val_acc: 0.9415
Epoch 204/500
133s 266ms/step - loss: 0.2342 - acc: 0.9878 - val_loss: 0.3996 - val_acc: 0.9410
Epoch 205/500
133s 266ms/step - loss: 0.2376 - acc: 0.9860 - val_loss: 0.3943 - val_acc: 0.9423
Epoch 206/500
133s 266ms/step - loss: 0.2315 - acc: 0.9869 - val_loss: 0.3860 - val_acc: 0.9441
Epoch 207/500
133s 266ms/step - loss: 0.2326 - acc: 0.9865 - val_loss: 0.3913 - val_acc: 0.9417
Epoch 208/500
133s 266ms/step - loss: 0.2282 - acc: 0.9875 - val_loss: 0.3957 - val_acc: 0.9422
Epoch 209/500
133s 266ms/step - loss: 0.2272 - acc: 0.9870 - val_loss: 0.4006 - val_acc: 0.9397
Epoch 210/500
133s 266ms/step - loss: 0.2279 - acc: 0.9867 - val_loss: 0.3860 - val_acc: 0.9437
Epoch 211/500
133s 266ms/step - loss: 0.2283 - acc: 0.9856 - val_loss: 0.3855 - val_acc: 0.9427
Epoch 212/500
133s 266ms/step - loss: 0.2238 - acc: 0.9870 - val_loss: 0.3897 - val_acc: 0.9409
Epoch 213/500
133s 266ms/step - loss: 0.2246 - acc: 0.9864 - val_loss: 0.3808 - val_acc: 0.9425
Epoch 214/500
133s 266ms/step - loss: 0.2247 - acc: 0.9861 - val_loss: 0.4011 - val_acc: 0.9375
Epoch 215/500
133s 266ms/step - loss: 0.2213 - acc: 0.9865 - val_loss: 0.3887 - val_acc: 0.9399
Epoch 216/500
133s 266ms/step - loss: 0.2193 - acc: 0.9867 - val_loss: 0.3850 - val_acc: 0.9420
Epoch 217/500
133s 266ms/step - loss: 0.2193 - acc: 0.9862 - val_loss: 0.3781 - val_acc: 0.9425
Epoch 218/500
133s 266ms/step - loss: 0.2194 - acc: 0.9861 - val_loss: 0.3863 - val_acc: 0.9399
Epoch 219/500
133s 266ms/step - loss: 0.2165 - acc: 0.9869 - val_loss: 0.3795 - val_acc: 0.9417
Epoch 220/500
133s 266ms/step - loss: 0.2149 - acc: 0.9874 - val_loss: 0.3749 - val_acc: 0.9443
Epoch 221/500
133s 266ms/step - loss: 0.2171 - acc: 0.9854 - val_loss: 0.3776 - val_acc: 0.9424
Epoch 222/500
133s 266ms/step - loss: 0.2183 - acc: 0.9845 - val_loss: 0.3854 - val_acc: 0.9397
Epoch 223/500
133s 266ms/step - loss: 0.2163 - acc: 0.9854 - val_loss: 0.3745 - val_acc: 0.9424
Epoch 224/500
133s 266ms/step - loss: 0.2138 - acc: 0.9861 - val_loss: 0.3695 - val_acc: 0.9425
Epoch 225/500
133s 266ms/step - loss: 0.2098 - acc: 0.9868 - val_loss: 0.3634 - val_acc: 0.9459
Epoch 226/500
133s 266ms/step - loss: 0.2120 - acc: 0.9863 - val_loss: 0.3709 - val_acc: 0.9431
Epoch 227/500
133s 266ms/step - loss: 0.2122 - acc: 0.9858 - val_loss: 0.3758 - val_acc: 0.9395
Epoch 228/500
133s 266ms/step - loss: 0.2103 - acc: 0.9861 - val_loss: 0.3628 - val_acc: 0.9423
Epoch 229/500
133s 266ms/step - loss: 0.2105 - acc: 0.9856 - val_loss: 0.3739 - val_acc: 0.9400
Epoch 230/500
133s 266ms/step - loss: 0.2109 - acc: 0.9854 - val_loss: 0.3757 - val_acc: 0.9399
Epoch 231/500
133s 266ms/step - loss: 0.2089 - acc: 0.9860 - val_loss: 0.3677 - val_acc: 0.9412
Epoch 232/500
133s 266ms/step - loss: 0.2062 - acc: 0.9875 - val_loss: 0.3596 - val_acc: 0.9430
Epoch 233/500
133s 266ms/step - loss: 0.2068 - acc: 0.9859 - val_loss: 0.3635 - val_acc: 0.9409
Epoch 234/500
133s 266ms/step - loss: 0.2060 - acc: 0.9863 - val_loss: 0.3792 - val_acc: 0.9381
Epoch 235/500
133s 266ms/step - loss: 0.2061 - acc: 0.9865 - val_loss: 0.3720 - val_acc: 0.9416
Epoch 236/500
133s 266ms/step - loss: 0.2066 - acc: 0.9853 - val_loss: 0.3862 - val_acc: 0.9353
Epoch 237/500
133s 266ms/step - loss: 0.2089 - acc: 0.9846 - val_loss: 0.3698 - val_acc: 0.9387
Epoch 238/500
133s 266ms/step - loss: 0.2065 - acc: 0.9853 - val_loss: 0.3611 - val_acc: 0.9405
Epoch 239/500
133s 266ms/step - loss: 0.2070 - acc: 0.9853 - val_loss: 0.3688 - val_acc: 0.9386
Epoch 240/500
133s 266ms/step - loss: 0.2044 - acc: 0.9858 - val_loss: 0.3689 - val_acc: 0.9398
Epoch 241/500
133s 266ms/step - loss: 0.2055 - acc: 0.9849 - val_loss: 0.3766 - val_acc: 0.9390
Epoch 242/500
133s 266ms/step - loss: 0.2028 - acc: 0.9861 - val_loss: 0.3592 - val_acc: 0.9414
Epoch 243/500
133s 266ms/step - loss: 0.2030 - acc: 0.9863 - val_loss: 0.3616 - val_acc: 0.9431
Epoch 244/500
133s 266ms/step - loss: 0.2024 - acc: 0.9861 - val_loss: 0.3722 - val_acc: 0.9379
Epoch 245/500
133s 266ms/step - loss: 0.2038 - acc: 0.9855 - val_loss: 0.3620 - val_acc: 0.9407
Epoch 246/500
133s 266ms/step - loss: 0.2014 - acc: 0.9865 - val_loss: 0.3740 - val_acc: 0.9376
Epoch 247/500
133s 266ms/step - loss: 0.2012 - acc: 0.9856 - val_loss: 0.3630 - val_acc: 0.9418
Epoch 248/500
133s 266ms/step - loss: 0.2045 - acc: 0.9845 - val_loss: 0.3644 - val_acc: 0.9401
Epoch 249/500
133s 266ms/step - loss: 0.2044 - acc: 0.9845 - val_loss: 0.3605 - val_acc: 0.9384
Epoch 250/500
133s 266ms/step - loss: 0.2066 - acc: 0.9842 - val_loss: 0.3684 - val_acc: 0.9383
Epoch 251/500
133s 266ms/step - loss: 0.2005 - acc: 0.9861 - val_loss: 0.3683 - val_acc: 0.9377
Epoch 252/500
133s 266ms/step - loss: 0.2019 - acc: 0.9856 - val_loss: 0.3663 - val_acc: 0.9382
Epoch 253/500
133s 266ms/step - loss: 0.2026 - acc: 0.9852 - val_loss: 0.3666 - val_acc: 0.9393
Epoch 254/500
133s 266ms/step - loss: 0.2034 - acc: 0.9841 - val_loss: 0.3627 - val_acc: 0.9409
Epoch 255/500
133s 266ms/step - loss: 0.1998 - acc: 0.9857 - val_loss: 0.3609 - val_acc: 0.9408
Epoch 256/500
133s 266ms/step - loss: 0.1987 - acc: 0.9859 - val_loss: 0.3629 - val_acc: 0.9414
Epoch 257/500
133s 266ms/step - loss: 0.1971 - acc: 0.9871 - val_loss: 0.3768 - val_acc: 0.9375
Epoch 258/500
133s 266ms/step - loss: 0.1985 - acc: 0.9860 - val_loss: 0.3706 - val_acc: 0.9375
Epoch 259/500
133s 266ms/step - loss: 0.2006 - acc: 0.9845 - val_loss: 0.3638 - val_acc: 0.9401
Epoch 260/500
133s 266ms/step - loss: 0.1991 - acc: 0.9851 - val_loss: 0.3629 - val_acc: 0.9392
Epoch 261/500
133s 265ms/step - loss: 0.1999 - acc: 0.9854 - val_loss: 0.3603 - val_acc: 0.9420
Epoch 262/500
133s 266ms/step - loss: 0.2021 - acc: 0.9841 - val_loss: 0.3610 - val_acc: 0.9391
Epoch 263/500
133s 265ms/step - loss: 0.2009 - acc: 0.9850 - val_loss: 0.3454 - val_acc: 0.9413
Epoch 264/500
133s 266ms/step - loss: 0.1956 - acc: 0.9866 - val_loss: 0.3662 - val_acc: 0.9382
Epoch 265/500
133s 266ms/step - loss: 0.2038 - acc: 0.9844 - val_loss: 0.3595 - val_acc: 0.9417
Epoch 266/500
133s 266ms/step - loss: 0.1982 - acc: 0.9854 - val_loss: 0.3578 - val_acc: 0.9396
Epoch 267/500
133s 266ms/step - loss: 0.1996 - acc: 0.9844 - val_loss: 0.3662 - val_acc: 0.9397
Epoch 268/500
133s 266ms/step - loss: 0.1978 - acc: 0.9854 - val_loss: 0.3551 - val_acc: 0.9437
Epoch 269/500
133s 266ms/step - loss: 0.1986 - acc: 0.9855 - val_loss: 0.3636 - val_acc: 0.9412
Epoch 270/500
133s 266ms/step - loss: 0.1982 - acc: 0.9851 - val_loss: 0.3495 - val_acc: 0.9415
Epoch 271/500
133s 266ms/step - loss: 0.2001 - acc: 0.9845 - val_loss: 0.3504 - val_acc: 0.9407
Epoch 272/500
133s 266ms/step - loss: 0.1982 - acc: 0.9850 - val_loss: 0.3496 - val_acc: 0.9432
Epoch 273/500
133s 266ms/step - loss: 0.1989 - acc: 0.9849 - val_loss: 0.3589 - val_acc: 0.9395
Epoch 274/500
133s 266ms/step - loss: 0.1954 - acc: 0.9862 - val_loss: 0.3517 - val_acc: 0.9440
Epoch 275/500
133s 266ms/step - loss: 0.1964 - acc: 0.9860 - val_loss: 0.3710 - val_acc: 0.9385
Epoch 276/500
133s 266ms/step - loss: 0.1991 - acc: 0.9848 - val_loss: 0.3572 - val_acc: 0.9375
Epoch 277/500
133s 266ms/step - loss: 0.1986 - acc: 0.9841 - val_loss: 0.3733 - val_acc: 0.9394
Epoch 278/500
133s 266ms/step - loss: 0.2004 - acc: 0.9846 - val_loss: 0.3516 - val_acc: 0.9408
Epoch 279/500
133s 266ms/step - loss: 0.1961 - acc: 0.9854 - val_loss: 0.3687 - val_acc: 0.9387
Epoch 280/500
133s 266ms/step - loss: 0.1952 - acc: 0.9855 - val_loss: 0.3724 - val_acc: 0.9397
Epoch 281/500
133s 266ms/step - loss: 0.1960 - acc: 0.9850 - val_loss: 0.3706 - val_acc: 0.9391
Epoch 282/500
133s 266ms/step - loss: 0.1961 - acc: 0.9855 - val_loss: 0.3711 - val_acc: 0.9376
Epoch 283/500
133s 266ms/step - loss: 0.1975 - acc: 0.9846 - val_loss: 0.3728 - val_acc: 0.9352
Epoch 284/500
133s 266ms/step - loss: 0.1980 - acc: 0.9847 - val_loss: 0.3666 - val_acc: 0.9361
Epoch 285/500
133s 266ms/step - loss: 0.1944 - acc: 0.9862 - val_loss: 0.3651 - val_acc: 0.9382
Epoch 286/500
133s 266ms/step - loss: 0.1932 - acc: 0.9862 - val_loss: 0.3679 - val_acc: 0.9366
Epoch 287/500
133s 266ms/step - loss: 0.1988 - acc: 0.9844 - val_loss: 0.3522 - val_acc: 0.9431
Epoch 288/500
133s 266ms/step - loss: 0.1947 - acc: 0.9860 - val_loss: 0.3574 - val_acc: 0.9394
Epoch 289/500
133s 266ms/step - loss: 0.1964 - acc: 0.9854 - val_loss: 0.3608 - val_acc: 0.9391
Epoch 290/500
133s 266ms/step - loss: 0.1955 - acc: 0.9853 - val_loss: 0.3663 - val_acc: 0.9373
Epoch 291/500
133s 266ms/step - loss: 0.1966 - acc: 0.9849 - val_loss: 0.3614 - val_acc: 0.9392
Epoch 292/500
133s 266ms/step - loss: 0.1951 - acc: 0.9856 - val_loss: 0.3698 - val_acc: 0.9350
Epoch 293/500
133s 266ms/step - loss: 0.1969 - acc: 0.9847 - val_loss: 0.3528 - val_acc: 0.9442
Epoch 294/500
133s 266ms/step - loss: 0.1981 - acc: 0.9846 - val_loss: 0.3644 - val_acc: 0.9395
Epoch 295/500
133s 266ms/step - loss: 0.1946 - acc: 0.9854 - val_loss: 0.3649 - val_acc: 0.9390
Epoch 296/500
133s 266ms/step - loss: 0.1973 - acc: 0.9840 - val_loss: 0.3725 - val_acc: 0.9378
Epoch 297/500
133s 266ms/step - loss: 0.1954 - acc: 0.9849 - val_loss: 0.3412 - val_acc: 0.9448
Epoch 298/500
133s 266ms/step - loss: 0.1889 - acc: 0.9877 - val_loss: 0.3610 - val_acc: 0.9377
Epoch 299/500
133s 266ms/step - loss: 0.1894 - acc: 0.9872 - val_loss: 0.3702 - val_acc: 0.9367
Epoch 300/500
133s 266ms/step - loss: 0.1957 - acc: 0.9846 - val_loss: 0.3614 - val_acc: 0.9400
Epoch 301/500
lr changed to 0.0009999999776482583
133s 266ms/step - loss: 0.1796 - acc: 0.9907 - val_loss: 0.3285 - val_acc: 0.9484
Epoch 302/500
133s 266ms/step - loss: 0.1694 - acc: 0.9946 - val_loss: 0.3244 - val_acc: 0.9480
Epoch 303/500
133s 266ms/step - loss: 0.1670 - acc: 0.9951 - val_loss: 0.3218 - val_acc: 0.9495
Epoch 304/500
133s 266ms/step - loss: 0.1638 - acc: 0.9963 - val_loss: 0.3213 - val_acc: 0.9500
Epoch 305/500
133s 266ms/step - loss: 0.1625 - acc: 0.9963 - val_loss: 0.3204 - val_acc: 0.9513
Epoch 306/500
133s 266ms/step - loss: 0.1614 - acc: 0.9967 - val_loss: 0.3201 - val_acc: 0.9507
Epoch 307/500
133s 266ms/step - loss: 0.1616 - acc: 0.9965 - val_loss: 0.3205 - val_acc: 0.9510
Epoch 308/500
133s 266ms/step - loss: 0.1600 - acc: 0.9968 - val_loss: 0.3193 - val_acc: 0.9518
Epoch 309/500
133s 266ms/step - loss: 0.1593 - acc: 0.9970 - val_loss: 0.3213 - val_acc: 0.9520
Epoch 310/500
133s 266ms/step - loss: 0.1574 - acc: 0.9975 - val_loss: 0.3204 - val_acc: 0.9517
Epoch 311/500
133s 266ms/step - loss: 0.1578 - acc: 0.9975 - val_loss: 0.3205 - val_acc: 0.9511
Epoch 312/500
133s 266ms/step - loss: 0.1575 - acc: 0.9972 - val_loss: 0.3199 - val_acc: 0.9518
Epoch 313/500
133s 266ms/step - loss: 0.1565 - acc: 0.9977 - val_loss: 0.3192 - val_acc: 0.9521
Epoch 314/500
133s 266ms/step - loss: 0.1563 - acc: 0.9977 - val_loss: 0.3186 - val_acc: 0.9528
Epoch 315/500
133s 266ms/step - loss: 0.1555 - acc: 0.9980 - val_loss: 0.3194 - val_acc: 0.9523
Epoch 316/500
133s 266ms/step - loss: 0.1551 - acc: 0.9978 - val_loss: 0.3190 - val_acc: 0.9529
Epoch 317/500
133s 266ms/step - loss: 0.1542 - acc: 0.9979 - val_loss: 0.3178 - val_acc: 0.9527
Epoch 318/500
133s 266ms/step - loss: 0.1542 - acc: 0.9981 - val_loss: 0.3160 - val_acc: 0.9535
Epoch 319/500
133s 266ms/step - loss: 0.1540 - acc: 0.9980 - val_loss: 0.3157 - val_acc: 0.9536
Epoch 320/500
133s 266ms/step - loss: 0.1536 - acc: 0.9980 - val_loss: 0.3171 - val_acc: 0.9528
Epoch 321/500
133s 266ms/step - loss: 0.1532 - acc: 0.9979 - val_loss: 0.3203 - val_acc: 0.9519
Epoch 322/500
133s 266ms/step - loss: 0.1522 - acc: 0.9982 - val_loss: 0.3214 - val_acc: 0.9526
Epoch 323/500
133s 266ms/step - loss: 0.1524 - acc: 0.9982 - val_loss: 0.3227 - val_acc: 0.9529
Epoch 324/500
133s 266ms/step - loss: 0.1519 - acc: 0.9983 - val_loss: 0.3229 - val_acc: 0.9527
Epoch 325/500
133s 266ms/step - loss: 0.1519 - acc: 0.9981 - val_loss: 0.3206 - val_acc: 0.9531
Epoch 326/500
133s 265ms/step - loss: 0.1510 - acc: 0.9983 - val_loss: 0.3202 - val_acc: 0.9528
Epoch 327/500
133s 266ms/step - loss: 0.1509 - acc: 0.9984 - val_loss: 0.3216 - val_acc: 0.9530
Epoch 328/500
133s 266ms/step - loss: 0.1513 - acc: 0.9979 - val_loss: 0.3222 - val_acc: 0.9537
Epoch 329/500
133s 266ms/step - loss: 0.1506 - acc: 0.9984 - val_loss: 0.3213 - val_acc: 0.9530
Epoch 330/500
133s 266ms/step - loss: 0.1496 - acc: 0.9985 - val_loss: 0.3221 - val_acc: 0.9527
Epoch 331/500
133s 266ms/step - loss: 0.1498 - acc: 0.9984 - val_loss: 0.3214 - val_acc: 0.9523
Epoch 332/500
133s 266ms/step - loss: 0.1484 - acc: 0.9989 - val_loss: 0.3201 - val_acc: 0.9520
Epoch 333/500
133s 266ms/step - loss: 0.1491 - acc: 0.9983 - val_loss: 0.3206 - val_acc: 0.9520
Epoch 334/500
133s 266ms/step - loss: 0.1491 - acc: 0.9984 - val_loss: 0.3196 - val_acc: 0.9530
Epoch 335/500
133s 266ms/step - loss: 0.1484 - acc: 0.9983 - val_loss: 0.3203 - val_acc: 0.9531
Epoch 336/500
133s 266ms/step - loss: 0.1476 - acc: 0.9986 - val_loss: 0.3195 - val_acc: 0.9527
Epoch 337/500
133s 266ms/step - loss: 0.1473 - acc: 0.9986 - val_loss: 0.3177 - val_acc: 0.9525
Epoch 338/500
133s 266ms/step - loss: 0.1470 - acc: 0.9987 - val_loss: 0.3186 - val_acc: 0.9537
Epoch 339/500
133s 266ms/step - loss: 0.1466 - acc: 0.9986 - val_loss: 0.3175 - val_acc: 0.9546
Epoch 340/500
133s 266ms/step - loss: 0.1465 - acc: 0.9986 - val_loss: 0.3148 - val_acc: 0.9540
Epoch 341/500
133s 265ms/step - loss: 0.1464 - acc: 0.9985 - val_loss: 0.3157 - val_acc: 0.9545
Epoch 342/500
133s 266ms/step - loss: 0.1464 - acc: 0.9983 - val_loss: 0.3149 - val_acc: 0.9540
Epoch 343/500
133s 266ms/step - loss: 0.1454 - acc: 0.9988 - val_loss: 0.3154 - val_acc: 0.9547
Epoch 344/500
133s 266ms/step - loss: 0.1449 - acc: 0.9990 - val_loss: 0.3146 - val_acc: 0.9536
Epoch 345/500
133s 266ms/step - loss: 0.1446 - acc: 0.9988 - val_loss: 0.3149 - val_acc: 0.9537
Epoch 346/500
133s 266ms/step - loss: 0.1439 - acc: 0.9992 - val_loss: 0.3148 - val_acc: 0.9540
Epoch 347/500
133s 266ms/step - loss: 0.1450 - acc: 0.9985 - val_loss: 0.3170 - val_acc: 0.9535
...
Epoch 427/500
133s 266ms/step - loss: 0.1260 - acc: 0.9988 - val_loss: 0.3015 - val_acc: 0.9541
Epoch 428/500
133s 266ms/step - loss: 0.1257 - acc: 0.9989 - val_loss: 0.3011 - val_acc: 0.9542
Epoch 429/500
133s 266ms/step - loss: 0.1247 - acc: 0.9992 - val_loss: 0.3011 - val_acc: 0.9541
Epoch 430/500
133s 266ms/step - loss: 0.1249 - acc: 0.9990 - val_loss: 0.3004 - val_acc: 0.9553
Epoch 431/500
133s 266ms/step - loss: 0.1247 - acc: 0.9991 - val_loss: 0.3010 - val_acc: 0.9542
Epoch 432/500
133s 265ms/step - loss: 0.1241 - acc: 0.9992 - val_loss: 0.3029 - val_acc: 0.9541
Epoch 433/500
133s 266ms/step - loss: 0.1244 - acc: 0.9990 - val_loss: 0.3011 - val_acc: 0.9538
Epoch 434/500
133s 265ms/step - loss: 0.1239 - acc: 0.9991 - val_loss: 0.3019 - val_acc: 0.9545
Epoch 435/500
133s 266ms/step - loss: 0.1235 - acc: 0.9992 - val_loss: 0.3008 - val_acc: 0.9548
Epoch 436/500
133s 266ms/step - loss: 0.1235 - acc: 0.9991 - val_loss: 0.2996 - val_acc: 0.9548
Epoch 437/500
133s 266ms/step - loss: 0.1232 - acc: 0.9991 - val_loss: 0.2999 - val_acc: 0.9545
Epoch 438/500
133s 266ms/step - loss: 0.1229 - acc: 0.9991 - val_loss: 0.3015 - val_acc: 0.9542
Epoch 439/500
133s 266ms/step - loss: 0.1225 - acc: 0.9992 - val_loss: 0.3014 - val_acc: 0.9538
Epoch 440/500
133s 266ms/step - loss: 0.1233 - acc: 0.9991 - val_loss: 0.3017 - val_acc: 0.9553
Epoch 441/500
133s 266ms/step - loss: 0.1225 - acc: 0.9992 - val_loss: 0.3018 - val_acc: 0.9546
Epoch 442/500
133s 266ms/step - loss: 0.1223 - acc: 0.9992 - val_loss: 0.3016 - val_acc: 0.9536
Epoch 443/500
133s 267ms/step - loss: 0.1221 - acc: 0.9992 - val_loss: 0.3026 - val_acc: 0.9546
Epoch 444/500
133s 266ms/step - loss: 0.1220 - acc: 0.9990 - val_loss: 0.3024 - val_acc: 0.9553
Epoch 445/500
133s 266ms/step - loss: 0.1220 - acc: 0.9990 - val_loss: 0.3016 - val_acc: 0.9540
Epoch 446/500
133s 266ms/step - loss: 0.1219 - acc: 0.9990 - val_loss: 0.2991 - val_acc: 0.9542
Epoch 447/500
133s 266ms/step - loss: 0.1215 - acc: 0.9990 - val_loss: 0.2986 - val_acc: 0.9546
Epoch 448/500
133s 266ms/step - loss: 0.1214 - acc: 0.9989 - val_loss: 0.3001 - val_acc: 0.9533
Epoch 449/500
133s 266ms/step - loss: 0.1211 - acc: 0.9991 - val_loss: 0.2969 - val_acc: 0.9560
Epoch 450/500
133s 266ms/step - loss: 0.1213 - acc: 0.9991 - val_loss: 0.2956 - val_acc: 0.9547
Epoch 451/500
lr changed to 9.999999310821295e-05
133s 266ms/step - loss: 0.1205 - acc: 0.9993 - val_loss: 0.2953 - val_acc: 0.9552
Epoch 452/500
133s 266ms/step - loss: 0.1205 - acc: 0.9993 - val_loss: 0.2953 - val_acc: 0.9551
Epoch 453/500
133s 265ms/step - loss: 0.1202 - acc: 0.9995 - val_loss: 0.2951 - val_acc: 0.9552
Epoch 454/500
133s 266ms/step - loss: 0.1205 - acc: 0.9991 - val_loss: 0.2949 - val_acc: 0.9550
Epoch 455/500
133s 266ms/step - loss: 0.1204 - acc: 0.9993 - val_loss: 0.2945 - val_acc: 0.9552
Epoch 456/500
133s 266ms/step - loss: 0.1203 - acc: 0.9993 - val_loss: 0.2947 - val_acc: 0.9546
Epoch 457/500
133s 266ms/step - loss: 0.1200 - acc: 0.9995 - val_loss: 0.2948 - val_acc: 0.9545
Epoch 458/500
133s 266ms/step - loss: 0.1204 - acc: 0.9993 - val_loss: 0.2948 - val_acc: 0.9548
Epoch 459/500
133s 266ms/step - loss: 0.1200 - acc: 0.9993 - val_loss: 0.2946 - val_acc: 0.9545
Epoch 460/500
133s 266ms/step - loss: 0.1203 - acc: 0.9992 - val_loss: 0.2942 - val_acc: 0.9551
Epoch 461/500
133s 266ms/step - loss: 0.1199 - acc: 0.9993 - val_loss: 0.2947 - val_acc: 0.9551
Epoch 462/500
133s 266ms/step - loss: 0.1201 - acc: 0.9993 - val_loss: 0.2946 - val_acc: 0.9547
Epoch 463/500
133s 266ms/step - loss: 0.1199 - acc: 0.9994 - val_loss: 0.2945 - val_acc: 0.9548
Epoch 464/500
133s 266ms/step - loss: 0.1200 - acc: 0.9994 - val_loss: 0.2946 - val_acc: 0.9549
Epoch 465/500
133s 266ms/step - loss: 0.1200 - acc: 0.9993 - val_loss: 0.2945 - val_acc: 0.9547
Epoch 466/500
133s 266ms/step - loss: 0.1202 - acc: 0.9990 - val_loss: 0.2941 - val_acc: 0.9552
Epoch 467/500
133s 266ms/step - loss: 0.1198 - acc: 0.9993 - val_loss: 0.2938 - val_acc: 0.9554
Epoch 468/500
133s 266ms/step - loss: 0.1195 - acc: 0.9994 - val_loss: 0.2938 - val_acc: 0.9552
Epoch 469/500
133s 266ms/step - loss: 0.1203 - acc: 0.9993 - val_loss: 0.2938 - val_acc: 0.9553
Epoch 470/500
133s 265ms/step - loss: 0.1199 - acc: 0.9994 - val_loss: 0.2941 - val_acc: 0.9553
Epoch 471/500
133s 266ms/step - loss: 0.1198 - acc: 0.9994 - val_loss: 0.2938 - val_acc: 0.9553
Epoch 472/500
133s 266ms/step - loss: 0.1199 - acc: 0.9993 - val_loss: 0.2938 - val_acc: 0.9551
Epoch 473/500
133s 266ms/step - loss: 0.1201 - acc: 0.9992 - val_loss: 0.2936 - val_acc: 0.9553
Epoch 474/500
133s 266ms/step - loss: 0.1197 - acc: 0.9993 - val_loss: 0.2937 - val_acc: 0.9549
Epoch 475/500
133s 265ms/step - loss: 0.1202 - acc: 0.9991 - val_loss: 0.2936 - val_acc: 0.9553
Epoch 476/500
133s 265ms/step - loss: 0.1201 - acc: 0.9992 - val_loss: 0.2935 - val_acc: 0.9554
Epoch 477/500
133s 265ms/step - loss: 0.1203 - acc: 0.9991 - val_loss: 0.2935 - val_acc: 0.9551
Epoch 478/500
133s 265ms/step - loss: 0.1198 - acc: 0.9992 - val_loss: 0.2938 - val_acc: 0.9553
Epoch 479/500
133s 265ms/step - loss: 0.1199 - acc: 0.9991 - val_loss: 0.2940 - val_acc: 0.9552
Epoch 480/500
133s 265ms/step - loss: 0.1199 - acc: 0.9992 - val_loss: 0.2938 - val_acc: 0.9553
Epoch 481/500
133s 266ms/step - loss: 0.1196 - acc: 0.9994 - val_loss: 0.2936 - val_acc: 0.9553
Epoch 482/500
133s 265ms/step - loss: 0.1196 - acc: 0.9994 - val_loss: 0.2938 - val_acc: 0.9552
Epoch 483/500
133s 265ms/step - loss: 0.1198 - acc: 0.9993 - val_loss: 0.2939 - val_acc: 0.9550
Epoch 484/500
133s 265ms/step - loss: 0.1196 - acc: 0.9994 - val_loss: 0.2942 - val_acc: 0.9549
Epoch 485/500
133s 265ms/step - loss: 0.1200 - acc: 0.9992 - val_loss: 0.2940 - val_acc: 0.9550
Epoch 486/500
133s 265ms/step - loss: 0.1194 - acc: 0.9994 - val_loss: 0.2941 - val_acc: 0.9552
Epoch 487/500
133s 265ms/step - loss: 0.1195 - acc: 0.9993 - val_loss: 0.2936 - val_acc: 0.9549
Epoch 488/500
133s 265ms/step - loss: 0.1196 - acc: 0.9993 - val_loss: 0.2936 - val_acc: 0.9551
Epoch 489/500
133s 265ms/step - loss: 0.1195 - acc: 0.9992 - val_loss: 0.2937 - val_acc: 0.9547
Epoch 490/500
133s 266ms/step - loss: 0.1195 - acc: 0.9992 - val_loss: 0.2936 - val_acc: 0.9550
Epoch 491/500
133s 266ms/step - loss: 0.1192 - acc: 0.9993 - val_loss: 0.2936 - val_acc: 0.9548
Epoch 492/500
133s 266ms/step - loss: 0.1197 - acc: 0.9993 - val_loss: 0.2933 - val_acc: 0.9551
Epoch 493/500
133s 266ms/step - loss: 0.1192 - acc: 0.9994 - val_loss: 0.2930 - val_acc: 0.9550
Epoch 494/500
133s 266ms/step - loss: 0.1195 - acc: 0.9994 - val_loss: 0.2929 - val_acc: 0.9553
Epoch 495/500
133s 266ms/step - loss: 0.1192 - acc: 0.9994 - val_loss: 0.2929 - val_acc: 0.9551
Epoch 496/500
133s 266ms/step - loss: 0.1192 - acc: 0.9993 - val_loss: 0.2930 - val_acc: 0.9553
Epoch 497/500
133s 266ms/step - loss: 0.1191 - acc: 0.9993 - val_loss: 0.2929 - val_acc: 0.9551
Epoch 498/500
133s 266ms/step - loss: 0.1192 - acc: 0.9994 - val_loss: 0.2928 - val_acc: 0.9552
Epoch 499/500
133s 266ms/step - loss: 0.1189 - acc: 0.9993 - val_loss: 0.2925 - val_acc: 0.9548
Epoch 500/500
133s 266ms/step - loss: 0.1197 - acc: 0.9991 - val_loss: 0.2927 - val_acc: 0.9547
Train loss: 0.11755225303769111
Train accuracy: 0.9996800003051758
Test loss: 0.29267876625061034
Test accuracy: 0.9547000050544738比调参记录21的95.12%高了一点。怎么样能够突破96%呢？
Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht, Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis, IEEE Transactions on Industrial Electronics, DOI: 10.1109/TIE.2020.2972458, Date of Publication: 13 February 2020
https://ieeexplore.ieee.org/document/8998530
