準備¶
Googleドライブのマウント¶
In [1]:
from google.colab import drive
drive.mount('/content/drive')
sys.pathの設定¶
以下では，Googleドライブのマイドライブ直下にDNN_codeフォルダを置くことを仮定しています．必要に応じて，パスを変更してください．
In [2]:
import sys
sys.path.append('/content/drive/My Drive/DNN_code_colab_lesson_1_2')
sys.path.append('/content/drive/My Drive/DNN_code_colab_lesson_1_2/lesson_2')
batch normalization¶
In [3]:
import numpy as np
from collections import OrderedDict
from common import layers
from data.mnist import load_mnist
import matplotlib.pyplot as plt
from multi_layer_net import MultiLayerNet
from common import optimizer
# バッチ正則化 layer
class BatchNormalization:
'''
gamma: スケール係数
beta: オフセット
momentum: 慣性
running_mean: テスト時に使用する平均
running_var: テスト時に使用する分散
'''
def __init__(self, gamma, beta, momentum=0.9, running_mean=None, running_var=None):
self.gamma = gamma
self.beta = beta
self.momentum = momentum
self.input_shape = None
self.running_mean = running_mean
self.running_var = running_var
# backward時に使用する中間データ
self.batch_size = None
self.xc = None
self.std = None
self.dgamma = None
self.dbeta = None
def forward(self, x, train_flg=True):
if self.running_mean is None:
N, D = x.shape
self.running_mean = np.zeros(D)
self.running_var = np.zeros(D)
if train_flg:
mu = x.mean(axis=0) # 平均
xc = x - mu # xをセンタリング
var = np.mean(xc**2, axis=0) # 分散
std = np.sqrt(var + 10e-7) # スケーリング
xn = xc / std
self.batch_size = x.shape[0]
self.xc = xc
self.xn = xn
self.std = std
self.running_mean = self.momentum * self.running_mean + (1-self.momentum) * mu # 平均値の加重平均
self.running_var = self.momentum * self.running_var + (1-self.momentum) * var #分散値の加重平均
else:
xc = x - self.running_mean
xn = xc / ((np.sqrt(self.running_var + 10e-7)))
out = self.gamma * xn + self.beta
return out
def backward(self, dout):
dbeta = dout.sum(axis=0)
dgamma = np.sum(self.xn * dout, axis=0)
dxn = self.gamma * dout
dxc = dxn / self.std
dstd = -np.sum((dxn * self.xc) / (self.std * self.std), axis=0)
dvar = 0.5 * dstd / self.std
dxc += (2.0 / self.batch_size) * self.xc * dvar
dmu = np.sum(dxc, axis=0)
dx = dxc - dmu / self.batch_size
self.dgamma = dgamma
self.dbeta = dbeta
return dx
In [42]:
def func ( batch ,iters, learning,axtive,weight):
(x_train, d_train), (x_test, d_test) = load_mnist(normalize=True)
print("データ読み込み完了")
# batch_normalizationの設定 =======================
use_batchnorm = batch
# use_batchnorm = False
# ====================================================
network = MultiLayerNet(input_size=784, hidden_size_list=[40, 20], output_size=10,
activation=axtive, weight_init_std=weight, use_batchnorm=use_batchnorm)
iters_num = iters
train_size = x_train.shape[0]
batch_size = 100
learning_rate=learning
train_loss_list = []
accuracies_train = []
accuracies_test = []
losses_train = []
losses_test = []
plot_interval=10
for i in range(iters_num):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
d_batch = d_train[batch_mask]
grad = network.gradient(x_batch, d_batch)
for key in ('W1', 'W2', 'W3', 'b1', 'b2', 'b3'):
network.params[key] -= learning_rate * grad[key]
loss = network.loss(x_batch, d_batch)
train_loss_list.append(loss)
if (i + 1) % plot_interval == 0:
accr_test = network.accuracy(x_test, d_test)
accuracies_test.append(accr_test)
accr_train = network.accuracy(x_batch, d_batch)
accuracies_train.append(accr_train)
losses_test.append(network.loss(x_test, d_test))
losses_train.append(network.loss(x_batch, d_batch))
#print('Generation: ' + str(i+1) + '. 正答率(トレーニング) = ' + str(accr_train))
#print(' : ' + str(i+1) + '. 正答率(テスト) = ' + str(accr_test))
lists = range(0, iters_num, plot_interval)
fig = plt.figure()
ax1 = fig.add_subplot(111)
ax1.plot(lists, accuracies_train, label="training_acc")
ax1.plot(lists, accuracies_test, label="test_acc")
ax2 = ax1.twinx()
ax2.plot(lists, losses_train, label="training_loss", color="b")
ax2.plot(lists, losses_test, label="test_loss", color="g")
h1, l1 = ax1.get_legend_handles_labels()
h2, l2 = ax2.get_legend_handles_labels()
ax1.legend(h1+h2, l1+l2, loc='lower right')
ax1.set_xlabel("count")
ax1.set_ylabel("accuracy")
ax2.set_ylabel("loss")
ax1.set_ylim(0, 1.0)
In [43]:
func( False ,1000,0.01,'sigmoid','Xavier')
#【コメント】sigmoid, Xaviel バッチ正規化なし
#【コメント】lossは下がっているがaccは上がり遅い。このまま続けたらacc上がっていきそうではあるか？
In [44]:
func( True ,1000,0.01,'sigmoid','Xavier')
#【コメント】sigmoid, Xaviel バッチ正規化あり⇒学習がうまくいっている
In [46]:
func( False ,5000,0.1,'sigmoid','Xavier')
#【コメント】sigmoid, Xaviel バッチ正規化なし
In [47]:
func( True ,5000,0.1,'sigmoid','Xavier')
#【コメント】sigmoid, Xaviel バッチ正規化あり
In [48]:
func( False ,1000,0.01,'relu','He')
In [49]:
func( True ,1000,0.01,'relu','He')
#【コメント】test_accが気持ち滑らかになっているような？気のせい？
