fc_mean,fc_var = tf.nn.moments(Wx_plus_b,axes=[0],# 想要 normalize 的维度, [0] 代表 batch 维度# 如果是图像数据, 可以传入 [0, 1, 2], 相当于求[batch, height, width] 的均值/方差, 注意不要加入 channel 维度
)
scale = tf.Variable(tf.ones([out_size]))
shift = tf.Variable(tf.zeros([out_size]))
epsilon = 0.001
Wx_plus_b = tf.nn.batch_normalization(Wx_plus_b,fc_mean,fc_var,shift,scale,epsilon)
# 上面那一步, 在做如下事情:
# Wx_plus_b = (Wx_plus_b - fc_mean) / tf.sqrt(fc_var + 0.001)
# Wx_plus_b = Wx_plus_b * scale + shift

class batch_norm():'''batch normalization层'''def __init__(self, epsilon=1e-5,momentum=0.9, name='batch_norm'):'''初始化:param epsilon:    防零极小值:param momentum:   滑动平均参数:param name:       节点名称'''with tf.variable_scope(name):self.epsilon = epsilonself.momentum = momentumself.name = namedef __call__(self, x, train=True):# 一个封装了的会在内部调用batch_normalization进行正则化的高级接口return tf.contrib.layers.batch_norm(x,decay=self.momentum,        # 滑动平均参数updates_collections=None,epsilon=self.epsilon,scale=True,is_training=train,          # 影响滑动平均scope=self.name)

def batch_norm(x,beta,gamma,phase_train,scope='bn',decay=0.9,eps=1e-5):with tf.variable_scope(scope):# beta = tf.get_variable(name='beta', shape=[n_out], initializer=tf.constant_initializer(0.0), trainable=True)# gamma = tf.get_variable(name='gamma', shape=[n_out],#                         initializer=tf.random_normal_initializer(1.0, stddev), trainable=True)batch_mean,batch_var = tf.nn.moments(x,[0,1,2],name='moments')ema = tf.train.ExponentialMovingAverage(decay=decay)def mean_var_with_update():ema_apply_op = ema.apply([batch_mean,batch_var])with tf.control_dependencies([ema_apply_op]):return tf.identity(batch_mean),tf.identity(batch_var)# identity之后会把Variable转换为Tensor并入图中，# 否则由于Variable是独立于Session的，不会被图控制control_dependencies限制mean,var = tf.cond(phase_train,mean_var_with_update,lambda: (ema.average(batch_mean),ema.average(batch_var)))normed = tf.nn.batch_normalization(x, mean, var, beta, gamma, eps)return normed

def batch_norm(x, size, training, decay=0.999):beta = tf.Variable(tf.zeros([size]), name='beta')scale = tf.Variable(tf.ones([size]), name='scale')pop_mean = tf.Variable(tf.zeros([size]))pop_var = tf.Variable(tf.ones([size]))epsilon = 1e-3batch_mean, batch_var = tf.nn.moments(x, [0, 1, 2])train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay))train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay))def batch_statistics():with tf.control_dependencies([train_mean, train_var]):return tf.nn.batch_normalization(x, batch_mean, batch_var, beta, scale, epsilon, name='batch_norm')def population_statistics():return tf.nn.batch_normalization(x, pop_mean, pop_var, beta, scale, epsilon, name='batch_norm')return tf.cond(training, batch_statistics, population_statistics)