class DBN(object):"""Deep Belief NetworkA deep belief network is obtained by stacking several RBMs on top of eachother. The hidden layer of the RBM at layer `i` becomes the input of theRBM at layer `i+1`. The first layer RBM gets as input the input of thenetwork, and the hidden layer of the last RBM represents the output. Whenused for classification, the DBN is treated as a MLP, by adding a logisticregression layer on top."""def __init__(self, numpy_rng, theano_rng=None, n_ins=784,hidden_layers_sizes=[500, 500], n_outs=10):"""This class is made to support a variable number of layers.:type numpy_rng: numpy.random.RandomState:param numpy_rng: numpy random number generator used to draw initialweights:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams:param theano_rng: Theano random generator; if None is given one isgenerated based on a seed drawn from `rng`:type n_ins: int:param n_ins: dimension of the input to the DBN:type hidden_layers_sizes: list of ints:param hidden_layers_sizes: intermediate layers size, must containat least one value:type n_outs: int:param n_outs: dimension of the output of the network"""self.sigmoid_layers = []self.rbm_layers = []self.params = []self.n_layers = len(hidden_layers_sizes)assert self.n_layers > 0if not theano_rng:theano_rng = MRG_RandomStreams(numpy_rng.randint(2 ** 30))# allocate symbolic variables for the data# the data is presented as rasterized imagesself.x = T.matrix('x')# the labels are presented as 1D vector of [int] labelsself.y = T.ivector('y')

 for i in range(self.n_layers):# construct the sigmoidal layer# the size of the input is either the number of hidden# units of the layer below or the input size if we are on# the first layerif i == 0:input_size = n_inselse:input_size = hidden_layers_sizes[i - 1]# the input to this layer is either the activation of the# hidden layer below or the input of the DBN if you are on# the first layerif i == 0:layer_input = self.xelse:layer_input = self.sigmoid_layers[-1].outputsigmoid_layer = HiddenLayer(rng=numpy_rng,input=layer_input,n_in=input_size,n_out=hidden_layers_sizes[i],activation=T.nnet.sigmoid)# add the layer to our list of layersself.sigmoid_layers.append(sigmoid_layer)# its arguably a philosophical question...  but we are# going to only declare that the parameters of the# sigmoid_layers are parameters of the DBN. The visible# biases in the RBM are parameters of those RBMs, but not# of the DBN.self.params.extend(sigmoid_layer.params)# Construct an RBM that shared weights with this layerrbm_layer = RBM(numpy_rng=numpy_rng,theano_rng=theano_rng,input=layer_input,n_visible=input_size,n_hidden=hidden_layers_sizes[i],W=sigmoid_layer.W,hbias=sigmoid_layer.b)self.rbm_layers.append(rbm_layer)

 self.logLayer = LogisticRegression(input=self.sigmoid_layers[-1].output,n_in=hidden_layers_sizes[-1],n_out=n_outs)self.params.extend(self.logLayer.params)# compute the cost for second phase of training, defined as the# negative log likelihood of the logistic regression (output) layerself.finetune_cost = self.logLayer.negative_log_likelihood(self.y)# compute the gradients with respect to the model parameters# symbolic variable that points to the number of errors made on the# minibatch given by self.x and self.yself.errors = self.logLayer.errors(self.y)

  def pretraining_functions(self, train_set_x, batch_size, k):'''Generates a list of functions, for performing one step ofgradient descent at a given layer. The function will requireas input the minibatch index, and to train an RBM you justneed to iterate, calling the corresponding function on allminibatch indexes.:type train_set_x: theano.tensor.TensorType:param train_set_x: Shared var. that contains all datapoints usedfor training the RBM:type batch_size: int:param batch_size: size of a [mini]batch:param k: number of Gibbs steps to do in CD-k / PCD-k'''# index to a [mini]batchindex = T.lscalar('index')  # index to a minibatch

  learning_rate = T.scalar('lr')  # learning rate to use# begining of a batch, given `index`batch_begin = index * batch_size# ending of a batch given `index`batch_end = batch_begin + batch_sizepretrain_fns = []for rbm in self.rbm_layers:# get the cost and the updates list# using CD-k here (persisent=None) for training each RBM.# TODO: change cost function to reconstruction errorcost, updates = rbm.get_cost_updates(learning_rate,persistent=None, k=k)# compile the theano functionfn = theano.function(inputs=[index, theano.In(learning_rate, value=0.1)],outputs=cost,updates=updates,givens={self.x: train_set_x[batch_begin:batch_end]})# append `fn` to the list of functionspretrain_fns.append(fn)return pretrain_fns

  def build_finetune_functions(self, datasets, batch_size, learning_rate):'''Generates a function `train` that implements one step offinetuning, a function `validate` that computes the error on abatch from the validation set, and a function `test` thatcomputes the error on a batch from the testing set:type datasets: list of pairs of theano.tensor.TensorType:param datasets: It is a list that contain all the datasets;the has to contain three pairs, `train`,`valid`, `test` in this order, where each pairis formed of two Theano variables, one for thedatapoints, the other for the labels:type batch_size: int:param batch_size: size of a minibatch:type learning_rate: float:param learning_rate: learning rate used during finetune stage'''(train_set_x, train_set_y) = datasets[0](valid_set_x, valid_set_y) = datasets[1](test_set_x, test_set_y) = datasets[2]# compute number of minibatches for training, validation and testingn_valid_batches = valid_set_x.get_value(borrow=True).shape[0]n_valid_batches //= batch_sizen_test_batches = test_set_x.get_value(borrow=True).shape[0]n_test_batches //= batch_sizeindex = T.lscalar('index')  # index to a [mini]batch# compute the gradients with respect to the model parametersgparams = T.grad(self.finetune_cost, self.params)# compute list of fine-tuning updatesupdates = []for param, gparam in zip(self.params, gparams):updates.append((param, param - gparam * learning_rate))train_fn = theano.function(inputs=[index],outputs=self.finetune_cost,updates=updates,givens={self.x: train_set_x[index * batch_size: (index + 1) * batch_size],self.y: train_set_y[index * batch_size: (index + 1) * batch_size]})test_score_i = theano.function([index],self.errors,givens={self.x: test_set_x[index * batch_size: (index + 1) * batch_size],self.y: test_set_y[index * batch_size: (index + 1) * batch_size]})valid_score_i = theano.function([index],self.errors,givens={self.x: valid_set_x[index * batch_size: (index + 1) * batch_size],self.y: valid_set_y[index * batch_size: (index + 1) * batch_size]})# Create a function that scans the entire validation setdef valid_score():return [valid_score_i(i) for i in range(n_valid_batches)]# Create a function that scans the entire test setdef test_score():return [test_score_i(i) for i in range(n_test_batches)]return train_fn, valid_score, test_score

  numpy_rng = numpy.random.RandomState(123)print('... building the model')# construct the Deep Belief Networkdbn = DBN(numpy_rng=numpy_rng, n_ins=28 * 28,hidden_layers_sizes=[1000, 1000, 1000],n_outs=10)

 ########################## PRETRAINING THE MODEL ##########################print('... getting the pretraining functions')pretraining_fns = dbn.pretraining_functions(train_set_x=train_set_x,batch_size=batch_size,k=k)print('... pre-training the model')start_time = timeit.default_timer()# Pre-train layer-wisefor i in range(dbn.n_layers):# go through pretraining epochsfor epoch in range(pretraining_epochs):# go through the training setc = []for batch_index in range(n_train_batches):c.append(pretraining_fns[i](index=batch_index,lr=pretrain_lr))print('Pre-training layer %i, epoch %d, cost ' % (i, epoch), end=' ')print(numpy.mean(c, dtype='float64'))end_time = timeit.default_timer()

python code/DBN.py