Random Forests builds various decision trees on bootstrap samples and random seleted features, prediction can be made by averaging

from __future__ import division
import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifierdata = pd.read_csv('data/sonar.all-data.csv', header=None)
X = data[data.columns[:-1]].as_matrix()
y = data[data.columns[-1]].as_matrix()class myRandomForest:def fit(self, X, y, n_estimators, max_depth):self.learners = []for n in range(n_estimators):idx_data = np.random.choice(X.shape[0],X.shape[0],replace=True)idx_feat = np.random.choice(X.shape[1],int(np.sqrt(X.shape[1])),replace=False)X_sub = X[idx_data][:,idx_feat]y_sub = y[idx_data]clf = DecisionTreeClassifier(max_depth=max_depth)clf.fit(X_sub, y_sub)self.learners.append({'tree': clf,'idx_feat': idx_feat})def predict(self, X):ps = []for i in range(len(self.learners)):clf = self.learners[i]['tree']idx_feat = self.learners[i]['idx_feat']p = clf.predict(X[:, idx_feat])ps.append(p)ps = np.transpose(ps).tolist()return [ max(p, key=p.count) for p in ps]def score(self, X, y):return np.count_nonzero(self.predict(X) == y)/len(y)rf = myRandomForest()
rf.fit(X,y,50,3)
print 'score:', rf.score(X,y)
#score: 0.942307692308dt = DecisionTreeClassifier(max_depth=3)
dt.fit(X,y)
print 'decision tree score:', np.count_nonzero(dt.predict(X) == y)/len(y)
#decision tree score: 0.884615384615