import numpy as npimport matplotlib.pyplot as pltfrom sklearn.model_selection import train_test_splitnp.random.seed(123)

# We will use a simple training setX = 2 * np.random.rand(500, 1)y = 5 + 3 * X + np.random.randn(500, 1)fig = plt.figure(figsize=(8,6))plt.scatter(X, y)plt.title("Dataset")plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

# Split the data into a training and test setX_train, X_test, y_train, y_test = train_test_split(X, y)print(f'Shape X_train: {X_train.shape}')print(f'Shape y_train: {y_train.shape}')print(f'Shape X_test: {X_test.shape}')print(f'Shape y_test: {y_test.shape}')

class LinearRegression:

    def __init__(self):        pass

    def train_gradient_descent(self, X, y, learning_rate=0.01, n_iters=100):        """        Trains a linear regression model using gradient descent        """        # Step 0: Initialize the parameters        n_samples, n_features = X.shape        self.weights = np.zeros(shape=(n_features,1))        self.bias = 0        costs = []

        for i in range(n_iters):            # Step 1: Compute a linear combination of the input features and weights            y_predict = np.dot(X, self.weights) + self.bias

            # Step 2: Compute cost over training set            cost = (1 / n_samples) * np.sum((y_predict - y)**2)            costs.append(cost)

            if i % 100 == 0:                print(f"Cost at iteration {i}: {cost}")

            # Step 3: Compute the gradients            dJ_dw = (2 / n_samples) * np.dot(X.T, (y_predict - y))            dJ_db = (2 / n_samples) * np.sum((y_predict - y)) 

            # Step 4: Update the parameters            self.weights = self.weights - learning_rate * dJ_dw            self.bias = self.bias - learning_rate * dJ_db

        return self.weights, self.bias, costs

    def train_normal_equation(self, X, y):        """        Trains a linear regression model using the normal equation        """        self.weights = np.dot(np.dot(np.linalg.inv(np.dot(X.T, X)), X.T), y)        self.bias = 0

        return self.weights, self.bias

    def predict(self, X):        return np.dot(X, self.weights) + self.bias

regressor = LinearRegression()w_trained, b_trained, costs = regressor.train_gradient_descent(X_train, y_train, learning_rate=0.005, n_iters=600)fig = plt.figure(figsize=(8,6))plt.plot(np.arange(n_iters), costs)plt.title("Development of cost during training")plt.xlabel("Number of iterations")plt.ylabel("Cost")plt.show()

n_samples, _ = X_train.shapen_samples_test, _ = X_test.shape

y_p_train = regressor.predict(X_train)y_p_test = regressor.predict(X_test)

error_train =  (1 / n_samples) * np.sum((y_p_train - y_train) ** 2)error_test =  (1 / n_samples_test) * np.sum((y_p_test - y_test) ** 2)

print(f"Error on training set: {np.round(error_train, 4)}")print(f"Error on test set: {np.round(error_test)}")

# To compute the parameters using the normal equation, we add a bias value of 1 to each input exampleX_b_train = np.c_[np.ones((n_samples)), X_train]X_b_test = np.c_[np.ones((n_samples_test)), X_test]

reg_normal = LinearRegression()w_trained = reg_normal.train_normal_equation(X_b_train, y_train)

y_p_train = reg_normal.predict(X_b_train)y_p_test = reg_normal.predict(X_b_test)

error_train =  (1 / n_samples) * np.sum((y_p_train - y_train) ** 2)error_test =  (1 / n_samples_test) * np.sum((y_p_test - y_test) ** 2)

print(f"Error on training set: {np.round(error_train, 4)}")print(f"Error on test set: {np.round(error_test, 4)}")

# Plot the test predictions

fig = plt.figure(figsize=(8,6))plt.scatter(X_train, y_train)plt.scatter(X_test, y_p_test)plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

import numpy as npfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import make_blobsimport matplotlib.pyplot as pltnp.random.seed(123)

% matplotlib inline

# We will perform logistic regression using a simple toy dataset of two classesX, y_true = make_blobs(n_samples= 1000, centers=2)

fig = plt.figure(figsize=(8,6))plt.scatter(X[:,0], X[:,1], c=y_true)plt.title("Dataset")plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

# Reshape targets to get column vector with shape (n_samples, 1)y_true = y_true[:, np.newaxis]# Split the data into a training and test setX_train, X_test, y_train, y_test = train_test_split(X, y_true)print(f'Shape X_train: {X_train.shape}')print(f'Shape y_train: {y_train.shape}')print(f'Shape X_test: {X_test.shape}')print(f'Shape y_test: {y_test.shape}')

class LogisticRegression:

    def __init__(self):        pass

    def sigmoid(self, a):        return 1 / (1 + np.exp(-a))

    def train(self, X, y_true, n_iters, learning_rate):        """        Trains the logistic regression model on given data X and targets y        """        # Step 0: Initialize the parameters        n_samples, n_features = X.shape        self.weights = np.zeros((n_features, 1))        self.bias = 0        costs = []

        for i in range(n_iters):            # Step 1 and 2: Compute a linear combination of the input features and weights,             # apply the sigmoid activation function            y_predict = self.sigmoid(np.dot(X, self.weights) + self.bias)

            # Step 3: Compute the cost over the whole training set.            cost = (- 1 / n_samples) * np.sum(y_true * np.log(y_predict) + (1 - y_true) * (np.log(1 - y_predict)))

            # Step 4: Compute the gradients            dw = (1 / n_samples) * np.dot(X.T, (y_predict - y_true))            db = (1 / n_samples) * np.sum(y_predict - y_true)

            # Step 5: Update the parameters            self.weights = self.weights - learning_rate * dw            self.bias = self.bias - learning_rate * db

            costs.append(cost)            if i % 100 == 0:                print(f"Cost after iteration {i}: {cost}")

        return self.weights, self.bias, costs

    def predict(self, X):        """        Predicts binary labels for a set of examples X.        """        y_predict = self.sigmoid(np.dot(X, self.weights) + self.bias)        y_predict_labels = [1 if elem > 0.5 else 0 for elem in y_predict]

        return np.array(y_predict_labels)[:, np.newaxis]

regressor = LogisticRegression()w_trained, b_trained, costs = regressor.train(X_train, y_train, n_iters=600, learning_rate=0.009)

fig = plt.figure(figsize=(8,6))plt.plot(np.arange(600), costs)plt.title("Development of cost over training")plt.xlabel("Number of iterations")plt.ylabel("Cost")plt.show()

y_p_train = regressor.predict(X_train)y_p_test = regressor.predict(X_test)

print(f"train accuracy: {100 - np.mean(np.abs(y_p_train - y_train)) * 100}%")print(f"test accuracy: {100 - np.mean(np.abs(y_p_test - y_test))}%")

import numpy as npimport matplotlib.pyplot as pltfrom sklearn.datasets import make_blobsfrom sklearn.model_selection import train_test_splitnp.random.seed(123)

% matplotlib inline

X, y = make_blobs(n_samples=1000, centers=2)fig = plt.figure(figsize=(8,6))plt.scatter(X[:,0], X[:,1], c=y)plt.title("Dataset")plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

y_true = y[:, np.newaxis]

X_train, X_test, y_train, y_test = train_test_split(X, y_true)print(f'Shape X_train: {X_train.shape}')print(f'Shape y_train: {y_train.shape})')print(f'Shape X_test: {X_test.shape}')print(f'Shape y_test: {y_test.shape}')

class Perceptron():

    def __init__(self):        pass

    def train(self, X, y, learning_rate=0.05, n_iters=100):        n_samples, n_features = X.shape

        # Step 0: Initialize the parameters        self.weights = np.zeros((n_features,1))        self.bias = 0

        for i in range(n_iters):            # Step 1: Compute the activation            a = np.dot(X, self.weights) + self.bias

            # Step 2: Compute the output            y_predict = self.step_function(a)

            # Step 3: Compute weight updates            delta_w = learning_rate * np.dot(X.T, (y - y_predict))            delta_b = learning_rate * np.sum(y - y_predict)

            # Step 4: Update the parameters            self.weights += delta_w            self.bias += delta_b

        return self.weights, self.bias

    def step_function(self, x):        return np.array([1 if elem >= 0 else 0 for elem in x])[:, np.newaxis]

    def predict(self, X):        a = np.dot(X, self.weights) + self.bias        return self.step_function(a)

p = Perceptron()w_trained, b_trained = p.train(X_train, y_train,learning_rate=0.05, n_iters=500)

y_p_train = p.predict(X_train)y_p_test = p.predict(X_test)

print(f"training accuracy: {100 - np.mean(np.abs(y_p_train - y_train)) * 100}%")print(f"test accuracy: {100 - np.mean(np.abs(y_p_test - y_test)) * 100}%")

def plot_hyperplane(X, y, weights, bias):    """    Plots the dataset and the estimated decision hyperplane    """    slope = - weights[0]/weights[1]    intercept = - bias/weights[1]    x_hyperplane = np.linspace(-10,10,10)    y_hyperplane = slope * x_hyperplane + intercept    fig = plt.figure(figsize=(8,6))    plt.scatter(X[:,0], X[:,1], c=y)    plt.plot(x_hyperplane, y_hyperplane, '-')    plt.title("Dataset and fitted decision hyperplane")    plt.xlabel("First feature")    plt.ylabel("Second feature")    plt.show()

plot_hyperplane(X, y, w_trained, b_trained)

import numpy as npimport matplotlib.pyplot as pltfrom sklearn.datasets import load_digitsfrom sklearn.model_selection import train_test_splitnp.random.seed(123)

% matplotlib inline

# We will use the digits dataset as an example. It consists of the 1797 images of hand-written digits. Each digit is# represented by a 64-dimensional vector of pixel values.

digits = load_digits()X, y = digits.data, digits.target

X_train, X_test, y_train, y_test = train_test_split(X, y)print(f'X_train shape: {X_train.shape}')print(f'y_train shape: {y_train.shape}')print(f'X_test shape: {X_test.shape}')print(f'y_test shape: {y_test.shape}')

# Example digitsfig = plt.figure(figsize=(10,8))for i in range(10):    ax = fig.add_subplot(2, 5, i+1)    plt.imshow(X[i].reshape((8,8)), cmap='gray')

class kNN():    def __init__(self):        pass

    def fit(self, X, y):        self.data = X        self.targets = y

    def euclidean_distance(self, X):        """        Computes the euclidean distance between the training data and        a new input example or matrix of input examples X        """        # input: single data point        if X.ndim == 1:            l2 = np.sqrt(np.sum((self.data - X)**2, axis=1))

        # input: matrix of data points        if X.ndim == 2:            n_samples, _ = X.shape            l2 = [np.sqrt(np.sum((self.data - X[i])**2, axis=1)) for i in range(n_samples)]

        return np.array(l2)

    def predict(self, X, k=1):        """        Predicts the classification for an input example or matrix of input examples X        """        # step 1: compute distance between input and training data        dists = self.euclidean_distance(X)

        # step 2: find the k nearest neighbors and their classifications        if X.ndim == 1:            if k == 1:                nn = np.argmin(dists)                return self.targets[nn]            else:                knn = np.argsort(dists)[:k]                y_knn = self.targets[knn]                max_vote = max(y_knn, key=list(y_knn).count)                return max_vote

        if X.ndim == 2:            knn = np.argsort(dists)[:, :k]            y_knn = self.targets[knn]            if k == 1:                return y_knn.T            else:                n_samples, _ = X.shape                max_votes = [max(y_knn[i], key=list(y_knn[i]).count) for i in range(n_samples)]                return max_votes

knn = kNN()knn.fit(X_train, y_train)

print("Testing one datapoint, k=1")print(f"Predicted label: {knn.predict(X_test[0], k=1)}")print(f"True label: {y_test[0]}")print()print("Testing one datapoint, k=5")print(f"Predicted label: {knn.predict(X_test[20], k=5)}")print(f"True label: {y_test[20]}")print()print("Testing 10 datapoint, k=1")print(f"Predicted labels: {knn.predict(X_test[5:15], k=1)}")print(f"True labels: {y_test[5:15]}")print()print("Testing 10 datapoint, k=4")print(f"Predicted labels: {knn.predict(X_test[5:15], k=4)}")print(f"True labels: {y_test[5:15]}")print()

# Compute accuracy on test sety_p_test1 = knn.predict(X_test, k=1)test_acc1= np.sum(y_p_test1[0] == y_test)/len(y_p_test1[0]) * 100print(f"Test accuracy with k = 1: {format(test_acc1)}")

y_p_test8 = knn.predict(X_test, k=5)test_acc8= np.sum(y_p_test8 == y_test)/len(y_p_test8) * 100print(f"Test accuracy with k = 8: {format(test_acc8)}")

import numpy as npimport matplotlib.pyplot as pltimport randomfrom sklearn.datasets import make_blobsnp.random.seed(123)

% matplotlib inline

X, y = make_blobs(centers=4, n_samples=1000)print(f'Shape of dataset: {X.shape}')

fig = plt.figure(figsize=(8,6))plt.scatter(X[:,0], X[:,1], c=y)plt.title("Dataset with 4 clusters")plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

class KMeans():    def __init__(self, n_clusters=4):        self.k = n_clusters

    def fit(self, data):        """        Fits the k-means model to the given dataset        """        n_samples, _ = data.shape        # initialize cluster centers        self.centers = np.array(random.sample(list(data), self.k))        self.initial_centers = np.copy(self.centers)

        # We will keep track of whether the assignment of data points        # to the clusters has changed. If it stops changing, we are         # done fitting the model        old_assigns = None        n_iters = 0

        while True:            new_assigns = [self.classify(datapoint) for datapoint in data]

            if new_assigns == old_assigns:                print(f"Training finished after {n_iters} iterations!")                return

            old_assigns = new_assigns            n_iters += 1

            # recalculate centers            for id_ in range(self.k):                points_idx = np.where(np.array(new_assigns) == id_)                datapoints = data[points_idx]                self.centers[id_] = datapoints.mean(axis=0)

    def l2_distance(self, datapoint):        dists = np.sqrt(np.sum((self.centers - datapoint)**2, axis=1))        return dists

    def classify(self, datapoint):        """        Given a datapoint, compute the cluster closest to the        datapoint. Return the cluster ID of that cluster.        """        dists = self.l2_distance(datapoint)        return np.argmin(dists)

    def plot_clusters(self, data):        plt.figure(figsize=(12,10))        plt.title("Initial centers in black, final centers in red")        plt.scatter(data[:, 0], data[:, 1], marker='.', c=y)        plt.scatter(self.centers[:, 0], self.centers[:,1], c='r')        plt.scatter(self.initial_centers[:, 0], self.initial_centers[:,1], c='k')        plt.show()

kmeans = KMeans(n_clusters=4)kmeans.fit(X)

kmeans.plot_clusters(X)

import numpy as npimport pandas as pdimport matplotlib.pyplot as pltfrom sklearn.datasets import make_circlesfrom sklearn.model_selection import train_test_splitnp.random.seed(123)% matplotlib inline

X, y = make_circles(n_samples=1000, factor=0.5, noise=.1)fig = plt.figure(figsize=(8,6))plt.scatter(X[:,0], X[:,1], c=y)plt.xlim([-1.5, 1.5])plt.ylim([-1.5, 1.5])plt.title("Dataset")plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

# reshape targets to get column vector with shape (n_samples, 1)y_true = y[:, np.newaxis]# Split the data into a training and test setX_train, X_test, y_train, y_test = train_test_split(X, y_true)print(f'Shape X_train: {X_train.shape}')print(f'Shape y_train: {y_train.shape}')print(f'Shape X_test: {X_test.shape}')print(f'Shape y_test: {y_test.shape}')

class NeuralNet():    def __init__(self, n_inputs, n_outputs, n_hidden):        self.n_inputs = n_inputs        self.n_outputs = n_outputs        self.hidden = n_hidden        # Initialize weight matrices and bias vectors        self.W_h = np.random.randn(self.n_inputs, self.hidden)        self.b_h = np.zeros((1, self.hidden))        self.W_o = np.random.randn(self.hidden, self.n_outputs)        self.b_o = np.zeros((1, self.n_outputs))    def sigmoid(self, a):        return 1 / (1 + np.exp(-a))    def forward_pass(self, X):        """        Propagates the given input X forward through the net.        Returns:            A_h: matrix with activations of all hidden neurons for all input examples            O_h: matrix with outputs of all hidden neurons for all input examples            A_o: matrix with activations of all output neurons for all input examples            O_o: matrix with outputs of all output neurons for all input examples        """        # Compute activations and outputs of hidden units        A_h = np.dot(X, self.W_h) + self.b_h        O_h = np.tanh(A_h)        # Compute activations and outputs of output units        A_o = np.dot(O_h, self.W_o) + self.b_o        O_o = self.sigmoid(A_o)        outputs = {                "A_h": A_h,                "A_o": A_o,                "O_h": O_h,                "O_o": O_o,                }        return outputs    def cost(self, y_true, y_predict, n_samples):        """        Computes and returns the cost over all examples        """        # same cost function as in logistic regression        cost = (- 1 / n_samples) * np.sum(y_true * np.log(y_predict) + (1 - y_true) * (np.log(1 - y_predict)))        cost = np.squeeze(cost)        assert isinstance(cost, float)        return cost    def backward_pass(self,  X, Y, n_samples, outputs):        """        Propagates the errors backward through the net.        Returns:            dW_h: partial derivatives of loss function w.r.t hidden weights            db_h: partial derivatives of loss function w.r.t hidden bias            dW_o: partial derivatives of loss function w.r.t output weights            db_o: partial derivatives of loss function w.r.t output bias        """        dA_o = (outputs["O_o"] - Y)        dW_o = (1 / n_samples) * np.dot(outputs["O_h"].T, dA_o)        db_o = (1 / n_samples) * np.sum(dA_o)        dA_h = (np.dot(dA_o, self.W_o.T)) * (1 - np.power(outputs["O_h"], 2))        dW_h = (1 / n_samples) * np.dot(X.T, dA_h)        db_h = (1 / n_samples) * np.sum(dA_h)        gradients = {                "dW_o": dW_o,                "db_o": db_o,                "dW_h": dW_h,                "db_h": db_h,                }        return gradients    def update_weights(self, gradients, eta):        """        Updates the model parameters using a fixed learning rate        """        self.W_o = self.W_o - eta * gradients["dW_o"]        self.W_h = self.W_h - eta * gradients["dW_h"]        self.b_o = self.b_o - eta * gradients["db_o"]        self.b_h = self.b_h - eta * gradients["db_h"]    def train(self, X, y, n_iters=500, eta=0.3):        """        Trains the neural net on the given input data        """        n_samples, _ = X.shape        for i in range(n_iters):            outputs = self.forward_pass(X)            cost = self.cost(y, outputs["O_o"], n_samples=n_samples)            gradients = self.backward_pass(X, y, n_samples, outputs)            if i % 100 == 0:                print(f'Cost at iteration {i}: {np.round(cost, 4)}')            self.update_weights(gradients, eta)    def predict(self, X):        """        Computes and returns network predictions for given dataset        """        outputs = self.forward_pass(X)        y_pred = [1 if elem >= 0.5 else 0 for elem in outputs["O_o"]]        return np.array(y_pred)[:, np.newaxis]

nn = NeuralNet(n_inputs=2, n_hidden=6, n_outputs=1)print("Shape of weight matrices and bias vectors:")print(f'W_h shape: {nn.W_h.shape}')print(f'b_h shape: {nn.b_h.shape}')print(f'W_o shape: {nn.W_o.shape}')print(f'b_o shape: {nn.b_o.shape}')print()print("Training:")nn.train(X_train, y_train, n_iters=2000, eta=0.7)

n_test_samples, _ = X_test.shapey_predict = nn.predict(X_test)print(f"Classification accuracy on test set: {(np.sum(y_predict == y_test)/n_test_samples)*100} %")

X_temp, y_temp = make_circles(n_samples=60000, noise=.5)y_predict_temp = nn.predict(X_temp)y_predict_temp = np.ravel(y_predict_temp)

fig = plt.figure(figsize=(8,12))ax = fig.add_subplot(2,1,1)plt.scatter(X[:,0], X[:,1], c=y)plt.xlim([-1.5, 1.5])plt.ylim([-1.5, 1.5])plt.xlabel("First feature")plt.ylabel("Second feature")plt.title("Training and test set")ax = fig.add_subplot(2,1,2)plt.scatter(X_temp[:,0], X_temp[:,1], c=y_predict_temp)plt.xlim([-1.5, 1.5])plt.ylim([-1.5, 1.5])plt.xlabel("First feature")plt.ylabel("Second feature")plt.title("Decision boundary")

from sklearn.datasets import load_irisimport numpy as npfrom sklearn.model_selection import train_test_splitfrom sklearn.datasets import make_blobsimport matplotlib.pyplot as pltnp.random.seed(13)

X, y_true = make_blobs(centers=4, n_samples = 5000)fig = plt.figure(figsize=(8,6))plt.scatter(X[:,0], X[:,1], c=y_true)plt.title("Dataset")plt.xlabel("First feature")plt.ylabel("Second feature")plt.show()

# reshape targets to get column vector with shape (n_samples, 1)y_true = y_true[:, np.newaxis]# Split the data into a training and test setX_train, X_test, y_train, y_test = train_test_split(X, y_true)print(f'Shape X_train: {X_train.shape}')print(f'Shape y_train: {y_train.shape}')print(f'Shape X_test: {X_test.shape}')print(f'Shape y_test: {y_test.shape}')

class SoftmaxRegressor:    def __init__(self):        pass    def train(self, X, y_true, n_classes, n_iters=10, learning_rate=0.1):        """        Trains a multinomial logistic regression model on given set of training data        """        self.n_samples, n_features = X.shape        self.n_classes = n_classes        self.weights = np.random.rand(self.n_classes, n_features)        self.bias = np.zeros((1, self.n_classes))        all_losses = []        for i in range(n_iters):            scores = self.compute_scores(X)            probs = self.softmax(scores)            y_predict = np.argmax(probs, axis=1)[:, np.newaxis]            y_one_hot = self.one_hot(y_true)            loss = self.cross_entropy(y_one_hot, probs)            all_losses.append(loss)            dw = (1 / self.n_samples) * np.dot(X.T, (probs - y_one_hot))            db = (1 / self.n_samples) * np.sum(probs - y_one_hot, axis=0)            self.weights = self.weights - learning_rate * dw.T            self.bias = self.bias - learning_rate * db            if i % 100 == 0:                print(f'Iteration number: {i}, loss: {np.round(loss, 4)}')        return self.weights, self.bias, all_losses    def predict(self, X):        """        Predict class labels for samples in X.        Args:            X: numpy array of shape (n_samples, n_features)        Returns:            numpy array of shape (n_samples, 1) with predicted classes        """        scores = self.compute_scores(X)        probs = self.softmax(scores)        return np.argmax(probs, axis=1)[:, np.newaxis]    def softmax(self, scores):        """        Tranforms matrix of predicted scores to matrix of probabilities        Args:            scores: numpy array of shape (n_samples, n_classes)            with unnormalized scores        Returns:            softmax: numpy array of shape (n_samples, n_classes)            with probabilities        """        exp = np.exp(scores)        sum_exp = np.sum(np.exp(scores), axis=1, keepdims=True)        softmax = exp / sum_exp        return softmax    def compute_scores(self, X):        """        Computes class-scores for samples in X        Args:            X: numpy array of shape (n_samples, n_features)        Returns:            scores: numpy array of shape (n_samples, n_classes)        """        return np.dot(X, self.weights.T) + self.bias    def cross_entropy(self, y_true, scores):        loss = - (1 / self.n_samples) * np.sum(y_true * np.log(scores))        return loss    def one_hot(self, y):        """        Tranforms vector y of labels to one-hot encoded matrix        """        one_hot = np.zeros((self.n_samples, self.n_classes))        one_hot[np.arange(self.n_samples), y.T] = 1        return one_hot

regressor = SoftmaxRegressor()w_trained, b_trained, loss = regressor.train(X_train, y_train, learning_rate=0.1, n_iters=800, n_classes=4)fig = plt.figure(figsize=(8,6))plt.plot(np.arange(800), loss)plt.title("Development of loss during training")plt.xlabel("Number of iterations")plt.ylabel("Loss")plt.show()Iteration number: 0, loss: 1.393

n_test_samples, _ = X_test.shapey_predict = regressor.predict(X_test)print(f"Classification accuracy on test set: {(np.sum(y_predict == y_test)/n_test_samples) * 100}%")