Numpy_implementation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/palVikram/Machine-Learning-using-Python/blob/master/Numpy_implementation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### Steps to create dropout\n",
        "import numpy as np\n",
        "\n",
        "# Example usage:\n",
        "X = np.random.rand(5, 5)\n",
        "X"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p1k4EpH65i0s",
        "outputId": "c0ca4b18-c98c-4cdb-f33b-d54884360e39"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.14635991, 0.01133066, 0.66539399, 0.74127252, 0.62321183],\n",
              "       [0.07453104, 0.70923138, 0.72681232, 0.4364586 , 0.44353092],\n",
              "       [0.82410157, 0.11618204, 0.92170095, 0.48524298, 0.87681611],\n",
              "       [0.01759614, 0.21426306, 0.10340425, 0.16864473, 0.91107617],\n",
              "       [0.76861997, 0.37522717, 0.672222  , 0.27148641, 0.3015718 ]])"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dropout_rate=0.5\n",
        "retain_prob=1-dropout_rate"
      ],
      "metadata": {
        "id": "0gQZBNoA5oNm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        " np.random.binomial(1, retain_prob, size=X.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3XVbXPJ56yNp",
        "outputId": "3dc89e4d-39b0-4060-cd2d-1f15a30eebc4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 0, 0, 1, 1],\n",
              "       [1, 1, 1, 0, 1],\n",
              "       [1, 0, 1, 0, 1],\n",
              "       [0, 0, 1, 1, 0],\n",
              "       [1, 0, 0, 0, 0]])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "mask=np.random.binomial(1, retain_prob, size=X.shape)/retain_prob\n",
        "mask"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mlN22lNr6PH1",
        "outputId": "f796814f-d6fc-492b-cf14-6edc44eb5483"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[2., 2., 0., 2., 2.],\n",
              "       [2., 2., 2., 0., 0.],\n",
              "       [0., 2., 2., 2., 2.],\n",
              "       [2., 2., 2., 2., 0.],\n",
              "       [2., 0., 0., 2., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X*mask"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tRIV2zSH6Zua",
        "outputId": "74f5143f-f79a-4401-e490-ff35eeb40887"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.        , 0.02266133, 0.        , 0.        , 0.        ],\n",
              "       [0.        , 1.41846276, 1.45362464, 0.        , 0.88706183],\n",
              "       [1.64820314, 0.        , 0.        , 0.        , 0.        ],\n",
              "       [0.03519229, 0.        , 0.        , 0.33728945, 1.82215235],\n",
              "       [0.        , 0.75045433, 1.34444399, 0.        , 0.60314359]])"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "7okuswCs6gcC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "eOb8DwW86_Yd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "qqTPHPz96_bq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "X=np.random.rand(5,5)\n",
        "\n",
        "gamma=np.ones(X.shape[1])\n",
        "beta=np.zeros(X.shape[1])\n",
        "\n",
        "gamma, beta"
      ],
      "metadata": {
        "id": "ABaKiz336_es",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8c0ee0a6-c72b-4948-b275-e3b91f6eda1f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([1., 1., 1., 1., 1.]), array([0., 0., 0., 0., 0.]))"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "### x=(X-mean)/sqrt(variance+e)\n",
        "### Gamma: No scaling\n",
        "### Beta: No Shifting\n",
        "mean=np.mean(X, axis=0)\n",
        "variance=np.var(X, axis=0)\n",
        "\n",
        "e=1e-8\n",
        "normalization=(X-mean)/np.sqrt(variance+e)\n",
        "\n",
        "gamma=np.ones(X.shape[1])\n",
        "beta=np.zeros(X.shape[1])\n",
        "\n",
        "print(normalization)\n",
        "normalization*gamma+beta"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vex_-NtO-XPp",
        "outputId": "26fcfab7-b51a-49f4-cb4c-a602887a4d65"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[-1.14602588  0.1007797  -1.68253673  0.85579513  1.73328787]\n",
            " [ 1.30300157  0.4509045   1.00244688 -1.16865403  0.01030518]\n",
            " [ 1.01623838 -1.12185311  0.22866793  0.61347388 -0.7603822 ]\n",
            " [-0.22064589 -1.01211007 -0.48491815 -1.26393374 -1.17500626]\n",
            " [-0.95256818  1.58227898  0.93634008  0.96331876  0.19179541]]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[-1.14602588,  0.1007797 , -1.68253673,  0.85579513,  1.73328787],\n",
              "       [ 1.30300157,  0.4509045 ,  1.00244688, -1.16865403,  0.01030518],\n",
              "       [ 1.01623838, -1.12185311,  0.22866793,  0.61347388, -0.7603822 ],\n",
              "       [-0.22064589, -1.01211007, -0.48491815, -1.26393374, -1.17500626],\n",
              "       [-0.95256818,  1.58227898,  0.93634008,  0.96331876,  0.19179541]])"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#The Adam optimizer combines the advantages of two other extensions of stochastic gradient descent: \\\n",
        "#Adaptive Gradient Algorithm (AdaGrad) and Root Mean Square Propagation (RMSProp).\n",
        "\n",
        "\"\"\"\n",
        "First Moment Estimate(Mean):\n",
        "mt=beta(1)*m(t-1)+(1-beta(1)).g(t)\n",
        "g(t) is the gradient at time step t.\n",
        "\n",
        "Bias Correct First Moment Estimate:\n",
        "m^(t)=m(t)/(1-beta(t))\n",
        "------------------------------------------\n",
        "Second Moment Estimate (uncentered variance).\n",
        "v(t)=beta(1)*m(t-1)+(1-beta(1)).g(t)^2\n",
        "\n",
        "v^(t)=v(t)/(1-beta(t))\n",
        "\n",
        "-------------------------------------\n",
        "Parameter update:\n",
        "theta=theta(t-1)- (alpha*m^(t)/(sqrt(v^t+epls)))\n",
        "\n",
        "Adam adjusts the learning rate based on the first and second moments of the gradients\n",
        "\"\"\""
      ],
      "metadata": {
        "id": "uRceViCcAKvj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "class AdamOptimizer:\n",
        "  def __init__(self, lr=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8):\n",
        "    self.lr=lr\n",
        "    self.beta1=beta1\n",
        "    self.beta2=beta\n",
        "    self.epsilon=epsilon\n",
        "    self.m=None\n",
        "    self.v=None\n",
        "    self.t=0\n",
        "\n",
        "  def update(self, params, grads):\n",
        "    if self.m is None:\n",
        "      self.m=np.zeros_like(params)\n",
        "      self.v=np.zeros_like(params)\n",
        "\n",
        "    self.t+=1\n",
        "\n",
        "    self.m=self.beta1*self.m+ (1-self.beta1)*grads\n",
        "    self.v=self.beta2*self.v+ (1-self.beta2)*(grads**2)\n",
        "\n",
        "    m_hat=self.m/(1-self.beta1*self.t)\n",
        "    v_hat=self.v/(1-self.beta2*self.t)\n",
        "\n",
        "    params -=self.lr* m_hat/ (np.sqrt(v_hat)+self.epsilon)\n",
        "\n",
        "    return params\n",
        "\n",
        "params = np.random.rand(5, 5)\n",
        "grads = np.random.rand(5, 5)\n",
        "adam = AdamOptimizer()\n",
        "updated_params = adam.update(params, grads)\n",
        "print(updated_params)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fxEKQ19lE2Ar",
        "outputId": "ed510c7a-fb02-44a2-fd6f-a90ae99f6f4a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.6160768  0.29453325 0.76647752 0.95353284 0.4898694 ]\n",
            " [0.54744692 0.18264579 0.54541644 0.60484666 0.0733511 ]\n",
            " [0.68868943 0.99086443 0.20968608 0.23156483 0.70653511]\n",
            " [0.46311339 0.40777608 0.48843327 0.00536044 0.48541724]\n",
            " [0.70512852 0.42231822 0.31508844 0.07083577 0.53763574]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Cross Entropy Loss\n",
        "#Cross-entropy loss measures the performance of a classification model whose output is a probability value between 0 and 1.\n",
        "#Cross-entropy loss is a commonly used loss function for classification problems. It measures the difference between two probability\\\n",
        "#distributions - the true labels and the predicted probabilities.\n",
        "\n",
        "'''\n",
        "y_true: True Label (one-hot encoded)\n",
        "y_pred: Predicted probabilities\n",
        "loss=- summation(y_true * log(y_pred))\n",
        "\n",
        "--------------------------------\n",
        "Clipping for Numerical Stability\n",
        "To avoid logarithm of zero (which is undefined), the predicted probabilities are clipped:\n",
        "\n",
        "\n",
        "np.clip limits the values in y_pred to be within the range [epsilon, 1 - epsilon].\n",
        "This means:\n",
        "Any value less than epsilon will be set to epsilon.\n",
        "Any value greater than 1 - epsilon will be set to 1 - epsilon.\n",
        "\n",
        "---------------------------------\n",
        "y_pred=clip(y_pred, epls, 1-epls)\n",
        "\n",
        "\n",
        "becuase log(0) is undefined, no value.\n",
        "'''"
      ],
      "metadata": {
        "id": "svf0wcyeJr2H"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def cross_entrophy(y_pred, y_true):\n",
        "  epsilon = 1e-15\n",
        "  y_clip=np.clip(y_pred, epsilon, 1-epsilon)\n",
        "\n",
        "  return -np.sum(y_true*np.log(y_clip))\n",
        "\n",
        "y_pred=np.array([0.7, 0.2,0])\n",
        "y_true= np.array([1,0,1])\n",
        "\n",
        "cross_entrophy(y_pred, y_true)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sEIkQtGJLMc7",
        "outputId": "9d4d03cc-7870-459e-a17b-b7b09761a5b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "34.89545133884942"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Sigmoid Activation Function\n",
        "#Sigmoid is a commonly used activation function in neural networks.\n",
        "\n",
        "\n",
        "'''\n",
        "σ(x): The sigmoid function applied to x\n",
        "x: The input to the function, which can be any real number.\n",
        "e: Euler's number, approximately equal to 2.71828.\n",
        "\n",
        "σ(x)= 1/1+e−x\n",
        "\n",
        "'''\n",
        "\n",
        "def sigmoid(x):\n",
        "  return 1/(1+np.exp(-x))\n",
        "\n",
        "\n",
        "x=np.rand(5)\n",
        "\n",
        "sigmoid_x=sigmoid(x)\n",
        "\n",
        "print(sigmoid_x)"
      ],
      "metadata": {
        "id": "Tn3Pka2-PjFo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "### Linear Regression Code\n",
        "def linear_regression(X, y, lr=0.01, epochs=1000):\n",
        "    m, n = X.shape\n",
        "    W = np.zeros(n)\n",
        "    b = 0\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        y_pred = np.dot(X, W) + b\n",
        "        error = y_pred - y\n",
        "\n",
        "        W -= lr * (1/m) * np.dot(X.T, error)\n",
        "        b -= lr * (1/m) * np.sum(error)\n",
        "\n",
        "    return W, b\n",
        "\n",
        "# Example usage:\n",
        "X = np.random.rand(100, 3)\n",
        "y = np.random.rand(100)\n",
        "W, b = linear_regression(X, y)\n",
        "print(W, b)\n"
      ],
      "metadata": {
        "id": "k8oBHqbZU4H9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "Normalization\n",
        "Normalization scales the input data to have a mean of 0 and a standard deviation of 1.\n",
        "'''\n",
        "def normalize(X):\n",
        "    mean = np.mean(X, axis=0)\n",
        "    std = np.std(X, axis=0)\n",
        "    return (X - mean) / std\n",
        "\n",
        "X = np.random.rand(5, 5)\n",
        "normalized_X = normalize(X)\n",
        "print(normalized_X)"
      ],
      "metadata": {
        "id": "cmth7UUUfTUM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def logistic_regression(X, y, lr=0.01, epochs=1000):\n",
        "    m, n = X.shape\n",
        "    W = np.zeros(n)\n",
        "    b = 0\n",
        "\n",
        "    print(m,n)\n",
        "    print(W)\n",
        "\n",
        "    for epoch in range(epochs):\n",
        "        z = np.dot(X, W) + b\n",
        "        y_pred = sigmoid(z)\n",
        "\n",
        "        error = y_pred - y\n",
        "        dW = (1/m) * np.dot(X.T, error)\n",
        "        db = (1/m) * np.sum(error)\n",
        "\n",
        "        W -= lr * dW\n",
        "        b -= lr * db\n",
        "\n",
        "    return W, b\n",
        "\n",
        "def predict(X, W, b):\n",
        "    z = np.dot(X, W) + b\n",
        "    probabilities = sigmoid(z)\n",
        "    return np.where(probabilities >= 0.5, 1, 0)\n",
        "\n",
        "np.random.seed(0)\n",
        "X = np.random.rand(100, 2)\n",
        "y = np.random.randint(0, 2, 100)\n",
        "\n",
        "W, b = logistic_regression(X, y, lr=0.1, epochs=1000)\n",
        "predictions = predict(X, W, b)\n",
        "\n",
        "print(\"Weights:\", W.flatten())\n",
        "print(\"Bias:\", b)\n",
        "print(\"First 10 Predictions:\", predictions[:10].flatten())\n",
        "print(\"First 10 True Labels:\", y[:10])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OU3keIJJR90Y",
        "outputId": "b09ab0fd-f8fd-4770-a566-84f2ca750c48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "100 2\n",
            "[0. 0.]\n",
            "Weights: [-0.49904328  0.81318917]\n",
            "Bias: -0.13626641366176775\n",
            "First 10 Predictions: [1 1 1 1 0 0 1 0 1 1]\n",
            "First 10 True Labels: [0 1 0 0 1 1 1 1 1 0]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def softmax(x):\n",
        "    \"\"\"Compute softmax values for each set of scores in x.\"\"\"\n",
        "    e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))\n",
        "    return e_x / np.sum(e_x, axis=-1, keepdims=True)\n",
        "\n",
        "def scaled_dot_product_attention(Q, K, V, mask=None):\n",
        "    \"\"\"Compute the scaled dot-product attention.\"\"\"\n",
        "    d_k = Q.shape[-1]\n",
        "    scores = np.dot(Q, K.transpose(-2, -1)) / np.sqrt(d_k)\n",
        "\n",
        "    if mask is not None:\n",
        "        scores += (mask * -1e9)  # Apply mask to scores (optional)\n",
        "\n",
        "    attention_weights = softmax(scores)\n",
        "    output = np.dot(attention_weights, V)\n",
        "    return output, attention_weights\n",
        "\n",
        "def multi_head_attention(Q, K, V, num_heads):\n",
        "    \"\"\"Compute the multi-head attention.\"\"\"\n",
        "    batch_size, seq_length, d_model = Q.shape\n",
        "    assert d_model % num_heads == 0, \"d_model must be divisible by num_heads\"\n",
        "    d_head = d_model // num_heads\n",
        "\n",
        "    # Initialize random projection matrices\n",
        "    W_Q = np.random.rand(d_model, d_model)\n",
        "    W_K = np.random.rand(d_model, d_model)\n",
        "    W_V = np.random.rand(d_model, d_model)\n",
        "    W_O = np.random.rand(d_model, d_model)\n",
        "\n",
        "    # Project Q, K, V\n",
        "    Q_proj = np.dot(Q, W_Q)\n",
        "    K_proj = np.dot(K, W_K)\n",
        "    V_proj = np.dot(V, W_V)\n",
        "\n",
        "    # Split Q, K, V into multiple heads\n",
        "    def split_heads(x):\n",
        "        x = x.reshape(batch_size, seq_length, num_heads, d_head)\n",
        "        return np.transpose(x, (0, 2, 1, 3))  # (batch_size, num_heads, seq_length, d_head)\n",
        "\n",
        "    Q_heads = split_heads(Q_proj)\n",
        "    K_heads = split_heads(K_proj)\n",
        "    V_heads = split_heads(V_proj)\n",
        "\n",
        "\n",
        "    # Compute attention for each head\n",
        "    attention_outputs = []\n",
        "    for i in range(num_heads):\n",
        "        output, _ = scaled_dot_product_attention(Q_heads[:, i], K_heads[:, i], V_heads[:, i])\n",
        "        attention_outputs.append(output)\n",
        "\n",
        "    # Concatenate heads\n",
        "    def concatenate_heads(x):\n",
        "        x = np.transpose(x, (0, 2, 1, 3))  # (batch_size, seq_length, num_heads, d_head)\n",
        "        return x.reshape(batch_size, seq_length, d_model)  # (batch_size, seq_length, d_model)\n",
        "\n",
        "    concatenated = concatenate_heads(np.stack(attention_outputs, axis=1))\n",
        "\n",
        "    # Final linear layer\n",
        "    output = np.dot(concatenated, W_O)\n",
        "    return output\n",
        "\n",
        "# Example usage:\n",
        "np.random.seed(0)\n",
        "batch_size = 2\n",
        "seq_length = 5\n",
        "d_model = 8\n",
        "num_heads = 2\n",
        "\n",
        "# Randomly initialize Q, K, V matrices\n",
        "Q = np.random.rand(batch_size, seq_length, d_model)\n",
        "K = np.random.rand(batch_size, seq_length, d_model)\n",
        "V = np.random.rand(batch_size, seq_length, d_model)\n",
        "\n",
        "# Compute multi-head attention\n",
        "output = multi_head_attention(Q, K, V, num_heads)\n",
        "print(\"Multi-Head Attention Output:\\n\", output)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IsUr1tlU0zip",
        "outputId": "83ee0bce-8935-4ec3-9ab2-c73c329cdd69"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0.65900114, 0.24243297, 0.09856589])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def sliding_window_attention(Q, K, V, window_size):\n",
        "    \"\"\"Compute sliding window attention.\"\"\"\n",
        "    batch_size, seq_length, d_model = Q.shape\n",
        "    output = np.zeros_like(Q)\n",
        "\n",
        "    for i in range(seq_length):\n",
        "        # Determine the start and end of the window\n",
        "        start = max(0, i - window_size)\n",
        "        end = i + 1\n",
        "\n",
        "        # Select the window of keys and values\n",
        "        Q_i = Q[:, i:i+1, :]  # Shape: (batch_size, 1, d_model)\n",
        "        K_window = K[:, start:end, :]  # Shape: (batch_size, window_length, d_model)\n",
        "        V_window = V[:, start:end, :]  # Shape: (batch_size, window_length, d_model)\n",
        "\n",
        "        # Compute attention scores\n",
        "        scores = np.matmul(Q_i, K_window.transpose(0, 2, 1)) / np.sqrt(d_model)\n",
        "        attention_weights = softmax(scores)\n",
        "\n",
        "        # Compute output for the current position\n",
        "        output[:, i:i+1, :] = np.matmul(attention_weights, V_window)\n",
        "\n",
        "    return output\n",
        "\n",
        "np.random.seed(0)\n",
        "batch_size = 2\n",
        "seq_length = 10\n",
        "d_model = 8\n",
        "window_size = 3\n",
        "\n",
        "# Randomly initialize Q, K, V matrices\n",
        "Q = np.random.rand(batch_size, seq_length, d_model)\n",
        "K = np.random.rand(batch_size, seq_length, d_model)\n",
        "V = np.random.rand(batch_size, seq_length, d_model)\n",
        "\n",
        "# Compute sliding window attention\n",
        "output = sliding_window_attention(Q, K, V, window_size)\n",
        "print(\"Sliding Window Attention Output:\\n\", output)\n"
      ],
      "metadata": {
        "id": "LqEEFS41TfZI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YPE20Lc_WpCU",
        "outputId": "27450e20-40d5-4d1d-f8d9-298dafa29439"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 1,  1,  1, -1, -1])"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "X = np.array([\n",
        "    [1, 2],\n",
        "    [2, 3],\n",
        "    [3, 3],\n",
        "    [2, 1],\n",
        "    [3, 2]\n",
        "])\n",
        "\n",
        "y = np.array([1, 1, 1, -1, -1])\n",
        "y_ = np.where(y <= 0, -1, 1)\n",
        "\n",
        "## Calculate the decision boundary distance: y_i * (w^T x_i + b)\n",
        "w=np.zeros(2)\n",
        "b=0\n",
        "\n",
        "learning_rate=0.001\n",
        "lambda_param=0.01\n",
        "n_iters=1000\n",
        "\n",
        "for iternation in range(n_iters):\n",
        "  for idx, x_i in enumerate(X):\n",
        "\n",
        "      condition=(y_[idx]*np.dot(w.T, x_i)+b)>=1\n",
        "\n",
        "      if condition:\n",
        "        # If the sample is correctly classified with a margin, update w using regularization\n",
        "        # Update rule: w = w - η * 2 * λ * w\n",
        "        w-=learning_rate*(2*lambda_param*w)\n",
        "      else:\n",
        "        # If the sample violates the margin (hinge loss), update w and b\n",
        "        # Update rule for weights: w = w - η * (2 * λ * w - y_i * x_i)\n",
        "        w-=learning_rate* (2*lambda_param* w-np.dot(y_[idx],x_i))\n",
        "        b-=learning_rate*y_[idx]\n",
        "\n",
        "def predict( X):\n",
        "        # Calculate the linear output: w^T x_i + b\n",
        "        linear_output = np.dot(X, w) + b\n",
        "\n",
        "        # Predict class labels based on the sign of the linear output\n",
        "        return np.sign(linear_output)\n",
        "\n",
        "# Make predictions on the dataset\n",
        "predictions = predict(X)\n",
        "print(\"Predictions:\", predictions)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "95Kqace3Wza4",
        "outputId": "304ce5ca-28e1-48fe-a053-da9ceb73dd81"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Predictions: [ 1.  1.  1. -1. -1.]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-E5wrXJjhJuf"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}