{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [] }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" } }, "cells": [ { "cell_type": "code", "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "RVNImn10HJcI", "outputId": "b01c1eb5-05dd-419b-faa8-c08ae8df50aa" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: tensorflow in /usr/local/lib/python3.11/dist-packages (2.18.0)\n", "Requirement already satisfied: keras in /usr/local/lib/python3.11/dist-packages (3.8.0)\n", "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.4.0)\n", "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.6.3)\n", "Requirement already satisfied: flatbuffers>=24.3.25 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (25.2.10)\n", "Requirement already satisfied: gast!=0.5.0,!=0.5.1,!=0.5.2,>=0.2.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.6.0)\n", "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.2.0)\n", "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (18.1.1)\n", "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.4.0)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from tensorflow) (24.2)\n", "Requirement already satisfied: protobuf!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<6.0.0dev,>=3.20.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (5.29.4)\n", "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.32.3)\n", "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from tensorflow) (75.2.0)\n", "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.17.0)\n", "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.0.1)\n", "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (4.13.2)\n", "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.17.2)\n", "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (1.71.0)\n", "Requirement already satisfied: tensorboard<2.19,>=2.18 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.18.0)\n", "Requirement already satisfied: numpy<2.1.0,>=1.26.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (2.0.2)\n", "Requirement already satisfied: h5py>=3.11.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (3.13.0)\n", "Requirement already satisfied: ml-dtypes<0.5.0,>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.4.1)\n", "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.11/dist-packages (from tensorflow) (0.37.1)\n", "Requirement already satisfied: rich in /usr/local/lib/python3.11/dist-packages (from keras) (13.9.4)\n", "Requirement already satisfied: namex in /usr/local/lib/python3.11/dist-packages (from keras) (0.0.9)\n", "Requirement already satisfied: optree in /usr/local/lib/python3.11/dist-packages (from keras) (0.15.0)\n", "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from astunparse>=1.6.0->tensorflow) (0.45.1)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorflow) (3.4.1)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorflow) (3.10)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorflow) (2.4.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests<3,>=2.21.0->tensorflow) (2025.4.26)\n", "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.19,>=2.18->tensorflow) (3.8)\n", "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.19,>=2.18->tensorflow) (0.7.2)\n", "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from tensorboard<2.19,>=2.18->tensorflow) (3.1.3)\n", "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.11/dist-packages (from rich->keras) (3.0.0)\n", "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.11/dist-packages (from rich->keras) (2.19.1)\n", "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.11/dist-packages (from markdown-it-py>=2.2.0->rich->keras) (0.1.2)\n", "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.11/dist-packages (from werkzeug>=1.0.1->tensorboard<2.19,>=2.18->tensorflow) (3.0.2)\n" ] } ], "source": [ "!pip install tensorflow keras\n", "import numpy as np\n", "import tensorflow as tf\n", "from tensorflow import keras\n", "\n", "# Data processing and visualization imports\n", "import string\n", "import pandas as pd\n", "import plotly.express as px\n", "import tensorflow.data as tfd\n", "import matplotlib.pyplot as plt\n", "from sklearn.preprocessing import LabelEncoder\n", "from sklearn.model_selection import train_test_split\n", "\n", "# Model building imports\n", "from sklearn.utils import class_weight\n", "from tensorflow.keras import callbacks\n", "from tensorflow.keras import Model, layers" ] }, { "cell_type": "code", "source": [ "num_heads = 4\n", "embed_dim = 256\n", "ff_dim = 128\n", "vocab_size = 10000\n", "max_seq_len = 40\n", "\n", "# Set constants\n", "learning_rate = 1e-3\n", "epochs = 100\n", "batch_size = 32\n", "\n", "# Define training callbacks\n", "callbacks = [\n", " keras.callbacks.EarlyStopping(patience=3, restore_best_weights=True),\n", " keras.callbacks.ModelCheckpoint(\"SpamDetector.h5\", save_best_only=True)\n", "]" ], "metadata": { "id": "KXZX-MiaIB9a" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "random_seed = 123\n", "np.random.seed(random_seed)\n", "tf.random.set_seed(random_seed)" ], "metadata": { "id": "AXke-pwyIZ_I" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "import pandas as pd\n", "\n", "# Load the dataset using pandas, specifying the encoding as 'latin-1'\n", "data_frame = pd.read_csv('spam.csv', encoding='latin-1')\n", "data_frame.rename(columns = {'v1':'Category','v2':'Messages'},inplace=True)\n", "\n", "# Print the first five rows of the dataset\n", "print(data_frame.head())" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "IWIFoa10Ien6", "outputId": "068ee878-6a88-4688-8830-81f6cddb26d0" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " Category Messages Unnamed: 2 \\\n", "0 ham Go until jurong point, crazy.. Available only ... NaN \n", "1 ham Ok lar... Joking wif u oni... NaN \n", "2 spam Free entry in 2 a wkly comp to win FA Cup fina... NaN \n", "3 ham U dun say so early hor... U c already then say... NaN \n", "4 ham Nah I don't think he goes to usf, he lives aro... NaN \n", "\n", " Unnamed: 3 Unnamed: 4 \n", "0 NaN NaN \n", "1 NaN NaN \n", "2 NaN NaN \n", "3 NaN NaN \n", "4 NaN NaN \n" ] } ] }, { "cell_type": "code", "source": [ "\n", "class_dis = data_frame.Category.value_counts()\n", "class_names = class_dis.index\n", "\n", "# Create the Pie Chart\n", "fig = px.pie(names=class_names,\n", " values=class_dis,\n", " color=class_names,\n", " hole=0.4,\n", " labels={'value': 'Count', 'names': 'Class'},\n", " title='Class Distribution of Spam Text Messages')\n", "\n", "# Customize the layout\n", "fig.update_layout(\n", " margin=dict(l=10, r=10, t=60, b=10),\n", " legend=dict(orientation=\"h\", yanchor=\"bottom\", y=1.02, xanchor=\"right\", x=1),\n", ")\n", "\n", "# Show the plot\n", "fig.show()\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 542 }, "id": "AytubNcyIiFE", "outputId": "fd32bc9d-0061-4778-ff22-52004bb71f5a" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/html": [ "\n", "
\n", "\n", "Model: \"TransformerNet\"\n",
"
\n"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n",
"┃\u001b[1m \u001b[0m\u001b[1mLayer (type) \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m Param #\u001b[0m\u001b[1m \u001b[0m┃\n",
"┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n",
"│ InputLayer (\u001b[38;5;33mInputLayer\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m40\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n",
"├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
"│ EmbeddingLayer │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m40\u001b[0m, \u001b[38;5;34m256\u001b[0m) │ \u001b[38;5;34m2,570,240\u001b[0m │\n",
"│ (\u001b[38;5;33mTokenAndPositionalEmbedding\u001b[0m) │ │ │\n",
"├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
"│ TransformerLayer │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m40\u001b[0m, \u001b[38;5;34m256\u001b[0m) │ \u001b[38;5;34m1,118,848\u001b[0m │\n",
"│ (\u001b[38;5;33mTransformerLayer\u001b[0m) │ │ │\n",
"├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
"│ GlobalAveragePooling │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n",
"│ (\u001b[38;5;33mGlobalAveragePooling1D\u001b[0m) │ │ │\n",
"├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
"│ Dropout (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m) │ \u001b[38;5;34m0\u001b[0m │\n",
"├─────────────────────────────────┼────────────────────────┼───────────────┤\n",
"│ OutputLayer (\u001b[38;5;33mDense\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m) │ \u001b[38;5;34m257\u001b[0m │\n",
"└─────────────────────────────────┴────────────────────────┴───────────────┘\n"
],
"text/html": [
"┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓\n", "┃ Layer (type) ┃ Output Shape ┃ Param # ┃\n", "┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩\n", "│ InputLayer (InputLayer) │ (None, 40) │ 0 │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ EmbeddingLayer │ (None, 40, 256) │ 2,570,240 │\n", "│ (TokenAndPositionalEmbedding) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ TransformerLayer │ (None, 40, 256) │ 1,118,848 │\n", "│ (TransformerLayer) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ GlobalAveragePooling │ (None, 256) │ 0 │\n", "│ (GlobalAveragePooling1D) │ │ │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ Dropout (Dropout) │ (None, 256) │ 0 │\n", "├─────────────────────────────────┼────────────────────────┼───────────────┤\n", "│ OutputLayer (Dense) │ (None, 1) │ 257 │\n", "└─────────────────────────────────┴────────────────────────┴───────────────┘\n", "\n" ] }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m3,689,345\u001b[0m (14.07 MB)\n" ], "text/html": [ "
Total params: 3,689,345 (14.07 MB)\n", "\n" ] }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m3,689,345\u001b[0m (14.07 MB)\n" ], "text/html": [ "
Trainable params: 3,689,345 (14.07 MB)\n", "\n" ] }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n" ], "text/html": [ "
Non-trainable params: 0 (0.00 B)\n", "\n" ] }, "metadata": {} } ] }, { "cell_type": "code", "source": [ "model.compile(\n", " loss='binary_crossentropy',\n", " optimizer='adam',\n", " metrics=[\n", " keras.metrics.BinaryAccuracy(name='accuracy'),\n", " keras.metrics.Precision(name='precision'),\n", " keras.metrics.Recall(name='recall'),\n", " keras.metrics.AUC(name='auc'),\n", " ]\n", ")\n", "\n", "# Train Model\n", "history = model.fit(\n", " X_train, y_train,\n", " validation_split=0.1,\n", " batch_size=batch_size,\n", " epochs=10,\n", " callbacks=callbacks,\n", " class_weight=class_weights\n", ")" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "oZYuIFJuOFuJ", "outputId": "4c17fa29-186a-4fba-bcb7-926b2293df03" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Epoch 1/10\n", "\u001b[1m126/126\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m70s\u001b[0m 512ms/step - accuracy: 0.9943 - auc: 0.9993 - loss: 0.0244 - precision: 0.9618 - recall: 0.9944 - val_accuracy: 0.9888 - val_auc: 0.9832 - val_loss: 0.0627 - val_precision: 0.9848 - val_recall: 0.9420\n", "Epoch 2/10\n", "\u001b[1m126/126\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m80s\u001b[0m 497ms/step - accuracy: 0.9985 - auc: 0.9997 - loss: 0.0080 - precision: 0.9882 - recall: 1.0000 - val_accuracy: 0.9865 - val_auc: 0.9779 - val_loss: 0.0860 - val_precision: 1.0000 - val_recall: 0.9130\n", "Epoch 3/10\n", "\u001b[1m126/126\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m81s\u001b[0m 491ms/step - accuracy: 0.9995 - auc: 1.0000 - loss: 0.0026 - precision: 0.9961 - recall: 1.0000 - val_accuracy: 0.9933 - val_auc: 0.9777 - val_loss: 0.0711 - val_precision: 1.0000 - val_recall: 0.9565\n", "Epoch 4/10\n", "\u001b[1m126/126\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m82s\u001b[0m 490ms/step - accuracy: 0.9997 - auc: 1.0000 - loss: 0.0015 - precision: 0.9980 - recall: 1.0000 - val_accuracy: 0.9865 - val_auc: 0.9777 - val_loss: 0.0906 - val_precision: 1.0000 - val_recall: 0.9130\n" ] } ] }, { "cell_type": "code", "source": [ "fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 8))\n", "plt.subplots_adjust(hspace=0.5)\n", "\n", "axes[0, 0].plot(history.history['loss'], label='Training Loss')\n", "axes[0, 0].plot(history.history['val_loss'], label='Validation Loss')\n", "axes[0, 0].set_title('Loss', fontsize=14)\n", "axes[0, 0].set_xlabel('Epoch', fontsize=12)\n", "axes[0, 0].set_ylabel('Loss', fontsize=12)\n", "axes[0, 0].grid(True)\n", "axes[0, 0].legend(fontsize=10)\n", "\n", "axes[0, 1].plot(history.history['accuracy'], label='Training Accuracy')\n", "axes[0, 1].plot(history.history['val_accuracy'], label='Validation Accuracy')\n", "axes[0, 1].set_title('Accuracy', fontsize=14)\n", "axes[0, 1].set_xlabel('Epoch', fontsize=12)\n", "axes[0, 1].set_ylabel('Accuracy', fontsize=12)\n", "axes[0, 1].grid(True)\n", "axes[0, 1].legend(fontsize=10)\n", "\n", "axes[1, 0].plot(history.history['precision'], label='Training Precision')\n", "axes[1, 0].plot(history.history['val_precision'], label='Validation Precision')\n", "axes[1, 0].set_title('Precision', fontsize=14)\n", "axes[1, 0].set_xlabel('Epoch', fontsize=12)\n", "axes[1, 0].set_ylabel('Precision', fontsize=12)\n", "axes[1, 0].grid(True)\n", "axes[1, 0].legend(fontsize=10)\n", "\n", "axes[1, 1].plot(history.history['recall'], label='Training Recall')\n", "axes[1, 1].plot(history.history['val_recall'], label='Validation Recall')\n", "axes[1, 1].set_title('Recall', fontsize=14)\n", "axes[1, 1].set_xlabel('Epoch', fontsize=12)\n", "axes[1, 1].set_ylabel('Recall', fontsize=12)\n", "axes[1, 1].grid(True)\n", "axes[1, 1].legend(fontsize=10)\n", "\n", "fig.suptitle('Model Performance Metrics', fontsize=16, y=1.05)\n", "plt.show()\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 630 }, "id": "QAiJS9mNOLIT", "outputId": "45dc8a9d-ef86-4363-8457-e52c8616b6a9" }, "execution_count": null, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "