Upload 8 files

# AAGCN-VSL-Detection
# Introction
In this project, we modify a model to better classify Vietnamese sign language (VSL). We use Mediapipe to extract keypoints. Then apply a bilinear preprocessing technique in order to reconstruct the missing keypoints. The model is built based on the Adaptive Graph Convolutional Networks by combining with Attention Mechanisms, including spatial, temporal, and channel attention.

The model is test on our self-collected VSL dataset, which was collected from the school for hearing-impaired children in Hanoi, Vietnam. The data consists of 5,572 videos of 28 actors, with 199 classes each, representing the most frequently used spoken Vietnamese. Before training on this dataset, we pre-train the model with the Ankara University Turkish Sign Language Dataset (AUTSL) first to obtain the weights, learning rate for further training.

# Setup
Download all the files and run the jupyter file Total (VSL). This file will do almost the entire training process, ranging from extracting the keypoints to performing interpolation, training the model. All the other files are the supplement for the model part.

Change the path of the input dataset. The code will read all videos in the folder.

In this coding file (Total (VSL)), there are 2 options:
1. Train new.
2. Train continuously from the model that was trained on the AUTSL dataset.

Uncomment the part of the code that you want to use.

# Evaluation
We use k-fold in this project with k is defined as 10. The 10 trained models will be stored in the "checkpoints" folder. The final output will print out the best accuracy.

# Acknoledgement
This project is built by iBME lab at Hanoi University of Science and Technology, Vietnam. It is funded by Hanoi University of Science and Technology (HUST) under project number T2023-PC-028.

Total (VSL).ipynb

@@ -0,0 +1,883 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "pre-train with AUTSL dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This is the code to train the model with the AUTSL dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'\\nfrom torch.utils.data import DataLoader\\nimport torch\\nfrom torchinfo import summary\\nfrom feeder import FeederINCLUDE\\nfrom aagcn import Model\\nimport pytorch_lightning as pl\\nfrom pytorch_lightning.loggers import WandbLogger  # Importing here\\nfrom pytorch_lightning.callbacks import ModelCheckpoint\\nimport wandb\\nfrom augumentation import Rotate, Compose\\nfrom torch.utils.data import random_split\\n\\n\\nif __name__ == \\'__main__\\':\\n\\n    # Hyper parameter tuning : batch_size, learning_rate, weight_decay\\n    config = {\\'batch_size\\': 150, \\'learning_rate\\': 0.0137296, \\'weight_decay\\': 0.000150403}\\n    \\n    # Load device\\n    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\\n\\n    # Initialize wandb\\n    wandb.finish()\\n    wandb.init(project=\"GCN_VSL\", config=config)  \\n    wandb_config = wandb.config  # Access the config parameters\\n\\n    try:\\n    # Your training or evaluation code here\\n        print(\"WandB initialized successfully.\")\\n\\n    finally:\\n    # Fnish the WandB run\\n        wandb.finish()\\n\\n    # Load model\\n    model = Model(num_class=226, num_point=46, num_person=1, in_channels=2,\\n                  graph_args={\"layout\": \"mediapipe_two_hand\", \"strategy\": \"spatial\"},\\n                  learning_rate=wandb_config.learning_rate, weight_decay=wandb_config.weight_decay)\\n\\n    # Callback PL\\n    callbacks = [\\n        ModelCheckpoint(\\n            dirpath=\"checkpoints\",\\n            monitor=\"valid_loss\",\\n            mode=\"min\",\\n            every_n_epochs=2,\\n            filename=\\'{epoch}-{valid_loss:.2f}-{valid_accuracy:.2f}-autsl-aagcn\\'\\n        ),\\n    ]\\n\\n    # Augmentation\\n    transforms = Compose([\\n        Rotate(15, 80, 25, (0.5, 0.5))\\n    ])\\n\\n    %cd /home/ibmelab/Documents/GG/VSLRecognition/AUTSL/AAGCN\\n    # Dataset class\\n    train_dataset = FeederINCLUDE(data_path=f\"autsl_train_data_preprocess.npy\", label_path=f\"train_label_preprocess.npy\",\\n                                   transform=transforms)\\n    test_dataset = FeederINCLUDE(data_path=f\"autsl_test_data_preprocess.npy\", label_path=f\"test_label_preprocess.npy\")\\n    valid_dataset = FeederINCLUDE(data_path=f\"autsl_valid_data_preprocess.npy\", label_path=f\"valid_label_preprocess.npy\")\\n\\n    # DataLoader\\n    train_dataloader = DataLoader(train_dataset, batch_size=wandb_config.batch_size, shuffle=True)\\n    test_dataloader = DataLoader(test_dataset, batch_size=wandb_config.batch_size, shuffle=False)\\n    val_dataloader = DataLoader(valid_dataset, batch_size=wandb_config.batch_size, shuffle=False)\\n\\n    # Wandb Logger\\n    wandb_logger = WandbLogger(log_model=\\'all\\')\\n\\n    %cd /media/ibmelab/ibme21/Test\\n    # Trainer PL\\n    trainer = pl.Trainer(max_epochs=120, accelerator=\"auto\", check_val_every_n_epoch=1,\\n                         devices=1, callbacks=callbacks, logger=wandb_logger)  # Added logger here\\n\\n    trainer.fit(model, train_dataloader, val_dataloader)\\n\\n    # Optional: Uncomment this when you want to test\\n    # trainer.test(model, test_dataloader, ckpt_path=\"checkpoints/your_checkpoint.ckpt\", verbose=True)\\n'"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "'''\n",
+    "from torch.utils.data import DataLoader\n",
+    "import torch\n",
+    "from torchinfo import summary\n",
+    "from feeder import FeederINCLUDE\n",
+    "from aagcn import Model\n",
+    "import pytorch_lightning as pl\n",
+    "from pytorch_lightning.loggers import WandbLogger  # Importing here\n",
+    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
+    "import wandb\n",
+    "from augumentation import Rotate, Compose\n",
+    "from torch.utils.data import random_split\n",
+    "\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "\n",
+    "    # Hyper parameter tuning : batch_size, learning_rate, weight_decay\n",
+    "    config = {'batch_size': 150, 'learning_rate': 0.0137296, 'weight_decay': 0.000150403}\n",
+    "    \n",
+    "    # Load device\n",
+    "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "\n",
+    "    # Initialize wandb\n",
+    "    wandb.finish()\n",
+    "    wandb.init(project=\"GCN_VSL\", config=config)  \n",
+    "    wandb_config = wandb.config  # Access the config parameters\n",
+    "\n",
+    "    try:\n",
+    "    # Your training or evaluation code here\n",
+    "        print(\"WandB initialized successfully.\")\n",
+    "\n",
+    "    finally:\n",
+    "    # Fnish the WandB run\n",
+    "        wandb.finish()\n",
+    "\n",
+    "    # Load model\n",
+    "    model = Model(num_class=226, num_point=46, num_person=1, in_channels=2,\n",
+    "                  graph_args={\"layout\": \"mediapipe_two_hand\", \"strategy\": \"spatial\"},\n",
+    "                  learning_rate=wandb_config.learning_rate, weight_decay=wandb_config.weight_decay)\n",
+    "\n",
+    "    # Callback PL\n",
+    "    callbacks = [\n",
+    "        ModelCheckpoint(\n",
+    "            dirpath=\"checkpoints\",\n",
+    "            monitor=\"valid_loss\",\n",
+    "            mode=\"min\",\n",
+    "            every_n_epochs=2,\n",
+    "            filename='{epoch}-{valid_loss:.2f}-{valid_accuracy:.2f}-autsl-aagcn'\n",
+    "        ),\n",
+    "    ]\n",
+    "\n",
+    "    # Augmentation\n",
+    "    transforms = Compose([\n",
+    "        Rotate(15, 80, 25, (0.5, 0.5))\n",
+    "    ])\n",
+    "\n",
+    "    %cd /home/ibmelab/Documents/GG/VSLRecognition/AUTSL/AAGCN\n",
+    "    # Dataset class\n",
+    "    train_dataset = FeederINCLUDE(data_path=f\"autsl_train_data_preprocess.npy\", label_path=f\"train_label_preprocess.npy\",\n",
+    "                                   transform=transforms)\n",
+    "    test_dataset = FeederINCLUDE(data_path=f\"autsl_test_data_preprocess.npy\", label_path=f\"test_label_preprocess.npy\")\n",
+    "    valid_dataset = FeederINCLUDE(data_path=f\"autsl_valid_data_preprocess.npy\", label_path=f\"valid_label_preprocess.npy\")\n",
+    "\n",
+    "    # DataLoader\n",
+    "    train_dataloader = DataLoader(train_dataset, batch_size=wandb_config.batch_size, shuffle=True)\n",
+    "    test_dataloader = DataLoader(test_dataset, batch_size=wandb_config.batch_size, shuffle=False)\n",
+    "    val_dataloader = DataLoader(valid_dataset, batch_size=wandb_config.batch_size, shuffle=False)\n",
+    "\n",
+    "    # Wandb Logger\n",
+    "    wandb_logger = WandbLogger(log_model='all')\n",
+    "\n",
+    "    %cd /media/ibmelab/ibme21/Test\n",
+    "    # Trainer PL\n",
+    "    trainer = pl.Trainer(max_epochs=120, accelerator=\"auto\", check_val_every_n_epoch=1,\n",
+    "                         devices=1, callbacks=callbacks, logger=wandb_logger)  # Added logger here\n",
+    "\n",
+    "    trainer.fit(model, train_dataloader, val_dataloader)\n",
+    "\n",
+    "    # Optional: Uncomment this when you want to test\n",
+    "    # trainer.test(model, test_dataloader, ckpt_path=\"checkpoints/your_checkpoint.ckpt\", verbose=True)\n",
+    "'''\n",
+    "    \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import mediapipe as mp\n",
+    "import cv2\n",
+    "from collections import defaultdict\n",
+    "from joblib import Parallel, delayed\n",
+    "from tqdm import tqdm\n",
+    "import ast\n",
+    "import os\n",
+    "import csv\n",
+    "import re\n",
+    "from sklearn.model_selection import KFold\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Load the videos to videos_list.csv (columns: file (path), label, gloss, video name, actor)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Video names have been written to videos_list.csv\n",
+      "Minimum label: 20\n",
+      "Labels have been updated and saved.\n"
+     ]
+    }
+   ],
+   "source": [
+    "folder_path = r'path_to_dataset_folder'\n",
+    "csv_file_path = 'videos_list.csv'\n",
+    "labels_file_path = '1_1000_label.csv'\n",
+    "final_file_path = 'temp_videos_list.csv'\n",
+    "\n",
+    "label_to_gloss = {}\n",
+    "with open(labels_file_path, mode='r', encoding='utf-8') as labels_file:\n",
+    "    csv_reader = csv.DictReader(labels_file)\n",
+    "    for row in csv_reader:\n",
+    "        label = int(row['id_label_in_documents'])\n",
+    "        gloss = row['name']\n",
+    "        label_to_gloss[label] = gloss\n",
+    "\n",
+    "with open(csv_file_path, mode='w', newline='', encoding='utf-8') as csv_file:\n",
+    "    csv_writer = csv.writer(csv_file)\n",
+    "    csv_writer.writerow(['file', 'label', 'gloss', 'video_name', 'actor'])\n",
+    "\n",
+    "    for filename in os.listdir(folder_path):\n",
+    "        if filename.lower().endswith(('.mp4', '.mkv', '.avi', '.mov', '.flv', '.wmv')):\n",
+    "            actor = filename.split('_')[0]\n",
+    "            \n",
+    "            match = re.search(r'_(\\d+)\\.', filename)\n",
+    "            if match:\n",
+    "                label = int(match.group(1))\n",
+    "                gloss = label_to_gloss.get(label, 'Unknown')\n",
+    "            else:\n",
+    "                label = 'N/A'\n",
+    "                gloss = 'Unknown'\n",
+    "\n",
+    "            if label != 200:\n",
+    "                full_filename = os.path.join(folder_path, filename)\n",
+    "                csv_writer.writerow([full_filename, label, gloss, filename, actor]) \n",
+    "\n",
+    "print(f'Video names have been written to {csv_file_path}')\n",
+    "\n",
+    "# Find min label\n",
+    "with open(csv_file_path, mode='r', newline='', encoding='utf-8') as csv_file:\n",
+    "    csv_reader = csv.DictReader(csv_file)\n",
+    "    labels = [int(row[\"label\"]) for row in csv_reader if row[\"label\"].isdigit()]  \n",
+    "    min_label = min(labels) if labels else None\n",
+    "\n",
+    "print(\"Minimum label:\", min_label)\n",
+    "\n",
+    "# Normalize labels\n",
+    "with open(csv_file_path, mode='r', newline='', encoding='utf-8') as csv_file, \\\n",
+    "     open(final_file_path, mode='w', newline='', encoding='utf-8') as final_file:\n",
+    "    \n",
+    "    csv_reader = csv.DictReader(csv_file)\n",
+    "    fieldnames = csv_reader.fieldnames\n",
+    "    \n",
+    "    csv_writer = csv.DictWriter(final_file, fieldnames=fieldnames)\n",
+    "    csv_writer.writeheader()\n",
+    "    \n",
+    "    for row in csv_reader:\n",
+    "        if row['label'].isdigit():  # Check if label is a digit before converting\n",
+    "            row['label'] = str(int(row['label']) - min_label)  \n",
+    "        csv_writer.writerow(row)\n",
+    "\n",
+    "# Replace the original file with the updated file\n",
+    "os.replace(final_file_path, csv_file_path)\n",
+    "\n",
+    "print(\"Labels have been updated and saved.\")\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Number of labels in the dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "10\n",
+      "{20, 21, 22, 23, 24, 25, 26, 27, 28, 29}\n"
+     ]
+    }
+   ],
+   "source": [
+    "num_labels = len(set(labels))\n",
+    "print(num_labels)\n",
+    "print(set(labels))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Extract keypoints"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                                    \r"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import mediapipe as mp\n",
+    "import cv2\n",
+    "import os\n",
+    "from collections import defaultdict\n",
+    "from joblib import Parallel, delayed\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "mp_holistic = mp.solutions.holistic\n",
+    "mp_drawing = mp.solutions.drawing_utils\n",
+    "\n",
+    "hand_landmarks = ['INDEX_FINGER_DIP', 'INDEX_FINGER_MCP', 'INDEX_FINGER_PIP', 'INDEX_FINGER_TIP', \n",
+    "                  'MIDDLE_FINGER_DIP', 'MIDDLE_FINGER_MCP', 'MIDDLE_FINGER_PIP', 'MIDDLE_FINGER_TIP', \n",
+    "                  'PINKY_DIP', 'PINKY_MCP', 'PINKY_PIP', 'PINKY_TIP', 'RING_FINGER_DIP', 'RING_FINGER_MCP', \n",
+    "                  'RING_FINGER_PIP', 'RING_FINGER_TIP', 'THUMB_CMC', 'THUMB_IP', 'THUMB_MCP', 'THUMB_TIP', 'WRIST']\n",
+    "pose_landmarks = ['LEFT_ANKLE', 'LEFT_EAR', 'LEFT_ELBOW', 'LEFT_EYE', 'LEFT_EYE_INNER', 'LEFT_EYE_OUTER', \n",
+    "                  'LEFT_FOOT_INDEX', 'LEFT_HEEL', 'LEFT_HIP', 'LEFT_INDEX', 'LEFT_KNEE', 'LEFT_PINKY', \n",
+    "                  'LEFT_SHOULDER', 'LEFT_THUMB', 'LEFT_WRIST', 'MOUTH_LEFT', 'MOUTH_RIGHT', 'NOSE', \n",
+    "                  'RIGHT_ANKLE', 'RIGHT_EAR', 'RIGHT_ELBOW', 'RIGHT_EYE', 'RIGHT_EYE_INNER', 'RIGHT_EYE_OUTER', \n",
+    "                  'RIGHT_FOOT_INDEX', 'RIGHT_HEEL', 'RIGHT_HIP', 'RIGHT_INDEX', 'RIGHT_KNEE', 'RIGHT_PINKY', \n",
+    "                  'RIGHT_SHOULDER', 'RIGHT_THUMB', 'RIGHT_WRIST']\n",
+    "\n",
+    "def extract_keypoint(video_path, label, actor):\n",
+    "    cap = cv2.VideoCapture(video_path)\n",
+    "    \n",
+    "    keypoint_dict = defaultdict(list)\n",
+    "    count = 0\n",
+    "\n",
+    "    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
+    "        while True:\n",
+    "            ret, frame = cap.read()\n",
+    "            if not ret:\n",
+    "                break\n",
+    "            \n",
+    "            count += 1\n",
+    "            image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n",
+    "            results = holistic.process(image)\n",
+    "\n",
+    "            if results.right_hand_landmarks:\n",
+    "                for idx, landmark in enumerate(results.right_hand_landmarks.landmark): \n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_right_x\"].append(landmark.x)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_right_y\"].append(landmark.y)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_right_z\"].append(landmark.z)\n",
+    "            else:\n",
+    "                for idx in range(len(hand_landmarks)):\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_right_x\"].append(0)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_right_y\"].append(0)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_right_z\"].append(0)\n",
+    "\n",
+    "            if results.left_hand_landmarks:\n",
+    "                for idx, landmark in enumerate(results.left_hand_landmarks.landmark): \n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_left_x\"].append(landmark.x)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_left_y\"].append(landmark.y)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_left_z\"].append(landmark.z)\n",
+    "            else:\n",
+    "                for idx in range(len(hand_landmarks)):\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_left_x\"].append(0)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_left_y\"].append(0)\n",
+    "                    keypoint_dict[f\"{hand_landmarks[idx]}_left_z\"].append(0)\n",
+    "\n",
+    "            if results.pose_landmarks:\n",
+    "                for idx, landmark in enumerate(results.pose_landmarks.landmark): \n",
+    "                    keypoint_dict[f\"{pose_landmarks[idx]}_x\"].append(landmark.x)\n",
+    "                    keypoint_dict[f\"{pose_landmarks[idx]}_y\"].append(landmark.y)\n",
+    "                    keypoint_dict[f\"{pose_landmarks[idx]}_z\"].append(landmark.z)\n",
+    "            else:\n",
+    "                for idx in range(len(pose_landmarks)):\n",
+    "                    keypoint_dict[f\"{pose_landmarks[idx]}_x\"].append(0)\n",
+    "                    keypoint_dict[f\"{pose_landmarks[idx]}_y\"].append(0)\n",
+    "                    keypoint_dict[f\"{pose_landmarks[idx]}_z\"].append(0)\n",
+    "\n",
+    "        keypoint_dict[\"frame\"] = count\n",
+    "        keypoint_dict[\"video_path\"] = video_path\n",
+    "        keypoint_dict[\"label\"] = label\n",
+    "        keypoint_dict[\"actor\"] = actor\n",
+    "\n",
+    "        return keypoint_dict\n",
+    "\n",
+    "def process_videos():\n",
+    "    csv_file = f\"videos_list.csv\"\n",
+    "    data = pd.read_csv(csv_file)\n",
+    "\n",
+    "    keypoints_list = Parallel(n_jobs=-1)( \n",
+    "        delayed(extract_keypoint)(row['file'], row['label'], row['actor']) for index, row in tqdm(data.iterrows(), total=len(data), desc=\"Processing videos\", leave=False)\n",
+    "    )\n",
+    "\n",
+    "    keypoints_df = pd.DataFrame(keypoints_list)\n",
+    "    keypoints_df.to_csv(f\"vsl{num_labels}_keypoints.csv\", index=False)\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    process_videos()\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Interpolation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 280/280 [00:04<00:00, 67.80it/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Interpolated keypoints saved to vsl10_interpolated_keypoints.csv\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 280/280 [00:03<00:00, 91.60it/s] "
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data processing and saving completed.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import ast\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "def find_index(array):\n",
+    "    for i, num in enumerate(array):\n",
+    "        if num != 0:\n",
+    "            return i\n",
+    "\n",
+    "def curl_skeleton(array):\n",
+    "    if sum(array) == 0:\n",
+    "        return array\n",
+    "    for i, location in enumerate(array):\n",
+    "        if location != 0:\n",
+    "            continue\n",
+    "        else:\n",
+    "            if i == 0 or i == len(array) - 1:\n",
+    "                continue\n",
+    "            else:\n",
+    "                if array[i + 1] != 0:\n",
+    "                    array[i] = float((array[i - 1] + array[i + 1]) / 2)\n",
+    "                else:\n",
+    "                    if sum(array[i:]) == 0:\n",
+    "                        continue\n",
+    "                    else:\n",
+    "                        j = find_index(array[i + 1:])\n",
+    "                        array[i] = float(((1 + j) * array[i - 1] + 1 * array[i + 1 + j]) / (2 + j))\n",
+    "    return array\n",
+    "\n",
+    "def interpolate_keypoints(input_file, output_file, body_identifiers):\n",
+    "    train_data = pd.read_csv(input_file)\n",
+    "    output_df = train_data.copy()\n",
+    "\n",
+    "    for index, video in tqdm(train_data.iterrows(), total=train_data.shape[0]):\n",
+    "        for identifier in body_identifiers:\n",
+    "            # Interpolate the x and y keypoints\n",
+    "            x_values = curl_skeleton(ast.literal_eval(video[identifier + \"_x\"]))\n",
+    "            y_values = curl_skeleton(ast.literal_eval(video[identifier + \"_y\"]))\n",
+    "\n",
+    "            output_df.at[index, identifier + \"_x\"] = str(x_values)\n",
+    "            output_df.at[index, identifier + \"_y\"] = str(y_values)\n",
+    "\n",
+    "    output_df.to_csv(output_file, index=False)\n",
+    "    print(f\"Interpolated keypoints saved to {output_file}\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    input_file_path = f\"vsl{num_labels}_keypoints.csv\"\n",
+    "    output_file_path = f\"vsl{num_labels}_interpolated_keypoints.csv\"\n",
+    "\n",
+    "    hand_landmarks = [\n",
+    "        'INDEX_FINGER_DIP', 'INDEX_FINGER_MCP', 'INDEX_FINGER_PIP', 'INDEX_FINGER_TIP', \n",
+    "        'MIDDLE_FINGER_DIP', 'MIDDLE_FINGER_MCP', 'MIDDLE_FINGER_PIP', 'MIDDLE_FINGER_TIP', \n",
+    "        'PINKY_DIP', 'PINKY_MCP', 'PINKY_PIP', 'PINKY_TIP', \n",
+    "        'RING_FINGER_DIP', 'RING_FINGER_MCP', 'RING_FINGER_PIP', 'RING_FINGER_TIP', \n",
+    "        'THUMB_CMC', 'THUMB_IP', 'THUMB_MCP', 'THUMB_TIP', 'WRIST'\n",
+    "    ]\n",
+    "    HAND_IDENTIFIERS = [id + \"_right\" for id in hand_landmarks] + [id + \"_left\" for id in hand_landmarks]\n",
+    "    POSE_IDENTIFIERS = [\"RIGHT_SHOULDER\", \"LEFT_SHOULDER\", \"LEFT_ELBOW\", \"RIGHT_ELBOW\"]\n",
+    "    body_identifiers = HAND_IDENTIFIERS + POSE_IDENTIFIERS \n",
+    "\n",
+    "    interpolate_keypoints(input_file_path, output_file_path, body_identifiers)\n",
+    "\n",
+    "    # Load interpolated data and store them in numpy files\n",
+    "    train_data = pd.read_csv(output_file_path)\n",
+    "    frames = 80\n",
+    "\n",
+    "    data = []\n",
+    "    labels = []\n",
+    "\n",
+    "    for video_index, video in tqdm(train_data.iterrows(), total=train_data.shape[0]):\n",
+    "        T = len(ast.literal_eval(video[\"INDEX_FINGER_DIP_right_x\"]))\n",
+    "        current_row = np.empty(shape=(2, T, len(body_identifiers), 1))\n",
+    "\n",
+    "        for index, identifier in enumerate(body_identifiers):\n",
+    "            data_keypoint_preprocess_x = ast.literal_eval(video[identifier + \"_x\"])\n",
+    "            current_row[0, :, index, :] = np.asarray(data_keypoint_preprocess_x).reshape(T, 1)\n",
+    "\n",
+    "            data_keypoint_preprocess_y = ast.literal_eval(video[identifier + \"_y\"])\n",
+    "            current_row[1, :, index, :] = np.asarray(data_keypoint_preprocess_y).reshape(T, 1)\n",
+    "\n",
+    "        if T < frames:\n",
+    "            target = np.zeros(shape=(2, frames, len(body_identifiers), 1))\n",
+    "            target[:, :T, :, :] = current_row\n",
+    "        else:\n",
+    "            target = current_row[:, :frames, :, :]\n",
+    "\n",
+    "        data.append(target)\n",
+    "        labels.append(int(video[\"label\"]))\n",
+    "\n",
+    "    keypoint_data = np.stack(data, axis=0)\n",
+    "    label_data = np.stack(labels, axis=0)\n",
+    "    np.save(f'vsl{num_labels}_data_preprocess.npy', keypoint_data)\n",
+    "    np.save(f'vsl{num_labels}_label_preprocess.npy', label_data)\n",
+    "    print(\"Data processing and saving completed.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(280, 2, 80, 46, 1)\n",
+      "(280,)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "a = np.load(f'vsl{num_labels}_data_preprocess.npy')\n",
+    "b = np.load(f'vsl{num_labels}_label_preprocess.npy')\n",
+    "\n",
+    "print(a.shape)\n",
+    "print(b.shape)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Do K-Folds and store the keypoints in numpy files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Number of actors: 28\n",
+      "-----------------------------------------------------\n",
+      "Fold 1: 30 test samples\n",
+      "Fold 2: 29 test samples\n",
+      "Fold 3: 30 test samples\n",
+      "Fold 4: 30 test samples\n",
+      "Fold 5: 30 test samples\n",
+      "Fold 6: 30 test samples\n",
+      "Fold 7: 31 test samples\n",
+      "Fold 8: 30 test samples\n",
+      "Fold 9: 20 test samples\n",
+      "Fold 10: 20 test samples\n",
+      "Processed and saved vsl10 fold 1 successfully.\n",
+      "Processed and saved vsl10 fold 2 successfully.\n",
+      "Processed and saved vsl10 fold 3 successfully.\n",
+      "Processed and saved vsl10 fold 4 successfully.\n",
+      "Processed and saved vsl10 fold 5 successfully.\n",
+      "Processed and saved vsl10 fold 6 successfully.\n",
+      "Processed and saved vsl10 fold 7 successfully.\n",
+      "Processed and saved vsl10 fold 8 successfully.\n",
+      "Processed and saved vsl10 fold 9 successfully.\n",
+      "Processed and saved vsl10 fold 10 successfully.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from sklearn.model_selection import KFold\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "def k_fold_cross_validation(train_data, keypoint_data, label_data, num_labels, k_folds, destination_folder=\"numpy_files\"):\n",
+    "    os.makedirs(destination_folder, exist_ok=True)\n",
+    "\n",
+    "    actors = train_data['actor'].unique()\n",
+    "    print(f\"Number of actors: {len(actors)}\")\n",
+    "    print('-----------------------------------------------------')\n",
+    "\n",
+    "    kf = KFold(n_splits=k_folds, shuffle=True, random_state=42)\n",
+    "\n",
+    "    actor_to_indices = {actor: train_data.index[train_data['actor'] == actor].tolist() for actor in actors}\n",
+    "    folds = [[] for _ in range(k_folds)]\n",
+    "\n",
+    "    for fold, (train_actors, test_actors) in enumerate(kf.split(actors)):\n",
+    "        train_actors = actors[train_actors]\n",
+    "        test_actors = actors[test_actors]\n",
+    "        \n",
+    "        for actor in test_actors:\n",
+    "            folds[fold].extend(actor_to_indices[actor])\n",
+    "\n",
+    "        tqdm.write(f\"Fold {fold+1}: {len(folds[fold])} test samples\")\n",
+    "\n",
+    "    # Iterate over each fold to create train-test splits\n",
+    "    for fold in range(k_folds):\n",
+    "        test_indices = folds[fold]\n",
+    "        train_indices = [idx for f in range(k_folds) if f != fold for idx in folds[f]]\n",
+    "\n",
+    "        X_train, X_test = keypoint_data[train_indices], keypoint_data[test_indices]\n",
+    "        y_train = np.array(label_data[train_indices], dtype=np.int64)\n",
+    "        y_test = np.array(label_data[test_indices], dtype=np.int64)\n",
+    "\n",
+    "        np.save(os.path.join(destination_folder, f'vsl{num_labels}_data_fold{fold+1}_train.npy'), X_train)\n",
+    "        np.save(os.path.join(destination_folder, f'vsl{num_labels}_label_fold{fold+1}_train.npy'), y_train)\n",
+    "        np.save(os.path.join(destination_folder, f'vsl{num_labels}_data_fold{fold+1}_test.npy'), X_test)\n",
+    "        np.save(os.path.join(destination_folder, f'vsl{num_labels}_label_fold{fold+1}_test.npy'), y_test)\n",
+    "\n",
+    "        tqdm.write(f\"Processed and saved vsl{num_labels} fold {fold+1} successfully.\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    input_file_path = f\"vsl{num_labels}_interpolated_keypoints.csv\"\n",
+    "    train_data = pd.read_csv(input_file_path)\n",
+    "\n",
+    "    keypoint_data = np.load(f'vsl{num_labels}_data_preprocess.npy')\n",
+    "    label_data = np.load(f'vsl{num_labels}_label_preprocess.npy')\n",
+    "\n",
+    "    num_labels = len(np.unique(label_data))\n",
+    "\n",
+    "    k_folds = 10\n",
+    "    k_fold_cross_validation(train_data, keypoint_data, label_data, num_labels, k_folds)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(29, 2, 80, 46, 1)\n",
+      "(251, 2, 80, 46, 1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "a = np.load(f'numpy_files/vsl{num_labels}_data_fold2_test.npy')\n",
+    "b = np.load(f'numpy_files/vsl{num_labels}_data_fold2_train.npy')\n",
+    "\n",
+    "print(a.shape)\n",
+    "print(b.shape)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "train directly with different folds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "'''\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch.utils.data import DataLoader\n",
+    "import pytorch_lightning as pl\n",
+    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
+    "from feeder import FeederINCLUDE\n",
+    "from aagcn import Model\n",
+    "from augumentation import Rotate, Compose\n",
+    "\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    k_folds = 10\n",
+    "    config = {'batch_size': 128, 'learning_rate': 0.0137296, 'weight_decay': 0.000150403}\n",
+    "    \n",
+    "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "\n",
+    "    best_accuracy = 0.0\n",
+    "    best_fold = -1\n",
+    "\n",
+    "    for fold in range(k_folds):\n",
+    "        print(f\"Starting fold {fold + 1}/{k_folds}\")\n",
+    "        train_data_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_data_fold{fold+1}_train.npy')\n",
+    "        train_label_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_label_fold{fold+1}_train.npy')\n",
+    "        val_data_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_data_fold{fold+1}_test.npy')\n",
+    "        val_label_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_label_fold{fold+1}_test.npy')\n",
+    "\n",
+    "        transforms = Compose([\n",
+    "            Rotate(15, 80, 25, (0.5, 0.5))\n",
+    "        ])\n",
+    "\n",
+    "        train_dataset = FeederINCLUDE(\n",
+    "            data_path=train_data_path,\n",
+    "            label_path=train_label_path,\n",
+    "            transform=transforms\n",
+    "        )\n",
+    "        val_dataset = FeederINCLUDE(\n",
+    "            data_path=val_data_path,\n",
+    "            label_path=val_label_path\n",
+    "        )\n",
+    "\n",
+    "        train_dataloader = DataLoader(train_dataset, batch_size=config['batch_size'], shuffle=True)\n",
+    "        val_dataloader = DataLoader(val_dataset, batch_size=config['batch_size'], shuffle=False)\n",
+    "\n",
+    "        model = Model(num_class=num_labels, num_point=46, num_person=1, in_channels=2,\n",
+    "                      graph_args={\"layout\": \"mediapipe_two_hand\", \"strategy\": \"spatial\"},\n",
+    "                      learning_rate=config['learning_rate'], weight_decay=config['weight_decay'])\n",
+    "\n",
+    "        callbacks = [\n",
+    "            ModelCheckpoint(\n",
+    "                dirpath=\"checkpoints\",\n",
+    "                monitor=\"valid_accuracy\",\n",
+    "                mode=\"max\",\n",
+    "                every_n_epochs=2,\n",
+    "                filename=f'vsl{num_labels}-aagcn-fold={fold+1}'\n",
+    "            ),\n",
+    "        ]\n",
+    "\n",
+    "        trainer = pl.Trainer(max_epochs=2, accelerator=\"auto\", check_val_every_n_epoch=1,\n",
+    "                             devices=1, callbacks=callbacks)\n",
+    "\n",
+    "        trainer.fit(model, train_dataloader, val_dataloader)\n",
+    "        val_accuracy = trainer.callback_metrics['valid_accuracy'].item()\n",
+    "        print(f\"Fold {fold + 1} finished with validation accuracy: {val_accuracy:.4f}\")\n",
+    "\n",
+    "        if val_accuracy > best_accuracy:\n",
+    "            best_accuracy = val_accuracy\n",
+    "            best_fold = fold + 1 \n",
+    "\n",
+    "    print(f\"The highest validation accuracy achieved is {best_accuracy:.4f} from fold {best_fold}.\")\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "print(f\"The highest validation accuracy achieved of vsl{num_labels} is {best_accuracy:.4f} from fold {best_fold}.\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "train based on AUTSL with different folds"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "import os\n",
+    "import numpy as np\n",
+    "import torch\n",
+    "from torch.utils.data import DataLoader\n",
+    "import pytorch_lightning as pl\n",
+    "from pytorch_lightning.callbacks import ModelCheckpoint\n",
+    "from feeder import FeederINCLUDE\n",
+    "from aagcn import Model\n",
+    "from augumentation import Rotate, Compose\n",
+    "from pytorch_lightning.utilities.migration import pl_legacy_patch\n",
+    "\n",
+    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
+    "\n",
+    "if __name__ == '__main__':\n",
+    "    k_folds = 10 \n",
+    "    config = {'batch_size': 128, 'learning_rate': 0.0137296, 'weight_decay': 0.000150403}\n",
+    "    \n",
+    "    device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
+    "\n",
+    "    best_accuracy = 0.0\n",
+    "    best_fold = -1\n",
+    "\n",
+    "    for fold in range(k_folds):\n",
+    "        print(f\"Starting fold {fold + 1}/{k_folds}\")\n",
+    "        train_data_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_data_fold{fold+1}_train.npy')\n",
+    "        train_label_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_label_fold{fold+1}_train.npy')\n",
+    "        val_data_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_data_fold{fold+1}_test.npy')\n",
+    "        val_label_path = os.path.join(\"numpy_files\", f'vsl{num_labels}_label_fold{fold+1}_test.npy')\n",
+    "\n",
+    "        transforms = Compose([\n",
+    "            Rotate(15, 80, 25, (0.5, 0.5))\n",
+    "        ])\n",
+    "\n",
+    "        train_dataset = FeederINCLUDE(\n",
+    "            data_path=train_data_path,\n",
+    "            label_path=train_label_path,\n",
+    "            transform=transforms\n",
+    "        )\n",
+    "        val_dataset = FeederINCLUDE(\n",
+    "            data_path=val_data_path,\n",
+    "            label_path=val_label_path\n",
+    "        )\n",
+    "\n",
+    "        train_dataloader = DataLoader(train_dataset, batch_size=config['batch_size'], shuffle=True)\n",
+    "        val_dataloader = DataLoader(val_dataset, batch_size=config['batch_size'], shuffle=False)\n",
+    "\n",
+    "        model = Model(num_class=num_labels, num_point=46, num_person=1, in_channels=2,\n",
+    "                      graph_args={\"layout\": \"mediapipe_two_hand\", \"strategy\": \"spatial\"},\n",
+    "                      learning_rate=config['learning_rate'], weight_decay=config['weight_decay'])\n",
+    "\n",
+    "        # Path pre-trained checkpoint file on AUTSL\n",
+    "        checkpoint_path = \"epoch=55-valid_loss=0.41-valid_accuracy=0.85-autsl-aagcn.ckpt\"\n",
+    "\n",
+    "        with pl_legacy_patch():\n",
+    "            checkpoint = torch.load(checkpoint_path, map_location=device)\n",
+    "\n",
+    "        state_dict = checkpoint['state_dict']\n",
+    "        filtered_state_dict = {k: v for k, v in state_dict.items() if not k.startswith('fc.')}\n",
+    "        model.load_state_dict(filtered_state_dict, strict=False)\n",
+    "\n",
+    "        callbacks = [\n",
+    "            ModelCheckpoint(\n",
+    "                dirpath=\"checkpoints\",\n",
+    "                monitor=\"valid_accuracy\",\n",
+    "                mode=\"max\",\n",
+    "                every_n_epochs=2,\n",
+    "                filename=f'autsl_vsl{num_labels}-aagcn-fold={fold+1}'\n",
+    "            ),\n",
+    "        ]\n",
+    "\n",
+    "        trainer = pl.Trainer(max_epochs=100, accelerator=\"auto\", check_val_every_n_epoch=1,\n",
+    "                             devices=1, callbacks=callbacks)\n",
+    "\n",
+    "        trainer.fit(model, train_dataloader, val_dataloader)\n",
+    "        val_accuracy = trainer.callback_metrics['valid_accuracy'].item() \n",
+    "        print(f\"Fold {fold + 1} finished with validation accuracy: {val_accuracy:.4f}\")\n",
+    "\n",
+    "        if val_accuracy > best_accuracy:\n",
+    "            best_accuracy = val_accuracy\n",
+    "            best_fold = fold + 1 \n",
+    "\n",
+    "    print(f\"The highest validation accuracy achieved is {best_accuracy:.4f} from fold {best_fold}.\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(f\"The highest validation accuracy achieved of autsl vsl{num_labels} is {best_accuracy:.4f} from fold {best_fold}.\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.11"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

aagcn.py

@@ -0,0 +1,440 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from graph import Graph
+import pytorch_lightning as pl
+from torchmetrics.classification import MulticlassAccuracy, BinaryAccuracy
+import torch.optim as optim
+
+def import_class(name):
+    components = name.split('.')
+    mod = __import__(components[0])
+    for comp in components[1:]:
+        mod = getattr(mod, comp)
+    return mod
+
+
+def conv_branch_init(conv, branches):
+    weight = conv.weight
+    n = weight.size(0)
+    k1 = weight.size(1)
+    k2 = weight.size(2)
+    nn.init.normal_(weight, 0, math.sqrt(2. / (n * k1 * k2 * branches)))
+    nn.init.constant_(conv.bias, 0)
+
+
+def conv_init(conv):
+    nn.init.kaiming_normal_(conv.weight, mode='fan_out')
+    nn.init.constant_(conv.bias, 0)
+
+
+def bn_init(bn, scale):
+    nn.init.constant_(bn.weight, scale)
+    nn.init.constant_(bn.bias, 0)
+
+
+class unit_tcn(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size=9, stride=1):
+        super(unit_tcn, self).__init__()
+        pad = int((kernel_size - 1) / 2)
+        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=(kernel_size, 1), padding=(pad, 0),
+                              stride=(stride, 1))
+
+        self.bn = nn.BatchNorm2d(out_channels)
+        self.relu = nn.ReLU(inplace=True)
+        conv_init(self.conv)
+        bn_init(self.bn, 1)
+
+    def forward(self, x):
+        x = self.bn(self.conv(x))
+        return x
+
+
+class unit_gcn(nn.Module):
+    def __init__(self, in_channels, out_channels, A, coff_embedding=4, num_subset=3, adaptive=True, attention=True):
+        super(unit_gcn, self).__init__()
+        inter_channels = out_channels // coff_embedding
+        self.inter_c = inter_channels
+        self.out_c = out_channels
+        self.in_c = in_channels
+        self.num_subset = num_subset
+        num_jpts = A.shape[-1]
+
+        self.conv_d = nn.ModuleList()
+        for i in range(self.num_subset):
+            self.conv_d.append(nn.Conv2d(in_channels, out_channels, 1))
+
+        if adaptive:
+            self.PA = nn.Parameter(torch.from_numpy(A.astype(np.float32)))
+            self.alpha = nn.Parameter(torch.zeros(1))
+            # self.beta = nn.Parameter(torch.ones(1))
+            # nn.init.constant_(self.PA, 1e-6)
+            # self.A = Variable(torch.from_numpy(A.astype(np.float32)), requires_grad=False)
+            # self.A = self.PA
+            self.conv_a = nn.ModuleList()
+            self.conv_b = nn.ModuleList()
+            for i in range(self.num_subset):
+                self.conv_a.append(nn.Conv2d(in_channels, inter_channels, 1))
+                self.conv_b.append(nn.Conv2d(in_channels, inter_channels, 1))
+        else:
+            self.A = Variable(torch.from_numpy(A.astype(np.float32)), requires_grad=False)
+        self.adaptive = adaptive
+
+        if attention:
+            # self.beta = nn.Parameter(torch.zeros(1))
+            # self.gamma = nn.Parameter(torch.zeros(1))
+            # unified attention
+            # self.Attention = nn.Parameter(torch.ones(num_jpts))
+
+            # temporal attention
+            self.conv_ta = nn.Conv1d(out_channels, 1, 9, padding=4)
+            nn.init.constant_(self.conv_ta.weight, 0)
+            nn.init.constant_(self.conv_ta.bias, 0)
+
+            # s attention
+            ker_jpt = num_jpts - 1 if not num_jpts % 2 else num_jpts
+            pad = (ker_jpt - 1) // 2
+            self.conv_sa = nn.Conv1d(out_channels, 1, ker_jpt, padding=pad)
+            nn.init.xavier_normal_(self.conv_sa.weight)
+            nn.init.constant_(self.conv_sa.bias, 0)
+
+            # channel attention
+            rr = 2
+            self.fc1c = nn.Linear(out_channels, out_channels // rr)
+            self.fc2c = nn.Linear(out_channels // rr, out_channels)
+            nn.init.kaiming_normal_(self.fc1c.weight)
+            nn.init.constant_(self.fc1c.bias, 0)
+            nn.init.constant_(self.fc2c.weight, 0)
+            nn.init.constant_(self.fc2c.bias, 0)
+
+            # self.bn = nn.BatchNorm2d(out_channels)
+            # bn_init(self.bn, 1)
+        self.attention = attention
+
+        if in_channels != out_channels:
+            self.down = nn.Sequential(
+                nn.Conv2d(in_channels, out_channels, 1),
+                nn.BatchNorm2d(out_channels)
+            )
+        else:
+            self.down = lambda x: x
+
+        self.bn = nn.BatchNorm2d(out_channels)
+        self.soft = nn.Softmax(-2)
+        self.tan = nn.Tanh()
+        self.sigmoid = nn.Sigmoid()
+        self.relu = nn.ReLU(inplace=True)
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                conv_init(m)
+            elif isinstance(m, nn.BatchNorm2d):
+                bn_init(m, 1)
+        bn_init(self.bn, 1e-6)
+        for i in range(self.num_subset):
+            conv_branch_init(self.conv_d[i], self.num_subset)
+
+    def forward(self, x):
+        N, C, T, V = x.size()
+
+        y = None
+        if self.adaptive:
+            A = self.PA
+            # A = A + self.PA
+            for i in range(self.num_subset):
+                A1 = self.conv_a[i](x).permute(0, 3, 1, 2).contiguous().view(N, V, self.inter_c * T)
+                A2 = self.conv_b[i](x).view(N, self.inter_c * T, V)
+                A1 = self.tan(torch.matmul(A1, A2) / A1.size(-1))  # N V V
+                A1 = A[i] + A1 * self.alpha
+                A2 = x.view(N, C * T, V)
+                z = self.conv_d[i](torch.matmul(A2, A1).view(N, C, T, V))
+                y = z + y if y is not None else z
+        else:
+            A = self.A.cuda(x.get_device()) * self.mask
+            for i in range(self.num_subset):
+                A1 = A[i]
+                A2 = x.view(N, C * T, V)
+                z = self.conv_d[i](torch.matmul(A2, A1).view(N, C, T, V))
+                y = z + y if y is not None else z
+
+        y = self.bn(y)
+        y += self.down(x)
+        y = self.relu(y)
+
+        if self.attention:
+            # spatial attention
+            se = y.mean(-2)  # N C V
+            se1 = self.sigmoid(self.conv_sa(se))
+            y = y * se1.unsqueeze(-2) + y
+            # a1 = se1.unsqueeze(-2)
+
+            # temporal attention
+            se = y.mean(-1)
+            se1 = self.sigmoid(self.conv_ta(se))
+            y = y * se1.unsqueeze(-1) + y
+            # a2 = se1.unsqueeze(-1)
+
+            # channel attention
+            se = y.mean(-1).mean(-1)
+            se1 = self.relu(self.fc1c(se))
+            se2 = self.sigmoid(self.fc2c(se1))
+            y = y * se2.unsqueeze(-1).unsqueeze(-1) + y
+            # a3 = se2.unsqueeze(-1).unsqueeze(-1)
+
+            # unified attention
+            # y = y * self.Attention + y
+            # y = y + y * ((a2 + a3) / 2)
+            # y = self.bn(y)
+        return y
+
+
+class TCN_GCN_unit(nn.Module):
+    def __init__(self, in_channels, out_channels, A, stride=1, residual=True, adaptive=True, attention=True):
+        super(TCN_GCN_unit, self).__init__()
+        self.gcn1 = unit_gcn(in_channels, out_channels, A, adaptive=adaptive, attention=attention)
+        self.tcn1 = unit_tcn(out_channels, out_channels, stride=stride)
+        self.relu = nn.ReLU(inplace=True)
+        # if attention:
+        # self.alpha = nn.Parameter(torch.zeros(1))
+        # self.beta = nn.Parameter(torch.ones(1))
+        # temporal attention
+        # self.conv_ta1 = nn.Conv1d(out_channels, out_channels//rt, 9, padding=4)
+        # self.bn = nn.BatchNorm2d(out_channels)
+        # bn_init(self.bn, 1)
+        # self.conv_ta2 = nn.Conv1d(out_channels, 1, 9, padding=4)
+        # nn.init.kaiming_normal_(self.conv_ta1.weight)
+        # nn.init.constant_(self.conv_ta1.bias, 0)
+        # nn.init.constant_(self.conv_ta2.weight, 0)
+        # nn.init.constant_(self.conv_ta2.bias, 0)
+
+        # rt = 4
+        # self.inter_c = out_channels // rt
+        # self.conv_ta1 = nn.Conv2d(out_channels, out_channels // rt, 1)
+        # self.conv_ta2 = nn.Conv2d(out_channels, out_channels // rt, 1)
+        # nn.init.constant_(self.conv_ta1.weight, 0)
+        # nn.init.constant_(self.conv_ta1.bias, 0)
+        # nn.init.constant_(self.conv_ta2.weight, 0)
+        # nn.init.constant_(self.conv_ta2.bias, 0)
+        # s attention
+        # num_jpts = A.shape[-1]
+        # ker_jpt = num_jpts - 1 if not num_jpts % 2 else num_jpts
+        # pad = (ker_jpt - 1) // 2
+        # self.conv_sa = nn.Conv1d(out_channels, 1, ker_jpt, padding=pad)
+        # nn.init.constant_(self.conv_sa.weight, 0)
+        # nn.init.constant_(self.conv_sa.bias, 0)
+
+        # channel attention
+        # rr = 16
+        # self.fc1c = nn.Linear(out_channels, out_channels // rr)
+        # self.fc2c = nn.Linear(out_channels // rr, out_channels)
+        # nn.init.kaiming_normal_(self.fc1c.weight)
+        # nn.init.constant_(self.fc1c.bias, 0)
+        # nn.init.constant_(self.fc2c.weight, 0)
+        # nn.init.constant_(self.fc2c.bias, 0)
+        #
+        # self.softmax = nn.Softmax(-2)
+        # self.sigmoid = nn.Sigmoid()
+        self.attention = attention
+
+        if not residual:
+            self.residual = lambda x: 0
+
+        elif (in_channels == out_channels) and (stride == 1):
+            self.residual = lambda x: x
+
+        else:
+            self.residual = unit_tcn(in_channels, out_channels, kernel_size=1, stride=stride)
+
+    def forward(self, x):
+        if self.attention:
+            y = self.relu(self.tcn1(self.gcn1(x)) + self.residual(x))
+
+            # spatial attention
+            # se = y.mean(-2)  # N C V
+            # se1 = self.sigmoid(self.conv_sa(se))
+            # y = y * se1.unsqueeze(-2) + y
+            # a1 = se1.unsqueeze(-2)
+
+            # temporal attention
+            # se = y.mean(-1)  # N C T
+            # # se1 = self.relu(self.bn(self.conv_ta1(se)))
+            # se2 = self.sigmoid(self.conv_ta2(se))
+            # # y = y * se1.unsqueeze(-1) + y
+            # a2 = se2.unsqueeze(-1)
+
+            # se = y  # NCTV
+            # N, C, T, V = y.shape
+            # se1 = self.conv_ta1(se).permute(0, 2, 1, 3).contiguous().view(N, T, self.inter_c * V)  # NTCV
+            # se2 = self.conv_ta2(se).permute(0, 1, 3, 2).contiguous().view(N, self.inter_c * V, T)  # NCVT
+            # a2 = self.softmax(torch.matmul(se1, se2) / np.sqrt(se1.size(-1)))  # N T T
+            # y = torch.matmul(y.permute(0, 1, 3, 2).contiguous().view(N, C * V, T), a2) \
+            #         .view(N, C, V, T).permute(0, 1, 3, 2) * self.alpha + y
+
+            # channel attention
+            # se = y.mean(-1).mean(-1)
+            # se1 = self.relu(self.fc1c(se))
+            # se2 = self.sigmoid(self.fc2c(se1))
+            # # y = y * se2.unsqueeze(-1).unsqueeze(-1) + y
+            # a3 = se2.unsqueeze(-1).unsqueeze(-1)
+            #
+            # y = y * ((a2 + a3) / 2) + y
+            # y = self.bn(y)
+        else:
+            y = self.relu(self.tcn1(self.gcn1(x)) + self.residual(x))
+        return y
+
+
+class Model(pl.LightningModule):
+    def __init__(self, num_class=60, num_point=25, num_person=2, graph=None, graph_args=dict(), in_channels=3,
+                 drop_out=0, adaptive=True, attention=True, learning_rate=1e-4, weight_decay=1e-4):
+        super(Model, self).__init__()
+
+        # if graph is None:
+            # raise ValueError()
+        # else:
+            # Graph = import_class(graph)
+        self.graph = Graph(**graph_args)
+
+        A = self.graph.A
+        self.num_class = num_class
+
+        self.data_bn = nn.BatchNorm1d(num_person * in_channels * num_point)
+
+        self.l1 = TCN_GCN_unit(in_channels, 64, A, residual=False, adaptive=adaptive, attention=attention)
+        self.l2 = TCN_GCN_unit(64, 64, A, adaptive=adaptive, attention=attention)
+        self.l3 = TCN_GCN_unit(64, 64, A, adaptive=adaptive, attention=attention)
+        self.l4 = TCN_GCN_unit(64, 64, A, adaptive=adaptive, attention=attention)
+        self.l5 = TCN_GCN_unit(64, 128, A, stride=2, adaptive=adaptive, attention=attention)
+        self.l6 = TCN_GCN_unit(128, 128, A, adaptive=adaptive, attention=attention)
+        self.l7 = TCN_GCN_unit(128, 128, A, adaptive=adaptive, attention=attention)
+        self.l8 = TCN_GCN_unit(128, 256, A, stride=2, adaptive=adaptive, attention=attention)
+        self.l9 = TCN_GCN_unit(256, 256, A, adaptive=adaptive, attention=attention)
+        self.l10 = TCN_GCN_unit(256, 256, A, adaptive=adaptive, attention=attention)
+        # self.l11 = TCN_GCN_unit(256, 512, A, stride=2, adaptive=adaptive, attention=attention)
+        # self.l12 = TCN_GCN_unit(512, 512, A, adaptive=adaptive, attention=attention)
+        # self.l13 = TCN_GCN_unit(512, 512, A, adaptive=adaptive, attention=attention)
+
+        self.fc = nn.Linear(256, num_class)
+        nn.init.normal_(self.fc.weight, 0, math.sqrt(2. / num_class))
+        bn_init(self.data_bn, 1)
+        if drop_out:
+            self.drop_out = nn.Dropout(drop_out)
+        else:
+            self.drop_out = lambda x: x
+        
+        self.loss = nn.CrossEntropyLoss()
+        self.metric = MulticlassAccuracy(num_class)
+        # self.metric = BinaryAccuracy()
+        self.learning_rate = learning_rate
+        self.weight_decay = weight_decay
+        self.validation_step_loss_outputs = []
+        self.validation_step_acc_outputs = []
+
+        self.save_hyperparameters()
+
+    def forward(self, x):
+        N, C, T, V, M = x.size()
+
+        x = x.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V * C, T)
+        x = self.data_bn(x.float())
+        x = x.view(N, M, V, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V)
+
+        x = self.l1(x)
+        x = self.l2(x)
+        x = self.l3(x)
+        x = self.l4(x)
+        x = self.l5(x)
+        x = self.l6(x)
+        x = self.l7(x)
+        x = self.l8(x)
+        x = self.l9(x)
+        x = self.l10(x)
+        # x = self.l11(x)
+        # x = self.l12(x)
+        # x = self.l13(x)
+
+        # N*M,C,T,V
+        c_new = x.size(1)
+        x = x.view(N, M, c_new, -1)
+        x = x.mean(3).mean(1)
+        x = self.drop_out(x)
+
+        return self.fc(x)
+
+    def training_step(self, batch, batch_idx):
+        inputs, targets = batch
+        outputs = self(inputs)
+        y_pred_class = torch.argmax(torch.softmax(outputs, dim=1), dim=1)
+        # print("Targets : ", targets)
+        # print("Preds : ", y_pred_class) 
+        train_accuracy = self.metric(y_pred_class, targets)
+        loss = self.loss(outputs, targets)
+        self.log('train_accuracy', train_accuracy, prog_bar=True, on_epoch=True)
+        self.log('train_loss', loss, prog_bar=True, on_epoch=True)
+        # return {"loss": loss, "train_accuracy" : train_accuracy}
+        return loss
+    
+    def validation_step(self, batch, batch_idx):
+        inputs, targets = batch
+        outputs = self.forward(inputs)
+        y_pred_class = torch.argmax(torch.softmax(outputs, dim=1), dim=1)
+        valid_accuracy = self.metric(y_pred_class, targets)
+        loss = self.loss(outputs, targets)
+        self.log('valid_accuracy', valid_accuracy, prog_bar=True, on_epoch=True)
+        self.log('valid_loss', loss, prog_bar=True, on_epoch=True)
+        self.validation_step_loss_outputs.append(loss)
+        self.validation_step_acc_outputs.append(valid_accuracy)
+        return {"valid_loss" : loss, "valid_accuracy" : valid_accuracy}
+    
+    def on_validation_epoch_end(self):
+        # avg_loss = torch.stack(
+            # [x["valid_loss"] for x in outputs]).mean()
+        # avg_acc = torch.stack(
+            # [x["valid_accuracy"] for x in outputs]).mean()
+        avg_loss = torch.stack(self.validation_step_loss_outputs).mean()
+        avg_acc = torch.stack(self.validation_step_acc_outputs).mean()
+        self.log("ptl/val_loss", avg_loss)
+        self.log("ptl/val_accuracy", avg_acc)
+        self.validation_step_loss_outputs.clear() 
+        self.validation_step_acc_outputs.clear()
+
+    def test_step(self, batch, batch_idx):
+        inputs, targets = batch
+        outputs = self.forward(inputs)
+        y_pred_class = torch.argmax(torch.softmax(outputs, dim=1), dim=1)
+        print("Targets : ", targets)
+        print("Preds : ", y_pred_class)
+        test_accuracy = self.metric(y_pred_class, targets)
+        loss = self.loss(outputs, targets)
+        self.log('test_accuracy', test_accuracy, prog_bar=True, on_epoch=True)
+        self.log('test_loss', loss, prog_bar=True, on_epoch=True)
+        return {"test_loss" : loss, "test_accuracy" : test_accuracy}
+
+    def configure_optimizers(self):
+        params = self.parameters()
+        optimizer = optim.Adam(params=params, lr = self.learning_rate, weight_decay = self.weight_decay)
+        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max')
+        return {"optimizer": optimizer, 
+                "lr_scheduler": {"scheduler": scheduler, "monitor": "valid_accuracy"}
+               }
+        # return optimizer  
+
+    def predict_step(self, batch, batch_idx):
+        return self(batch)
+
+if __name__ == "__main__":  
+    import os
+    from torchinfo import summary
+    print(os.getcwd())  
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = Model(num_class=20, num_point=18, num_person=1, 
+                  graph_args={}, in_channels=2).to(device)
+    # print(model.device)
+    # N, C, T, V, M
+    summary(model)
+    x = torch.randn((1, 2, 80, 18, 1)).to(device)
+    y = model(x)    
+    print(y.shape)

agcn.py

@@ -0,0 +1,275 @@
+import math
+
+import numpy as np
+import torch
+import torch.nn as nn
+from torch.autograd import Variable
+from graph import Graph
+import pytorch_lightning as pl
+from torchmetrics.classification import MulticlassAccuracy
+import torch.optim as optim
+
+def import_class(name):
+    components = name.split('.')
+    mod = __import__(components[0])
+    for comp in components[1:]:
+        mod = getattr(mod, comp)
+    return mod
+
+
+def conv_branch_init(conv, branches):
+    weight = conv.weight
+    n = weight.size(0)
+    k1 = weight.size(1)
+    k2 = weight.size(2)
+    nn.init.normal_(weight, 0, math.sqrt(2. / (n * k1 * k2 * branches)))
+    nn.init.constant_(conv.bias, 0)
+
+
+def conv_init(conv):
+    nn.init.kaiming_normal_(conv.weight, mode='fan_out')
+    nn.init.constant_(conv.bias, 0)
+
+
+def bn_init(bn, scale):
+    nn.init.constant_(bn.weight, scale)
+    nn.init.constant_(bn.bias, 0)
+
+
+class unit_tcn(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size=9, stride=1):
+        super(unit_tcn, self).__init__()
+        pad = int((kernel_size - 1) / 2)
+        self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=(kernel_size, 1), padding=(pad, 0),
+                              stride=(stride, 1))
+
+        self.bn = nn.BatchNorm2d(out_channels)
+        self.relu = nn.ReLU()
+        conv_init(self.conv)
+        bn_init(self.bn, 1)
+
+    def forward(self, x):
+        x = self.bn(self.conv(x))
+        return x
+
+
+class unit_gcn(nn.Module):
+    def __init__(self, in_channels, out_channels, A, coff_embedding=4, num_subset=3):
+        super(unit_gcn, self).__init__()
+        inter_channels = out_channels // coff_embedding
+        self.inter_c = inter_channels
+        self.PA = nn.Parameter(torch.from_numpy(A.astype(np.float32)))
+        nn.init.constant_(self.PA, 1e-6)
+        self.A = Variable(torch.from_numpy(A.astype(np.float32)), requires_grad=False)
+        self.num_subset = num_subset
+
+        self.conv_a = nn.ModuleList()
+        self.conv_b = nn.ModuleList()
+        self.conv_d = nn.ModuleList()
+        for i in range(self.num_subset):
+            self.conv_a.append(nn.Conv2d(in_channels, inter_channels, 1))
+            self.conv_b.append(nn.Conv2d(in_channels, inter_channels, 1))
+            self.conv_d.append(nn.Conv2d(in_channels, out_channels, 1))
+
+        if in_channels != out_channels:
+            self.down = nn.Sequential(
+                nn.Conv2d(in_channels, out_channels, 1),
+                nn.BatchNorm2d(out_channels)
+            )
+        else:
+            self.down = lambda x: x
+
+        self.bn = nn.BatchNorm2d(out_channels)
+        self.soft = nn.Softmax(-2)
+        self.relu = nn.ReLU()
+
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                conv_init(m)
+            elif isinstance(m, nn.BatchNorm2d):
+                bn_init(m, 1)
+        bn_init(self.bn, 1e-6)
+        for i in range(self.num_subset):
+            conv_branch_init(self.conv_d[i], self.num_subset)
+
+    def forward(self, x):
+        N, C, T, V = x.size()
+        A = self.A.cuda(x.get_device())
+        A = A + self.PA
+
+        y = None
+        for i in range(self.num_subset):
+            A1 = self.conv_a[i](x).permute(0, 3, 1, 2).contiguous().view(N, V, self.inter_c * T)
+            A2 = self.conv_b[i](x).view(N, self.inter_c * T, V)
+            A1 = self.soft(torch.matmul(A1, A2) / A1.size(-1))  # N V V
+            A1 = A1 + A[i]
+            A2 = x.view(N, C * T, V)
+            z = self.conv_d[i](torch.matmul(A2, A1).view(N, C, T, V))
+            y = z + y if y is not None else z
+
+        y = self.bn(y)
+        y += self.down(x)
+        return self.relu(y)
+
+
+class TCN_GCN_unit(nn.Module):
+    def __init__(self, in_channels, out_channels, A, stride=1, residual=True):
+        super(TCN_GCN_unit, self).__init__()
+        self.gcn1 = unit_gcn(in_channels, out_channels, A)
+        self.tcn1 = unit_tcn(out_channels, out_channels, stride=stride)
+        self.relu = nn.ReLU()
+        if not residual:
+            self.residual = lambda x: 0
+
+        elif (in_channels == out_channels) and (stride == 1):
+            self.residual = lambda x: x
+
+        else:
+            self.residual = unit_tcn(in_channels, out_channels, kernel_size=1, stride=stride)
+
+    def forward(self, x):
+        x = self.tcn1(self.gcn1(x)) + self.residual(x)
+        return self.relu(x)
+
+
+class Model(pl.LightningModule):
+    def __init__(self, num_class=60, num_point=25, num_person=2, graph=None, graph_args=dict(), in_channels=3,
+                 learning_rate = 1e-4, weight_decay = 1e-4):
+        super(Model, self).__init__()
+
+        # if graph is None:
+        #     raise ValueError()
+        # else:
+        # Graph = import_class(graph)
+        self.graph = Graph(**graph_args)
+
+        A = self.graph.A
+        # print(num_person * in_channels * num_point)
+        self.data_bn = nn.BatchNorm1d(num_person * in_channels * num_point)
+
+        self.l1 = TCN_GCN_unit(in_channels, 64, A, residual=False)
+        self.l2 = TCN_GCN_unit(64, 64, A)
+        self.l3 = TCN_GCN_unit(64, 64, A)
+        self.l4 = TCN_GCN_unit(64, 64, A)
+        self.l5 = TCN_GCN_unit(64, 128, A, stride=2)
+        self.l6 = TCN_GCN_unit(128, 128, A)
+        self.l7 = TCN_GCN_unit(128, 128, A)
+        self.l8 = TCN_GCN_unit(128, 256, A, stride=2)
+        self.l9 = TCN_GCN_unit(256, 256, A)
+        self.l10 = TCN_GCN_unit(256, 256, A)
+
+        self.fc = nn.Linear(256, num_class)
+        nn.init.normal_(self.fc.weight, 0, math.sqrt(2. / num_class))
+        bn_init(self.data_bn, 1)
+
+        self.loss = nn.CrossEntropyLoss()
+        self.metric = MulticlassAccuracy(num_class)
+        self.learning_rate = learning_rate
+        self.weight_decay = weight_decay
+        self.validation_step_loss_outputs = []
+        self.validation_step_acc_outputs = []
+
+        self.save_hyperparameters()
+
+    def forward(self, x):
+        # 0, 1, 2, 3, 4 
+        N, C, T, V, M = x.size()
+        # print(f"N {N}, C {C}, T {T}, V {V}, M {M}")
+        # N, M, V, C, T
+        x = x.permute(0, 4, 3, 1, 2).contiguous().view(N, M * V * C, T)
+        # print(M*V*C)
+        x = self.data_bn(x)
+        x = x.view(N, M, V, C, T).permute(0, 1, 3, 4, 2).contiguous().view(N * M, C, T, V)
+
+        x = self.l1(x)
+        x = self.l2(x)
+        x = self.l3(x)
+        x = self.l4(x)
+        x = self.l5(x)
+        x = self.l6(x)
+        x = self.l7(x)
+        x = self.l8(x)
+        x = self.l9(x)
+        x = self.l10(x)
+
+        # N*M,C,T,V
+        c_new = x.size(1)
+        x = x.view(N, M, c_new, -1)
+        x = x.mean(3).mean(1)
+
+        return self.fc(x)
+    
+    def training_step(self, batch, batch_idx):
+        inputs, targets = batch
+        outputs = self(inputs)
+        y_pred_class = torch.argmax(torch.softmax(outputs, dim=1), dim=1)
+        # print("Targets : ", targets)
+        # print("Preds : ", y_pred_class) 
+        train_accuracy = self.metric(y_pred_class, targets)
+        loss = self.loss(outputs, targets)
+        self.log('train_accuracy', train_accuracy, prog_bar=True, on_epoch=True)
+        self.log('train_loss', loss, prog_bar=True, on_epoch=True)
+        # return {"loss": loss, "train_accuracy" : train_accuracy}
+        return loss
+    
+    def validation_step(self, batch, batch_idx):
+        inputs, targets = batch
+        outputs = self.forward(inputs)
+        y_pred_class = torch.argmax(torch.softmax(outputs, dim=1), dim=1)
+        valid_accuracy = self.metric(y_pred_class, targets)
+        loss = self.loss(outputs, targets)
+        self.log('valid_accuracy', valid_accuracy, prog_bar=True, on_epoch=True)
+        self.log('valid_loss', loss, prog_bar=True, on_epoch=True)
+        self.validation_step_loss_outputs.append(loss)
+        self.validation_step_acc_outputs.append(valid_accuracy)
+        return {"valid_loss" : loss, "valid_accuracy" : valid_accuracy}
+    
+    def on_validation_epoch_end(self):
+        # avg_loss = torch.stack(
+            # [x["valid_loss"] for x in outputs]).mean()
+        # avg_acc = torch.stack(
+            # [x["valid_accuracy"] for x in outputs]).mean()
+        avg_loss = torch.stack(self.validation_step_loss_outputs).mean()
+        avg_acc = torch.stack(self.validation_step_acc_outputs).mean()
+        self.log("ptl/val_loss", avg_loss)
+        self.log("ptl/val_accuracy", avg_acc)
+        self.validation_step_loss_outputs.clear() 
+        self.validation_step_acc_outputs.clear()
+
+    def test_step(self, batch, batch_idx):
+        inputs, targets = batch
+        outputs = self.forward(inputs)
+        y_pred_class = torch.argmax(torch.softmax(outputs, dim=1), dim=1)
+        print("Targets : ", targets)
+        print("Preds : ", y_pred_class)
+        test_accuracy = self.metric(y_pred_class, targets)
+        loss = self.loss(outputs, targets)
+        self.log('test_accuracy', test_accuracy, prog_bar=True, on_epoch=True)
+        self.log('test_loss', loss, prog_bar=True, on_epoch=True)
+        return {"test_loss" : loss, "test_accuracy" : test_accuracy}
+
+    def configure_optimizers(self):
+        params = self.parameters()
+        optimizer = optim.Adam(params=params, lr = self.learning_rate, weight_decay = self.weight_decay)
+        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max')
+        return {"optimizer": optimizer, 
+                "lr_scheduler": {"scheduler": scheduler, "monitor": "valid_accuracy"}
+               }
+        # return optimizer  
+
+    def predict_step(self, batch, batch_idx):
+        return self(batch)
+
+if __name__ == "__main__":  
+    import os
+    from torchinfo import summary
+    print(os.getcwd())  
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    model = Model(num_class=20, num_point=25, num_person=1, 
+                  graph_args={"layout":"mediapipe", "strategy":"spatial"}, in_channels=2).to(device)
+    # print(model.device)
+    # N, C, T, V, M
+    summary(model)
+    x = torch.randn((1, 2, 80, 25, 1)).to(device)
+    y = model(x)    
+    print(y.shape)

augumentation.py

@@ -0,0 +1,73 @@
+import math
+import numpy as np
+from numpy import random
+from einops import einsum
+import torch
+class Rotate(object):
+    def __init__(self, range_angle, num_frames, num_nodes, point):
+        self.range_angle = range_angle
+        self.num_frames = num_frames
+        self.num_nodes = num_nodes
+        self.point = point
+    def __call__(self, data, label):
+        # data, label = sample
+        data = data.double()
+        angle = math.radians(random.uniform((-1)*self.range_angle, self.range_angle))
+        rotation_matrix = torch.Tensor([[math.cos(angle), (-1)*math.sin(angle)], 
+                                    [math.sin(angle), math.cos(angle)]])
+        # print(type(rotation_matrix))
+        ox, oy = self.point
+        data[0, :, :] -= ox
+        data[1, :, :] -= oy
+        
+        result = einsum(rotation_matrix.double(), data, "a b, b c d e -> a c d e") + 0.5
+
+        return result, label
+
+
+class Left(object):
+    def __init__(self, width):
+        self.width = width
+    def __call__(self, data, label):
+        idx = find_frames(data)
+        p = random.random()
+        if p > 0.5:
+            data[0, :idx, :] -= self.width
+        return data, label
+    
+class Right(object):
+    def __init__(self, width):
+        self.width = width
+    def __call__(self, data, label):
+        idx = find_frames(data)
+        p = random.random()
+        if p > 0.5:
+            data[0, :idx, :] += self.width
+        return data, label
+    
+class GaussianNoise(object):
+    def __init__(self, mean, var):
+        self.mean = mean
+        self.var = var
+    def __call__(self, data, label):
+        # C, T, V, 1
+        print(data.size())
+        noise = torch.randn(size = data.size())
+        data = data + noise
+        return data, label
+
+class Compose(object):
+    def __init__(self, transforms):
+        self.transforms = transforms
+        
+    def __call__(self, data, label):
+        for t in self.transforms:
+            data, label = t(data, label)
+        return data, label
+    
+def find_frames(data):
+    for i in range(data.shape[1]):
+        if(data[:, i, :][0][0] == 0):
+            # print(i)
+            return i
+    return data.shape[1]

epoch=55-valid_loss=0.41-valid_accuracy=0.85-autsl-aagcn.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ede54d8c48503897bdad5d8f680cfe1d24d1960242d11e1c6a704fb7bbb1dbb
+size 47248523

feeder.py

@@ -0,0 +1,72 @@
+import torch
+from torch.utils.data import Dataset
+from pathlib import Path
+import numpy as np
+import random
+from einops import rearrange
+from augumentation import Rotate
+from torch.utils.data import random_split
+
+# Class read npy and pickle file to make data and label in couple
+class FeederINCLUDE(Dataset):
+    """ Feeder for skeleton-based action recognition
+    Arguments:
+        data_path: the path to '.npy' data, the shape of data should be (N, C, T, V, M)
+        label_path: the path to label
+        window_size: The length of the output sequence
+    """
+    def __init__(self, data_path: Path, label_path: Path, transform = None):
+        super(FeederINCLUDE, self).__init__
+        self.data_path = data_path
+        self.label_path = label_path
+        self.transform = transform
+        self.load_data()
+    
+    def load_data(self):
+        # data: N C V T M
+        # Load label with numpy
+        self.label = np.load(self.label_path)
+        # load data
+        self.data = np.load(self.data_path)     
+        self.N, self.C, self.T, self.V, self.M = self.data.shape
+
+    def __getitem__(self, index):
+        """
+        Input shape (N, C, V, T, M)
+        N : batch size
+        C : numbers of features
+        V : numbers of joints (as nodes)
+        T : numbers of frames
+        M : numbers of people (should delete)
+        
+        Output shape (C, V, T, M)
+        C : numbers of features
+        V : numbers of joints (as nodes)
+        T : numbers of frames
+        label : label of videos
+        """
+        data_numpy = torch.tensor(self.data[index]).float()
+        # Delete one dimension
+        # data_numpy = data_numpy[:, :, :2]
+        # data_numpy = rearrange(data_numpy, ' t v c 1 -> c t v 1')
+        label = self.label[index]
+        p = random.random()
+        if self.transform and p > 0.5: 
+            data_numpy, label = self.transform(data_numpy, label)
+        return data_numpy, label
+    
+    def __len__(self):
+        return len(self.label)
+    
+if __name__ == '__main__':
+    file, label = np.load("wsl100_train_data_preprocess.npy"), np.load("wsl100_train_label_preprocess.npy")
+    print(file.shape, label.shape)
+    data = FeederINCLUDE(data_path=f"wsl100_train_data_preprocess.npy", label_path=f"wsl100_train_data_preprocess.npy",
+                            transform=None)
+    # test_dataset = FeederINCLUDE(data_path=f"data/vsl100_test_data_preprocess.npy", label_path=f"data/vsl100_test_label_preprocess.npy")
+    # valid_dataset = FeederINCLUDE(data_path=f"data/vsl100_valid_data_preprocess.npy", label_path=f"data/vsl100_valid_label_preprocess.npy")
+    # data = FeederINCLUDE(data_path=f"data/vsl100_test_data_preprocess.npy", label_path=f"data/vsl100_test_label_preprocess.npy", 
+                        # transform=None)
+    print(data.N, data.C, data.T, data.V, data.M)
+    print(data.data.shape)
+    print(data.__len__())

graph.py

@@ -0,0 +1,287 @@
+import numpy as np
+from enum import Enum
+
+class BodyIdentifier(Enum):
+    INDEX_FINGER_DIP_right = 0
+    INDEX_FINGER_MCP_right = 1
+    INDEX_FINGER_PIP_right = 2
+    INDEX_FINGER_TIP_right = 3
+    MIDDLE_FINGER_DIP_right = 4
+    MIDDLE_FINGER_MCP_right = 5
+    MIDDLE_FINGER_PIP_right = 6
+    MIDDLE_FINGER_TIP_right = 7
+    PINKY_DIP_right = 8
+    PINKY_MCP_right = 9
+    PINKY_PIP_right = 10
+    PINKY_TIP_right = 11
+    RING_FINGER_DIP_right = 12
+    RING_FINGER_MCP_right = 13
+    RING_FINGER_PIP_right = 14
+    RING_FINGER_TIP_right = 15
+    THUMB_CMC_right = 16
+    THUMB_IP_right = 17
+    THUMB_MCP_right = 18
+    THUMB_TIP_right = 19
+    WRIST_right = 20
+    INDEX_FINGER_DIP_left = 21
+    INDEX_FINGER_MCP_left = 22
+    INDEX_FINGER_PIP_left = 23
+    INDEX_FINGER_TIP_left = 24
+    MIDDLE_FINGER_DIP_left = 25
+    MIDDLE_FINGER_MCP_left = 26
+    MIDDLE_FINGER_PIP_left = 27
+    MIDDLE_FINGER_TIP_left = 28
+    PINKY_DIP_left = 29
+    PINKY_MCP_left = 30
+    PINKY_PIP_left = 31
+    PINKY_TIP_left = 32
+    RING_FINGER_DIP_left = 33
+    RING_FINGER_MCP_left = 34
+    RING_FINGER_PIP_left = 35
+    RING_FINGER_TIP_left = 36
+    THUMB_CMC_left = 37
+    THUMB_IP_left = 38
+    THUMB_MCP_left = 39
+    THUMB_TIP_left = 40
+    WRIST_left = 41
+    RIGHT_SHOULDER = 42
+    LEFT_SHOULDER = 43
+    LEFT_ELBOW = 44
+    RIGHT_ELBOW = 45
+
+class Graph():
+    """ The Graph to model the skeletons extracted by the openpose
+
+    Args:
+        strategy (string): must be one of the follow candidates
+        - uniform: Uniform Labeling
+        - distance: Distance Partitioning
+        - spatial: Spatial Configuration
+        For more information, please refer to the section 'Partition Strategies'
+            in our paper (https://arxiv.org/abs/1801.07455).
+
+        layout (string): must be one of the follow candidates
+        - openpose: Is consists of 18 joints. For more information, please
+            refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose#output
+        - ntu-rgb+d: Is consists of 25 joints. For more information, please
+            refer to https://github.com/shahroudy/NTURGB-D
+
+        max_hop (int): the maximal distance between two connected nodes
+        dilation (int): controls the spacing between the kernel points
+
+    """
+
+    def __init__(self,
+                 layout='openpose',
+                 strategy='uniform',
+                 max_hop=1,
+                 dilation=1):
+        self.max_hop = max_hop
+        self.dilation = dilation
+
+        self.get_edge(layout)
+        self.hop_dis = get_hop_distance(
+            self.num_node, self.edge, max_hop=max_hop)
+        self.get_adjacency(strategy)
+
+    def __str__(self):
+        return self.A
+
+    def get_edge(self, layout):
+        if layout == 'openpose':
+            self.num_node = 18
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_link = [(4, 3), (3, 2), (7, 6), (6, 5), (13, 12), (12,
+                                                                        11),
+                             (10, 9), (9, 8), (11, 5), (8, 2), (5, 1), (2, 1),
+                             (0, 1), (15, 0), (14, 0), (17, 15), (16, 14)]
+            self.edge = self_link + neighbor_link
+            self.center = 1
+        elif layout == 'ntu-rgb+d':
+            self.num_node = 25
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [(1, 2), (2, 21), (3, 21), (4, 3), (5, 21),
+                              (6, 5), (7, 6), (8, 7), (9, 21), (10, 9),
+                              (11, 10), (12, 11), (13, 1), (14, 13), (15, 14),
+                              (16, 15), (17, 1), (18, 17), (19, 18), (20, 19),
+                              (22, 23), (23, 8), (24, 25), (25, 12)]
+            neighbor_link = [(i - 1, j - 1) for (i, j) in neighbor_1base]
+            self.edge = self_link + neighbor_link
+            self.center = 21 - 1
+        elif layout == 'ntu_edge':
+            self.num_node = 24
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [(1, 2), (3, 2), (4, 3), (5, 2), (6, 5), (7, 6),
+                              (8, 7), (9, 2), (10, 9), (11, 10), (12, 11),
+                              (13, 1), (14, 13), (15, 14), (16, 15), (17, 1),
+                              (18, 17), (19, 18), (20, 19), (21, 22), (22, 8),
+                              (23, 24), (24, 12)]
+            neighbor_link = [(i - 1, j - 1) for (i, j) in neighbor_1base]
+            self.edge = self_link + neighbor_link
+            self.center = 2
+        elif layout == 'mediapipe':
+            self.num_node = 25
+            self_link = [(i, i) for i in range(self.num_node)]
+            neighbor_1base = [(20, 18), (18, 16), (20, 16), (16, 22), (16, 14), (14, 12),
+                              (19, 17), (17, 15), (19, 15), (15, 21), (15, 13), (13, 11),
+                              (12, 11), (12, 24), (24, 23), (23, 11),
+                              (10, 9), 
+                              (0, 4), (4, 5), (5, 6), (6, 8),
+                              (0, 1), (1, 2), (2, 3), (3, 7)]
+            neighbor_link = [(i - 1, j - 1) for (i, j) in neighbor_1base]
+            self.edge = self_link + neighbor_link
+            self.center = 10
+
+        elif layout == "mediapipe_two_hand":
+            self.num_node = 46
+            self_link = [(i, i) for i in range(self.num_node)]            
+            neighbor_1base = [(BodyIdentifier.WRIST_left.value, BodyIdentifier.THUMB_CMC_left.value), 
+                              (BodyIdentifier.THUMB_CMC_left.value, BodyIdentifier.THUMB_MCP_left.value),
+                              (BodyIdentifier.THUMB_MCP_left.value, BodyIdentifier.THUMB_IP_left.value),
+                              (BodyIdentifier.THUMB_IP_left.value, BodyIdentifier.THUMB_TIP_left.value),
+
+                              (BodyIdentifier.WRIST_left.value, BodyIdentifier.INDEX_FINGER_MCP_left.value),
+                              (BodyIdentifier.INDEX_FINGER_MCP_left.value, BodyIdentifier.INDEX_FINGER_PIP_left.value),
+                              (BodyIdentifier.INDEX_FINGER_PIP_left.value, BodyIdentifier.INDEX_FINGER_DIP_left.value),
+                              (BodyIdentifier.INDEX_FINGER_DIP_left.value, BodyIdentifier.INDEX_FINGER_TIP_left.value),
+
+                              (BodyIdentifier.INDEX_FINGER_MCP_left.value, BodyIdentifier.MIDDLE_FINGER_MCP_left.value),
+                              (BodyIdentifier.MIDDLE_FINGER_MCP_left.value, BodyIdentifier.MIDDLE_FINGER_PIP_left.value),
+                              (BodyIdentifier.MIDDLE_FINGER_PIP_left.value, BodyIdentifier.MIDDLE_FINGER_DIP_left.value),
+                              (BodyIdentifier.MIDDLE_FINGER_DIP_left.value, BodyIdentifier.MIDDLE_FINGER_TIP_left.value),
+
+                              (BodyIdentifier.MIDDLE_FINGER_MCP_left.value, BodyIdentifier.RING_FINGER_MCP_left.value),
+                              (BodyIdentifier.RING_FINGER_MCP_left.value, BodyIdentifier.RING_FINGER_PIP_left.value),
+                              (BodyIdentifier.RING_FINGER_PIP_left.value, BodyIdentifier.RING_FINGER_DIP_left.value),
+                              (BodyIdentifier.RING_FINGER_DIP_left.value, BodyIdentifier.RING_FINGER_TIP_left.value),
+
+                              (BodyIdentifier.WRIST_left.value, BodyIdentifier.PINKY_MCP_left.value),
+                              (BodyIdentifier.PINKY_MCP_left.value, BodyIdentifier.PINKY_PIP_left.value),
+                              (BodyIdentifier.PINKY_PIP_left.value, BodyIdentifier.PINKY_DIP_left.value),
+                              (BodyIdentifier.PINKY_DIP_left.value, BodyIdentifier.PINKY_TIP_left.value),
+
+                              # RIGHT HAND
+                              (BodyIdentifier.WRIST_right.value, BodyIdentifier.THUMB_CMC_right.value), 
+                              (BodyIdentifier.THUMB_CMC_right.value, BodyIdentifier.THUMB_MCP_right.value),
+                              (BodyIdentifier.THUMB_MCP_right.value, BodyIdentifier.THUMB_IP_right.value),
+                              (BodyIdentifier.THUMB_IP_right.value, BodyIdentifier.THUMB_TIP_right.value),
+
+                              (BodyIdentifier.WRIST_right.value, BodyIdentifier.INDEX_FINGER_MCP_right.value),
+                              (BodyIdentifier.INDEX_FINGER_MCP_right.value, BodyIdentifier.INDEX_FINGER_PIP_right.value),
+                              (BodyIdentifier.INDEX_FINGER_PIP_right.value, BodyIdentifier.INDEX_FINGER_DIP_right.value),
+                              (BodyIdentifier.INDEX_FINGER_DIP_right.value, BodyIdentifier.INDEX_FINGER_TIP_right.value),
+
+                              (BodyIdentifier.INDEX_FINGER_MCP_right.value, BodyIdentifier.MIDDLE_FINGER_MCP_right.value),
+                              (BodyIdentifier.MIDDLE_FINGER_MCP_right.value, BodyIdentifier.MIDDLE_FINGER_PIP_right.value),
+                              (BodyIdentifier.MIDDLE_FINGER_PIP_right.value, BodyIdentifier.MIDDLE_FINGER_DIP_right.value),
+                              (BodyIdentifier.MIDDLE_FINGER_DIP_right.value, BodyIdentifier.MIDDLE_FINGER_TIP_right.value),
+
+                              (BodyIdentifier.MIDDLE_FINGER_MCP_right.value, BodyIdentifier.RING_FINGER_MCP_right.value),
+                              (BodyIdentifier.RING_FINGER_MCP_right.value, BodyIdentifier.RING_FINGER_PIP_right.value),
+                              (BodyIdentifier.RING_FINGER_PIP_right.value, BodyIdentifier.RING_FINGER_DIP_right.value),
+                              (BodyIdentifier.RING_FINGER_DIP_right.value, BodyIdentifier.RING_FINGER_TIP_right.value),
+
+                              (BodyIdentifier.WRIST_right.value, BodyIdentifier.PINKY_MCP_right.value),
+                              (BodyIdentifier.PINKY_MCP_right.value, BodyIdentifier.PINKY_PIP_right.value),
+                              (BodyIdentifier.PINKY_PIP_right.value, BodyIdentifier.PINKY_DIP_right.value),
+                              (BodyIdentifier.PINKY_DIP_right.value, BodyIdentifier.PINKY_TIP_right.value),
+
+                              # 2 HAND + SHOULDER + ELBOW
+                              (BodyIdentifier.RIGHT_SHOULDER.value, BodyIdentifier.RIGHT_ELBOW.value),
+                              (BodyIdentifier.RIGHT_ELBOW.value, BodyIdentifier.WRIST_right.value),
+
+                              (BodyIdentifier.RIGHT_SHOULDER.value, BodyIdentifier.LEFT_SHOULDER.value),
+
+                              (BodyIdentifier.LEFT_SHOULDER.value, BodyIdentifier.LEFT_ELBOW.value),
+                              (BodyIdentifier.LEFT_ELBOW.value, BodyIdentifier.WRIST_left.value)]
+            
+            neighbor_link = [(i, j) for (i, j) in neighbor_1base]
+            self.edge = self_link + neighbor_link
+            self.center = BodyIdentifier.RIGHT_SHOULDER.value
+        # elif layout=='customer settings'
+        #     pass
+        else:
+            raise ValueError("Do Not Exist This Layout.")
+
+    def get_adjacency(self, strategy):
+        valid_hop = range(0, self.max_hop + 1, self.dilation)
+        adjacency = np.zeros((self.num_node, self.num_node))
+        for hop in valid_hop:
+            adjacency[self.hop_dis == hop] = 1
+        normalize_adjacency = normalize_digraph(adjacency)
+
+        if strategy == 'uniform':
+            A = np.zeros((1, self.num_node, self.num_node))
+            A[0] = normalize_adjacency
+            self.A = A
+        elif strategy == 'distance':
+            A = np.zeros((len(valid_hop), self.num_node, self.num_node))
+            for i, hop in enumerate(valid_hop):
+                A[i][self.hop_dis == hop] = normalize_adjacency[self.hop_dis ==
+                                                                hop]
+            self.A = A
+        elif strategy == 'spatial':
+            A = []
+            for hop in valid_hop:
+                a_root = np.zeros((self.num_node, self.num_node))
+                a_close = np.zeros((self.num_node, self.num_node))
+                a_further = np.zeros((self.num_node, self.num_node))
+                for i in range(self.num_node):
+                    for j in range(self.num_node):
+                        if self.hop_dis[j, i] == hop:
+                            if self.hop_dis[j, self.center] == self.hop_dis[
+                                    i, self.center]:
+                                a_root[j, i] = normalize_adjacency[j, i]
+                            elif self.hop_dis[j, self.
+                                              center] > self.hop_dis[i, self.
+                                                                     center]:
+                                a_close[j, i] = normalize_adjacency[j, i]
+                            else:
+                                a_further[j, i] = normalize_adjacency[j, i]
+                if hop == 0:
+                    A.append(a_root)
+                else:
+                    A.append(a_root + a_close)
+                    A.append(a_further)
+            A = np.stack(A)
+            self.A = A
+        else:
+            raise ValueError("Do Not Exist This Strategy")
+
+
+def get_hop_distance(num_node, edge, max_hop=1):
+    A = np.zeros((num_node, num_node))
+    print(edge)
+    for i, j in edge:
+        A[j, i] = 1
+        A[i, j] = 1
+
+    # compute hop steps
+    hop_dis = np.zeros((num_node, num_node)) + np.inf
+    transfer_mat = [np.linalg.matrix_power(A, d) for d in range(max_hop + 1)]
+    arrive_mat = (np.stack(transfer_mat) > 0)
+    for d in range(max_hop, -1, -1):
+        hop_dis[arrive_mat[d]] = d
+    return hop_dis
+
+
+def normalize_digraph(A):
+    Dl = np.sum(A, 0)
+    num_node = A.shape[0]
+    Dn = np.zeros((num_node, num_node))
+    for i in range(num_node):
+        if Dl[i] > 0:
+            Dn[i, i] = Dl[i]**(-1)
+    AD = np.dot(A, Dn)
+    return AD
+
+
+def normalize_undigraph(A):
+    Dl = np.sum(A, 0)
+    num_node = A.shape[0]
+    Dn = np.zeros((num_node, num_node))
+    for i in range(num_node):
+        if Dl[i] > 0:
+            Dn[i, i] = Dl[i]**(-0.5)
+    DAD = np.dot(np.dot(Dn, A), Dn)
+    return DAD