{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import lightning as L\n",
    "\n",
    "modelR50 = torchvision.models.resnet50(weights=torchvision.models.ResNet50_Weights.DEFAULT)\n",
    "modelEV2 = torchvision.models.efficientnet_v2_s(weights=torchvision.models.EfficientNet_V2_S_Weights.DEFAULT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import lightning as L\n",
    "import van\n",
    "\n",
    "modelR50 = torchvision.models.resnet50(weights=torchvision.models.ResNet50_Weights.DEFAULT)\n",
    "modelEV2 = van.van_b2()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dummyLoad = torch.rand(1, 3, 224, 224)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a variable to store the output\n",
    "last_conv_output = None\n",
    "\n",
    "# Define a hook function to capture the output\n",
    "def hook_fn(module, input, output):\n",
    "    global last_conv_output\n",
    "    last_conv_output = output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2.eval()\n",
    "modelEV2.block4[-1].mlp.dwconv.register_forward_hook(hook_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2(dummyLoad).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "last_conv_output.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelR50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.nn.functional.adaptive_avg_pool2d()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelR50.layer4[-1].conv3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2.eval()\n",
    "salida = modelEV2(dummyLoad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2.forward_features(dummyLoad).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2Sub = torch.nn.Sequential(*list(modelEV2.children())[:-1])\n",
    "modelEV2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "salida.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelR50.eval()\n",
    "subModel = torch.nn.Sequential(*list(modelR50.children())[:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "subModel(dummyLoad).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for params in modelEV2.parameters():\n",
    "    print(params)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.norm(params, p=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.linalg.norm(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelR50_Decoder = torch.nn.Sequential(*list(modelR50.children())[:-2])\n",
    "modelEV2_Decoder = modelEV2.features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import timm\n",
    "\n",
    "model = timm.create_model(\"davit_small.msft_in1k\", pretrained=True, num_classes=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelEV2.fc2= torch.nn.Linear(12,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "pred = torch.tensor([0.3137, 0.3161, 0.4395, 0.4636, 0.4427, 0.3616, 0.3256, 0.3367])\n",
    "trueVal = torch.tensor([0.1577, 0.8148, 0.4295, 0.4841, 0.5133, 0.2474, 0.6306, 0.3618])\n",
    "\n",
    "PredBach = torch.tensor([[0.3137, 0.3161, 0.4395, 0.4636, 0.4427, 0.3616, 0.3256, 0.3367],\n",
    "        [0.2372, 0.0121, 0.5123, 0.4597, 0.3809, 0.3405, 0.3988, 0.3922],\n",
    "        [0.0436, 0.0425, 0.2698, 0.5095, 0.5610, 0.2571, 0.6514, 0.4645]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = torch.abs(PredBach - PredBach )\n",
    "print(res)\n",
    "res = torch.sum(res, 1)\n",
    "\n",
    "mean = res.mean(1).unsqueeze(1)\n",
    "\n",
    "f = (res <= mean).float()\n",
    "\n",
    "f.split\n",
    "\n",
    "f = f / f.count_nonzero(dim=1).unsqueeze(1)\n",
    "\n",
    "torch.sum(PredBach * f, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CalcAgreement(x:torch):\n",
    "    deltas = torch.sum(torch.abs(x.unsqueeze(1) - x), 1)\n",
    "    deltasMean = deltas.mean()\n",
    "    \n",
    "    factor = (deltas <= deltasMean).float()\n",
    "    factor = factor / factor.count_nonzero()\n",
    "    \n",
    "    return torch.dot(factor, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.tensor([CalcAgreement(d.squeeze()) for d in PredBach.split(1, 0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = []\n",
    "for d in PredBach.split(1, 0):\n",
    "    res.append(CalcAgreement(d.squeeze()))\n",
    "\n",
    "torch.tensor(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "CalcAgreement(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.abs(pred.unsqueeze(1) - pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.sum(torch.abs(pred.unsqueeze(1) - pred), 1)\n",
    "aMean = a.mean()\n",
    "aMean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factor = (a <= aMean).float()\n",
    "factor = factor / factor.count_nonzero()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "factor, a, pred, factor * pred, torch.dot(factor, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def kl_diver(a, b):\n",
    "    return torch.sum(a * torch.log(a/b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kl_diver(pred, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dkl = torch.nn.KLDivLoss(reduction=\"sum\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dkl(pred, trueVal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Models\n",
    "\n",
    "modelo = Models.RARP_NVB_EfficientNetV2_Deep()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "modelo.model.classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RARP_NVB_NN(L.LightningModule):\n",
    "    def __init__(self, input_size:int=4) -> None:\n",
    "        super().__init__()\n",
    "        \n",
    "        self.model = torch.nn.Sequential(\n",
    "            torch.nn.Linear(input_size, 3),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(3, 1)\n",
    "        )\n",
    "        \n",
    "    def forward(self, data):\n",
    "        x = self.model(data)\n",
    "        return x\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RARP_NVB_ResNet50_CAM(L.LightningModule):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "        \n",
    "        self.model = torchvision.models.resnet50()\n",
    "        tempFC_ft = self.model.fc.in_features \n",
    "        self.model.fc = torch.nn.Linear(in_features=tempFC_ft, out_features=1)\n",
    "        \n",
    "        self.feature_map = torch.nn.Sequential(*list(self.model.children())[:-2])\n",
    "        \n",
    "    def forward(self, data):\n",
    "        featureMap = self.feature_map(data)\n",
    "        Cont_Net = torch.nn.functional.adaptive_avg_pool2d(input=featureMap, output_size=(1, 1)) \n",
    "        Cont_Net = torch.flatten(Cont_Net)\n",
    "        \n",
    "        pred = self.model.fc(Cont_Net)\n",
    "        \n",
    "        return pred, featureMap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import Any\n",
    "\n",
    "\n",
    "class Modelo_er(L.LightningModule):\n",
    "    def __init__(self, *args: Any, **kwargs: Any) -> None:\n",
    "        super().__init__(*args, **kwargs)\n",
    "               \n",
    "        self.feature_map1 = torch.nn.Sequential(*list(modelR50.children())[:-2])\n",
    "        self.feature_map2 = modelEV2.features\n",
    "        \n",
    "    def forward(self, data):\n",
    "        featureMap1 = self.feature_map1(data)\n",
    "        featureMap2 = self.feature_map2(data)\n",
    "        \n",
    "        Cont_Net1 = torch.nn.functional.adaptive_avg_pool2d(input=featureMap1, output_size=(1, 1)) \n",
    "        Cont_Net2 = torch.nn.functional.adaptive_avg_pool2d(input=featureMap2, output_size=(1, 1)) \n",
    "        \n",
    "        Cont_Net1 = torch.flatten(Cont_Net1)\n",
    "        #Cont_Net2 = torch.flatten(Cont_Net2)\n",
    "        \n",
    "        return Cont_Net1, Cont_Net2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchvision import transforms\n",
    "import defs\n",
    "import Loaders\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "Dataset = Loaders.RARP_DatasetCreator(\n",
    "    \"./DataSet_big_2\",\n",
    "    FoldSeed=505,\n",
    "    createFile=True,\n",
    "    SavePath=\"./DatasetBig2\",\n",
    "    Fold=5,\n",
    "    removeBlackBar=True,\n",
    ")\n",
    "\n",
    "Dataset.CreateFolds()\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "Dataset.mean, Dataset.std = ([30.38144216, 42.03988769, 97.8896116], [40.63141752, 44.26910074, 50.29294373])\n",
    "batchSize = 17\n",
    "Fold = 0\n",
    "rootFile = Dataset.CVS_File.parent.parent/f\"fold_{Fold}\"\n",
    "\n",
    "testtransform =  torch.nn.Sequential(\n",
    "    transforms.Resize(256, antialias=True),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.Normalize(Dataset.mean, Dataset.std)\n",
    ").to(device)\n",
    "\n",
    "testDataset = torchvision.datasets.DatasetFolder(\n",
    "    str (rootFile/\"test\"),\n",
    "    loader=defs.load_file_tensor,\n",
    "    extensions=\"npy\",\n",
    "    transform=testtransform\n",
    ")\n",
    "\n",
    "Test_DataLoader = DataLoader(\n",
    "    testDataset, \n",
    "    batch_size=batchSize, \n",
    "    num_workers=0, \n",
    "    shuffle=False, \n",
    "    pin_memory=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ModeloObj = Modelo_er()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ModeloObj.to(device)\n",
    "ModeloObj.eval()\n",
    "\n",
    "with torch.no_grad():\n",
    "    for data, label in iter(Test_DataLoader):\n",
    "        data = data.float().to(device)\n",
    "        label = label.to(device)\n",
    "        pred = ModeloObj(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import Models as M\n",
    "\n",
    "lsita = [M.RARP_NVB_ResNet50(None, M.TypeLossFunction.BCEWithLogits), M.RARP_NVB_ResNet18(None, M.TypeLossFunction.BCEWithLogits)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lsita[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.flatten(pred[1]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "\n",
    "DumpCSV = pd.read_csv(Path(\"./DatasetSmallBalanced_seed_505/dump/dataset.csv\"))\n",
    "Extradata = pd.read_excel(Path(\"./DataSet_smallBalaced/data.xlsx\"))\n",
    "\n",
    "Extradata[\"name\"] =  Extradata[\"列1\"].astype(str) + \".tiff\"\n",
    "Extradata = Extradata.drop(columns=[\"列1\"])\n",
    "\n",
    "DumpCSV[\"raw_name\"] = \"Img0-\" + DumpCSV[\"id\"].astype(str) + \".npy\"\n",
    "DumpCSV = DumpCSV.drop(columns=[\"id\", \"path\", \"mean_1\", \"mean_2\", \"mean_3\", \"std_1\", \"std_2\", \"std_3\"])\n",
    "\n",
    "NewData = pd.merge(Extradata, DumpCSV, on=\"name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Path(\"./DatasetSmallBalanced_seed_505/dump/dataset.csv\").name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Extradata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DumpCSV.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NewData.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "NewData[NewData[\"name\"] == \"51.tiff\"].values.flatten().tolist()[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoFeatureExtractor, VanForImageClassification\n",
    "\n",
    "model = VanForImageClassification.from_pretrained(\"Visual-Attention-Network/van-base\")\n",
    "feature_extractor = AutoFeatureExtractor.from_pretrained(\"Visual-Attention-Network/van-base\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from timm.models.layers import DropPath, to_2tuple, trunc_normal_\n",
    "from timm.models.registry import register_model\n",
    "from timm.models.vision_transformer import _cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "checkpoint = torch.hub.load_state_dict_from_url(\n",
    "    url=\"https://huggingface.co/Visual-Attention-Network/VAN-Small-original/resolve/main/van_small_811.pth.tar\", map_location=\"cpu\", check_hash=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import van\n",
    "\n",
    "model = van.van_b1(pretrained = True)\n",
    "model.head.in_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "\n",
    "#resnet50 = torch.hub.load('facebookresearch/dino:main', 'dino_resnet50')\n",
    "resnet50 = torchvision.models.resnet50(weights=torchvision.models.ResNet50_Weights.DEFAULT)\n",
    "vit = torchvision.models.vit_b_16()\n",
    "\n",
    "vitb8 = torch.hub.load('facebookresearch/dino:main', 'dino_vitb8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vitb8\n",
    "#vit\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lr=0.0001, L1=None\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import torch\n",
    "import cv2\n",
    "import numpy as np\n",
    "import Models as M\n",
    "\n",
    "import torchvision.transforms as T\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "def _removeBlackBorder(image):\n",
    "    image = np.array(image)\n",
    "    \n",
    "    copyImg = cv2.cvtColor(image.copy(), cv2.COLOR_BGR2HSV)\n",
    "    h = copyImg[:,:,0]\n",
    "    mask = np.ones(h.shape, dtype=np.uint8) * 255\n",
    "    th = (25, 175)\n",
    "    mask[(h > th[0]) & (h < th[1])] = 0\n",
    "    copyImg = cv2.cvtColor(copyImg, cv2.COLOR_HSV2BGR)\n",
    "    resROI = cv2.bitwise_and(copyImg, copyImg, mask=mask)\n",
    "        \n",
    "    image_gray = cv2.cvtColor(resROI, cv2.COLOR_BGR2GRAY)\n",
    "    _, thresh = cv2.threshold(image_gray, 0, 255, cv2.THRESH_BINARY)\n",
    "    kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (15, 15))\n",
    "    morph = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)\n",
    "    contours = cv2.findContours(morph, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
    "    contours = contours[0] if len(contours) == 2 else contours[1]\n",
    "    bigCont = max(contours, key=cv2.contourArea)\n",
    "    x, y, w, h = cv2.boundingRect(bigCont)\n",
    "    crop = image[y : y + h, x : x + w]\n",
    "    return crop\n",
    "\n",
    "    \n",
    "\n",
    "#RN50ModelToEval = M.RARP_NVB_ResNet50_VAN.load_from_checkpoint(\"./log_X12_VAN_Review/lightning_logs/version_17/checkpoints/RARP-epoch=29.ckpt\")\n",
    "RN50ModelToEval = M.RARP_NVB_DINO_MultiTask.load_from_checkpoint(\"./log_X13_van_DINO/lightning_logs/version_20/checkpoints/RARP-epoch=21.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean, std = ([30.38144216, 42.03988769, 97.8896116], [40.63141752, 44.26910074, 50.29294373])\n",
    "transforms = T.Compose([\n",
    "    T.Resize((256,256), antialias=True, interpolation=T.InterpolationMode.BICUBIC),\n",
    "    T.CenterCrop(224),\n",
    "    T.Normalize(mean, std)\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.0550], device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "frameToFind = cv2.imread(str(Path(\"D:/Users/user/Documents/postata/RARP/Clasification/DataSet_big/NOT_NVB/213.tiff\")), cv2.IMREAD_COLOR)\n",
    "frameToFind = _removeBlackBorder(frameToFind)\n",
    "frameToFind = torch.Tensor(frameToFind)\n",
    "frameToFind = frameToFind.permute(2, 0, 1).float()\n",
    "\n",
    "originalSize = frameToFind.shape[-2:]\n",
    "originalImg = frameToFind\n",
    "\n",
    "frameToFind = transforms(frameToFind)\n",
    "\n",
    "frameToFind = frameToFind.repeat(1, 1, 1, 1)\n",
    "\n",
    "torch.set_float32_matmul_precision('high')\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "RN50ModelToEval.to(device)\n",
    "RN50ModelToEval.eval()\n",
    "\n",
    "with torch.no_grad():\n",
    "    frameToFind = frameToFind.to(device)\n",
    "    \n",
    "    Doutput, _, _ = RN50ModelToEval(frameToFind)\n",
    "    \n",
    "    Doutput = Doutput.flatten()\n",
    "    \n",
    "    print (torch.sigmoid(Doutput))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pytorch_grad_cam import GradCAM, GradCAMPlusPlus\n",
    "from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget\n",
    "from pytorch_grad_cam.utils.image import show_cam_on_image\n",
    "\n",
    "targetL = [RN50ModelToEval.student.backbone.block4[-1]]\n",
    "\n",
    "CAM = GradCAMPlusPlus(model=RN50ModelToEval, target_layers=targetL)#GradCAM(model=RN50ModelToEval, target_layers=targetL)\n",
    "tar = [ClassifierOutputTarget(0)]\n",
    "\n",
    "\n",
    "gi = CAM(frameToFind, targets=tar)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Denorlalize (img:torch.Tensor, std, mean):\n",
    "    ImgNumpy = img.numpy().transpose((1, 2, 0))\n",
    "    ImgNumpy = np.clip((std * ImgNumpy + mean) / 255, 0, 1)\n",
    "    ImgNumpy = ImgNumpy[...,::-1].copy()\n",
    "    \n",
    "    return ImgNumpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 1918568 characters]",
      "text/plain": [
       "<Figure size 2000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "rgb_img = Denorlalize(frameToFind.cpu()[0], std, mean)\n",
    "oriImg = Denorlalize(originalImg.cpu(), [1, 1, 1], [0, 0, 0])\n",
    "\n",
    "grayscale_cam = gi[0]\n",
    "\n",
    "visualization = show_cam_on_image(rgb_img, grayscale_cam, use_rgb=True)\n",
    "\n",
    "smallH, smallW, _ = rgb_img.shape\n",
    "x, y = (16, 16) #(256 - 224) // 2\n",
    "over = np.ones((256, 256), dtype=np.uint8)\n",
    "layer2 = np.zeros((224, 224), dtype=np.uint8)\n",
    "over[y:y + smallH, x:x + smallW] = layer2\n",
    "over = cv2.resize(over, (originalSize[1], originalSize[0]))\n",
    "over = cv2.bitwise_and(oriImg, oriImg, mask=over)\n",
    "\n",
    "bkMask = cv2.inRange(over, (0,0,0), (0,0,0))\n",
    "contours, _ = cv2.findContours(bkMask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n",
    "\n",
    "xBK, yBK, w, h = cv2.boundingRect(max(contours, key=cv2.contourArea)) \n",
    "\n",
    "layer2 = cv2.resize(visualization / 255, (w, h))\n",
    "\n",
    "over[yBK:yBK + h, xBK:xBK + w] = layer2\n",
    "\n",
    "fig, ax = plt.subplots(1, 2, figsize=(20, 8))\n",
    "\n",
    "ax[0].imshow(oriImg)\n",
    "ax[0].set_title(\"Original Image\")\n",
    "ax[0].axis(\"off\")\n",
    "\n",
    "ax[1].imshow(over)\n",
    "ax[1].set_title(\"CAM\")\n",
    "ax[1].axis(\"off\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 41100 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "copyImg = cv2.cvtColor((oriImg * 255).astype(np.uint8), cv2.COLOR_RGB2HSV)\n",
    "h_channel = copyImg[:, :, 0]\n",
    "\n",
    "hist = cv2.calcHist([h_channel], [0], None, [180], [0, 180])\n",
    "\n",
    "Dominant_hue = np.argmax(hist)\n",
    "\n",
    "lower_tool = max(0, Dominant_hue - 5)\n",
    "upper_tool = min(179, Dominant_hue + 5)\n",
    "\n",
    "plt.figure(figsize=(10, 5))\n",
    "for i in range(180):\n",
    "    # Normalize the histogram values for display purposes\n",
    "    plt.bar(i, hist[i][0], color=plt.cm.hsv(i / 180), edgecolor='none', width=1.0)\n",
    "\n",
    "plt.plot(hist, color='purple')\n",
    "plt.title('Hue Histogram')\n",
    "plt.xlabel('Hue Value')\n",
    "plt.ylabel('Frequency')\n",
    "plt.xlim([0, 180]) \n",
    "plt.ylim([0, 10000]) \n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 9, 4)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lower_tool, upper_tool, Dominant_hue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x14d432de6d0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 121552 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mask = cv2.inRange(h_channel, 0, 7)\n",
    "plt.imshow(mask, cmap=\"gray\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x14d43316dd0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 44356 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "copyImg = cv2.cvtColor((oriImg * 255).astype(np.uint8), cv2.COLOR_RGB2HSV)\n",
    "\n",
    "h = copyImg[:,:,0]\n",
    "mask = np.zeros(h.shape, dtype=np.uint8)\n",
    "th = (25, 175)\n",
    "mask[(h > th[0])] = 1\n",
    "\n",
    "plt.imshow(mask, cmap=\"gray\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 1271836 characters]",
      "text/plain": [
       "<Figure size 2000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(1, 2, figsize=(20, 8))\n",
    "lower_tool = np.array([0, 0, 115])\n",
    "upper_tool = np.array([180, 77, 255])\n",
    "\n",
    "#175, 180; 9, 20\n",
    "\n",
    "\n",
    "\n",
    "copyImg = cv2.cvtColor((oriImg * 255).astype(np.uint8), cv2.COLOR_RGB2HSV)\n",
    "\n",
    "mask = cv2.inRange(copyImg, lower_tool, upper_tool)\n",
    "\n",
    "kernel = np.ones((5, 5), dtype=np.uint8)\n",
    "\n",
    "\n",
    "mask = cv2.bitwise_not(mask)\n",
    "\n",
    "copyImg = cv2.cvtColor(copyImg, cv2.COLOR_HSV2RGB)\n",
    "resROI = cv2.bitwise_and(copyImg, copyImg, mask=mask)\n",
    "\n",
    "ax[0].imshow(mask, cmap=\"gray\")\n",
    "ax[0].set_title(\"Original Image\")\n",
    "ax[0].axis(\"off\")\n",
    "\n",
    "ax[1].imshow(resROI)\n",
    "ax[1].set_title(\"CAM\")\n",
    "ax[1].axis(\"off\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pyRARP",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
