bring dvats & requirements & entrypoint

dvats/.ipynb_checkpoints/__init__-checkpoint.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

dvats/.ipynb_checkpoints/_modidx-checkpoint.py

@@ -0,0 +1,14 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'master',
+                'doc_baseurl': '/dvats/',
+                'doc_host': 'https://vrodriguezf.github.io',
+                'git_url': 'https://github.com/vrodriguezf/deepvats',
+                'lib_path': 'dvats'},
+  'syms': { 'dvats.all': {},
+            'dvats.dr': {},
+            'dvats.encoder': {},
+            'dvats.imports': {},
+            'dvats.load': {},
+            'dvats.utils': {},
+            'dvats.visualization': {}}}

dvats/.ipynb_checkpoints/_nbdev-checkpoint.py

@@ -0,0 +1,39 @@
+# AUTOGENERATED BY NBDEV! DO NOT EDIT!
+
+__all__ = ["index", "modules", "custom_doc_links", "git_url"]
+
+index = {"check_compatibility": "dr.ipynb",
+         "get_UMAP_prjs": "dr.ipynb",
+         "get_PCA_prjs": "dr.ipynb",
+         "get_TSNE_prjs": "dr.ipynb",
+         "DCAE_torch": "encoder.ipynb",
+         "ENCODER_EMBS_MODULE_NAME": "encoder.ipynb",
+         "get_enc_embs": "encoder.ipynb",
+         "TSArtifact": "load.ipynb",
+         "wandb.apis.public.Artifact.to_df": "load.ipynb",
+         "wandb.apis.public.Artifact.to_tsartifact": "load.ipynb",
+         "infer_or_inject_freq": "load.ipynb",
+         "generate_TS_df": "utils.ipynb",
+         "normalize_columns": "utils.ipynb",
+         "remove_constant_columns": "utils.ipynb",
+         "ReferenceArtifact": "utils.ipynb",
+         "wandb.apis.public.Artifact.to_obj": "utils.ipynb",
+         "PrintLayer": "utils.ipynb",
+         "Learner.export_and_get": "utils.ipynb",
+         "get_wandb_artifacts": "utils.ipynb",
+         "get_pickle_artifact": "utils.ipynb",
+         "plot_TS": "visualization.ipynb",
+         "plot_validation_ts_ae": "visualization.ipynb",
+         "plot_mask": "visualization.ipynb"}
+
+modules = ["dr.py",
+           "encoder.py",
+           "load.py",
+           "utils.py",
+           "visualization.py"]
+
+doc_url = "https://vrodriguezf.github.io/tchub/"
+
+git_url = "https://gitlab.geist.re/pml/x_timecluster_extension/tree/master/"
+
+def custom_doc_links(name): return None

dvats/.ipynb_checkpoints/all-checkpoint.py

@@ -0,0 +1,8 @@
+import dvats
+from .imports import *
+from .load import *
+from .utils import *
+from .dr import *
+from .encoder import *
+from .visualization import *
+from .xai import *

dvats/.ipynb_checkpoints/dr-checkpoint.py

@@ -0,0 +1,166 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/dr.ipynb.
+
+# %% auto 0
+__all__ = ['get_gpu_memory', 'color_for_percentage', 'create_bar', 'gpu_memory_status', 'check_compatibility', 'get_UMAP_prjs',
+           'get_PCA_prjs', 'get_TSNE_prjs', 'cluster_score']
+
+# %% ../nbs/dr.ipynb 2
+import subprocess
+def get_gpu_memory(device = 0):
+    total_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=" + str(device)])
+    total_memory = int(total_memory.decode().split('\n')[0])
+    used_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits",  "--id=" + str(device)])
+    used_memory = int(used_memory.decode().split('\n')[0])
+
+    percentage = round((used_memory / total_memory) * 100)
+    return used_memory, total_memory, percentage
+
+def color_for_percentage(percentage):
+    if percentage < 20:
+        return "\033[90m"  # Gray
+    elif percentage < 40:
+        return "\033[94m"  # Blue
+    elif percentage < 60:
+        return "\033[92m"  # Green
+    elif percentage < 80:
+        return "\033[93m"  # Orange
+    else:
+        return "\033[91m"  # Red
+        
+def create_bar(percentage, color_code, length=20):
+    filled_length = int(length * percentage // 100)
+    bar = "█" * filled_length + "-" * (length - filled_length)
+    return color_code + bar + "\033[0m"  # Apply color and reset after bar
+
+def gpu_memory_status(device=0):
+    used, total, percentage = get_gpu_memory(device)
+    color_code = color_for_percentage(percentage)
+    bar = create_bar(percentage, color_code)
+    print(f"Used mem: {used}")
+    print(f"Used mem: {total}")
+    print(f"Memory Usage: [{bar}] {color_code}{percentage}%\033[0m")
+
+# %% ../nbs/dr.ipynb 4
+import umap
+import cudf
+import cuml
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from .imports import *
+from .load import TSArtifact
+
+# %% ../nbs/dr.ipynb 5
+def check_compatibility(dr_ar:TSArtifact, enc_ar:TSArtifact):
+    "Function to check that the artifact used by the encoder model and the artifact that is \
+    going to be passed through the DR are compatible"
+    try:
+        # Check that both artifacts have the same variables
+        chk_vars = dr_ar.metadata['TS']['vars'] == enc_ar.metadata['TS']['vars']
+        # Check that both artifacts have the same freq
+        chk_freq = dr_ar.metadata['TS']['freq'] == enc_ar.metadata['TS']['freq']
+        # Check that the dr artifact is not normalized (not normalized data has not the key normalization)
+        chk_norm = dr_ar.metadata['TS'].get('normalization') is None
+        # Check that the dr artifact has not missing values
+        chk_miss = dr_ar.metadata['TS']['has_missing_values'] == "False"
+        # Check all logical vars.
+        if chk_vars and chk_freq and chk_norm and chk_miss:
+            print("Artifacts are compatible.")
+        else:
+            raise Exception
+    except Exception as e:
+        print("Artifacts are not compatible.")
+        raise e
+    return None
+
+# %% ../nbs/dr.ipynb 7
+#Comment this part after 4_seconds debugged
+import hashlib
+
+# %% ../nbs/dr.ipynb 8
+import warnings
+import sys
+from numba.core.errors import NumbaPerformanceWarning
+@delegates(cuml.UMAP)
+def get_UMAP_prjs(
+    input_data, 
+    cpu=True, 
+    print_flag = False, 
+    check_memory_usage = True,
+    **kwargs
+):
+    "Compute the projections of `input_data` using UMAP, with a configuration contained in `**kwargs`."
+    if print_flag: 
+        print("--> get_UMAP_prjs")
+        print("kwargs: ", kwargs)
+        sys.stdout.flush()
+        ####
+        checksum = hashlib.md5(input_data.tobytes()).hexdigest()
+        print(checksum)
+        ####
+        
+    if check_memory_usage: gpu_memory_status()
+    
+    warnings.filterwarnings("ignore", category=NumbaPerformanceWarning) # silence NumbaPerformanceWarning
+    
+    #reducer = umap.UMAP(**kwargs) if cpu else cuml.UMAP(**kwargs)
+    if cpu:
+        print("-- umap.UMAP --", cpu)
+        sys.stdout.flush()
+        reducer = umap.UMAP(**kwargs)
+    else:
+        print("-- cuml.UMAP --", cpu)
+        sys.stdout.flush()
+        if 'random_state' in kwargs:
+            kwargs['random_state'] = np.uint64(kwargs['random_state'])
+        reducer = cuml.UMAP(**kwargs)
+    
+    if print_flag: 
+        print("------- reducer --------")
+        print(reducer)
+        print(reducer.get_params())
+        print("------- reducer --------")
+        sys.stdout.flush()
+    
+    projections = reducer.fit_transform(input_data)
+    
+    if check_memory_usage: gpu_memory_status()
+    if print_flag: 
+        checksum = hashlib.md5(projections.tobytes()).hexdigest()
+        print("prjs checksum ", checksum)
+        print("get_UMAP_prjs -->")
+        sys.stdout.flush()
+    return projections
+
+# %% ../nbs/dr.ipynb 13
+@delegates(cuml.PCA)
+def get_PCA_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes PCA projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.PCA(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 15
+@delegates(cuml.TSNE)
+def get_TSNE_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes TSNE projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.TSNE(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 18
+from sklearn.metrics import silhouette_score
+def cluster_score(prjs, clusters_labels, print_flag):
+    score = silhouette_score(prjs, clusters_labels)
+    if print_flag: print("Silhouette_score:", score)
+    return score

dvats/.ipynb_checkpoints/encoder-checkpoint.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+"""encoder.ipynb
+
+Automatically generated.
+
+Original file is located at:
+    /home/macu/work/nbs/encoder.ipynb
+"""
+
+#default_exp encoder
+
+#hide
+%load_ext autoreload
+%autoreload 2
+
+#export
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from tsai.callback.MVP import *
+from tsai.imports import *
+from tsai.models.InceptionTimePlus import InceptionTimePlus
+from tsai.models.explainability import get_acts_and_grads
+from tsai.models.layers import *
+from tsai.data.validation import combine_split_data
+
+#hide
+from tsai.all import *
+
+#export 
+class DCAE_torch(Module):
+    def __init__(self, c_in, seq_len, delta, nfs=[64, 32, 12], kss=[10, 5, 5],
+                 pool_szs=[2,2,3], output_fsz=10):
+        """
+        Create a Deep Convolutional Autoencoder for multivariate time series of `d` dimensions,
+        sliced with a window size of `w`. The parameter `delta` sets the number of latent features that will be
+        contained in the Dense layer of the network. The the number of features
+        maps (filters), the filter size and the pool size can also be adjusted."
+        """
+        assert all_equal([len(x) for x in [nfs, kss, pool_szs]], np.repeat(len(nfs), 3)), \
+            'nfs, kss, and pool_szs must have the same length'
+        assert np.prod(pool_szs) == nfs[-1], \
+            'The number of filters in the last conv layer must be equal to the product of pool sizes'
+        assert seq_len % np.prod(pool_szs) == 0, \
+            'The product of pool sizes must be a divisor of the window size'
+        layers = []
+        for i in range_of(kss):
+            layers += [Conv1d(ni=nfs[i-1] if i>0 else c_in, nf=nfs[i], ks=kss[i]),
+                       nn.MaxPool1d(kernel_size=pool_szs[i])]
+        self.downsample = nn.Sequential(*layers)
+        self.bottleneck = nn.Sequential(OrderedDict([
+            ('flatten', nn.Flatten()),
+            ('latent_in', nn.Linear(seq_len, delta)),
+            ('latent_out', nn.Linear(delta, seq_len)),
+            ('reshape', Reshape(nfs[-1], seq_len // np.prod(pool_szs)))
+        ]))
+        layers = []
+        for i in reversed(range_of(kss)):
+            layers += [Conv1d(ni=nfs[i+1] if i != (len(nfs)-1) else nfs[-1],
+                              nf=nfs[i], ks=kss[i]),
+                       nn.Upsample(scale_factor=pool_szs[i])]
+        layers += [Conv1d(ni=nfs[0], nf=c_in, kernel_size=output_fsz)]
+        self.upsample = nn.Sequential(*layers)
+
+    def forward(self, x):
+        x = self.downsample(x)
+        x = self.bottleneck(x)
+        x = self.upsample(x)
+        return x
+
+#hide
+foo = torch.rand(3, 1, 48)
+m = DCAE_torch(c_in=foo.shape[1], seq_len=foo.shape[2], delta=12)
+m(foo).shape
+
+#export
+ENCODER_EMBS_MODULE_NAME = {
+    InceptionTimePlus: 'backbone', # for mvp based models
+    DCAE_torch: 'bottleneck.latent_in'
+}
+
+#export
+def get_enc_embs(X, enc_learn, module=None, cpu=False, average_seq_dim=True, to_numpy=True):
+    """
+        Get the embeddings of X from an encoder, passed in `enc_learn as a fastai
+        learner. By default, the embeddings are obtained from the last layer
+        before the model head, although any layer can be passed to `model`.
+        Input
+        - `cpu`: Whether to do the model inference in cpu of gpu (GPU recommended)
+        - `average_seq_dim`: Whether to aggregate the embeddings in the sequence dimensions
+        - `to_numpy`: Whether to return the result as a numpy array (if false returns a tensor)
+    """
+    if cpu:
+        print("--> Get enc embs CPU")
+        enc_learn.dls.cpu()
+        enc_learn.cpu()
+    else:
+        print("--> Use CUDA |Get enc embs GPU")
+        enc_learn.dls.cuda()
+        enc_learn.cuda()
+        print("devices: ", enc_learn.dls.device, enc_learn.model.device)
+        print("Use CUDA -->")
+    if enc_learn.dls.bs == 0: enc_learn.dls.bs = 64
+    print("--> Get enc embs bs: ", enc_learn.dls.bs)
+    aux_dl = enc_learn.dls.valid.new_dl(X=X)
+    aux_dl.bs = enc_learn.dls.bs if enc_learn.dls.bs>0 else 64
+    module = nested_attr(enc_learn.model,
+                         ENCODER_EMBS_MODULE_NAME[type(enc_learn.model)]) \
+                if module is None else module
+    embs = [get_acts_and_grads(model=enc_learn.model,
+                               modules=module,
+                               x=xb[0], cpu=cpu)[0] for xb in aux_dl]
+    embs = to_concat(embs)
+    if embs.ndim == 3 and average_seq_dim: embs = embs.mean(axis=2)
+    if to_numpy: embs = embs.numpy() if cpu else embs.cpu().numpy()
+    return embs
+
+#hide
+import wandb
+from dvats.utils import *
+wandb_api = wandb.Api()
+enc_artifact = wandb_api.artifact('deepvats/mvp:latest')
+enc_learner = enc_artifact.to_obj()
+X = torch.rand(9, 1, 48)
+
+#hide
+#slow
+#%%time
+embs = get_enc_embs(X, enc_learner, cpu=True)
+test_eq(embs.shape[0], X.shape[0])
+embs.shape, embs.__class__
+
+#hide
+%%time
+embs = get_enc_embs(X, enc_learner, cpu=False, to_numpy=False)
+test_eq(embs.shape[0], X.shape[0])
+embs.shape, embs.__class__, embs.device
+
+#hide 
+%%time
+embs = get_enc_embs(X, enc_learner, cpu=False, to_numpy=True)
+test_eq(embs.shape[0], X.shape[0])
+embs.shape, embs.__class__
+
+#hide
+
+#from nbdev.export import notebook2script
+
+#notebook2script()
+
+#from tsai import nb2py
+#nb2py
+#beep(1)

dvats/.ipynb_checkpoints/imports-checkpoint.py

@@ -0,0 +1,24 @@
+from IPython.display import Audio, display, HTML, Javascript, clear_output # from tsai
+import importlib
+import numpy as np
+import time
+import sys
+
+##
+# Constants
+##
+WANDB_ARTIFACTS_DIR = 'data/wandb_artifacts'
+
+# General purpose functions
+def beep(inp=1, duration=.1, n=1):
+    rate = 10000
+    mult = 1.6 * inp if inp else .08
+    wave = np.sin(mult*np.arange(rate*duration))
+    for i in range(n): 
+        display(Audio(wave, rate=10000, autoplay=True))
+        time.sleep(duration / .1)
+        
+def m_reload(package_name):
+    for k,v in sys.modules.items():
+        if k.startswith(package_name):
+            importlib.reload(v)

dvats/.ipynb_checkpoints/load-checkpoint.py

@@ -0,0 +1,166 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/load.ipynb.
+
+# %% auto 0
+__all__ = ['TSArtifact', 'infer_or_inject_freq']
+
+# %% ../nbs/load.ipynb 2
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+import wandb
+from datetime import datetime, timedelta
+from .imports import *
+from .utils import *
+import pickle
+import pyarrow.feather as ft
+
+# %% ../nbs/load.ipynb 7
+class TSArtifact(wandb.Artifact):
+
+    default_storage_path = Path(Path.home()/'data/wandb_artifacts/')
+    date_format = '%Y-%m-%d %H:%M:%S' # TODO add milliseconds
+    handle_missing_values_techniques = {
+        'linear_interpolation': lambda df : df.interpolate(method='linear', limit_direction='both'),
+        'overall_mean': lambda df : df.fillna(df.mean()),
+        'overall_median': lambda df : df.fillna(df.median()),
+        'backward_fill' : lambda df : df.fillna(method='bfill'),
+        'forward_fill' : lambda df : df.fillna(method='ffill')
+    }
+
+    "Class that represents a wandb artifact containing time series data. sd stands for start_date \
+    and ed for end_date. Both should be pd.Timestamps"
+
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, name, sd:pd.Timestamp, ed:pd.Timestamp, **kwargs):
+        super().__init__(type='dataset', name=name, **kwargs)
+        self.sd = sd
+        self.ed = ed
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['TS'] = dict(sd = self.sd.strftime(self.date_format),
+                                   ed = self.ed.strftime(self.date_format))
+
+
+    @classmethod
+    def from_daily_csv_files(cls, root_path, fread=pd.read_csv, start_date=None, end_date=None, metadata=None, **kwargs):
+
+        "Create a wandb artifact of type `dataset`, containing the CSV files from `start_date` \
+        to `end_date`. Dates must be pased as `datetime.datetime` objects. If a `wandb_run` is \
+        defined, the created artifact will be logged to that run, using the longwall name as \
+        artifact name, and the date range as version."
+
+        return None
+
+
+    @classmethod
+    @delegates(__init__)
+    def from_df(cls, df:pd.DataFrame, name:str, path:str=None, sd:pd.Timestamp=None, ed:pd.Timestamp=None,
+                normalize:bool=False, missing_values_technique:str=None, resampling_freq:str=None, **kwargs):
+
+        """
+        Create a TSArtifact of type `dataset`, using the DataFrame `df` samples from \
+        `sd` (start date) to `ed` (end date). Dates must be passed as `datetime.datetime` \
+        objects. The transformed DataFrame is stored as a pickle file in the path `path` \
+        and its reference is added to the artifact entries. Additionally, the dataset can \
+        be normalized (see `normalize` argument) or transformed using missing values \
+        handling techniques (see `missing_values_technique` argument) or resampling (see \
+        `resampling_freq` argument).
+
+        Arguments:
+            df: (DataFrame) The dataframe you want to convert into an artifact.
+            name: (str) The artifact name.
+            path: (str, optional) The path where the file, containing the new transformed \
+                dataframe, is saved. Default None.
+            sd: (sd, optional) Start date. By default, the first index of `df` is taken.
+            ed: (ed, optional) End date. By default, the last index of `df` is taken.
+            normalize: (bool, optional) If the dataset values should be normalized. Default\
+                False.
+            missing_values_technique: (str, optional) The technique used to handle missing \
+                values. Options: "linear_iterpolation", "overall_mean", "overall_median" or \
+                None. Default None.
+            resampling_freq: (str, optional) The offset string or object representing \
+                frequency conversion for time series resampling. Default None.
+
+        Returns:
+            TSArtifact object.
+        """
+        sd = df.index[0] if sd is None else sd
+        ed = df.index[-1] if ed is None else ed
+        obj = cls(name, sd=sd, ed=ed, **kwargs)
+        df = df.query('@obj.sd <= index <= @obj.ed')
+        obj.metadata['TS']['created'] = 'from-df'
+        obj.metadata['TS']['n_vars'] = df.columns.__len__()
+
+        # Handle Missing Values
+        df = obj.handle_missing_values_techniques[missing_values_technique](df) if missing_values_technique is not None else df
+        obj.metadata['TS']['handle_missing_values_technique'] = missing_values_technique.__str__()
+        obj.metadata['TS']['has_missing_values'] = np.any(df.isna().values).__str__()
+
+        # Indexing and Resampling
+        if resampling_freq: df = df.resample(resampling_freq).mean()
+        obj.metadata['TS']['n_samples'] = len(df)
+        obj.metadata['TS']['freq'] = str(df.index.freq)
+
+        # Time Series Variables
+        obj.metadata['TS']['vars'] = list(df.columns)
+
+        # Normalization - Save the previous means and stds
+        if normalize:
+            obj.metadata['TS']['normalization'] = dict(means = df.describe().loc['mean'].to_dict(),
+                                                       stds = df.describe().loc['std'].to_dict())
+            df = normalize_columns(df)
+
+        # Hash and save
+        hash_code = str(pd.util.hash_pandas_object(df).sum()) # str(hash(df.values.tobytes()))
+        path = obj.default_storage_path/f'{hash_code}' if path is None else Path(path)/f'{hash_code}.feather'
+        ft.write_feather(df, path)
+        obj.metadata['TS']['hash'] = hash_code
+        obj.add_file(str(path))
+
+        return obj
+
+# %% ../nbs/load.ipynb 11
+@patch
+def to_df(self:wandb.apis.public.Artifact):
+    "Download the files of a saved wandb artifact and process them as a single dataframe. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."
+    # The way we have to ensure that the argument comes from a TS arfitact is the metadata
+    if self.metadata.get('TS') is None:
+        print(f'ERROR:{self} does not come from a logged TSArtifact')
+        return None
+    dir = Path(self.download())
+    if self.metadata['TS']['created'] == 'from-df':
+        # Call read_pickle with the single file from dir
+        #return pd.read_pickle(dir.ls()[0])
+        return ft.read_feather(dir.ls()[0])
+    else:
+        print("ERROR: Only from_df method is allowed yet")
+
+# %% ../nbs/load.ipynb 13
+@patch
+def to_tsartifact(self:wandb.apis.public.Artifact):
+    "Cast an artifact as a TS artifact. The artifact must have been created from one of the \
+    class creation methods of the class `TSArtifact`. This is useful to go back to a TSArtifact \
+    after downloading an artifact through the wand API"
+    return TSArtifact(name=self.digest, #TODO change this
+                      sd=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      ed=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      description=self.description,
+                      metadata=self.metadata)
+
+# %% ../nbs/load.ipynb 15
+@delegates(pd.to_datetime)
+def infer_or_inject_freq(df, injected_freq='1s', start_date=None, **kwargs):
+    """
+        Infer index frequency. If there's not a proper time index, create fake timestamps,
+        keeping the desired `injected_freq`. If that is None, set a default one of 1 second.
+        start_date: the first date of the index (int or string).
+    """
+    inferred_freq = pd.infer_freq(df.index)
+    if inferred_freq == 'N':
+        timedelta = pd.to_timedelta(injected_freq)
+        df.index = pd.to_datetime(ifnone(start_date, 0), **kwargs) + timedelta*df.index
+        df.index.freq = pd.infer_freq(df.index)
+    else:
+        df.index.freq = inferred_freq
+    return df

dvats/.ipynb_checkpoints/utils-checkpoint.py

@@ -0,0 +1,134 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: nbs/utils.ipynb (unless otherwise specified).
+
+__all__ = ['generate_TS_df', 'normalize_columns', 'remove_constant_columns', 'ReferenceArtifact', 'PrintLayer',
+           'get_wandb_artifacts', 'get_pickle_artifact']
+
+# Cell
+from .imports import *
+from fastcore.all import *
+import wandb
+import pickle
+import pandas as pd
+import numpy as np
+#import tensorflow as tf
+import torch.nn as nn
+from fastai.basics import *
+
+# Cell
+def generate_TS_df(rows, cols):
+    "Generates a dataframe containing a multivariate time series, where each column \
+    represents a variable and each row a time point (sample). The timestamp is in the \
+    index of the dataframe, and it is created with a even space of 1 second between samples"
+    index = np.arange(pd.Timestamp.now(),
+                      pd.Timestamp.now() + pd.Timedelta(rows-1, 'seconds'),
+                      pd.Timedelta(1, 'seconds'))
+    data = np.random.randn(len(index), cols)
+    return pd.DataFrame(data, index=index)
+
+# Cell
+def normalize_columns(df:pd.DataFrame):
+    "Normalize columns from `df` to have 0 mean and 1 standard deviation"
+    mean = df.mean()
+    std = df.std() + 1e-7
+    return (df-mean)/std
+
+# Cell
+def remove_constant_columns(df:pd.DataFrame):
+    return df.loc[:, (df != df.iloc[0]).any()]
+
+# Cell
+class ReferenceArtifact(wandb.Artifact):
+    default_storage_path = Path('data/wandb_artifacts/') # * this path is relative to Path.home()
+    "This class is meant to create an artifact with a single reference to an object \
+    passed as argument in the contructor. The object will be pickled, hashed and stored \
+    in a specified folder."
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, obj, name, type='object', folder=None, **kwargs):
+        super().__init__(type=type, name=name, **kwargs)
+        # pickle dumps the object and then hash it
+        hash_code = str(hash(pickle.dumps(obj)))
+        folder = Path(ifnone(folder, Path.home()/self.default_storage_path))
+        with open(f'{folder}/{hash_code}', 'wb') as f:
+            pickle.dump(obj, f)
+        self.add_reference(f'file://{folder}/{hash_code}')
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['ref'] = dict()
+        self.metadata['ref']['hash'] = hash_code
+        self.metadata['ref']['type'] = str(obj.__class__)
+
+# Cell
+@patch
+def to_obj(self:wandb.apis.public.Artifact):
+    """Download the files of a saved ReferenceArtifact and get the referenced object. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."""
+    if self.metadata.get('ref') is None:
+        print(f'ERROR:{self} does not come from a saved ReferenceArtifact')
+        return None
+    original_path = ReferenceArtifact.default_storage_path/self.metadata['ref']['hash']
+    path = original_path if original_path.exists() else Path(self.download()).ls()[0]
+    with open(path, 'rb') as f:
+        obj = pickle.load(f)
+    return obj
+
+# Cell
+import torch.nn as nn
+class PrintLayer(nn.Module):
+    def __init__(self):
+        super(PrintLayer, self).__init__()
+
+    def forward(self, x):
+        # Do your print / debug stuff here
+        print(x.shape)
+        return x
+
+# Cell
+@patch
+def export_and_get(self:Learner, keep_exported_file=False):
+    """
+        Export the learner into an auxiliary file, load it and return it back.
+    """
+    aux_path = Path('aux.pkl')
+    self.export(fname='aux.pkl')
+    aux_learn = load_learner('aux.pkl')
+    if not keep_exported_file: aux_path.unlink()
+    return aux_learn
+
+# Cell
+def get_wandb_artifacts(project_path, type=None, name=None, last_version=True):
+    """
+        Get the artifacts logged in a wandb project.
+        Input:
+        - `project_path` (str): entity/project_name
+        - `type` (str): whether to return only one type of artifacts
+        - `name` (str): Leave none to have all artifact names
+        - `last_version`: whether to return only the last version of each artifact or not
+
+        Output: List of artifacts
+    """
+    public_api = wandb.Api()
+    if type is not None:
+        types = [public_api.artifact_type(type, project_path)]
+    else:
+        types = public_api.artifact_types(project_path)
+
+    res = L()
+    for kind in types:
+        for collection in kind.collections():
+            if name is None or name == collection.name:
+                versions = public_api.artifact_versions(
+                    kind.type,
+                    "/".join([kind.entity, kind.project, collection.name]),
+                    per_page=1,
+                )
+                if last_version: res += next(versions)
+                else: res += L(versions)
+    return list(res)
+
+# Cell
+def get_pickle_artifact(filename):
+
+    with open(filename, "rb") as f:
+        df = pickle.load(f)
+
+    return df

dvats/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

dvats/_modidx.py

@@ -0,0 +1,105 @@
+# Autogenerated by nbdev
+
+d = { 'settings': { 'branch': 'master',
+                'doc_baseurl': '/dvats/',
+                'doc_host': 'https://vrodriguezf.github.io',
+                'git_url': 'https://github.com/vrodriguezf/deepvats',
+                'lib_path': 'dvats'},
+  'syms': { 'dvats.all': {},
+            'dvats.dr': { 'dvats.dr.check_compatibility': ('dr.html#check_compatibility', 'dvats/dr.py'),
+                          'dvats.dr.cluster_score': ('dr.html#cluster_score', 'dvats/dr.py'),
+                          'dvats.dr.color_for_percentage': ('dr.html#color_for_percentage', 'dvats/dr.py'),
+                          'dvats.dr.create_bar': ('dr.html#create_bar', 'dvats/dr.py'),
+                          'dvats.dr.get_PCA_prjs': ('dr.html#get_pca_prjs', 'dvats/dr.py'),
+                          'dvats.dr.get_TSNE_prjs': ('dr.html#get_tsne_prjs', 'dvats/dr.py'),
+                          'dvats.dr.get_UMAP_prjs': ('dr.html#get_umap_prjs', 'dvats/dr.py'),
+                          'dvats.dr.get_gpu_memory': ('dr.html#get_gpu_memory', 'dvats/dr.py'),
+                          'dvats.dr.gpu_memory_status': ('dr.html#gpu_memory_status', 'dvats/dr.py')},
+            'dvats.encoder': { 'dvats.encoder.DCAE_torch': ('encoder.html#dcae_torch', 'dvats/encoder.py'),
+                               'dvats.encoder.DCAE_torch.__init__': ('encoder.html#__init__', 'dvats/encoder.py'),
+                               'dvats.encoder.DCAE_torch.forward': ('encoder.html#forward', 'dvats/encoder.py'),
+                               'dvats.encoder.color_for_percentage': ('encoder.html#color_for_percentage', 'dvats/encoder.py'),
+                               'dvats.encoder.create_bar': ('encoder.html#create_bar', 'dvats/encoder.py'),
+                               'dvats.encoder.get_enc_embs': ('encoder.html#get_enc_embs', 'dvats/encoder.py'),
+                               'dvats.encoder.get_enc_embs_set_stride_set_batch_size': ( 'encoder.html#get_enc_embs_set_stride_set_batch_size',
+                                                                                         'dvats/encoder.py'),
+                               'dvats.encoder.get_gpu_memory_': ('encoder.html#get_gpu_memory_', 'dvats/encoder.py'),
+                               'dvats.encoder.gpu_memory_status_': ('encoder.html#gpu_memory_status_', 'dvats/encoder.py')},
+            'dvats.imports': {},
+            'dvats.load': { 'dvats.load.TSArtifact': ('load.html#tsartifact', 'dvats/load.py'),
+                            'dvats.load.TSArtifact.__init__': ('load.html#__init__', 'dvats/load.py'),
+                            'dvats.load.TSArtifact.from_daily_csv_files': ('load.html#from_daily_csv_files', 'dvats/load.py'),
+                            'dvats.load.TSArtifact.from_df': ('load.html#from_df', 'dvats/load.py'),
+                            'dvats.load.infer_or_inject_freq': ('load.html#infer_or_inject_freq', 'dvats/load.py'),
+                            'dvats.load.wandb.apis.public.Artifact.to_df': ('load.html#wandb.apis.public.artifact.to_df', 'dvats/load.py'),
+                            'dvats.load.wandb.apis.public.Artifact.to_tsartifact': ( 'load.html#wandb.apis.public.artifact.to_tsartifact',
+                                                                                     'dvats/load.py')},
+            'dvats.utils': { 'dvats.utils.Learner.export_and_get': ('utils.html#learner.export_and_get', 'dvats/utils.py'),
+                             'dvats.utils.PrintLayer': ('utils.html#printlayer', 'dvats/utils.py'),
+                             'dvats.utils.PrintLayer.__init__': ('utils.html#__init__', 'dvats/utils.py'),
+                             'dvats.utils.PrintLayer.forward': ('utils.html#forward', 'dvats/utils.py'),
+                             'dvats.utils.ReferenceArtifact': ('utils.html#referenceartifact', 'dvats/utils.py'),
+                             'dvats.utils.ReferenceArtifact.__init__': ('utils.html#__init__', 'dvats/utils.py'),
+                             'dvats.utils.exec_with_and_feather_k_output': ('utils.html#exec_with_and_feather_k_output', 'dvats/utils.py'),
+                             'dvats.utils.exec_with_feather': ('utils.html#exec_with_feather', 'dvats/utils.py'),
+                             'dvats.utils.exec_with_feather_k_output': ('utils.html#exec_with_feather_k_output', 'dvats/utils.py'),
+                             'dvats.utils.generate_TS_df': ('utils.html#generate_ts_df', 'dvats/utils.py'),
+                             'dvats.utils.get_pickle_artifact': ('utils.html#get_pickle_artifact', 'dvats/utils.py'),
+                             'dvats.utils.get_wandb_artifacts': ('utils.html#get_wandb_artifacts', 'dvats/utils.py'),
+                             'dvats.utils.learner_module_leaves': ('utils.html#learner_module_leaves', 'dvats/utils.py'),
+                             'dvats.utils.learner_module_leaves_subtables': ( 'utils.html#learner_module_leaves_subtables',
+                                                                              'dvats/utils.py'),
+                             'dvats.utils.normalize_columns': ('utils.html#normalize_columns', 'dvats/utils.py'),
+                             'dvats.utils.py_function': ('utils.html#py_function', 'dvats/utils.py'),
+                             'dvats.utils.remove_constant_columns': ('utils.html#remove_constant_columns', 'dvats/utils.py'),
+                             'dvats.utils.wandb.apis.public.Artifact.to_obj': ( 'utils.html#wandb.apis.public.artifact.to_obj',
+                                                                                'dvats/utils.py')},
+            'dvats.visualization': { 'dvats.visualization.plot_TS': ('visualization.html#plot_ts', 'dvats/visualization.py'),
+                                     'dvats.visualization.plot_mask': ('visualization.html#plot_mask', 'dvats/visualization.py'),
+                                     'dvats.visualization.plot_validation_ts_ae': ( 'visualization.html#plot_validation_ts_ae',
+                                                                                    'dvats/visualization.py')},
+            'dvats.xai': { 'dvats.xai.InteractiveAnomalyPlot': ('xai.html#interactiveanomalyplot', 'dvats/xai.py'),
+                           'dvats.xai.InteractiveAnomalyPlot.__init__': ('xai.html#__init__', 'dvats/xai.py'),
+                           'dvats.xai.InteractiveAnomalyPlot.plot_projections_clusters_interactive': ( 'xai.html#plot_projections_clusters_interactive',
+                                                                                                       'dvats/xai.py'),
+                           'dvats.xai.InteractiveTSPlot': ('xai.html#interactivetsplot', 'dvats/xai.py'),
+                           'dvats.xai.InteractiveTSPlot.__init__': ('xai.html#__init__', 'dvats/xai.py'),
+                           'dvats.xai.add_movement_buttons': ('xai.html#add_movement_buttons', 'dvats/xai.py'),
+                           'dvats.xai.add_selected_features': ('xai.html#add_selected_features', 'dvats/xai.py'),
+                           'dvats.xai.add_windows': ('xai.html#add_windows', 'dvats/xai.py'),
+                           'dvats.xai.anomaly_score': ('xai.html#anomaly_score', 'dvats/xai.py'),
+                           'dvats.xai.calculate_cluster_stats': ('xai.html#calculate_cluster_stats', 'dvats/xai.py'),
+                           'dvats.xai.delta_x_bigger': ('xai.html#delta_x_bigger', 'dvats/xai.py'),
+                           'dvats.xai.delta_x_lower': ('xai.html#delta_x_lower', 'dvats/xai.py'),
+                           'dvats.xai.delta_y_bigger': ('xai.html#delta_y_bigger', 'dvats/xai.py'),
+                           'dvats.xai.delta_y_lower': ('xai.html#delta_y_lower', 'dvats/xai.py'),
+                           'dvats.xai.detector': ('xai.html#detector', 'dvats/xai.py'),
+                           'dvats.xai.get_anomalies': ('xai.html#get_anomalies', 'dvats/xai.py'),
+                           'dvats.xai.get_anomaly_styles': ('xai.html#get_anomaly_styles', 'dvats/xai.py'),
+                           'dvats.xai.get_dataset': ('xai.html#get_dataset', 'dvats/xai.py'),
+                           'dvats.xai.get_dateformat': ('xai.html#get_dateformat', 'dvats/xai.py'),
+                           'dvats.xai.get_df_selected': ('xai.html#get_df_selected', 'dvats/xai.py'),
+                           'dvats.xai.get_embeddings': ('xai.html#get_embeddings', 'dvats/xai.py'),
+                           'dvats.xai.get_prjs': ('xai.html#get_prjs', 'dvats/xai.py'),
+                           'dvats.xai.initial_plot': ('xai.html#initial_plot', 'dvats/xai.py'),
+                           'dvats.xai.merge_overlapping_windows': ('xai.html#merge_overlapping_windows', 'dvats/xai.py'),
+                           'dvats.xai.move_down': ('xai.html#move_down', 'dvats/xai.py'),
+                           'dvats.xai.move_left': ('xai.html#move_left', 'dvats/xai.py'),
+                           'dvats.xai.move_right': ('xai.html#move_right', 'dvats/xai.py'),
+                           'dvats.xai.move_up': ('xai.html#move_up', 'dvats/xai.py'),
+                           'dvats.xai.plot_anomaly_scores_distribution': ('xai.html#plot_anomaly_scores_distribution', 'dvats/xai.py'),
+                           'dvats.xai.plot_clusters_with_anomalies': ('xai.html#plot_clusters_with_anomalies', 'dvats/xai.py'),
+                           'dvats.xai.plot_clusters_with_anomalies_interactive_plot': ( 'xai.html#plot_clusters_with_anomalies_interactive_plot',
+                                                                                        'dvats/xai.py'),
+                           'dvats.xai.plot_initial_config': ('xai.html#plot_initial_config', 'dvats/xai.py'),
+                           'dvats.xai.plot_projections': ('xai.html#plot_projections', 'dvats/xai.py'),
+                           'dvats.xai.plot_projections_clusters': ('xai.html#plot_projections_clusters', 'dvats/xai.py'),
+                           'dvats.xai.plot_save': ('xai.html#plot_save', 'dvats/xai.py'),
+                           'dvats.xai.set_features_buttons': ('xai.html#set_features_buttons', 'dvats/xai.py'),
+                           'dvats.xai.setup_boxes': ('xai.html#setup_boxes', 'dvats/xai.py'),
+                           'dvats.xai.setup_style': ('xai.html#setup_style', 'dvats/xai.py'),
+                           'dvats.xai.shift_datetime': ('xai.html#shift_datetime', 'dvats/xai.py'),
+                           'dvats.xai.show': ('xai.html#show', 'dvats/xai.py'),
+                           'dvats.xai.toggle_trace': ('xai.html#toggle_trace', 'dvats/xai.py'),
+                           'dvats.xai.umap_parameters': ('xai.html#umap_parameters', 'dvats/xai.py'),
+                           'dvats.xai.update_plot': ('xai.html#update_plot', 'dvats/xai.py')}}}

dvats/_nbdev.py

@@ -0,0 +1,39 @@
+# AUTOGENERATED BY NBDEV! DO NOT EDIT!
+
+__all__ = ["index", "modules", "custom_doc_links", "git_url"]
+
+index = {"check_compatibility": "dr.ipynb",
+         "get_UMAP_prjs": "dr.ipynb",
+         "get_PCA_prjs": "dr.ipynb",
+         "get_TSNE_prjs": "dr.ipynb",
+         "DCAE_torch": "encoder.ipynb",
+         "ENCODER_EMBS_MODULE_NAME": "encoder.ipynb",
+         "get_enc_embs": "encoder.ipynb",
+         "TSArtifact": "load.ipynb",
+         "wandb.apis.public.Artifact.to_df": "load.ipynb",
+         "wandb.apis.public.Artifact.to_tsartifact": "load.ipynb",
+         "infer_or_inject_freq": "load.ipynb",
+         "generate_TS_df": "utils.ipynb",
+         "normalize_columns": "utils.ipynb",
+         "remove_constant_columns": "utils.ipynb",
+         "ReferenceArtifact": "utils.ipynb",
+         "wandb.apis.public.Artifact.to_obj": "utils.ipynb",
+         "PrintLayer": "utils.ipynb",
+         "Learner.export_and_get": "utils.ipynb",
+         "get_wandb_artifacts": "utils.ipynb",
+         "get_pickle_artifact": "utils.ipynb",
+         "plot_TS": "visualization.ipynb",
+         "plot_validation_ts_ae": "visualization.ipynb",
+         "plot_mask": "visualization.ipynb"}
+
+modules = ["dr.py",
+           "encoder.py",
+           "load.py",
+           "utils.py",
+           "visualization.py"]
+
+doc_url = "https://vrodriguezf.github.io/tchub/"
+
+git_url = "https://gitlab.geist.re/pml/x_timecluster_extension/tree/master/"
+
+def custom_doc_links(name): return None

dvats/all.py

@@ -0,0 +1,8 @@
+import dvats
+from .imports import *
+from .load import *
+from .utils import *
+from .dr import *
+from .encoder import *
+from .visualization import *
+from .xai import *

dvats/dr.py

@@ -0,0 +1,166 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/dr.ipynb.
+
+# %% auto 0
+__all__ = ['get_gpu_memory', 'color_for_percentage', 'create_bar', 'gpu_memory_status', 'check_compatibility', 'get_UMAP_prjs',
+           'get_PCA_prjs', 'get_TSNE_prjs', 'cluster_score']
+
+# %% ../nbs/dr.ipynb 2
+import subprocess
+def get_gpu_memory(device = 0):
+    total_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=" + str(device)])
+    total_memory = int(total_memory.decode().split('\n')[0])
+    used_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits",  "--id=" + str(device)])
+    used_memory = int(used_memory.decode().split('\n')[0])
+
+    percentage = round((used_memory / total_memory) * 100)
+    return used_memory, total_memory, percentage
+
+def color_for_percentage(percentage):
+    if percentage < 20:
+        return "\033[90m"  # Gray
+    elif percentage < 40:
+        return "\033[94m"  # Blue
+    elif percentage < 60:
+        return "\033[92m"  # Green
+    elif percentage < 80:
+        return "\033[93m"  # Orange
+    else:
+        return "\033[91m"  # Red
+        
+def create_bar(percentage, color_code, length=20):
+    filled_length = int(length * percentage // 100)
+    bar = "█" * filled_length + "-" * (length - filled_length)
+    return color_code + bar + "\033[0m"  # Apply color and reset after bar
+
+def gpu_memory_status(device=0):
+    used, total, percentage = get_gpu_memory(device)
+    color_code = color_for_percentage(percentage)
+    bar = create_bar(percentage, color_code)
+    print(f"GPU | Used mem: {used}")
+    print(f"GPU | Used mem: {total}")
+    print(f"GPU | Memory Usage: [{bar}] {color_code}{percentage}%\033[0m")
+
+# %% ../nbs/dr.ipynb 4
+import umap
+import cudf
+import cuml
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from .imports import *
+from .load import TSArtifact
+
+# %% ../nbs/dr.ipynb 5
+def check_compatibility(dr_ar:TSArtifact, enc_ar:TSArtifact):
+    "Function to check that the artifact used by the encoder model and the artifact that is \
+    going to be passed through the DR are compatible"
+    try:
+        # Check that both artifacts have the same variables
+        chk_vars = dr_ar.metadata['TS']['vars'] == enc_ar.metadata['TS']['vars']
+        # Check that both artifacts have the same freq
+        chk_freq = dr_ar.metadata['TS']['freq'] == enc_ar.metadata['TS']['freq']
+        # Check that the dr artifact is not normalized (not normalized data has not the key normalization)
+        chk_norm = dr_ar.metadata['TS'].get('normalization') is None
+        # Check that the dr artifact has not missing values
+        chk_miss = dr_ar.metadata['TS']['has_missing_values'] == "False"
+        # Check all logical vars.
+        if chk_vars and chk_freq and chk_norm and chk_miss:
+            print("Artifacts are compatible.")
+        else:
+            raise Exception
+    except Exception as e:
+        print("Artifacts are not compatible.")
+        raise e
+    return None
+
+# %% ../nbs/dr.ipynb 7
+#Comment this part after 4_seconds debugged
+import hashlib
+
+# %% ../nbs/dr.ipynb 8
+import warnings
+import sys
+from numba.core.errors import NumbaPerformanceWarning
+@delegates(cuml.UMAP)
+def get_UMAP_prjs(
+    input_data, 
+    cpu=True, 
+    print_flag = False, 
+    check_memory_usage = True,
+    **kwargs
+):
+    "Compute the projections of `input_data` using UMAP, with a configuration contained in `**kwargs`."
+    if print_flag: 
+        print("--> get_UMAP_prjs")
+        print("kwargs: ", kwargs)
+        sys.stdout.flush()
+        ####
+        checksum = hashlib.md5(input_data.tobytes()).hexdigest()
+        print(checksum)
+        ####
+        
+    if check_memory_usage: gpu_memory_status()
+    
+    warnings.filterwarnings("ignore", category=NumbaPerformanceWarning) # silence NumbaPerformanceWarning
+    
+    #reducer = umap.UMAP(**kwargs) if cpu else cuml.UMAP(**kwargs)
+    if cpu:
+        print("-- umap.UMAP --", cpu)
+        sys.stdout.flush()
+        reducer = umap.UMAP(**kwargs)
+    else:
+        print("-- cuml.UMAP --", cpu)
+        sys.stdout.flush()
+        if 'random_state' in kwargs:
+            kwargs['random_state'] = np.uint64(kwargs['random_state'])
+        reducer = cuml.UMAP(**kwargs)
+    
+    if print_flag: 
+        print("------- reducer --------")
+        print(reducer)
+        print(reducer.get_params())
+        print("------- reducer --------")
+        sys.stdout.flush()
+    
+    projections = reducer.fit_transform(input_data)
+    
+    if check_memory_usage: gpu_memory_status()
+    if print_flag: 
+        checksum = hashlib.md5(projections.tobytes()).hexdigest()
+        print("prjs checksum ", checksum)
+        print("get_UMAP_prjs -->")
+        sys.stdout.flush()
+    return projections
+
+# %% ../nbs/dr.ipynb 13
+@delegates(cuml.PCA)
+def get_PCA_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes PCA projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.PCA(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 15
+@delegates(cuml.TSNE)
+def get_TSNE_prjs(X, cpu=False, **kwargs):
+    r"""
+    Computes TSNE projections of X
+    """
+    if cpu:
+        raise NotImplementedError
+    else:
+        reducer = cuml.TSNE(**kwargs)
+    projections = reducer.fit_transform(X)
+    return projections
+
+# %% ../nbs/dr.ipynb 18
+from sklearn.metrics import silhouette_score
+def cluster_score(prjs, clusters_labels, print_flag):
+    score = silhouette_score(prjs, clusters_labels)
+    if print_flag: print("Silhouette_score:", score)
+    return score

dvats/encoder.py

@@ -0,0 +1,301 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/encoder.ipynb.
+
+# %% auto 0
+__all__ = ['ENCODER_EMBS_MODULE_NAME', 'get_gpu_memory_', 'color_for_percentage', 'create_bar', 'gpu_memory_status_',
+           'DCAE_torch', 'get_enc_embs', 'get_enc_embs_set_stride_set_batch_size']
+
+# %% ../nbs/encoder.ipynb 2
+import subprocess
+def get_gpu_memory_(device = 0):
+    total_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits", "--id=" + str(device)])
+    total_memory = int(total_memory.decode().split('\n')[0])
+    used_memory = subprocess.check_output(["nvidia-smi", "--query-gpu=memory.used", "--format=csv,noheader,nounits",  "--id=" + str(device)])
+    used_memory = int(used_memory.decode().split('\n')[0])
+
+    percentage = round((used_memory / total_memory) * 100)
+    return used_memory, total_memory, percentage
+
+def color_for_percentage(percentage):
+    if percentage < 20:
+        return "\033[90m"  # Gray
+    elif percentage < 40:
+        return "\033[94m"  # Blue
+    elif percentage < 60:
+        return "\033[92m"  # Green
+    elif percentage < 80:
+        return "\033[93m"  # Orange
+    else:
+        return "\033[91m"  # Red
+        
+def create_bar(percentage, color_code, length=20):
+    filled_length = int(length * percentage // 100)
+    bar = "█" * filled_length + "-" * (length - filled_length)
+    return color_code + bar + "\033[0m"  # Apply color and reset after bar
+
+def gpu_memory_status_(device=0):
+    used, total, percentage = get_gpu_memory_(device)
+    color_code = color_for_percentage(percentage)
+    bar = create_bar(percentage, color_code)
+    print(f"GPU | Used mem: {used}")
+    print(f"GPU | Used mem: {total}")
+    print(f"GPU | Memory Usage: [{bar}] {color_code}{percentage}%\033[0m")
+
+
+# %% ../nbs/encoder.ipynb 4
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+from tsai.callback.MVP import *
+from tsai.imports import *
+from tsai.models.InceptionTimePlus import InceptionTimePlus
+from tsai.models.explainability import get_acts_and_grads
+from tsai.models.layers import *
+from tsai.data.validation import combine_split_data
+import time
+
+# %% ../nbs/encoder.ipynb 7
+class DCAE_torch(Module):
+    def __init__(self, c_in, seq_len, delta, nfs=[64, 32, 12], kss=[10, 5, 5],
+                 pool_szs=[2,2,3], output_fsz=10):
+        """
+        Create a Deep Convolutional Autoencoder for multivariate time series of `d` dimensions,
+        sliced with a window size of `w`. The parameter `delta` sets the number of latent features that will be
+        contained in the Dense layer of the network. The the number of features
+        maps (filters), the filter size and the pool size can also be adjusted."
+        """
+        assert all_equal([len(x) for x in [nfs, kss, pool_szs]], np.repeat(len(nfs), 3)), \
+            'nfs, kss, and pool_szs must have the same length'
+        assert np.prod(pool_szs) == nfs[-1], \
+            'The number of filters in the last conv layer must be equal to the product of pool sizes'
+        assert seq_len % np.prod(pool_szs) == 0, \
+            'The product of pool sizes must be a divisor of the window size'
+        layers = []
+        for i in range_of(kss):
+            layers += [Conv1d(ni=nfs[i-1] if i>0 else c_in, nf=nfs[i], ks=kss[i]),
+                       nn.MaxPool1d(kernel_size=pool_szs[i])]
+        self.downsample = nn.Sequential(*layers)
+        self.bottleneck = nn.Sequential(OrderedDict([
+            ('flatten', nn.Flatten()),
+            ('latent_in', nn.Linear(seq_len, delta)),
+            ('latent_out', nn.Linear(delta, seq_len)),
+            ('reshape', Reshape(nfs[-1], seq_len // np.prod(pool_szs)))
+        ]))
+        layers = []
+        for i in reversed(range_of(kss)):
+            layers += [Conv1d(ni=nfs[i+1] if i != (len(nfs)-1) else nfs[-1],
+                              nf=nfs[i], ks=kss[i]),
+                       nn.Upsample(scale_factor=pool_szs[i])]
+        layers += [Conv1d(ni=nfs[0], nf=c_in, kernel_size=output_fsz)]
+        self.upsample = nn.Sequential(*layers)
+
+
+    def forward(self, x):
+        x = self.downsample(x)
+        x = self.bottleneck(x)
+        x = self.upsample(x)
+        return x
+
+# %% ../nbs/encoder.ipynb 10
+ENCODER_EMBS_MODULE_NAME = {
+    InceptionTimePlus: 'backbone', # for mvp based models
+    DCAE_torch: 'bottleneck.latent_in'
+}
+
+# %% ../nbs/encoder.ipynb 12
+def get_enc_embs(X, enc_learn, module=None, cpu=False, average_seq_dim=True, to_numpy=True):
+    """
+        Get the embeddings of X from an encoder, passed in `enc_learn as a fastai
+        learner. By default, the embeddings are obtained from the last layer
+        before the model head, although any layer can be passed to `model`.
+        Input
+        - `cpu`: Whether to do the model inference in cpu of gpu (GPU recommended)
+        - `average_seq_dim`: Whether to aggregate the embeddings in the sequence dimensions
+        - `to_numpy`: Whether to return the result as a numpy array (if false returns a tensor)
+    """
+    print("--> Check CUDA")
+    if cpu:
+        print("--> Get enc embs CPU")
+        enc_learn.dls.cpu()
+        enc_learn.cpu()
+    else:
+        print("--> Ensure empty cache")
+        torch.cuda.empty_cache()
+        print("--> Use CUDA |Get enc embs GPU ")
+        enc_learn.dls.cuda()
+        enc_learn.cuda()
+        if torch.cuda.is_available():
+            print("CUDA está disponible")
+            print("Dispositivo CUDA actual: ", torch.cuda.current_device())
+            print("Nombre del dispositivo CUDA actual: ", torch.cuda.get_device_name(torch.cuda.current_device()))
+            
+        else:
+            print("CUDA no está disponible ")
+            print("Use CUDA -->")
+    if enc_learn.dls.bs == 0: enc_learn.dls.bs = 64
+    
+    print("--> Set dataset from X (enc_learn does not contain dls)")
+    aux_dl = enc_learn.dls.valid.new_dl(X=X)
+    aux_dl.bs = enc_learn.dls.bs if enc_learn.dls.bs>0 else 64
+    print("--> Get module")
+    module = nested_attr(enc_learn.model,ENCODER_EMBS_MODULE_NAME[type(enc_learn.model)]) if module is None else module
+    
+    print("--> Get enc embs bs: ", aux_dl.bs)
+    embs = [
+        get_acts_and_grads(
+            model=enc_learn.model,
+            modules=module,
+            x=xb[0], 
+            cpu=cpu
+        )[0] 
+        for xb in aux_dl
+    ]
+    print("--> Concat")
+    if not cpu:
+        total_emb_size = sum([emb.element_size() * emb.nelement() for emb in embs])
+        free_memory = torch.cuda.get_device_properties(0).total_memory - torch.cuda.memory_allocated()
+        if (total_emb_size < free_memory):
+            print("Fit in GPU")
+            embs=[emb.cuda() for emb in embs]
+        else:
+            print("Dont fit in GPU --> Go to CPU")
+            embs=[emb.cpu() for emb in embs]
+    embs = to_concat(embs)
+    print("--> reduce")
+    if embs.ndim == 3 and average_seq_dim: embs = embs.mean(axis=2)
+    print("--> 2 numpy")
+    if to_numpy: embs = embs.numpy() if cpu else embs.cpu().numpy()
+    return embs
+
+# %% ../nbs/encoder.ipynb 13
+def get_enc_embs_set_stride_set_batch_size(
+    X, enc_learn, stride, batch_size, module=None, cpu=False, average_seq_dim=True, to_numpy=True, 
+    print_flag = False, time_flag=False, chunk_size = 0, check_memory_usage = False
+):
+    """
+        Get the embeddings of X from an encoder, passed in `enc_learn as a fastai
+        learner. By default, the embeddings are obtained from the last layer
+        before the model head, although any layer can be passed to `model`.
+        Input
+        - `cpu`: Whether to do the model inference in cpu of gpu (GPU recommended)
+        - `average_seq_dim`: Whether to aggregate the embeddings in the sequence dimensions
+        - `to_numpy`: Whether to return the result as a numpy array (if false returns a tensor)
+    """
+    if time_flag:
+        t_start = time.time()
+    if print_flag:
+        print("--> get_enc_embs_set_stride_set_batch_size")
+    if check_memory_usage: gpu_memory_status_()
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions")
+        #import sys
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions | Python version", sys.version)
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions | PyTorch version", torch.__version__)
+        #print("get_enc_embs_set_stride_set_batch_size | Check versions | CUDA version", torch.version.cuda)
+        #print("get_enc_embs_set_stride_set_batch_size | Apply stride & batch size")
+    
+    X = X[::stride]
+    enc_learn.dls.bs = batch_size 
+    
+    if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Check CUDA | X ~ ", X.shape[0])
+    if cpu:
+        if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Get enc embs CPU")
+        enc_learn.dls.cpu()
+        enc_learn.cpu()
+    else:
+        if torch.cuda.is_available():
+            if (print_flag): 
+                print("get_enc_embs_set_stride_set_batch_size | CUDA device id:", torch.cuda.current_device())
+                print("get_enc_embs_set_stride_set_batch_size | CUDA device name: ", torch.cuda.get_device_name(torch.cuda.current_device()))
+                print("get_enc_embs_set_stride_set_batch_size | Ensure empty cache & move 2 GPU")
+            torch.cuda.empty_cache()
+            enc_learn.dls.cuda()
+            enc_learn.cuda()
+        else:
+            if (print_flag): print("get_enc_embs_set_stride_set_batch_size | No cuda available. Set CPU = true")
+            cpu = True
+    
+    if enc_learn.dls.bs is None or enc_learn.dls.bs == 0: enc_learn.dls.bs = 64
+
+    if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Set dataset from X (enc_learn does not contain dls)")
+    aux_dl = enc_learn.dls.valid.new_dl(X=X)
+    aux_dl.bs = enc_learn.dls.bs if enc_learn.dls.bs>0 else 64
+    if (print_flag): print("get_enc_embs_set_stride_set_batch_size | Get module")
+    module = nested_attr(enc_learn.model,ENCODER_EMBS_MODULE_NAME[type(enc_learn.model)]) if module is None else module
+    
+    if (print_flag): 
+        #print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | module ", module)
+        print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl len", len(aux_dl))
+        print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl.batch_len ", len(next(iter(aux_dl))))
+        print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl.bs ", aux_dl.bs)
+        if (not cpu):
+            total = torch.cuda.get_device_properties(device).total_memory
+            used = torch.cuda.memory_allocated(torch.cuda.current_device())
+            reserved = torch.cuda.memory_reserved(torch.cuda.current_device())
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | total_mem ", total)
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | used_mem ", used)
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | reserved_mem ", reserved)
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | available_mem ", total-reserved)
+            sys.stdout.flush()
+                                              
+    if (cpu or ( chunk_size == 0 )):
+        embs = [
+            get_acts_and_grads(
+                model=enc_learn.model,
+                modules=module, 
+                x=xb[0], 
+                cpu=cpu
+            )[0] 
+            for xb in aux_dl
+        ]
+        if not cpu: embs=[emb.cpu() for emb in embs]
+    else:
+        embs = []
+        total_chunks=max(1,round(len(X)/chunk_size))
+        if print_flag: print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | aux_dl len | " + str(len(X)) + " chunk size: " + str(chunk_size) + " => " + str(total_chunks) + " chunks")
+        for i in range(0, total_chunks):
+            if print_flag: 
+                print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | Chunk [ " + str(i) + "/"+str(total_chunks)+"] => " + str(round(i*100/total_chunks)) + "%")
+                sys.stdout.flush()
+            chunk = [batch for (n, batch) in enumerate(aux_dl) if (chunk_size*i <= n  and chunk_size*(i+1) > n) ]
+            chunk_embs = [
+                get_acts_and_grads(
+                    model=enc_learn.model,
+                    modules=module,
+                    x=xb[0], 
+                    cpu=cpu
+                )[0]
+                for xb in chunk
+            ]
+            # Mueve los embeddings del bloque a la CPU
+            chunk_embs = [emb.cpu() for emb in chunk_embs]
+            embs.extend(chunk_embs)
+            torch.cuda.empty_cache()
+        if print_flag: 
+            print("get_enc_embs_set_stride_set_batch_size | Get acts and grads | 100%")
+            sys.stdout.flush()
+    
+    if print_flag: print("get_enc_embs_set_stride_set_batch_size | concat embeddings")
+    
+    embs = to_concat(embs)
+    
+    if print_flag: print("get_enc_embs_set_stride_set_batch_size | Reduce")
+    
+    if embs.ndim == 3 and average_seq_dim: embs = embs.mean(axis=2)
+    
+    if print_flag: print("get_enc_embs_set_stride_set_batch_size | Convert to numpy")
+    
+    if to_numpy: 
+        if cpu or chunk_size > 0:
+            embs = embs.numpy() 
+        else: 
+            embs = embs.cpu().numpy()
+            torch.cuda.empty_cache()
+    if time_flag:
+        t = time.time()-t_start
+        if print_flag:
+            print("get_enc_embs_set_stride_set_batch_size " + str(t) + " seconds -->")
+        else:
+            print("get_enc_embs_set_stride_set_batch_size " + str(t) + " seconds")
+    if check_memory_usage: gpu_memory_status_()
+    if print_flag: 
+        print("get_enc_embs_set_stride_set_batch_size -->")
+    return embs

dvats/imports.py

@@ -0,0 +1,24 @@
+from IPython.display import Audio, display, HTML, Javascript, clear_output # from tsai
+import importlib
+import numpy as np
+import time
+import sys
+
+##
+# Constants
+##
+WANDB_ARTIFACTS_DIR = 'data/wandb_artifacts'
+
+# General purpose functions
+def beep(inp=1, duration=.1, n=1):
+    rate = 10000
+    mult = 1.6 * inp if inp else .08
+    wave = np.sin(mult*np.arange(rate*duration))
+    for i in range(n): 
+        display(Audio(wave, rate=10000, autoplay=True))
+        time.sleep(duration / .1)
+        
+def m_reload(package_name):
+    for k,v in sys.modules.items():
+        if k.startswith(package_name):
+            importlib.reload(v)

dvats/load.py

@@ -0,0 +1,168 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/load.ipynb.
+
+# %% auto 0
+__all__ = ['TSArtifact', 'infer_or_inject_freq']
+
+# %% ../nbs/load.ipynb 2
+import pandas as pd
+import numpy as np
+from fastcore.all import *
+import wandb
+from datetime import datetime, timedelta
+from .imports import *
+from .utils import *
+import pickle
+import pyarrow.feather as ft
+
+# %% ../nbs/load.ipynb 7
+class TSArtifact(wandb.Artifact):
+
+    default_storage_path = Path(Path.home()/'data/wandb_artifacts/')
+    date_format = '%Y-%m-%d %H:%M:%S' # TODO add milliseconds
+    handle_missing_values_techniques = {
+        'linear_interpolation': lambda df : df.interpolate(method='linear', limit_direction='both'),
+        'overall_mean': lambda df : df.fillna(df.mean()),
+        'overall_median': lambda df : df.fillna(df.median()),
+        'backward_fill' : lambda df : df.fillna(method='bfill'),
+        'forward_fill' : lambda df : df.fillna(method='ffill')
+    }
+
+    "Class that represents a wandb artifact containing time series data. sd stands for start_date \
+    and ed for end_date. Both should be pd.Timestamps"
+
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, name, sd:pd.Timestamp, ed:pd.Timestamp, **kwargs):
+        super().__init__(type='dataset', name=name, **kwargs)
+        self.sd = sd
+        self.ed = ed
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['TS'] = dict(sd = self.sd.strftime(self.date_format),
+                                   ed = self.ed.strftime(self.date_format))
+
+
+    @classmethod
+    def from_daily_csv_files(cls, root_path, fread=pd.read_csv, start_date=None, end_date=None, metadata=None, **kwargs):
+
+        "Create a wandb artifact of type `dataset`, containing the CSV files from `start_date` \
+        to `end_date`. Dates must be pased as `datetime.datetime` objects. If a `wandb_run` is \
+        defined, the created artifact will be logged to that run, using the longwall name as \
+        artifact name, and the date range as version."
+
+        return None
+
+
+    @classmethod
+    @delegates(__init__)
+    def from_df(cls, df:pd.DataFrame, name:str, path:str=None, sd:pd.Timestamp=None, ed:pd.Timestamp=None,
+                normalize:bool=False, missing_values_technique:str=None, resampling_freq:str=None, **kwargs):
+
+        """
+        Create a TSArtifact of type `dataset`, using the DataFrame `df` samples from \
+        `sd` (start date) to `ed` (end date). Dates must be passed as `datetime.datetime` \
+        objects. The transformed DataFrame is stored as a pickle file in the path `path` \
+        and its reference is added to the artifact entries. Additionally, the dataset can \
+        be normalized (see `normalize` argument) or transformed using missing values \
+        handling techniques (see `missing_values_technique` argument) or resampling (see \
+        `resampling_freq` argument).
+
+        Arguments:
+            df: (DataFrame) The dataframe you want to convert into an artifact.
+            name: (str) The artifact name.
+            path: (str, optional) The path where the file, containing the new transformed \
+                dataframe, is saved. Default None.
+            sd: (sd, optional) Start date. By default, the first index of `df` is taken.
+            ed: (ed, optional) End date. By default, the last index of `df` is taken.
+            normalize: (bool, optional) If the dataset values should be normalized. Default\
+                False.
+            missing_values_technique: (str, optional) The technique used to handle missing \
+                values. Options: "linear_iterpolation", "overall_mean", "overall_median" or \
+                None. Default None.
+            resampling_freq: (str, optional) The offset string or object representing \
+                frequency conversion for time series resampling. Default None.
+
+        Returns:
+            TSArtifact object.
+        """
+        sd = df.index[0] if sd is None else sd
+        ed = df.index[-1] if ed is None else ed
+        obj = cls(name, sd=sd, ed=ed, **kwargs)
+        df = df.query('@obj.sd <= index <= @obj.ed')
+        obj.metadata['TS']['created'] = 'from-df'
+        obj.metadata['TS']['n_vars'] = df.columns.__len__()
+
+        # Handle Missing Values
+        df = obj.handle_missing_values_techniques[missing_values_technique](df) if missing_values_technique is not None else df
+        obj.metadata['TS']['handle_missing_values_technique'] = missing_values_technique.__str__()
+        obj.metadata['TS']['has_missing_values'] = np.any(df.isna().values).__str__()
+
+        # Indexing and Resampling
+        if resampling_freq: df = df.resample(resampling_freq).mean()
+        obj.metadata['TS']['n_samples'] = len(df)
+        obj.metadata['TS']['freq'] = str(df.index.freq)
+
+        # Time Series Variables
+        obj.metadata['TS']['vars'] = list(df.columns)
+
+        # Normalization - Save the previous means and stds
+        if normalize:
+            obj.metadata['TS']['normalization'] = dict(means = df.describe().loc['mean'].to_dict(),
+                                                       stds = df.describe().loc['std'].to_dict())
+            df = normalize_columns(df)
+
+        # Hash and save
+        hash_code = str(pd.util.hash_pandas_object(df).sum()) # str(hash(df.values.tobytes()))
+        path = obj.default_storage_path/f'{hash_code}' if path is None else Path(path)/f'{hash_code}'
+        print("About to write df to ", path)
+        ft.write_feather(df, path, compression = 'lz4')
+        #feather.write_dataframe
+        obj.metadata['TS']['hash'] = hash_code
+        obj.add_file(str(path))
+
+        return obj
+
+# %% ../nbs/load.ipynb 14
+@patch
+def to_df(self:wandb.apis.public.Artifact):
+    "Download the files of a saved wandb artifact and process them as a single dataframe. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."
+    # The way we have to ensure that the argument comes from a TS arfitact is the metadata
+    if self.metadata.get('TS') is None:
+        print(f'ERROR:{self} does not come from a logged TSArtifact')
+        return None
+    dir = Path(self.download())
+    if self.metadata['TS']['created'] == 'from-df':
+        # Call read_pickle with the single file from dir
+        #return pd.read_pickle(dir.ls()[0])
+        return ft.read_feather(dir.ls()[0])
+    else:
+        print("ERROR: Only from_df method is allowed yet")
+
+# %% ../nbs/load.ipynb 16
+@patch
+def to_tsartifact(self:wandb.apis.public.Artifact):
+    "Cast an artifact as a TS artifact. The artifact must have been created from one of the \
+    class creation methods of the class `TSArtifact`. This is useful to go back to a TSArtifact \
+    after downloading an artifact through the wand API"
+    return TSArtifact(name=self.digest, #TODO change this
+                      sd=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      ed=pd.to_datetime(self.metadata['TS']['sd'], format=TSArtifact.date_format),
+                      description=self.description,
+                      metadata=self.metadata)
+
+# %% ../nbs/load.ipynb 18
+@delegates(pd.to_datetime)
+def infer_or_inject_freq(df, injected_freq='1s', start_date=None, **kwargs):
+    """
+        Infer index frequency. If there's not a proper time index, create fake timestamps,
+        keeping the desired `injected_freq`. If that is None, set a default one of 1 second.
+        start_date: the first date of the index (int or string).
+    """
+    inferred_freq = pd.infer_freq(df.index)
+    if inferred_freq == 'N':
+        timedelta = pd.to_timedelta(injected_freq)
+        df.index = pd.to_datetime(ifnone(start_date, 0), **kwargs) + timedelta*df.index
+        df.index.freq = pd.infer_freq(df.index)
+    else:
+        df.index.freq = inferred_freq
+    return df

dvats/utils.py

@@ -0,0 +1,245 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/utils.ipynb.
+
+# %% auto 0
+__all__ = ['generate_TS_df', 'normalize_columns', 'remove_constant_columns', 'ReferenceArtifact', 'PrintLayer',
+           'get_wandb_artifacts', 'get_pickle_artifact', 'exec_with_feather', 'py_function',
+           'exec_with_feather_k_output', 'exec_with_and_feather_k_output', 'learner_module_leaves',
+           'learner_module_leaves_subtables']
+
+# %% ../nbs/utils.ipynb 3
+from .imports import *
+from fastcore.all import *
+import wandb
+import pickle
+import pandas as pd
+import numpy as np
+#import tensorflow as tf
+import torch.nn as nn
+from fastai.basics import *
+
+# %% ../nbs/utils.ipynb 5
+def generate_TS_df(rows, cols):
+    "Generates a dataframe containing a multivariate time series, where each column \
+    represents a variable and each row a time point (sample). The timestamp is in the \
+    index of the dataframe, and it is created with a even space of 1 second between samples"
+    index = np.arange(pd.Timestamp.now(),
+                      pd.Timestamp.now() + pd.Timedelta(rows-1, 'seconds'),
+                      pd.Timedelta(1, 'seconds'))
+    data = np.random.randn(len(index), cols)
+    return pd.DataFrame(data, index=index)
+
+# %% ../nbs/utils.ipynb 10
+def normalize_columns(df:pd.DataFrame):
+    "Normalize columns from `df` to have 0 mean and 1 standard deviation"
+    mean = df.mean()
+    std = df.std() + 1e-7
+    return (df-mean)/std
+
+# %% ../nbs/utils.ipynb 16
+def remove_constant_columns(df:pd.DataFrame):
+    return df.loc[:, (df != df.iloc[0]).any()]
+
+# %% ../nbs/utils.ipynb 21
+class ReferenceArtifact(wandb.Artifact):
+    default_storage_path = Path('data/wandb_artifacts/') # * this path is relative to Path.home()
+    "This class is meant to create an artifact with a single reference to an object \
+    passed as argument in the contructor. The object will be pickled, hashed and stored \
+    in a specified folder."
+    @delegates(wandb.Artifact.__init__)
+    def __init__(self, obj, name, type='object', folder=None, **kwargs):
+        super().__init__(type=type, name=name, **kwargs)
+        # pickle dumps the object and then hash it
+        hash_code = str(hash(pickle.dumps(obj)))
+        folder = Path(ifnone(folder, Path.home()/self.default_storage_path))
+        with open(f'{folder}/{hash_code}', 'wb') as f:
+            pickle.dump(obj, f)
+        self.add_reference(f'file://{folder}/{hash_code}')
+        if self.metadata is None:
+            self.metadata = dict()
+        self.metadata['ref'] = dict()
+        self.metadata['ref']['hash'] = hash_code
+        self.metadata['ref']['type'] = str(obj.__class__)
+
+# %% ../nbs/utils.ipynb 24
+@patch
+def to_obj(self:wandb.apis.public.Artifact):
+    """Download the files of a saved ReferenceArtifact and get the referenced object. The artifact must \
+    come from a call to `run.use_artifact` with a proper wandb run."""
+    if self.metadata.get('ref') is None:
+        print(f'ERROR:{self} does not come from a saved ReferenceArtifact')
+        return None
+    original_path = ReferenceArtifact.default_storage_path/self.metadata['ref']['hash']
+    path = original_path if original_path.exists() else Path(self.download()).ls()[0]
+    with open(path, 'rb') as f:
+        obj = pickle.load(f)
+    return obj
+
+# %% ../nbs/utils.ipynb 33
+import torch.nn as nn
+class PrintLayer(nn.Module):
+    def __init__(self):
+        super(PrintLayer, self).__init__()
+
+    def forward(self, x):
+        # Do your print / debug stuff here
+        print(x.shape)
+        return x
+
+# %% ../nbs/utils.ipynb 34
+@patch
+def export_and_get(self:Learner, keep_exported_file=False):
+    """
+        Export the learner into an auxiliary file, load it and return it back.
+    """
+    aux_path = Path('aux.pkl')
+    self.export(fname='aux.pkl')
+    aux_learn = load_learner('aux.pkl')
+    if not keep_exported_file: aux_path.unlink()
+    return aux_learn
+
+# %% ../nbs/utils.ipynb 35
+def get_wandb_artifacts(project_path, type=None, name=None, last_version=True):
+    """
+        Get the artifacts logged in a wandb project.
+        Input:
+        - `project_path` (str): entity/project_name
+        - `type` (str): whether to return only one type of artifacts
+        - `name` (str): Leave none to have all artifact names
+        - `last_version`: whether to return only the last version of each artifact or not
+
+        Output: List of artifacts
+    """
+    public_api = wandb.Api()
+    if type is not None:
+        types = [public_api.artifact_type(type, project_path)]
+    else:
+        types = public_api.artifact_types(project_path)
+
+    res = L()
+    for kind in types:
+        for collection in kind.collections():
+            if name is None or name == collection.name:
+                versions = public_api.artifact_versions(
+                    kind.type,
+                    "/".join([kind.entity, kind.project, collection.name]),
+                    per_page=1,
+                )
+                if last_version: res += next(versions)
+                else: res += L(versions)
+    return list(res)
+
+# %% ../nbs/utils.ipynb 39
+def get_pickle_artifact(filename):
+
+    with open(filename, "rb") as f:
+        df = pickle.load(f)
+    
+    return df
+
+# %% ../nbs/utils.ipynb 41
+import pyarrow.feather as ft
+import pickle
+
+# %% ../nbs/utils.ipynb 42
+def exec_with_feather(function, path = None, print_flag = False, *args, **kwargs):
+    result = None
+    if not (path is none):
+        if print_flag: print("--> Exec with feather | reading input from ", path)
+        input = ft.read_feather(path)
+        if print_flag: print("--> Exec with feather | Apply function ", path)
+        result = function(input, *args, **kwargs)
+        if print_flag: print("Exec with feather --> ", path)
+    return result
+
+# %% ../nbs/utils.ipynb 43
+def py_function(module_name, function_name, print_flag = False):
+    try:
+        function = getattr(__import__('__main__'), function_name)
+    except:
+        module = __import__(module_name, fromlist=[''])
+        function = getattr(module, function_name)
+    print("py function: ", function_name, ": ", function)
+    return function
+
+# %% ../nbs/utils.ipynb 46
+import time
+def exec_with_feather_k_output(function_name, module_name = "main", path = None, k_output = 0, print_flag = False, time_flag = False, *args, **kwargs):
+    result = None
+    function = py_function(module_name, function_name, print_flag)
+    if time_flag: t_start = time.time()
+    if not (path is None):
+        if print_flag: print("--> Exec with feather | reading input from ", path)
+        input = ft.read_feather(path)
+        if print_flag: print("--> Exec with feather | Apply function ", path)
+        result = function(input, *args, **kwargs)[k_output]
+    if time_flag:
+        t_end = time.time()
+        print("Exec with feather | time: ", t_end-t_start)
+    if print_flag: print("Exec with feather --> ", path)
+    return result
+
+# %% ../nbs/utils.ipynb 48
+def exec_with_and_feather_k_output(function_name, module_name = "main", path_input = None, path_output = None, k_output = 0, print_flag = False, time_flag = False, *args, **kwargs):
+    result = None
+    function = py_function(module_name, function_name, print_flag)
+    if time_flag: t_start = time.time()
+    if not (path_input is None):
+        if print_flag: print("--> Exec with feather | reading input from ", path_input)
+        input = ft.read_feather(path_input)
+        if print_flag: 
+            print("--> Exec with feather | Apply function ", function_name, "input type: ", type(input))
+        
+        result = function(input, *args, **kwargs)[k_output]
+        ft.write_feather(df, path, compression = 'lz4')
+    if time_flag:
+        t_end = time.time()
+        print("Exec with feather | time: ", t_end-t_start)
+    if print_flag: print("Exec with feather --> ", path_output)
+    return path_output
+
+# %% ../nbs/utils.ipynb 52
+def learner_module_leaves(learner):
+    modules = list(learner.modules())[0]  # Obtener el módulo raíz
+    rows = []
+
+    def find_leave_modules(module, path=[]):
+        for name, sub_module in module.named_children():
+            current_path = path + [f"{type(sub_module).__name__}"]
+            if not list(sub_module.children()):
+                leave_name = ' -> '.join(current_path)
+                leave_params = str(sub_module).strip()  
+                rows.append([
+                    leave_name,
+                    f"{type(sub_module).__name__}",
+                    name,
+                    leave_params
+                ]
+                )
+
+            find_leave_modules(sub_module, current_path)
+
+    find_leave_modules(modules)
+    
+    df = pd.DataFrame(rows, columns=['Path', 'Module_type', 'Module_name', 'Module'])
+    return df
+
+# %% ../nbs/utils.ipynb 56
+def learner_module_leaves_subtables(learner, print_flag = False):
+    df = pd.DataFrame(columns=['Path', 'Module_type', 'Module_name', 'Module'])
+    md = learner_module_leaves(learner).drop(
+            'Path', axis = 1
+        ).sort_values(
+            by = 'Module_type'
+        )
+    if print_flag: print("The layers are of this types:")
+
+    md_types = pd.DataFrame(md['Module_type'].drop_duplicates())
+    if print_flag: 
+        display(md_types)
+        print("And they are called with this parameters:")
+    
+    md_modules = pd.DataFrame(md['Module'].drop_duplicates())
+    
+    if print_flag: display(md_modules)
+    
+    return md_types, md_modules

dvats/visualization.py

@@ -0,0 +1,63 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/visualization.ipynb.
+
+# %% auto 0
+__all__ = ['plot_TS', 'plot_validation_ts_ae', 'plot_mask']
+
+# %% ../nbs/visualization.ipynb 3
+from fastcore.all import *
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import torch
+
+# %% ../nbs/visualization.ipynb 6
+@delegates(pd.DataFrame.plot)
+def plot_TS(df:pd.core.frame.DataFrame, **kwargs):
+    df.plot(subplots=True, **kwargs)
+    plt.show()
+
+# %% ../nbs/visualization.ipynb 8
+def plot_validation_ts_ae(prediction:np.array, original:np.array, title_str = "Validation plot", fig_size = (15,15), anchor = (-0.01, 0.89), window_num = 0, return_fig=True, title_pos = 0.9):
+    # Create the figure
+    fig = plt.figure(figsize=(fig_size[0],fig_size[1]))
+    # Create the subplot axes
+    axes = fig.subplots(nrows=original.shape[2], ncols=1)
+    # We iterate over the sensor data and plot both the original and the prediction
+    for i,ax in zip(range(original.shape[2]),fig.axes):
+        ax.plot(original[window_num,:,i], label='Original Data')
+        ax.plot(prediction[window_num,:,i], label='Prediction')
+    # Handle the legend configuration and position
+    lines, labels = fig.axes[-1].get_legend_handles_labels()
+    fig.legend(lines, labels,loc='upper left', ncol=2)
+    # Write the plot title (and position it closer to the top of the graph)
+    fig.suptitle(title_str, y = title_pos)
+    # Tight results:
+    fig.tight_layout()
+    # Returns
+    if return_fig:
+        return fig
+    fig
+    return None
+
+# %% ../nbs/visualization.ipynb 12
+def plot_mask(mask, i=0, fig_size=(10,10), title_str="Mask", return_fig=False):
+    """
+    Plot the mask passed as argument. The mask is a 3D boolean tensor. The first 
+    dimension is the window number (or item index), the second is the variable, and the third is the time step.
+    Input:
+        mask: 3D boolean tensor
+        i: index of the window to plot
+        fig_size: size of the figure
+        title_str: title of the plot
+        return_fig: if True, returns the figure
+    Output:
+        if return_fig is True, returns the figure, otherwise, it does not return anything
+    """
+    plt.figure(figsize=fig_size)
+    plt.pcolormesh(mask[i], cmap='cool')
+    plt.title(f'{title_str} {i}, mean: {mask[0].float().mean().item():.3f}')
+    if return_fig:
+        return plt.gcf()
+    else:
+        plt.show()
+        return None

dvats/xai.py

@@ -0,0 +1,964 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/xai.ipynb.
+
+# %% auto 0
+__all__ = ['get_embeddings', 'get_dataset', 'umap_parameters', 'get_prjs', 'plot_projections', 'plot_projections_clusters',
+           'calculate_cluster_stats', 'anomaly_score', 'detector', 'plot_anomaly_scores_distribution',
+           'plot_clusters_with_anomalies', 'update_plot', 'plot_clusters_with_anomalies_interactive_plot',
+           'get_df_selected', 'shift_datetime', 'get_dateformat', 'get_anomalies', 'get_anomaly_styles',
+           'InteractiveAnomalyPlot', 'plot_save', 'plot_initial_config', 'merge_overlapping_windows',
+           'InteractiveTSPlot', 'add_selected_features', 'add_windows', 'setup_style', 'toggle_trace',
+           'set_features_buttons', 'move_left', 'move_right', 'move_down', 'move_up', 'delta_x_bigger',
+           'delta_y_bigger', 'delta_x_lower', 'delta_y_lower', 'add_movement_buttons', 'setup_boxes', 'initial_plot',
+           'show']
+
+# %% ../nbs/xai.ipynb 1
+#Weight & Biases
+import wandb
+
+#Yaml
+from yaml import load, FullLoader
+
+#Embeddings
+from .all import *
+from tsai.data.preparation import prepare_forecasting_data
+from tsai.data.validation import get_forecasting_splits
+from fastcore.all import *
+
+#Dimensionality reduction
+from tsai.imports import *
+
+#Clustering
+import hdbscan
+import time
+from .dr import get_PCA_prjs, get_UMAP_prjs, get_TSNE_prjs
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import ipywidgets as widgets
+from IPython.display import display
+from functools import partial
+
+from IPython.display import display, clear_output, HTML as IPHTML
+from ipywidgets import Button, Output, VBox, HBox, HTML, Layout, FloatSlider
+
+import plotly.graph_objs as go
+import plotly.offline as py
+import plotly.io as pio
+#! pip install kaleido
+import kaleido
+
+
+# %% ../nbs/xai.ipynb 4
+def get_embeddings(config_lrp, run_lrp, api, print_flag = False):
+    artifacts_gettr = run_lrp.use_artifact if config_lrp.use_wandb else api.artifact
+    emb_artifact = artifacts_gettr(config_lrp.emb_artifact, type='embeddings')
+    if print_flag: print(emb_artifact.name)
+    emb_config = emb_artifact.logged_by().config
+    return emb_artifact.to_obj(), emb_artifact, emb_config
+
+# %% ../nbs/xai.ipynb 5
+def get_dataset(
+    config_lrp, 
+    config_emb,
+    config_dr,
+    run_lrp,
+    api, 
+    print_flag = False
+):
+    # Botch to use artifacts offline
+    artifacts_gettr = run_lrp.use_artifact if config_lrp.use_wandb else api.artifact
+    enc_artifact = artifacts_gettr(config_emb['enc_artifact'], type='learner')
+    if print_flag: print (enc_artifact.name)
+    ## TODO: This only works when you run it two timeS! WTF?
+    try:
+        enc_learner = enc_artifact.to_obj()
+    except:
+        enc_learner = enc_artifact.to_obj()
+    
+    ## Restore artifact
+    enc_logger = enc_artifact.logged_by()
+    enc_artifact_train = artifacts_gettr(enc_logger.config['train_artifact'], type='dataset')
+    #cfg_.show_attrdict(enc_logger.config)
+    if enc_logger.config['valid_artifact'] is not None:
+        enc_artifact_valid = artifacts_gettr(enc_logger.config['valid_artifact'], type='dataset')
+        if print_flag: print("enc_artifact_valid:", enc_artifact_valid.name)
+    if print_flag: print("enc_artifact_train: ", enc_artifact_train.name)
+    
+    if config_dr['dr_artifact'] is not None:
+        print("Is not none")
+        dr_artifact = artifacts_gettr(config_dr['enc_artifact'])
+    else:
+        dr_artifact = enc_artifact_train
+    if print_flag: print("DR artifact train: ", dr_artifact.name)
+    if print_flag: print("--> DR artifact name", dr_artifact.name)
+    dr_artifact
+    df = dr_artifact.to_df()
+    if print_flag: print("--> DR After to df", df.shape)
+    if print_flag: display(df.head())
+    return df, dr_artifact, enc_artifact, enc_learner
+
+# %% ../nbs/xai.ipynb 6
+def umap_parameters(config_dr, config):
+    umap_params_cpu = {
+        'n_neighbors' : config_dr.n_neighbors,
+        'min_dist' : config_dr.min_dist,
+        'random_state': np.uint64(822569775), 
+        'metric': config_dr.metric,
+        #'a': 1.5769434601962196,
+        #'b': 0.8950608779914887,
+        #'metric_kwds': {'p': 2}, #No debería ser necesario, just in case
+        #'output_metric': 'euclidean',
+        'verbose': 4,
+        #'n_epochs': 200
+    }
+    umap_params_gpu = {
+        'n_neighbors' : config_dr.n_neighbors,
+        'min_dist' : config_dr.min_dist,
+        'random_state': np.uint64(1234), 
+        'metric': config_dr.metric,
+        'a': 1.5769434601962196,
+        'b': 0.8950608779914887,
+        'target_metric': 'euclidean',
+        'target_n_neighbors': config_dr.n_neighbors,
+        'verbose': 4, #6, #CUML_LEVEL_TRACE
+        'n_epochs': 200*3*2,
+        'init': 'random',
+        'hash_input': True
+    }
+    if config_dr.cpu_flag: 
+        umap_params = umap_params_cpu
+    else:
+        umap_params = umap_params_gpu
+    return umap_params
+
+# %% ../nbs/xai.ipynb 7
+def get_prjs(embs_no_nan, config_dr, config, print_flag = False):
+    umap_params = umap_parameters(config_dr, config)
+    prjs_pca = get_PCA_prjs(
+        X    = embs_no_nan, 
+        cpu  = False,  
+        print_flag = print_flag, 
+        **umap_params
+    )
+    if print_flag: 
+        print(prjs_pca.shape)
+    prjs_umap = get_UMAP_prjs(
+        input_data = prjs_pca, 
+        cpu =  config_dr.cpu_flag, #config_dr.cpu, 
+        print_flag = print_flag,         
+        **umap_params
+    )
+    if print_flag: prjs_umap.shape
+    return prjs_umap
+
+# %% ../nbs/xai.ipynb 9
+def plot_projections(prjs, umap_params, fig_size = (25,25)):
+    "Plot 2D projections thorugh a connected scatter plot"
+    df_prjs = pd.DataFrame(prjs, columns = ['x1', 'x2'])
+    fig = plt.figure(figsize=(fig_size[0],fig_size[1]))
+    ax = fig.add_subplot(111)
+    ax.scatter(df_prjs['x1'], df_prjs['x2'], marker='o', facecolors='none', edgecolors='b', alpha=0.1)
+    ax.plot(df_prjs['x1'], df_prjs['x2'], alpha=0.5, picker=1)
+    plt.title('DR params -  n_neighbors:{:d} min_dist:{:f}'.format(
+        umap_params['n_neighbors'],umap_params['min_dist']))
+    return ax
+
+# %% ../nbs/xai.ipynb 10
+def plot_projections_clusters(prjs, clusters_labels, umap_params, fig_size = (25,25)):
+    "Plot 2D projections thorugh a connected scatter plot"
+    df_prjs = pd.DataFrame(prjs, columns = ['x1', 'x2'])
+    df_prjs['cluster'] = clusters_labels
+    
+    fig = plt.figure(figsize=(fig_size[0],fig_size[1]))
+    ax = fig.add_subplot(111)
+    
+    # Create a scatter plot for each cluster with different colors
+    unique_labels = df_prjs['cluster'].unique()
+    print(unique_labels)
+    for label in unique_labels:
+        cluster_data = df_prjs[df_prjs['cluster'] == label]
+        ax.scatter(cluster_data['x1'], cluster_data['x2'], label=f'Cluster {label}')
+        #ax.scatter(df_prjs['x1'], df_prjs['x2'], marker='o', facecolors='none', edgecolors='b', alpha=0.1)
+    
+    #ax.plot(df_prjs['x1'], df_prjs['x2'], alpha=0.5, picker=1)
+    plt.title('DR params -  n_neighbors:{:d} min_dist:{:f}'.format(
+        umap_params['n_neighbors'],umap_params['min_dist']))
+    return ax
+
+# %% ../nbs/xai.ipynb 11
+def calculate_cluster_stats(data, labels):
+    """Computes the media and the standard deviation for every cluster."""
+    cluster_stats = {}
+    for label in np.unique(labels):
+        #members = data[labels == label]
+        members = data
+        mean = np.mean(members, axis = 0)
+        std = np.std(members, axis = 0)
+        cluster_stats[label] = (mean, std)
+    return cluster_stats
+
+# %% ../nbs/xai.ipynb 12
+def anomaly_score(point, cluster_stats, label):
+    """Computes an anomaly score for each point."""
+    mean, std = cluster_stats[label]
+    return np.linalg.norm((point - mean) / std)
+
+# %% ../nbs/xai.ipynb 13
+def detector(data, labels):
+    """Anomaly detection function."""
+    cluster_stats = calculate_cluster_stats(data, labels)
+    scores = []
+    for point, label in zip(data, labels):
+        score = anomaly_score(point, cluster_stats, label)
+        scores.append(score)
+    return np.array(scores)
+
+# %% ../nbs/xai.ipynb 15
+def plot_anomaly_scores_distribution(anomaly_scores):
+    "Plot the distribution of anomaly scores to check for normality"
+    plt.figure(figsize=(10, 6))
+    sns.histplot(anomaly_scores, kde=True, bins=30)
+    plt.title("Distribución de Anomaly Scores")
+    plt.xlabel("Anomaly Score")
+    plt.ylabel("Frecuencia")
+    plt.show()
+
+# %% ../nbs/xai.ipynb 16
+def plot_clusters_with_anomalies(prjs, clusters_labels, anomaly_scores, threshold, fig_size=(25, 25)):
+    "Plot 2D projections of clusters and superimpose anomalies"
+    df_prjs = pd.DataFrame(prjs, columns=['x1', 'x2'])
+    df_prjs['cluster'] = clusters_labels
+    df_prjs['anomaly'] = anomaly_scores > threshold
+
+    fig = plt.figure(figsize=(fig_size[0], fig_size[1]))
+    ax = fig.add_subplot(111)
+
+    # Plot each cluster with different colors
+    unique_labels = df_prjs['cluster'].unique()
+    for label in unique_labels:
+        cluster_data = df_prjs[df_prjs['cluster'] == label]
+        ax.scatter(cluster_data['x1'], cluster_data['x2'], label=f'Cluster {label}', alpha=0.7)
+
+    # Superimpose anomalies
+    anomalies = df_prjs[df_prjs['anomaly']]
+    ax.scatter(anomalies['x1'], anomalies['x2'], color='red', label='Anomalies', edgecolor='k', s=50)
+
+    plt.title('Clusters and anomalies')
+    plt.legend()
+    plt.show()
+
+def update_plot(threshold, prjs_umap, clusters_labels, anomaly_scores, fig_size):
+    plot_clusters_with_anomalies(prjs_umap, clusters_labels, anomaly_scores, threshold, fig_size)
+
+def plot_clusters_with_anomalies_interactive_plot(threshold, prjs_umap, clusters_labels, anomaly_scores, fig_size):
+    threshold_slider = widgets.FloatSlider(value=threshold, min=0.001, max=3, step=0.001, description='Threshold')
+    interactive_plot =  widgets.interactive(update_plot, threshold = threshold_slider, 
+                              prjs_umap = widgets.fixed(prjs_umap), 
+                              clusters_labels = widgets.fixed(clusters_labels),
+                              anomaly_scores = widgets.fixed(anomaly_scores),
+                              fig_size = widgets.fixed((25,25)))
+    display(interactive_plot)
+    
+
+# %% ../nbs/xai.ipynb 18
+import plotly.express as px
+from datetime import timedelta
+
+# %% ../nbs/xai.ipynb 19
+def get_df_selected(df, selected_indices, w, stride = 1): #Cuidado con stride
+    '''Links back the selected points to the original dataframe and returns the associated windows indices'''
+    n_windows = len(selected_indices)
+    window_ranges = [(id*stride, (id*stride)+w) for id in selected_indices]    
+    #window_ranges = [(id*w, (id+1)*w+1) for id in selected_indices]    
+    #window_ranges = [(id*stride, (id*stride)+w) for id in selected_indices]
+    #print(window_ranges)
+    valores_tramos = [df.iloc[inicio:fin+1] for inicio, fin in window_ranges]
+    df_selected = pd.concat(valores_tramos, ignore_index=False)
+    return window_ranges, n_windows, df_selected
+
+# %% ../nbs/xai.ipynb 20
+def shift_datetime(dt, seconds, sign, dateformat="%Y-%m-%d %H:%M:%S.%f", print_flag = False):
+    """
+    This function gets a datetime dt, a number of seconds, 
+    a sign and moves the date such number of seconds to the future 
+    if sign is '+' and to the past if sing is '-'.
+    """
+    
+    if print_flag: print(dateformat)
+    dateformat2= "%Y-%m-%d %H:%M:%S.%f"
+    dateformat3 = "%Y-%m-%d"
+    ok = False
+
+    try: 
+        if print_flag: print("dt ", dt, "seconds", seconds, "sign", sign)
+        new_dt = datetime.strptime(dt, dateformat)
+        if print_flag: print("ndt", new_dt)
+        ok = True
+    except ValueError as e:
+        if print_flag: 
+            print("Error: ", e)
+        
+    if (not ok):
+        try:
+            if print_flag: print("Parsing alternative dataformat", dt, "seconds", seconds, "sign", sign, dateformat2)
+            new_dt = datetime.strptime(dt, dateformat3)
+            if print_flag: print("2ndt", new_dt)            
+        except ValueError as e:
+            print("Error: ", e)
+    if print_flag: print(new_dt)
+    try:
+            
+        if new_dt.hour == 0 and new_dt.minute == 0 and new_dt.second == 0:
+            if print_flag: "Aqui"
+            new_dt = new_dt.replace(hour=0, minute=0, second=0, microsecond=0)
+            if print_flag: print(new_dt)
+
+        if print_flag: print("ndt", new_dt)
+                
+        if (sign == '+'):
+            if print_flag: print("Aqui")
+            new_dt = new_dt + timedelta(seconds = seconds)
+            if print_flag: print(new_dt)
+        else: 
+            if print_flag: print(sign, type(dt))
+            new_dt = new_dt - timedelta(seconds = seconds)
+            if print_flag: print(new_dt)            
+        if new_dt.hour == 0 and new_dt.minute == 0 and new_dt.second == 0:
+            if print_flag: print("replacing")
+            new_dt = new_dt.replace(hour=0, minute=0, second=0, microsecond=0)
+        
+        new_dt_str = new_dt.strftime(dateformat2)
+        if print_flag: print("new dt ", new_dt)
+    except ValueError as e:
+        if print_flag: print("Aqui3")
+        shift_datetime(dt, 0, sign, dateformat = "%Y-%m-%d", print_flag = False)
+        return str(e)
+    return new_dt_str
+
+
+
+# %% ../nbs/xai.ipynb 21
+def get_dateformat(text_date):
+    dateformat1 = "%Y-%m-%d %H:%M:%S"
+    dateformat2 = "%Y-%m-%d %H:%M:%S.%f"
+    dateformat3 = "%Y-%m-%d"
+    dateformat = ""
+    parts = text_date.split()
+
+    if len(parts) == 2:
+        time_parts = parts[1].split(':')
+        if len(time_parts) == 3:
+            sec_parts = time_parts[2].split('.')
+            if len(sec_parts) == 2:
+                dateformat = dateformat2
+            else:
+                dateformat = dateformat1
+        else:
+            dateformat = "unknown format 1"
+    elif len(parts) == 1:
+        dateformat = dateformat3
+    else:
+        dateformat = "unknown format 2"
+
+    return dateformat
+
+# %% ../nbs/xai.ipynb 23
+def get_anomalies(df, threshold, flag):
+    df['anomaly'] = [ (score > threshold) and flag for score in df['anomaly_score']]
+    
+def get_anomaly_styles(df, threshold, anomaly_scores, flag = False, print_flag = False):
+        if print_flag: print("Threshold: ", threshold)
+        if print_flag: print("Flag", flag)
+        if print_flag: print("df ~", df.shape)
+        df['anomaly'] = [ (score > threshold) and flag for score in df['anomaly_score'] ]
+        if print_flag: print(df)
+        get_anomalies(df, threshold, flag)
+        anomalies = df[df['anomaly']]
+        if flag:
+            df['anomaly'] = [ 
+                (score > threshold) and flag 
+                for score in anomaly_scores 
+            ]
+            symbols = [
+                'x' if is_anomaly else 'circle' 
+                for is_anomaly in df['anomaly']
+            ]
+            line_colors = [
+                'black'
+                if (is_anomaly and flag) else 'rgba(0,0,0,0)'
+                for is_anomaly in df['anomaly']
+            ]
+        else:
+            symbols = ['circle' for _ in df['x1']]
+            line_colors = ['rgba(0,0,0,0)' for _ in df['x1']]
+        if print_flag: print(anomalies)
+        return symbols, line_colors
+### Example of use
+#prjs_df = pd.DataFrame(prjs_umap, columns = ['x1', 'x2'])
+#prjs_df['anomaly_score'] = anomaly_scores
+#s, l = get_anomaly_styles(prjs_df, 1, True)
+
+# %% ../nbs/xai.ipynb 24
+class InteractiveAnomalyPlot():
+    def __init__(
+        self, selected_indices = [], 
+        threshold = 0.15, 
+        anomaly_flag = False,
+        path = "../imgs", w = 0
+    ):
+        self.selected_indices = selected_indices
+        self.selected_indices_tmp = selected_indices
+        self.threshold = threshold
+        self.threshold_ = threshold
+        self.anomaly_flag = anomaly_flag
+        self.w = w
+        self.name = f"w={self.w}"
+        self.path = f"{path}{self.name}.png"
+        self.interaction_enabled = True
+    
+
+    def plot_projections_clusters_interactive(
+        self, prjs, cluster_labels, umap_params, anomaly_scores=[], fig_size=(7,7), print_flag = False
+    ):
+        self.selected_indices_tmp = self.selected_indices
+        py.init_notebook_mode()
+
+        prjs_df, cluster_colors = plot_initial_config(prjs, cluster_labels, anomaly_scores)
+        legend_items = [widgets.HTML(f'<b>Cluster {cluster}:</b> <span style="color:{color};">■</span>')
+                        for cluster, color in cluster_colors.items()]
+        legend = widgets.VBox(legend_items)
+
+        marker_colors = prjs_df['cluster'].map(cluster_colors)
+    
+        symbols, line_colors = get_anomaly_styles(prjs_df, self.threshold_, anomaly_scores, self.anomaly_flag, print_flag)
+    
+        fig = go.FigureWidget(
+            [
+                go.Scatter(
+                    x=prjs_df['x1'], y=prjs_df['x2'], 
+                    mode="markers", 
+                    marker= {
+                        'color': marker_colors,
+                        'line': { 'color': line_colors, 'width': 1 },
+                        'symbol': symbols
+                    },
+                    text = prjs_df.index
+                )
+            ]
+        )
+
+        line_trace = go.Scatter(
+            x=prjs_df['x1'], 
+            y=prjs_df['x2'], 
+            mode="lines",  
+            line=dict(color='rgba(128, 128, 128, 0.5)', width=1)#,
+            #showlegend=False  # Puedes configurar si deseas mostrar esta línea en la leyenda
+        )
+    
+        fig.add_trace(line_trace)
+
+        sca = fig.data[0]
+        
+        fig.update_layout(
+            dragmode='lasso',
+            width=700, 
+            height=500,
+            title={
+                'text': '<span style="font-weight:bold">DR params - n_neighbors:{:d} min_dist:{:f}</span>'.format(
+                         umap_params['n_neighbors'], umap_params['min_dist']),
+                'y':0.98,
+                'x':0.5,
+                'xanchor': 'center',
+                'yanchor': 'top'
+            },
+            plot_bgcolor='white',
+            paper_bgcolor='#f0f0f0',
+            xaxis=dict(gridcolor='lightgray', zerolinecolor='black', title = 'x'), 
+            yaxis=dict(gridcolor='lightgray', zerolinecolor='black', title = 'y'),
+            margin=dict(l=10, r=20, t=30, b=10)
+        
+        
+        )
+    
+        output_tmp = Output()
+        output_button = Output()
+        output_anomaly = Output()
+        output_threshold = Output()
+        output_width = Output()
+    
+        def select_action(trace, points, selector):
+            self.selected_indices_tmp = points.point_inds
+            with output_tmp:
+                output_tmp.clear_output(wait=True)
+                if print_flag: print("Selected indices tmp:", self.selected_indices_tmp)
+        
+        def button_action(b):
+            self.selected_indices = self.selected_indices_tmp 
+            with output_button: 
+                output_button.clear_output(wait = True)
+                if print_flag: print("Selected indices:", self.selected_indices)
+
+    
+        def update_anomalies():           
+            if print_flag: print("About to update anomalies")
+                
+            symbols, line_colors = get_anomaly_styles(prjs_df, self.threshold_, anomaly_scores, self.anomaly_flag, print_flag)
+
+            if print_flag: print("Anomaly styles got")
+
+            with fig.batch_update():
+                fig.data[0].marker.symbol = symbols
+                fig.data[0].marker.line.color = line_colors
+            if print_flag: print("Anomalies updated")
+            if print_flag: print("Threshold: ", self.threshold_)
+            if print_flag: print("Scores: ", anomaly_scores)
+        
+              
+        def anomaly_action(b):
+            with output_anomaly:  # Cambia output_flag a output_anomaly
+                output_anomaly.clear_output(wait=True)
+                if print_fllag: print("Negate anomaly flag")
+                self.anomaly_flag = not self.anomaly_flag
+                if print_flag: print("Show anomalies:", self.anomaly_flag)
+                update_anomalies()
+                              
+        sca.on_selection(select_action)
+        layout = widgets.Layout(width='auto', height='40px')
+        button = Button(
+            description="Update selected_indices",
+            style = {'button_color': 'lightblue'},
+            display = 'flex',
+            flex_row = 'column',
+            align_items = 'stretch',
+            layout = layout
+        )
+        anomaly_button = Button(
+            description = "Show anomalies",
+            style = {'button_color': 'lightgray'},
+            display = 'flex',
+            flex_row = 'column',
+            align_items = 'stretch',
+            layout = layout
+        )
+        
+        button.on_click(button_action)
+        anomaly_button.on_click(anomaly_action)
+    
+        ##### Reactivity buttons
+        pause_button = Button(
+            description = "Pause interactiveness",
+            style = {'button_color': 'pink'},
+            display = 'flex',
+            flex_row = 'column',
+            align_items = 'stretch',
+            layout = layout
+        )
+        resume_button = Button(
+            description = "Resume interactiveness",
+            style = {'button_color': 'lightgreen'},
+            display = 'flex',
+            flex_row = 'column',
+            align_items = 'stretch',
+            layout = layout
+        )
+
+    
+        threshold_slider = FloatSlider(
+            value=self.threshold_,
+            min=0.0,
+            max=float(np.ceil(self.threshold+5)),
+            step=0.0001,
+            description='Anomaly threshold:',
+            continuous_update=False
+        )
+    
+        def pause_interaction(b):
+            self.interaction_enabled = False
+            fig.update_layout(dragmode='pan')
+    
+        def resume_interaction(b):
+            self.interaction_enabled = True
+            fig.update_layout(dragmode='lasso')
+
+    
+        def update_threshold(change):
+            with output_threshold: 
+                output_threshold.clear_output(wait = True)
+                if print_flag: print("Update threshold")
+                self.threshold_ = change.new
+                if print_flag: print("Update anomalies threshold = ", self.threshold_)
+                update_anomalies()
+
+        #### Width
+        width_slider = FloatSlider(
+            value = 0.5,  
+            min   = 0.0,  
+            max   = 1.0,  
+            step  = 0.0001, 
+            description = 'Line width:',  
+            continuous_update = False  
+        )
+
+        def update_width(change):
+            with output_width:
+                try:
+                    output_width.clear_output(wait = True)
+                    if print_flag: 
+                        print("Change line width")
+                        print("Trace to update:", fig.data[1])  
+                    with fig.batch_update():
+                        fig.data[1].line.width = change.new  # Actualiza la opacidad de la línea
+                    if print_flag: print("ChangeD line width")
+                except Exception as e:
+                    print("Error updating line width:", e)
+                    
+
+
+        pause_button.on_click(pause_interaction)
+        resume_button.on_click(resume_interaction)
+    
+        threshold_slider.observe(update_threshold, 'value')
+    
+        #### 
+        width_slider.observe(update_width, names = 'value')
+        
+        #####
+        space = HTML("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;") 
+    
+        vbox = VBox((output_tmp, output_button, output_anomaly, output_threshold, fig))
+        hbox = HBox((space, button, space, pause_button, space, resume_button, anomaly_button))
+    
+        # Centrar las dos cajas horizontalmente en el VBox
+
+        box_layout = widgets.Layout(display='flex',
+                    flex_flow='column',
+                    align_items='center',
+                    width='100%')
+
+        if self.anomaly_flag:
+            box = VBox((hbox,threshold_slider,width_slider, output_width, vbox), layout = box_layout)
+        else: 
+            box = VBox((hbox, width_slider, output_width, vbox), layout = box_layout)
+        box.add_class("layout")
+        plot_save(fig, self.w)
+    
+        display(box)
+
+
+# %% ../nbs/xai.ipynb 25
+def plot_save(fig, w):
+    image_bytes = pio.to_image(fig, format='png')
+    with open(f"../imgs/w={w}.png", 'wb') as f:
+        f.write(image_bytes)
+    
+
+# %% ../nbs/xai.ipynb 26
+def plot_initial_config(prjs, cluster_labels, anomaly_scores):
+    prjs_df = pd.DataFrame(prjs, columns = ['x1', 'x2'])
+    prjs_df['cluster'] = cluster_labels
+    prjs_df['anomaly_score'] = anomaly_scores
+    
+    cluster_colors_df = pd.DataFrame({'cluster': cluster_labels}).drop_duplicates()
+    cluster_colors_df['color'] = px.colors.qualitative.Set1[:len(cluster_colors_df)]
+    cluster_colors = dict(zip(cluster_colors_df['cluster'], cluster_colors_df['color']))
+    return prjs_df, cluster_colors
+
+# %% ../nbs/xai.ipynb 27
+def merge_overlapping_windows(windows):
+    if not windows:
+        return []
+
+    # Order
+    sorted_windows = sorted(windows, key=lambda x: x[0])
+
+    merged_windows = [sorted_windows[0]]
+
+    for window in sorted_windows[1:]:
+        if window[0] <= merged_windows[-1][1]:
+            # Merge!
+            merged_windows[-1] = (merged_windows[-1][0], max(window[1], merged_windows[-1][1]))
+        else:
+            merged_windows.append(window)
+
+    return merged_windows
+
+# %% ../nbs/xai.ipynb 29
+class InteractiveTSPlot: 
+    def __init__(
+        self, 
+        df, 
+        selected_indices, 
+        meaningful_features_subset_ids, 
+        w, 
+        stride=1, 
+        print_flag=False, 
+        num_points=10000, 
+        dateformat='%Y-%m-%d %H:%M:%S',
+        delta_x = 10, 
+        delta_y = 0.1  
+    ):
+        self.df = df
+        self.selected_indices = selected_indices
+        self.meaningful_features_subset_ids = meaningful_features_subset_ids
+        self.w = w
+        self.stride = stride
+        self.print_flag = print_flag
+        self.num_points = num_points
+        self.dateformat = dateformat
+        self.fig = go.FigureWidget()
+        self.buttons = []
+        self.print_flag = print_flag   
+    
+        self.delta_x = delta_x
+        self.delta_y = delta_y
+
+        self.window_ranges, self.n_windows, self.df_selected = get_df_selected(
+            self.df, self.selected_indices, self.w, self.stride
+        )
+        # Ensure the small possible number of windows to plot (like in R Shiny App)
+        self.window_ranges = merge_overlapping_windows(self.window_ranges)
+
+        #Num points no va bien...
+        #num_points = min(df_selected.shape[0], num_points)
+
+        if self.print_flag: 
+            print("windows: ", self.n_windows, self.window_ranges)
+            print("selected id: ", self.df_selected.index)
+            print("points: ", self.num_points)
+
+        self.df.index = self.df.index.astype(str)
+        self.fig = go.FigureWidget()
+        self.colors = [
+            f'rgb({np.random.randint(0, 256)}, {np.random.randint(0, 256)}, {np.random.randint(0, 256)})' 
+            for _ in range(self.n_windows)
+        ]
+
+        ##############################
+        # Outputs for debug printing #
+        ##############################
+        self.output_windows = Output()
+        self.output_move = Output()
+        self.output_delta_x = Output()
+        self.output_delta_y = Output()
+
+        
+        
+        
+
+# %% ../nbs/xai.ipynb 30
+def add_selected_features(self: InteractiveTSPlot):
+    # Add features time series
+    for feature_id in self.df.columns:
+        feature_pos = self.df.columns.get_loc(feature_id)
+        trace = go.Scatter(
+            #x=df.index[:num_points],
+            #y=df[feature_id][:num_points],
+            x = self.df.index,
+            y = self.df[feature_id],
+            mode='lines',
+            name=feature_id,
+            visible=feature_pos in self.meaningful_features_subset_ids,
+            text=self.df.index
+            #text=[f'{i}-{val}' for i, val in enumerate(df.index)]
+        )
+        self.fig.add_trace(trace)
+        
+InteractiveTSPlot.add_selected_features = add_selected_features
+
+# %% ../nbs/xai.ipynb 31
+def add_windows(self: InteractiveTSPlot):
+    for i, (start, end) in enumerate(self.window_ranges):
+        self.fig.add_shape(
+            type="rect",
+            x0=self.df.index[start],
+            x1=self.df.index[end],
+            y0= 0,
+            y1= 1,
+            yref = "paper",
+            fillcolor=self.colors[i], #"LightSalmon",
+            opacity=0.25,
+            layer="below",
+            line=dict(color=self.colors[i], width=1),
+            name = f"w_{i}"
+        )
+        with self.output_windows:
+            print("w[" + str( self.selected_indices[i] )+ "]="+str(self.df.index[start])+", "+str(self.df.index[end])+")")
+
+InteractiveTSPlot.add_windows = add_windows
+
+# %% ../nbs/xai.ipynb 32
+def setup_style(self: InteractiveTSPlot):
+    self.fig.update_layout(
+        title='Time Series with time window plot',
+        xaxis_title='Datetime',
+        yaxis_title='Value',
+        legend_title='Variables',
+        margin=dict(l=10, r=10, t=30, b=10),
+        xaxis=dict(
+            tickformat = '%d-' + self.dateformat,
+            #tickvals=list(range(len(df.index))),
+            #ticktext = [f'{i}-{val}' for i, val in enumerate(df.index)]
+            #grid_color = 'lightgray', zerolinecolor='black', title = 'x'
+        ),
+        #yaxis = dict(grid_color = 'lightgray', zerolinecolor='black', title = 'y'),
+        #plot_color = 'white',
+        paper_bgcolor='#f0f0f0'
+    )
+    self.fig.update_yaxes(fixedrange=True)
+
+InteractiveTSPlot.setup_style = setup_style
+
+# %% ../nbs/xai.ipynb 34
+def toggle_trace(self : InteractiveTSPlot, button : Button):
+    idx = button.description
+    trace = self.fig.data[self.df.columns.get_loc(idx)]
+    trace.visible = not trace.visible
+
+InteractiveTSPlot.toggle_trace = toggle_trace
+
+# %% ../nbs/xai.ipynb 35
+def set_features_buttons(self):
+    self.buttons = [
+        Button(
+            description=str(feature_id),
+            button_style='success' if self.df.columns.get_loc(feature_id) in self.meaningful_features_subset_ids else ''
+        ) 
+        for feature_id in self.df.columns
+    ]
+    for button in self.buttons:
+        button.on_click(self.toggle_trace)
+InteractiveTSPlot.set_features_buttons = set_features_buttons
+
+# %% ../nbs/xai.ipynb 36
+def move_left(self : InteractiveTSPlot, button : Button):
+    with self.output_move:
+        self.output_move.clear_output(wait=True)
+        start_date, end_date = self.fig.layout.xaxis.range
+        new_start_date = shift_datetime(start_date, self.delta_x, '-', self.dateformat, self.print_flag) 
+        new_end_date = shift_datetime(end_date, self.delta_x, '-', self.dateformat, self.print_flag) 
+        with self.fig.batch_update():
+            self.fig.layout.xaxis.range = [new_start_date, new_end_date]
+
+def move_right(self : InteractiveTSPlot, button : Button):
+    self.output_move.clear_output(wait=True)
+    with self.output_move:
+        start_date, end_date = self.fig.layout.xaxis.range
+        new_start_date = shift_datetime(start_date, self.delta_x, '+', self.dateformat, self.print_flag) 
+        new_end_date = shift_datetime(end_date, self.delta_x, '+', self.dateformat, self.print_flag) 
+        with self.fig.batch_update():
+            self.fig.layout.xaxis.range = [new_start_date, new_end_date]
+    
+def move_down(self: InteractiveTSPlot, button : Button):
+    with self.output_move:
+        self.output_move.clear_output(wait=True)
+        start_y, end_y = self.fig.layout.yaxis.range
+        with self.fig.batch_update():
+            self.ig.layout.yaxis.range = [start_y-self.delta_y, end_y-self.delta_y]
+def move_up(self: InteractiveTSPlot, button : Button):
+    with self.output_move:
+        self.output_move.clear_output(wait=True)
+        start_y, end_y = self.fig.layout.yaxis.range
+        with self.fig.batch_update():
+            self.fig.layout.yaxis.range = [start_y+self.delta_y, end_y+self.delta_y]
+
+InteractiveTSPlot.move_left = move_left
+InteractiveTSPlot.move_right = move_right
+InteractiveTSPlot.move_down = move_down
+InteractiveTSPlot.move_up = move_up
+
+# %% ../nbs/xai.ipynb 37
+def delta_x_bigger(self: InteractiveTSPlot):
+    with self.output_delta_x: 
+        self.output_delta_x.clear_output(wait = True)
+        if self.print_flag: print("Delta before", self.delta_x)
+        self.delta_x *= 10
+        if self.print_flag: print("delta_x:", self.delta_x)
+
+def delta_y_bigger(self: InteractiveTSPlot):
+    with self.output_delta_y: 
+        self.output_delta_y.clear_output(wait = True)
+        if self.print_flag: print("Delta before", self.delta_y)
+        self.delta_y *= 10
+        if self.print_flag: print("delta_y:", self.delta_y)
+
+def delta_x_lower(self:InteractiveTSPlot):
+    with self.output_delta_x:
+        self.output_delta_x.clear_output(wait = True)
+        if self.print_flag: print("Delta before", self.delta_x)
+        self.delta_x /= 10
+        if self.print_flag: print("delta_x:", self.delta_x)
+
+def delta_y_lower(self:InteractiveTSPlot):
+    with self.output_delta_y: 
+        self.output_delta_y.clear_output(wait = True)
+        print("Delta before", self.delta_y)
+        self.delta_y = self.delta_y * 10
+        print("delta_y:", self.delta_y)
+InteractiveTSPlot.delta_x_bigger = delta_x_bigger
+InteractiveTSPlot.delta_y_bigger = delta_y_bigger
+InteractiveTSPlot.delta_x_lower = delta_x_lower
+InteractiveTSPlot.delta_y_lower = delta_y_lower
+
+# %% ../nbs/xai.ipynb 38
+def add_movement_buttons(self: InteractiveTSPlot):
+    self.button_left = Button(description="←")
+    self.button_right = Button(description="→")
+    self.button_up = Button(description="↑")
+    self.button_down = Button(description="↓")
+    
+    self.button_step_x_up = Button(description="dx ↑")
+    self.button_step_x_down = Button(description="dx ↓")
+    self.button_step_y_up = Button(description="dy↑")
+    self.button_step_y_down = Button(description="dy↓")
+
+
+    # TODO: Arreglar que se pueda modificar el paso con el que se avanza. No se ve el output y no se modifica el valor
+    self.button_step_x_up.on_click(self.delta_x_bigger)
+    self.button_step_x_down.on_click(self.delta_x_lower)
+    self.button_step_y_up.on_click(self.delta_y_bigger)
+    self.button_step_y_down.on_click(self.delta_y_lower)
+
+    self.button_left.on_click(self.move_left)
+    self.button_right.on_click(self.move_right)
+    self.button_up.on_click(self.move_up)
+    self.button_down.on_click(self.move_down)
+
+InteractiveTSPlot.add_movement_buttons = add_movement_buttons
+
+# %% ../nbs/xai.ipynb 40
+def setup_boxes(self: InteractiveTSPlot):
+    self.steps_x = VBox([self.button_step_x_up, self.button_step_x_down])
+    self.steps_y = VBox([self.button_step_y_up, self.button_step_y_down])
+    arrow_buttons = HBox([self.button_left, self.button_right, self.button_up, self.button_down, self.steps_x, self.steps_y])
+    hbox_layout = widgets.Layout(display='flex', flex_flow='row wrap', align_items='flex-start')
+    hbox = HBox(self.buttons, layout=hbox_layout)
+    box_layout = widgets.Layout(
+        display='flex',
+        flex_flow='column',
+        align_items='center',
+        width='100%'
+    )
+    if self.print_flag:
+        self.box = VBox([hbox, arrow_buttons, self.output_move, self.output_delta_x, self.output_delta_y, self.fig, self.output_windows], layout=box_layout)
+    else: 
+        self.box = VBox([hbox, arrow_buttons, self.fig, self.output_windows], layout=box_layout)
+
+InteractiveTSPlot.setup_boxes = setup_boxes
+
+
+# %% ../nbs/xai.ipynb 41
+def initial_plot(self: InteractiveTSPlot):
+    self.add_selected_features()
+    self.add_windows()
+    self.setup_style()
+    self.set_features_buttons()
+    self.add_movement_buttons()
+    self.setup_boxes()
+InteractiveTSPlot.initial_plot = initial_plot
+
+# %% ../nbs/xai.ipynb 42
+def show(self : InteractiveTSPlot):
+    self.initial_plot()
+    display(self.box)
+InteractiveTSPlot.show = show

entrypoint-rstudio.sh

@@ -0,0 +1,25 @@
+#!/bin/bash --login
+set -e
+
+echo ". ${HOME}/miniconda3/etc/profile.d/conda.sh" >> ${HOME}/.bashrc
+# Make bash automatically activate the conda environment
+echo "conda activate ${ENV_PREFIX}" >> ~/.bashrc
+#echo "export WANDB_ENTITY=${WANDB_ENTITY:-default}" >> ${HOME}/.bashrc
+# echo "WANDB_ENTITY=${WANDB_ENTITY:-default}" >> ${HOME}/.Renviron
+
+# Define an array of environment variable names from the ENV_VARS Compose variable
+IFS=',' read -ra ENV_VAR_NAMES <<< "$ENV_VARS"
+
+echo "ENV_VAR_NAMES=${ENV_VAR_NAMES[@]}"
+
+# Loop through the array of environment variable names and set the variables
+for ENV_VAR_NAME in "${ENV_VAR_NAMES[@]}"; do
+  ENV_VAR_VALUE="${!ENV_VAR_NAME:-default}"
+  echo "$ENV_VAR_NAME=$ENV_VAR_VALUE" >> ${HOME}/.Renviron
+done
+
+ulimit -s 16384
+
+# start rstudio server
+/init
+exec "$@"

requirements.txt

@@ -1,3 +1 @@
-gradio==3.23.0
-uvicorn
-fastapi
+