Taha Aksu commited on
Commit
fe3e749
·
1 Parent(s): 1895436

Add code links for all models with replication code available

Browse files
results/Kairos_10m/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/mldi-lab/Kairos_10m",
6
- "code_link": "https://github.com/foundation-model-research/Kairos",
7
  "org": "ShanghaiTech University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/mldi-lab/Kairos_10m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/kairos.ipynb",
7
  "org": "ShanghaiTech University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/Kairos_23m/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/mldi-lab/Kairos_23m",
6
- "code_link": "https://github.com/foundation-model-research/Kairos",
7
  "org": "ShanghaiTech University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/mldi-lab/Kairos_23m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/kairos.ipynb",
7
  "org": "ShanghaiTech University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/Kairos_50m/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/mldi-lab/Kairos_50m",
6
- "code_link": "https://github.com/foundation-model-research/Kairos",
7
  "org": "ShanghaiTech University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/mldi-lab/Kairos_50m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/kairos.ipynb",
7
  "org": "ShanghaiTech University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/Lag-Llama/config.json CHANGED
@@ -5,5 +5,5 @@
5
  "model_link": "https://huggingface.co/time-series-foundation-models/Lag-Llama",
6
  "org": "Morgan Stanley & Service Now",
7
  "testdata_leakage": "Yes",
8
- "replication_code_available": "Yes"
9
  }
 
5
  "model_link": "https://huggingface.co/time-series-foundation-models/Lag-Llama",
6
  "org": "Morgan Stanley & Service Now",
7
  "testdata_leakage": "Yes",
8
+ "replication_code_available": "No"
9
  }
results/Moirai2/config.json CHANGED
@@ -3,7 +3,7 @@
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/Salesforce/moirai-2.0-R-small",
6
- "code_link": "https://github.com/SalesforceAIResearch/uni2ts",
7
  "org": "Salesforce AI Research",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
 
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/Salesforce/moirai-2.0-R-small",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai2.ipynb",
7
  "org": "Salesforce AI Research",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/TTM-R1-Pretrained/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r1",
 
6
  "org": "IBM Research",
7
  "testdata_leakage": "Yes",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r1",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb",
7
  "org": "IBM Research",
8
  "testdata_leakage": "Yes",
9
  "replication_code_available": "Yes"
results/TTM-R2-Pretrained/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2",
 
6
  "org": "IBM Research",
7
  "testdata_leakage": "Yes",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb",
7
  "org": "IBM Research",
8
  "testdata_leakage": "Yes",
9
  "replication_code_available": "Yes"
results/TiRex/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/NX-AI/TiRex-1.1-gifteval",
 
6
  "org": "NXAI",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/NX-AI/TiRex-1.1-gifteval",
6
+ "code_link":"https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/tirex.ipynb",
7
  "org": "NXAI",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/TimeCopilot/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "agentic",
4
  "model_dtype": "float32",
5
  "model_link": "https://github.com/AzulGarza/TimeCopilot",
 
6
  "testdata_leakage": "No",
7
  "replication_code_available": "Yes"
8
  }
 
3
  "model_type": "agentic",
4
  "model_dtype": "float32",
5
  "model_link": "https://github.com/AzulGarza/TimeCopilot",
6
+ "code_link": "https://github.com/AzulGarza/timecopilot/tree/main/experiments/gift-eval",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
9
  }
results/Toto_Open_Base_1.0/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/Datadog/Toto-Open-Base-1.0",
 
6
  "org": "Datadog",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/Datadog/Toto-Open-Base-1.0",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/toto.ipynb",
7
  "org": "Datadog",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/YingLong_110m/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_110m",
 
6
  "org": "Alibaba",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_110m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb",
7
  "org": "Alibaba",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/YingLong_300m/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_300m",
 
6
  "org": "Alibaba",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_300m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb",
7
  "org": "Alibaba",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/YingLong_50m/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_50m",
 
6
  "org": "Alibaba",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_50m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb",
7
  "org": "Alibaba",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/YingLong_6m/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_6m",
 
6
  "org": "Alibaba",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "bf16",
5
  "model_link": "https://huggingface.co/qcw2333/YingLong_6m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/yinglong.ipynb",
7
  "org": "Alibaba",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/naive/config.json CHANGED
@@ -3,5 +3,7 @@
3
  "model_type": "statistical",
4
  "model_dtype": "float32",
5
  "testdata_leakage": "No",
 
 
6
  "replication_code_available": "Yes"
7
  }
 
3
  "model_type": "statistical",
4
  "model_dtype": "float32",
5
  "testdata_leakage": "No",
6
+ "model_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb",
7
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb",
8
  "replication_code_available": "Yes"
9
  }
results/seasonal_naive/config.json CHANGED
@@ -2,6 +2,8 @@
2
  "model": "Seasonal_Naive",
3
  "model_type": "statistical",
4
  "model_dtype": "float32",
 
 
5
  "testdata_leakage": "No",
6
  "replication_code_available": "Yes"
7
  }
 
2
  "model": "Seasonal_Naive",
3
  "model_type": "statistical",
4
  "model_dtype": "float32",
5
+ "model_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/naive.ipynb",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
9
  }
results/sundial_base_128m/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/thuml/sundial-base-128m",
 
6
  "org": "Tsinghua University",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/thuml/sundial-base-128m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/sundial.ipynb",
7
  "org": "Tsinghua University",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/tabpfn_ts/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://github.com/liam-sbhoo/tabpfn-time-series/tree/main",
 
6
  "org": "PriorLabs",
7
  "testdata_leakage": "No",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "zero-shot",
4
  "model_dtype": "float32",
5
  "model_link": "https://github.com/liam-sbhoo/tabpfn-time-series/tree/main",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/tabpfn_ts.ipynb",
7
  "org": "PriorLabs",
8
  "testdata_leakage": "No",
9
  "replication_code_available": "Yes"
results/timesfm/config.json CHANGED
@@ -3,6 +3,7 @@
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/google/timesfm-1.0-200m",
 
6
  "org": "Google Research",
7
  "testdata_leakage": "Yes",
8
  "replication_code_available": "Yes"
 
3
  "model_type": "pretrained",
4
  "model_dtype": "float32",
5
  "model_link": "https://huggingface.co/google/timesfm-1.0-200m",
6
+ "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm.ipynb",
7
  "org": "Google Research",
8
  "testdata_leakage": "Yes",
9
  "replication_code_available": "Yes"
results/visionts/config.json CHANGED
@@ -5,5 +5,5 @@
5
  "model_link": "https://github.com/Keytoyze/VisionTS",
6
  "org": "Zhejiang University",
7
  "testdata_leakage": "No",
8
- "replication_code_available": "Yes"
9
  }
 
5
  "model_link": "https://github.com/Keytoyze/VisionTS",
6
  "org": "Zhejiang University",
7
  "testdata_leakage": "No",
8
+ "replication_code_available": "No"
9
  }
src/about.py CHANGED
@@ -45,7 +45,7 @@ LLM_BENCHMARKS_TEXT = f"""
45
  ## Update Log
46
 
47
  ### 2025-10-17
48
- - Added new column: Replication Code to indicate whether the model's evaluation code is made available. This column is a binary indicator specifying whether the model's evaluation code is made available to the public by the submission author. The preferable way to share the evaluation code is to share a notebook in the GIFT-Eval github repository (as many previous submissions have done), but a standalone repo for the evaluation code is also acceptable as long as it is accessible to the public and the link is provided in the config.json file.
49
 
50
  ### 2025-08-25
51
  - Added new model type: Zero-shot to distinguish between foundation model submissions that don't use training data of GIFT-Eval. Now models tagged with zero-shot indicate that the model is not trained on the GIFT-Eval training data. Test data leakage is still separately tracked with the TestData Leakage column. For a model be tagged as `zero-shot`, it must both not have test data leakage and not use any training split from GIFT-Eval.
 
45
  ## Update Log
46
 
47
  ### 2025-10-17
48
+ - Added new column: Replication Code to indicate whether the model' evaluation replication code is made available. This column is a binary indicator specifying whether the evaluation code is made available to the public by the submission author. The preferable way to share the evaluation code is to share a notebook in the GIFT-Eval github repository (as many previous submissions have done), but a standalone repo for the evaluation code is also acceptable as long as it is accessible to the public and the link is provided in the config.json file through the `code_link` field.
49
 
50
  ### 2025-08-25
51
  - Added new model type: Zero-shot to distinguish between foundation model submissions that don't use training data of GIFT-Eval. Now models tagged with zero-shot indicate that the model is not trained on the GIFT-Eval training data. Test data leakage is still separately tracked with the TestData Leakage column. For a model be tagged as `zero-shot`, it must both not have test data leakage and not use any training split from GIFT-Eval.