diff --git a/app.py b/app.py index 0356bba71cf70f62864a1f0c02f726fcb68c9549..91899449cdec5a2c0301b2a8d6f5aa9a84e66618 100644 --- a/app.py +++ b/app.py @@ -90,7 +90,7 @@ grouped_dfs = get_grouped_dfs() domain_df, freq_df, term_length_df, variate_type_df, overall_df = grouped_dfs['domain'], grouped_dfs['frequency'], grouped_dfs['term_length'], grouped_dfs['univariate'], grouped_dfs['overall'] overall_df = rename_metrics(overall_df) overall_df = format_df(overall_df) -overall_df = overall_df.sort_values(by=['MASE_Rank']) +overall_df = overall_df.sort_values(by=['Rank']) domain_df = pivot_existed_df(domain_df, tab_name='domain') print(f'Domain dataframe is {domain_df}') freq_df = pivot_existed_df(freq_df, tab_name='frequency') @@ -107,7 +107,7 @@ model_info_df = get_model_info_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH) # ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS) -def init_leaderboard(ori_dataframe, model_info_df, sort_val: str | list | None = None): +def init_leaderboard(ori_dataframe, model_info_df, sort_val: str|None = None): if ori_dataframe is None or ori_dataframe.empty: raise ValueError("Leaderboard DataFrame is empty or None.") model_info_col_list = [c.name for c in fields(ModelInfoColumn) if c.displayed_by_default if c.name not in ['#Params (B)', 'available_on_hub', 'hub', 'Model sha','Hub License']] @@ -120,19 +120,11 @@ def init_leaderboard(ori_dataframe, model_info_df, sort_val: str | list | None = new_cols = ['T'] + [col for col in merged_df.columns if col != 'T'] merged_df = merged_df[new_cols] if sort_val: - if isinstance(sort_val, list): - assert sort_val[0] == 'TestData Leakage' - # ipdb.set_trace() - leakage_order = pd.Categorical(merged_df[sort_val[0]], categories=['No', 'Yes', 'N/A'], ordered=True) - merged_df['leakage_order'] = leakage_order - merged_df = merged_df.sort_values(by=['leakage_order', sort_val[1]]) - merged_df = merged_df.drop(columns=['leakage_order']) - elif sort_val in merged_df.columns: + if sort_val in merged_df.columns: merged_df = merged_df.sort_values(by=[sort_val]) else: print(f'Warning: cannot sort by {sort_val}') print('Merged df: ', merged_df) - # ipdb.set_trace() # get the data type datatype_list = [col2type_dict[col] if col in col2type_dict else 'number' for col in merged_df.columns] # print('datatype_list: ', datatype_list) @@ -170,13 +162,13 @@ def init_leaderboard(ori_dataframe, model_info_df, sort_val: str | list | None = # ], filter_columns=[ ColumnFilter(ModelInfoColumn.model_type.name, type="checkboxgroup", label="Model types"), - ColumnFilter(ModelInfoColumn.testdata_leakage.name, type="checkboxgroup", label="TestData Leakage"), ], # bool_checkboxgroup_label="", column_widths=[40, 150] + [180 for _ in range(len(merged_df.columns)-2)], interactive=False, ) + demo = gr.Blocks(css=custom_css) with demo: gr.HTML(TITLE) @@ -184,8 +176,7 @@ with demo: with gr.Tabs(elem_classes="tab-buttons") as tabs: with gr.TabItem('๐Ÿ… Overall', elem_id="llm-benchmark-tab-table", id=5): - # leaderboard = init_leaderboard(overall_df, model_info_df, sort_val='Rank') - leaderboard = init_leaderboard(overall_df, model_info_df, sort_val=['TestData Leakage', 'MASE_Rank']) + leaderboard = init_leaderboard(overall_df, model_info_df, sort_val='Rank') print(f'FINAL Overall LEADERBOARD {overall_df}') with gr.TabItem("๐Ÿ… By Domain", elem_id="llm-benchmark-tab-table", id=0): leaderboard = init_leaderboard(domain_df, model_info_df) @@ -205,26 +196,6 @@ with demo: with gr.TabItem("๐Ÿ“ About", elem_id="llm-benchmark-tab-table", id=4): gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text") - # Trigger the column filters once on initial load so default selections take effect - demo.load( - js=""" - () => { - // Make the JS fire one legitimate `input` event once the checkboxgroup - // component is ready. `querySelector` looks for the *wrapper* div Gradio - // puts around the checkbox-group. - const target = document.querySelector( - 'div[data-testid="checkboxgroup-model types"]'); - - if (!target) { return []; } // safety guard - - // Ask Gradioโ€™s front-end to re-compute its filters: - target.dispatchEvent(new Event('input', { bubbles: true })); - - return []; // load() must return something - } - """ - ) - with gr.Row(): with gr.Accordion("๐Ÿ“™ Citation", open=False): citation_button = gr.Textbox( diff --git a/results/Chronos_small/config.json b/results/Chronos_small/config.json index 49390dd99c513889e3f41141e353da6b2ab0eed5..22561f50905857de747befdc29fbb85aab6b2c64 100644 --- a/results/Chronos_small/config.json +++ b/results/Chronos_small/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-t5-small", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/DLinear/config.json b/results/DLinear/config.json index 39fb17b2e093f8f982249d15325c5ebfd72e2b9f..ca288eba8c06216a3f58190bf5f32d86a8b842cf 100644 --- a/results/DLinear/config.json +++ b/results/DLinear/config.json @@ -1,7 +1,5 @@ { "model": "DLinear", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "The Chinese University of Hong Kong", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/Lag-Llama/config.json b/results/Lag-Llama/config.json index d8e4759aafe0cdfea544085d0f1965d6c8ad6f12..0a1fe2a9369b4e0a96c9b2162114f1f6abf2db1e 100644 --- a/results/Lag-Llama/config.json +++ b/results/Lag-Llama/config.json @@ -2,7 +2,5 @@ "model": "Lag-Llama", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://huggingface.co/time-series-foundation-models/Lag-Llama", - "org": "Morgan Stanley & Service Now", - "testdata_leakage": "Yes" + "model_link": "https://huggingface.co/time-series-foundation-models/Lag-Llama" } \ No newline at end of file diff --git a/results/Moirai2/all_results.csv b/results/Moirai2/all_results.csv deleted file mode 100644 index 9464cb03c82869e4b034430bed97ce79ecb59b07..0000000000000000000000000000000000000000 --- a/results/Moirai2/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,Moirai2,3594920.75,3594920.75,890.2982177734375,3.3204338550567627,0.1615607738494873,0.14871330559253693,31.004405975341797,1896.027587890625,0.30403682589530945,0.14276345074176788,0.11582298576831818,Econ/Fin,1 -m4_quarterly/Q/short,Moirai2,1823365.0,1823365.0,563.21240234375,1.1834304332733154,0.11163925379514694,0.10256534814834595,9.126250267028809,1350.3203125,0.22601984441280365,0.09427183121442795,0.0754113420844078,Econ/Fin,1 -m4_monthly/M/short,Moirai2,1941434.375,1941434.375,570.535400390625,0.9396260380744934,0.1640520691871643,0.1344050168991089,7.7516584396362305,1393.3536376953125,0.28958672285079956,0.11857683956623077,0.0947224348783493,Econ/Fin,1 -m4_weekly/W/short,Moirai2,320634.75,320634.75,278.2564697265625,2.12347149848938,0.0672045573592186,0.06834465265274048,14.761430740356445,566.2462158203125,0.1031612828373909,0.050694018602371216,0.04001309722661972,Econ/Fin,1 -m4_daily/D/short,Moirai2,370240.09375,370240.09375,162.80653381347656,3.0710153579711914,0.0357334166765213,0.028526095673441887,25.159442901611328,608.4735717773438,0.09399235248565674,0.025149110704660416,0.02005612850189209,Econ/Fin,1 -m4_hourly/H/short,Moirai2,1835445.0,1835445.0,234.19720458984375,0.8085440993309021,0.09139207750558853,0.0857313722372055,6.641612529754639,1354.7860107421875,0.18495818972587585,0.03197309002280235,0.023612583056092262,Econ/Fin,1 -electricity/15T/short,Moirai2,126713.3984375,126713.3984375,46.910743713378906,0.8698201179504395,0.15082401037216187,0.15161579847335815,9.704551696777344,355.96826171875,0.7016835808753967,0.09247032552957535,0.07707162946462631,Energy,1 -electricity/H/short,Moirai2,1653415.0,1653415.0,166.158447265625,0.8867659568786621,0.17141255736351013,0.1225140392780304,10.239829063415527,1285.851806640625,0.6070138812065125,0.0784386545419693,0.06450968980789185,Energy,1 -electricity/D/short,Moirai2,1252630400.0,1252630400.0,4144.4931640625,1.4215503931045532,0.3179091513156891,0.09655529260635376,11.699943542480469,35392.51953125,0.5816166400909424,0.06810778379440308,0.054104726761579514,Energy,1 -electricity/W/short,Moirai2,119910146048.0,119910146048.0,38409.51171875,1.6595454216003418,0.2552073895931244,0.10663959383964539,12.825586318969727,346280.4375,0.790097713470459,0.08763783425092697,0.06640905886888504,Energy,1 -solar/10T/short,Moirai2,16.64216423034668,16.64216423034668,1.5266270637512207,0.6575393676757812,3.457381010055542,1.4877198934555054,4.72634220123291,4.07948112487793,1.186720609664917,0.444095641374588,0.35037222504615784,Energy,1 -solar/H/short,Moirai2,802.4385986328125,802.4385986328125,11.403338432312012,0.8793642520904541,5.135405540466309,1.4044291973114014,7.138167858123779,28.327346801757812,1.044155478477478,0.42033085227012634,0.34237420558929443,Energy,1 -solar/D/short,Moirai2,170813.640625,170813.640625,271.90545654296875,1.0638502836227417,1.260561227798462,0.43576809763908386,5.254066467285156,413.2960510253906,0.5970720648765564,0.39281076192855835,0.302970290184021,Energy,1 -solar/W/short,Moirai2,1808328.625,1808328.625,1091.556396484375,1.1966747045516968,0.24777232110500336,0.2126481980085373,6.803486347198486,1344.7410888671875,0.27449527382850647,0.2228139340877533,0.16269271075725555,Energy,1 -hospital/M/short,Moirai2,2882.404541015625,2882.404541015625,17.830530166625977,0.7649638056755066,0.19130390882492065,0.17565345764160156,5.211375713348389,53.68803024291992,0.1949000209569931,0.06472896039485931,0.051829639822244644,Healthcare,1 -covid_deaths/D/short,Moirai2,252106.03125,252106.03125,90.99989318847656,36.958065032958984,0.09614738076925278,0.37670475244522095,903.7923583984375,502.10162353515625,0.18891817331314087,0.03423915058374405,0.028077920898795128,Healthcare,1 -us_births/D/short,Moirai2,190028.15625,190028.15625,253.57337951660156,0.3733060956001282,0.024424130097031593,0.024021783843636513,3.786815881729126,435.92218017578125,0.04086416959762573,0.023770447820425034,0.019569363445043564,Healthcare,1 -us_births/M/short,Moirai2,78014528.0,78014528.0,7187.57177734375,0.811284065246582,0.022174924612045288,0.022547760978341103,4.347311973571777,8832.5830078125,0.02743394300341606,0.02232454903423786,0.016870463266968727,Healthcare,1 -us_births/W/short,Moirai2,1552936.875,1552936.875,957.8921508789062,0.8713688850402832,0.013035018928349018,0.01307821087539196,8.680670738220215,1246.1688232421875,0.01691661961376667,0.013003292493522167,0.010818051174283028,Healthcare,1 -saugeen/D/short,Moirai2,1027.0377197265625,1027.0377197265625,12.161438941955566,2.7013986110687256,0.27170529961586,0.31519150733947754,28.517648696899414,32.047428131103516,1.038023829460144,0.3939118981361389,0.32843467593193054,Nature,1 -saugeen/M/short,Moirai2,395.2047119140625,395.2047119140625,12.477724075317383,0.7310303449630737,0.34958547353744507,0.36250320076942444,4.56439733505249,19.879756927490234,0.5968499183654785,0.3746187090873718,0.291035532951355,Nature,1 -saugeen/W/short,Moirai2,1292.568603515625,1292.568603515625,17.306808471679688,1.4041545391082764,0.43147262930870056,0.4447954297065735,11.917930603027344,35.952308654785156,1.0852503776550293,0.5224204063415527,0.41932234168052673,Nature,1 -temperature_rain/D/short,Moirai2,182.9368438720703,182.9368438720703,5.879575729370117,1.3483240604400635,26.145727157592773,1.5040239095687866,20.553316116333008,13.52541446685791,1.5922988653182983,0.6921814680099487,0.5607564449310303,Nature,1 -kdd_cup_2018/H/short,Moirai2,5302.97900390625,5302.97900390625,25.09421157836914,1.006497859954834,0.7897746562957764,0.5716216564178467,8.732624053955078,72.8215560913086,1.5242869853973389,0.5252673029899597,0.4258558750152588,Nature,1 -kdd_cup_2018/D/short,Moirai2,3038.3671875,3038.3671875,22.03321647644043,1.241451621055603,0.5251110196113586,0.49087435007095337,9.494770050048828,55.12138366699219,1.234777569770813,0.49356743693351746,0.38876861333847046,Nature,1 -car_parts/M/short,Moirai2,1.3793753385543823,1.3793753385543823,0.452247679233551,0.8265860080718994,0.84610915184021,1.9076740741729736,15.108853340148926,1.1744681596755981,2.816254138946533,1.0844435691833496,0.9362376928329468,Sales,1 -restaurant/D/short,Moirai2,145.2991180419922,145.2991180419922,7.219489574432373,0.6959659457206726,0.5734684467315674,0.4184328019618988,4.903660774230957,12.054008483886719,0.5547534227371216,0.33225762844085693,0.26036787033081055,Sales,1 -hierarchical_sales/D/short,Moirai2,29.79330825805664,29.79330825805664,2.318131446838379,0.7479611039161682,0.6207244992256165,1.0917775630950928,7.330531597137451,5.458324432373047,1.6745535135269165,0.7111770510673523,0.5769974589347839,Sales,1 -hierarchical_sales/W/short,Moirai2,479.5635070800781,479.5635070800781,9.145384788513184,0.7380942106246948,0.5495446920394897,0.4673403799533844,6.649598598480225,21.898937225341797,1.005391001701355,0.4198690950870514,0.35205039381980896,Sales,1 -loop_seattle/5T/short,Moirai2,39.24714279174805,39.24714279174805,3.421999216079712,0.5417709946632385,0.08640541136264801,0.0729915127158165,3.9730300903320312,6.264753818511963,0.1074267029762268,0.0586797334253788,0.04626261815428734,Transport,1 -loop_seattle/H/short,Moirai2,61.74736022949219,61.74736022949219,4.191839694976807,0.8503301739692688,0.11133775115013123,0.0967826396226883,9.166641235351562,7.8579487800598145,0.13909368216991425,0.07419982552528381,0.06182778254151344,Transport,1 -loop_seattle/D/short,Moirai2,18.762725830078125,18.762725830078125,2.959958553314209,0.8935706615447998,0.054861217737197876,0.05470937117934227,7.584428787231445,4.3315958976745605,0.07741141319274902,0.05289841443300247,0.04312709718942642,Transport,1 -sz_taxi/15T/short,Moirai2,16.780986785888672,16.780986785888672,2.739116907119751,0.5463942885398865,1337784270848.0,0.39846450090408325,3.9365270137786865,4.096460342407227,0.3830365538597107,0.25611916184425354,0.20139019191265106,Transport,1 -sz_taxi/H/short,Moirai2,7.243575096130371,7.243575096130371,1.8513975143432617,0.5632746815681458,1.0763623714447021,0.2980678081512451,4.026472091674805,2.6913890838623047,0.25071948766708374,0.1724691092967987,0.13598862290382385,Transport,1 -m_dense/H/short,Moirai2,46547.97265625,46547.97265625,89.18578338623047,0.7929656505584717,0.2878399193286896,0.22047041356563568,9.276103973388672,215.74978637695312,0.3826863765716553,0.15819334983825684,0.1318719983100891,Transport,1 -m_dense/D/short,Moirai2,9495.8916015625,9495.8916015625,49.772308349609375,0.6916432976722717,0.1258518397808075,0.10052505135536194,7.0636701583862305,97.44686889648438,0.16869494318962097,0.086163230240345,0.07009396702051163,Transport,1 -ett1/15T/short,Moirai2,5.021040439605713,5.021040439605713,1.079026699066162,0.6870396733283997,0.508618950843811,0.2342115193605423,5.1140007972717285,2.24076771736145,0.42359837889671326,0.203980952501297,0.1601320505142212,Energy,7 -ett1/H/short,Moirai2,110.48771667480469,110.48771667480469,5.065814971923828,0.8362465500831604,0.5074601173400879,0.2602593004703522,5.890653133392334,10.511313438415527,0.49060600996017456,0.23644231259822845,0.1851363480091095,Energy,7 -ett1/D/short,Moirai2,45395.75390625,45395.75390625,142.25717163085938,1.7477439641952515,1.1709531545639038,0.4903564453125,11.691117286682129,213.0627899169922,0.5596165060997009,0.373643159866333,0.28650546073913574,Energy,7 -ett1/W/short,Moirai2,1070556.875,1070556.875,834.5147094726562,1.4887369871139526,0.6270838379859924,0.5270519852638245,8.900157928466797,1034.67724609375,0.41172274947166443,0.3320733308792114,0.2488435059785843,Energy,7 -ett2/15T/short,Moirai2,8.050238609313965,8.050238609313965,1.7549543380737305,0.7664317488670349,0.11144769191741943,0.1304120570421219,5.862476825714111,2.837294101715088,0.13469645380973816,0.08331392705440521,0.06601111590862274,Energy,7 -ett2/H/short,Moirai2,111.35823822021484,111.35823822021484,6.3867340087890625,0.7354120016098022,0.11914452165365219,0.10967425256967545,5.209497928619385,10.552640914916992,0.13383685052394867,0.08100155740976334,0.06421768665313721,Energy,7 -ett2/D/short,Moirai2,112720.8125,112720.8125,200.43377685546875,1.3028600215911865,0.42615124583244324,0.13562987744808197,11.184418678283691,335.73919677734375,0.19215241074562073,0.11471354216337204,0.09300631284713745,Energy,7 -ett2/W/short,Moirai2,3018818.25,3018818.25,1220.6822509765625,0.8076348304748535,0.13835936784744263,0.15684010088443756,8.835420608520508,1737.4747314453125,0.14574748277664185,0.10239652544260025,0.08512651920318604,Energy,7 -jena_weather/10T/short,Moirai2,1263.07080078125,1263.07080078125,7.481215953826904,0.3154330551624298,0.3750856816768646,0.5614975690841675,2.2623369693756104,35.53970718383789,0.22074612975120544,0.04646772891283035,0.03595806285738945,Nature,21 -jena_weather/H/short,Moirai2,1116.116455078125,1116.116455078125,8.453888893127441,0.536453366279602,1.3488954305648804,0.6119290590286255,4.829719066619873,33.408329010009766,0.2047909051179886,0.05182179436087608,0.04214661940932274,Nature,21 -jena_weather/D/short,Moirai2,341.45184326171875,341.45184326171875,9.141987800598145,1.0152313709259033,0.49779126048088074,0.4462507367134094,6.8829264640808105,18.47841453552246,0.11127026379108429,0.05504970625042915,0.043448418378829956,Nature,21 -bitbrains_fast_storage/5T/short,Moirai2,2142125.25,2142125.25,171.67364501953125,0.6864748001098633,2.1051151752471924,0.7426697015762329,13.77940845489502,1463.60009765625,4.59534215927124,0.5390127897262573,0.42746859788894653,Web/CloudOps,2 -bitbrains_fast_storage/H/short,Moirai2,2942650.5,2942650.5,292.0852355957031,1.0675305128097534,3.8830220699310303,0.5814351439476013,22.434349060058594,1715.41552734375,4.889491558074951,0.8325377702713013,0.6148646473884583,Web/CloudOps,2 -bitbrains_rnd/5T/short,Moirai2,1660531.75,1660531.75,115.86862182617188,1.6520637273788452,1.0863659381866455,0.6535171270370483,53.418670654296875,1288.6162109375,5.272645473480225,0.4741009473800659,0.4042632579803467,Web/CloudOps,2 -bitbrains_rnd/H/short,Moirai2,2235573.0,2235573.0,196.98313903808594,5.809325695037842,3.0100300312042236,0.5586869716644287,197.04698181152344,1495.1832275390625,6.54365348815918,0.8620945811271667,0.6697196960449219,Web/CloudOps,2 -bizitobs_application/10S/short,Moirai2,621851.5625,621851.5625,406.2403259277344,1.438624620437622,0.019857333973050117,0.020211493596434593,18.09433364868164,788.57568359375,0.030412212014198303,0.015667065978050232,0.01300627738237381,Web/CloudOps,2 -bizitobs_service/10S/short,Moirai2,4797.98974609375,4797.98974609375,22.429920196533203,0.8738541007041931,0.07667911797761917,0.05649945139884949,9.500022888183594,69.26752471923828,0.05131559818983078,0.016616802662611008,0.01355437096208334,Web/CloudOps,2 -bizitobs_l2c/5T/short,Moirai2,20.624032974243164,20.624032974243164,2.863663911819458,0.2905579209327698,0.1645745486021042,0.20925605297088623,4.3934006690979,4.5413689613342285,0.15596899390220642,0.09834981709718704,0.08443012833595276,Web/CloudOps,7 -bizitobs_l2c/H/short,Moirai2,67.85011291503906,67.85011291503906,5.013020038604736,0.5027103424072266,0.4784625768661499,0.636954665184021,7.536383152008057,8.237117767333984,0.4440017640590668,0.27021461725234985,0.23466992378234863,Web/CloudOps,7 -electricity/15T/medium,Moirai2,283338.0,283338.0,55.66805648803711,0.8413034677505493,0.13227048516273499,0.13056059181690216,9.948001861572266,532.2950439453125,0.9171979427337646,0.09592166543006897,0.07995456457138062,Energy,1 -electricity/H/medium,Moirai2,6361778.5,6361778.5,248.01693725585938,1.0840357542037964,0.23050330579280853,0.13177354633808136,13.117870330810547,2522.256591796875,0.9845198392868042,0.09680917859077454,0.0803634449839592,Energy,1 -solar/10T/medium,Moirai2,29.05948257446289,29.05948257446289,2.3172357082366943,0.9995384812355042,3.0454554557800293,1.504119634628296,8.617097854614258,5.390684604644775,1.1962143182754517,0.5142037868499756,0.41236650943756104,Energy,1 -solar/H/medium,Moirai2,714.8151245117188,714.8151245117188,11.081192016601562,0.8482193350791931,4.51237154006958,1.3964426517486572,9.920930862426758,26.736026763916016,0.9630941152572632,0.3991703987121582,0.3328731954097748,Energy,1 -kdd_cup_2018/H/medium,Moirai2,5786.36328125,5786.36328125,28.03462028503418,1.1240187883377075,0.9898953437805176,0.6367859840393066,13.927753448486328,76.06815338134766,1.5922441482543945,0.586815357208252,0.4954785704612732,Nature,1 -loop_seattle/5T/medium,Moirai2,91.64151000976562,91.64151000976562,5.445966720581055,0.864562451839447,0.1969592571258545,0.12233935296535492,10.540971755981445,9.572957038879395,0.17032258212566376,0.09689493477344513,0.08025791496038437,Transport,1 -loop_seattle/H/medium,Moirai2,71.71796417236328,71.71796417236328,4.58095121383667,0.9398095607757568,0.11857419461011887,0.10557787865400314,13.948471069335938,8.468646049499512,0.14968235790729523,0.08096779882907867,0.06914147734642029,Transport,1 -sz_taxi/15T/medium,Moirai2,17.261032104492188,17.261032104492188,2.8276829719543457,0.551992654800415,10671182839808.0,0.41678428649902344,6.010798931121826,4.154639720916748,0.38665857911109924,0.26316311955451965,0.21361444890499115,Transport,1 -m_dense/H/medium,Moirai2,39792.32421875,39792.32421875,81.75311279296875,0.7038857936859131,0.2560286521911621,0.2057977169752121,10.532341957092285,199.48013305664062,0.34707024693489075,0.14224009215831757,0.12004674971103668,Transport,1 -ett1/15T/medium,Moirai2,10.665780067443848,10.665780067443848,1.6653506755828857,1.0389811992645264,0.8133009672164917,0.3889414370059967,11.519312858581543,3.26585054397583,0.6224743723869324,0.3174174726009369,0.2604987323284149,Energy,7 -ett1/H/medium,Moirai2,167.32998657226562,167.32998657226562,7.205399036407471,1.3042583465576172,3247734521856.0,0.4289178252220154,16.902162551879883,12.935608863830566,0.6195850372314453,0.3451215326786041,0.2872616946697235,Energy,7 -ett2/15T/medium,Moirai2,12.762186050415039,12.762186050415039,2.2088661193847656,0.9195600748062134,0.14112639427185059,0.1747804433107376,11.09008502960205,3.572420120239258,0.1903807371854782,0.11771447211503983,0.0978662297129631,Energy,7 -ett2/H/medium,Moirai2,253.32736206054688,253.32736206054688,10.055752754211426,1.0852832794189453,0.18437251448631287,0.1749981790781021,11.153921127319336,15.916260719299316,0.21448184549808502,0.135507732629776,0.11071720719337463,Energy,7 -jena_weather/10T/medium,Moirai2,1942.8153076171875,1942.8153076171875,11.685406684875488,0.6510283946990967,0.8314501047134399,0.6661898493766785,9.578142166137695,44.07737731933594,0.27050212025642395,0.07171314209699631,0.058668095618486404,Nature,21 -jena_weather/H/medium,Moirai2,1340.6402587890625,1340.6402587890625,10.796601295471191,0.8172689080238342,2.6193902492523193,0.6811408400535583,10.437860488891602,36.61475372314453,0.22405217587947845,0.06606631726026535,0.05496005341410637,Nature,21 -bitbrains_fast_storage/5T/medium,Moirai2,3831930.5,3831930.5,260.03118896484375,0.9918146729469299,2.2075326442718506,0.8067377805709839,25.72391128540039,1957.53173828125,5.947522163391113,0.7900465726852417,0.6938542127609253,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Moirai2,2298316.0,2298316.0,147.26242065429688,4.3996782302856445,0.5052017569541931,0.7080380916595459,163.8717498779297,1516.019775390625,6.378668308258057,0.6196081042289734,0.5960801839828491,Web/CloudOps,2 -bizitobs_application/10S/medium,Moirai2,3613374.25,3613374.25,1043.581787109375,2.478975534439087,0.03553047776222229,0.036169908940792084,53.43565368652344,1900.8878173828125,0.07464227080345154,0.040978383272886276,0.03691835328936577,Web/CloudOps,2 -bizitobs_service/10S/medium,Moirai2,56963.7421875,56963.7421875,49.91252899169922,1.131983995437622,0.10594205558300018,0.08129119127988815,18.115341186523438,238.67079162597656,0.17962925136089325,0.03756534680724144,0.03391871601343155,Web/CloudOps,2 -bizitobs_l2c/5T/medium,Moirai2,77.68477630615234,77.68477630615234,5.3431549072265625,0.5360136032104492,0.37753960490226746,0.6504175662994385,14.239152908325195,8.813896179199219,0.46374058723449707,0.28112852573394775,0.2606326937675476,Web/CloudOps,7 -bizitobs_l2c/H/medium,Moirai2,73.60548400878906,73.60548400878906,5.040341854095459,0.5085751414299011,0.561661422252655,0.7950536608695984,10.486610412597656,8.579363822937012,0.5194935202598572,0.30520036816596985,0.27381497621536255,Web/CloudOps,7 -electricity/15T/long,Moirai2,391376.21875,391376.21875,63.17216110229492,0.9070706963539124,0.13897426426410675,0.134074404835701,10.943906784057617,625.6007080078125,0.9869617819786072,0.09966183453798294,0.08316168934106827,Energy,1 -electricity/H/long,Moirai2,9811671.0,9811671.0,291.53839111328125,1.262234091758728,0.30803436040878296,0.1474093645811081,17.574865341186523,3132.35888671875,1.2354013919830322,0.11498266458511353,0.09748584777116776,Energy,1 -solar/10T/long,Moirai2,29.88235855102539,29.88235855102539,2.3448755741119385,1.01083505153656,3.0021917819976807,1.503761887550354,9.01179027557373,5.466475963592529,1.1814912557601929,0.5068073272705078,0.40836524963378906,Energy,1 -solar/H/long,Moirai2,695.760498046875,695.760498046875,11.615360260009766,0.8833699226379395,5.532365322113037,1.3997015953063965,10.741609573364258,26.37727165222168,0.9145480990409851,0.4027256965637207,0.3319021165370941,Energy,1 -kdd_cup_2018/H/long,Moirai2,4539.05078125,4539.05078125,25.811946868896484,1.0698148012161255,0.9702631831169128,0.6511303782463074,15.044140815734863,67.37247467041016,1.5808305740356445,0.6056525707244873,0.5183228254318237,Nature,1 -loop_seattle/5T/long,Moirai2,102.15283966064453,102.15283966064453,5.852677822113037,0.928242027759552,0.20338046550750732,0.12979671359062195,12.898492813110352,10.10706901550293,0.17869503796100616,0.10347653925418854,0.08709994703531265,Transport,1 -loop_seattle/H/long,Moirai2,63.72977828979492,63.72977828979492,4.314446926116943,0.9038061499595642,0.12193074822425842,0.1005469411611557,13.63674259185791,7.98309326171875,0.14266857504844666,0.07710494101047516,0.06565489619970322,Transport,1 -sz_taxi/15T/long,Moirai2,17.477622985839844,17.477622985839844,2.822140693664551,0.5302703380584717,7329904328704.0,0.4138064384460449,6.833168029785156,4.180624961853027,0.38695117831230164,0.2612123191356659,0.21665456891059875,Transport,1 -m_dense/H/long,Moirai2,37458.58203125,37458.58203125,82.49683380126953,0.7182016968727112,0.26045969128608704,0.2135668247938156,10.146082878112793,193.54220581054688,0.33505016565322876,0.14281421899795532,0.11958861351013184,Transport,1 -ett1/15T/long,Moirai2,10.191858291625977,10.191858291625977,1.6781684160232544,1.0669130086898804,0.8645601868629456,0.3900018632411957,14.423590660095215,3.1924688816070557,0.6084877252578735,0.3198605477809906,0.2684479057788849,Energy,7 -ett1/H/long,Moirai2,203.1143341064453,203.1143341064453,7.873347282409668,1.4649949073791504,2660186456064.0,0.47378653287887573,24.58342933654785,14.251818656921387,0.682488739490509,0.37703752517700195,0.3230729103088379,Energy,7 -ett2/15T/long,Moirai2,13.122858047485352,13.122858047485352,2.279362916946411,0.959116518497467,0.14799295365810394,0.17831845581531525,13.057952880859375,3.6225485801696777,0.19305217266082764,0.12147137522697449,0.10234040766954422,Energy,7 -ett2/H/long,Moirai2,237.56210327148438,237.56210327148438,9.41628360748291,1.0515315532684326,0.18637529015541077,0.17694972455501556,14.501702308654785,15.413049697875977,0.21106985211372375,0.12894874811172485,0.10860944539308548,Energy,7 -jena_weather/10T/long,Moirai2,1825.4752197265625,1825.4752197265625,11.603591918945312,0.7330641150474548,0.9823510646820068,0.6760526895523071,13.818408012390137,42.725582122802734,0.26144516468048096,0.07100436836481094,0.05961557850241661,Nature,21 -jena_weather/H/long,Moirai2,1278.8531494140625,1278.8531494140625,11.639805793762207,1.0389355421066284,2.1917943954467773,0.6806634664535522,18.089033126831055,35.76105499267578,0.21532118320465088,0.07008452713489532,0.05799703299999237,Nature,21 -bitbrains_fast_storage/5T/long,Moirai2,5638371.5,5638371.5,367.75579833984375,0.9175259470939636,3.341670274734497,0.8233354091644287,21.859882354736328,2374.525390625,6.275162220001221,0.9718688130378723,0.8065826892852783,Web/CloudOps,2 -bitbrains_rnd/5T/long,Moirai2,2340055.5,2340055.5,154.8313751220703,3.3333687782287598,0.7227475643157959,0.6934060454368591,121.20797729492188,1529.7239990234375,5.860028266906738,0.5931240916252136,0.5697691440582275,Web/CloudOps,2 -bizitobs_application/10S/long,Moirai2,6483073.0,6483073.0,1533.4085693359375,3.4278934001922607,0.050070058554410934,0.05103800445795059,101.53668975830078,2546.18798828125,0.09819629043340683,0.05913744121789932,0.056454192847013474,Web/CloudOps,2 -bizitobs_service/10S/long,Moirai2,128119.78125,128119.78125,76.06694030761719,1.37122642993927,0.11781994998455048,0.09284570068120956,30.744794845581055,357.938232421875,0.2651720941066742,0.05635282024741173,0.053895868360996246,Web/CloudOps,2 -bizitobs_l2c/5T/long,Moirai2,85.3919677734375,85.3919677734375,5.707359790802002,0.5799223184585571,0.40167975425720215,0.7719800472259521,16.739368438720703,9.240777015686035,0.5188558101654053,0.3204597234725952,0.30009692907333374,Web/CloudOps,7 -bizitobs_l2c/H/long,Moirai2,99.5688705444336,99.5688705444336,5.755020618438721,0.6107828617095947,0.7120028138160706,0.8073437213897705,15.002673149108887,9.97842025756836,0.6095107197761536,0.35153326392173767,0.3212907910346985,Web/CloudOps,7 diff --git a/results/Moirai2/config.json b/results/Moirai2/config.json deleted file mode 100644 index 8123823ebad705235d4a5d013f273e838de5104c..0000000000000000000000000000000000000000 --- a/results/Moirai2/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "model": "Moirai2", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Salesforce/moirai-2.0-R-small", - "code_link": "https://github.com/SalesforceAIResearch/uni2ts", - "org": "Salesforce AI Research", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/Moirai_base/config.json b/results/Moirai_base/config.json index 0e960aa206fc65c11bae39562cf9b572171d8255..f29995c47e4cec59b8afe69863dd78883db13420 100644 --- a/results/Moirai_base/config.json +++ b/results/Moirai_base/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/Salesforce/moirai-1.1-R-base", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb" } \ No newline at end of file diff --git a/results/Moirai_large/config.json b/results/Moirai_large/config.json index 7c056c4018841076af65d49281af958c9b47d31e..fa603bd566a01af6fde3d47879cef1265d05b6af 100644 --- a/results/Moirai_large/config.json +++ b/results/Moirai_large/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/Salesforce/moirai-1.1-R-large", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb" } \ No newline at end of file diff --git a/results/Moirai_small/config.json b/results/Moirai_small/config.json index 8d8aa9ef467c2c3a8a6856853d627703ae03dd83..a21cb00fc74c0a950e5796e844ae7d8fe6cb63a2 100644 --- a/results/Moirai_small/config.json +++ b/results/Moirai_small/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/Salesforce/moirai-1.1-R-large", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb", - "org": "Salesforce AI Research", - "testdata_leakage": "No" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/moirai.ipynb" } \ No newline at end of file diff --git a/results/N-BEATS/config.json b/results/N-BEATS/config.json index 11b57c5fea5e3dfca5a47f831bb56705c6a62bfb..f08e6a72d696eeb49059663f377f17282f1ee742 100644 --- a/results/N-BEATS/config.json +++ b/results/N-BEATS/config.json @@ -1,7 +1,5 @@ { "model": "N-BEATS", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "ServiceNow", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/PatchTST/config.json b/results/PatchTST/config.json index 2db0c808b61603f5f1e38155e7f3e7e5612d9509..cee967e019ae586779525f1d408cacba18e8810e 100644 --- a/results/PatchTST/config.json +++ b/results/PatchTST/config.json @@ -1,7 +1,5 @@ { "model": "PatchTST", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Princeton University", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/TTM-R1-Pretrained/all_results.csv b/results/TTM-R1-Pretrained/all_results.csv deleted file mode 100644 index c25ff4929171059b08eaa7e4565f49dadaf906cb..0000000000000000000000000000000000000000 --- a/results/TTM-R1-Pretrained/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TTM-R1-Pretrained,4839935.5,4839935.5,1196.179443359375,5.112272262573242,0.232824757695198,0.2032455503940582,204.4908905029297,2199.9853515625,0.3527778387069702,0.1918129175901413,0.1918129175901413,Econ/Fin,1.0 -m4_quarterly/Q/short,TTM-R1-Pretrained,2172303.25,2172303.25,762.6624145507812,1.9847232103347776,0.1766680777072906,0.1416244357824325,79.38892364501953,1473.87353515625,0.2467004805803299,0.1276562511920929,0.1276562511920929,Econ/Fin,1.0 -m4_monthly/M/short,TTM-R1-Pretrained,2287464.25,2287464.25,733.611572265625,1.4358307123184204,0.2279524058103561,0.1735519915819168,57.4332275390625,1512.4365234375,0.3143362402915954,0.152469664812088,0.152469664812088,Econ/Fin,1.0 -m4_weekly/W/short,TTM-R1-Pretrained,422039.78125,422039.78125,342.1238708496094,2.8267741203308105,0.0932142734527587,0.0928001403808593,113.07096099853516,649.6458740234375,0.1183554157614708,0.0623296685516834,0.0623296722769737,Econ/Fin,1.0 -m4_daily/D/short,TTM-R1-Pretrained,411048.875,411048.875,216.05897521972656,4.402057647705078,0.0544536747038364,0.040205355733633,176.08229064941406,641.1309204101562,0.0990370139479637,0.0333751402795314,0.0333751402795314,Econ/Fin,1.0 -m4_hourly/H/short,TTM-R1-Pretrained,1733123.125,1733123.125,274.3367919921875,2.284323215484619,0.1546705365180969,0.1287814080715179,91.3729248046875,1316.4813232421875,0.1797287613153457,0.0374530255794525,0.0374530255794525,Econ/Fin,1.0 -electricity/15T/short,TTM-R1-Pretrained,257502.15625,257502.15625,88.42657470703125,1.5933691263198853,0.2426293641328811,0.2388309836387634,63.73476409912109,507.44671630859375,1.0002775192260742,0.1743062138557434,0.1743062138557434,Energy,1.0 -electricity/H/short,TTM-R1-Pretrained,1847637.375,1847637.375,209.80812072753903,1.1588753461837769,0.2957945764064789,0.1582918018102646,46.35501480102539,1359.2783203125,0.6416764259338379,0.099044419825077,0.099044419825077,Energy,1.0 -electricity/D/short,TTM-R1-Pretrained,2313620736.0,2313620736.0,5347.56494140625,1.626549243927002,0.6676267981529236,0.1234149485826492,65.06197357177734,48100.11328125,0.7904446125030518,0.0878782495856285,0.0878782570362091,Energy,1.0 -electricity/W/short,TTM-R1-Pretrained,401613684736.0,401613684736.0,71897.9140625,2.820096492767334,0.3193942010402679,0.1503859013319015,112.8038558959961,633729.9375,1.44596266746521,0.1640473157167434,0.1640473008155822,Energy,1.0 -solar/10T/short,TTM-R1-Pretrained,24.345251083374023,24.345251083374023,2.662130355834961,1.1646230220794678,5.660563945770264,1.5489561557769775,46.584922790527344,4.934090614318848,1.4353264570236206,0.7744134068489075,0.7744134068489075,Energy,1.0 -solar/H/short,TTM-R1-Pretrained,620.20068359375,620.20068359375,12.117817878723145,0.9356493949890136,4.952815055847168,1.4163239002227783,37.42597579956055,24.903827667236328,0.9179633855819702,0.446666806936264,0.446666806936264,Energy,1.0 -solar/D/short,TTM-R1-Pretrained,118481.9609375,118481.9609375,253.32144165039065,0.9883174300193788,1.0210517644882202,0.4342885315418243,39.53269577026367,344.21209716796875,0.4972692430019378,0.3659631907939911,0.3659631907939911,Energy,1.0 -solar/W/short,TTM-R1-Pretrained,7674999.5,7674999.5,2448.83447265625,2.70544981956482,0.5475783348083496,0.4141778945922851,108.2179946899414,2770.37890625,0.5655035972595215,0.4998683333396911,0.4998683035373688,Energy,1.0 -hospital/M/short,TTM-R1-Pretrained,12836.142578125,12836.142578125,32.64528274536133,0.9913527369499208,0.2491465359926223,0.203258529305458,39.654109954833984,113.29669952392578,0.4112933278083801,0.1185099557042121,0.1185099706053733,Healthcare,1.0 -covid_deaths/D/short,TTM-R1-Pretrained,1131897.75,1131897.75,222.81866455078125,44.56435394287109,0.1296754330396652,0.4075009226799011,1782.57421875,1063.9068603515625,0.4003001153469085,0.0838366076350212,0.0838366150856018,Healthcare,1.0 -us_births/D/short,TTM-R1-Pretrained,1401044.5,1401044.5,1088.5601806640625,1.6032034158706665,0.1065103560686111,0.1045429110527038,64.12813568115234,1183.6572265625,0.1109582632780075,0.1020436882972717,0.1020436808466911,Healthcare,1.0 -us_births/M/short,TTM-R1-Pretrained,368808928.0,368808928.0,14531.2119140625,1.643504619598389,0.0464245378971099,0.0449309945106506,65.74018096923828,19204.3984375,0.0596487298607826,0.0451338477432727,0.0451338477432727,Healthcare,1.0 -us_births/W/short,TTM-R1-Pretrained,6060797.0,6060797.0,1816.890869140625,1.6510907411575315,0.0242670644074678,0.0244892444461584,66.04362487792969,2461.86865234375,0.0334196239709854,0.0246641151607036,0.0246641170233488,Healthcare,1.0 -saugeen/D/short,TTM-R1-Pretrained,1217.6488037109375,1217.6488037109375,18.195556640625,4.038330078125,0.736946702003479,0.5322720408439636,161.533203125,34.89482498168945,1.1302517652511597,0.589358389377594,0.589358389377594,Nature,1.0 -saugeen/M/short,TTM-R1-Pretrained,403.4919128417969,403.4919128417969,12.90295696258545,0.7559629678726196,0.430253803730011,0.3774499595165252,30.2385196685791,20.087108612060547,0.6030752658843994,0.3873854875564575,0.3873854577541351,Nature,1.0 -saugeen/W/short,TTM-R1-Pretrained,1417.9427490234375,1417.9427490234375,22.10640335083008,1.7919658422470093,0.9105833172798156,0.621161162853241,71.67863464355469,37.65558242797852,1.1366649866104126,0.6673001646995544,0.6673002243041992,Nature,1.0 -temperature_rain/D/short,TTM-R1-Pretrained,164.8345947265625,164.8345947265625,6.6282057762146,1.636677384376526,95.407470703125,1.397904634475708,65.46709442138672,12.838791847229004,1.511465311050415,0.7803150415420532,0.7803150415420532,Nature,1.0 -kdd_cup_2018/H/short,TTM-R1-Pretrained,4423.197265625,4423.197265625,24.123720169067383,0.9970476627349854,1.096936583518982,0.5092742443084717,39.88190841674805,66.50711822509766,1.392114520072937,0.504953145980835,0.504953145980835,Nature,1.0 -kdd_cup_2018/D/short,TTM-R1-Pretrained,2660.199462890625,2660.199462890625,20.4514102935791,1.1857279539108276,0.5912871360778809,0.4502138793468475,47.42911911010742,51.57712173461914,1.15538227558136,0.4581333100795746,0.4581332802772522,Nature,1.0 -car_parts/M/short,TTM-R1-Pretrained,1.949265718460083,1.949265718460083,0.7429021596908569,1.3475180864334106,0.5989811420440674,1.742508053779602,53.900726318359375,1.3961610794067385,3.347851037979126,1.7814031839370728,1.7814030647277832,Sales,1.0 -restaurant/D/short,TTM-R1-Pretrained,189.1906890869141,189.1906890869141,8.840666770935059,0.8296910524368286,0.9069666266441344,0.4570354223251343,33.18764114379883,13.754660606384276,0.633021354675293,0.4068679809570312,0.4068679809570312,Sales,1.0 -hierarchical_sales/D/short,TTM-R1-Pretrained,28.03666305541992,28.03666305541992,2.56404709815979,0.8271127343177795,0.8048070073127747,0.9882603287696838,33.084510803222656,5.294965744018555,1.624436855316162,0.7866212129592896,0.7866212129592896,Sales,1.0 -hierarchical_sales/W/short,TTM-R1-Pretrained,564.8724365234375,564.8724365234375,12.2916898727417,0.9056759476661682,1.0096296072006226,0.5597119331359863,36.22703552246094,23.76704406738281,1.0911566019058228,0.564317524433136,0.564317524433136,Sales,1.0 -loop_seattle/5T/short,TTM-R1-Pretrained,82.84699249267578,82.84699249267578,5.735431671142578,0.8927099704742432,0.1390590518712997,0.117562897503376,35.708396911621094,9.102032661437988,0.1560797691345214,0.0983499959111213,0.0983499959111213,Transport,1.0 -loop_seattle/H/short,TTM-R1-Pretrained,78.17028045654297,78.17028045654297,5.784073352813721,1.1600669622421265,0.1653453558683395,0.1292116194963455,46.402679443359375,8.841395378112793,0.1565016955137252,0.102383978664875,0.102383978664875,Transport,1.0 -loop_seattle/D/short,TTM-R1-Pretrained,39.735931396484375,39.735931396484375,5.312212944030762,1.637539029121399,0.100871056318283,0.0958108827471733,65.50155639648438,6.303644180297852,0.1126545518636703,0.0949363484978675,0.0949363484978675,Transport,1.0 -sz_taxi/15T/short,TTM-R1-Pretrained,18.08315086364746,18.08315086364746,2.9162304401397705,0.5838834047317505,1241597870080.0,0.4137653708457947,23.355337142944336,4.252429008483887,0.397620290517807,0.2726800143718719,0.2726800143718719,Transport,1.0 -sz_taxi/H/short,TTM-R1-Pretrained,7.639774799346924,7.639774799346924,1.977967381477356,0.6016649603843689,1.2737908363342283,0.3176651895046234,24.06659889221192,2.76401424407959,0.2574849724769592,0.1842598617076873,0.1842598617076873,Transport,1.0 -m_dense/H/short,TTM-R1-Pretrained,61866.4765625,61866.4765625,125.69860076904295,1.0748567581176758,0.4138478338718414,0.2934266030788421,42.99427032470703,248.729721069336,0.4411845505237579,0.2229579836130142,0.2229579985141754,Transport,1.0 -m_dense/D/short,TTM-R1-Pretrained,18425.615234375,18425.615234375,86.44058990478516,1.195383071899414,0.2065937370061874,0.1755319833755493,47.81532287597656,135.74098205566406,0.2349877208471298,0.1496414542198181,0.1496414393186569,Transport,1.0 -ett1/15T/short,TTM-R1-Pretrained,7.1254191398620605,7.1254191398620605,1.507292866706848,0.929407835006714,0.4586927890777588,0.3414961099624634,37.17631530761719,2.6693480014801025,0.5046178698539734,0.2849410772323608,0.2849410772323608,Energy,7.0 -ett1/H/short,TTM-R1-Pretrained,106.6740493774414,106.6740493774414,5.530933856964111,0.9026903510093688,0.49075448513031,0.2867715656757355,36.10761260986328,10.328312873840332,0.4820646345615387,0.2581512928009033,0.2581512928009033,Energy,7.0 -ett1/D/short,TTM-R1-Pretrained,56256.45703125,56256.45703125,158.85206604003906,1.9342973232269287,1.07516610622406,0.5437031984329224,77.37189483642578,237.18443298339844,0.6229727864265442,0.4172302186489105,0.4172302186489105,Energy,7.0 -ett1/W/short,TTM-R1-Pretrained,1469617.875,1469617.875,879.71875,1.447729229927063,0.7957108616828918,0.4806247055530548,57.9091682434082,1212.2779541015625,0.4823943376541137,0.3500610888004303,0.3500610888004303,Energy,7.0 -ett2/15T/short,TTM-R1-Pretrained,9.67288589477539,9.67288589477539,2.0208580493927,0.8757458925247192,0.122892290353775,0.1461191624403,35.02983856201172,3.110126256942749,0.1476487666368484,0.0959373190999031,0.0959373190999031,Energy,7.0 -ett2/H/short,TTM-R1-Pretrained,120.1141357421875,120.1141357421875,6.845362663269043,0.7833001017570496,0.1272993832826614,0.1162665784358978,31.332002639770508,10.959659576416016,0.1389989703893661,0.0868182405829429,0.0868182480335235,Energy,7.0 -ett2/D/short,TTM-R1-Pretrained,116203.3046875,116203.3046875,213.6118621826172,1.5472772121429443,0.4215363562107086,0.1568193733692169,61.891090393066406,340.88604736328125,0.1950980871915817,0.1222556978464126,0.1222556978464126,Energy,7.0 -ett2/W/short,TTM-R1-Pretrained,8909286.0,8909286.0,2076.41357421875,1.2180850505828855,0.2469210624694824,0.2033569812774658,48.72340393066406,2984.8427734375,0.2503825426101684,0.1741792559623718,0.1741792559623718,Energy,7.0 -jena_weather/10T/short,TTM-R1-Pretrained,782.1746826171875,782.1746826171875,7.494993209838867,0.3400033116340637,0.4026773571968078,0.5611201524734497,13.60013198852539,27.96738624572754,0.1737125217914581,0.0465533025562763,0.046553298830986,Nature,21.0 -jena_weather/H/short,TTM-R1-Pretrained,1094.249755859375,1094.249755859375,9.976097106933594,0.5863804817199707,1.7231212854385376,0.6161960363388062,23.455219268798828,33.079444885253906,0.2027748823165893,0.0611528344452381,0.0611528381705284,Nature,21.0 -jena_weather/D/short,TTM-R1-Pretrained,718.0636596679688,718.0636596679688,15.289955139160156,2.276055097579956,1.530358076095581,0.4862256646156311,91.04220581054688,26.796709060668945,0.1613599956035614,0.0920705199241638,0.0920705199241638,Nature,21.0 -bitbrains_fast_storage/5T/short,TTM-R1-Pretrained,1628531.125,1628531.125,187.12472534179688,1.171704649925232,2.403828859329224,0.7738839983940125,46.86818313598633,1276.13916015625,4.00676155090332,0.5875253677368164,0.5875253677368164,Web/CloudOps,2.0 -bitbrains_fast_storage/H/short,TTM-R1-Pretrained,2192248.25,2192248.25,295.1697998046875,1.3008264303207395,4.625766277313232,0.6319354772567749,52.03305816650391,1480.624267578125,4.220260143280029,0.8413297533988953,0.8413297533988953,Web/CloudOps,2.0 -bitbrains_rnd/5T/short,TTM-R1-Pretrained,1510926.75,1510926.75,147.6888885498047,2.705741167068481,1.7197303771972656,0.7114022970199585,108.22964477539062,1229.1976318359375,5.029521465301514,0.604300320148468,0.604300320148468,Web/CloudOps,2.0 -bitbrains_rnd/H/short,TTM-R1-Pretrained,1933281.5,1933281.5,244.4737243652344,5.975783824920654,5.3333740234375,0.6560726165771484,239.0313415527344,1390.4249267578125,6.085179805755615,1.0699366331100464,1.0699366331100464,Web/CloudOps,2.0 -bizitobs_application/10S/short,TTM-R1-Pretrained,6637856.5,6637856.5,1252.2286376953125,4.147694110870361,0.059621375054121,0.0600129775702953,165.9077606201172,2576.40380859375,0.0993615984916687,0.0482934527099132,0.0482934489846229,Web/CloudOps,2.0 -bizitobs_service/10S/short,TTM-R1-Pretrained,27536.26171875,27536.26171875,59.36934661865234,2.4597558975219727,0.4024703204631805,0.1219735294580459,98.3902359008789,165.94053649902344,0.1229340583086013,0.0439827106893062,0.0439827106893062,Web/CloudOps,2.0 -bizitobs_l2c/5T/short,TTM-R1-Pretrained,23.40999794006348,23.40999794006348,2.9665751457214355,0.308409571647644,0.1606444716453552,0.2220607250928878,12.336382865905762,4.838387966156006,0.1661698222160339,0.1018841937184333,0.1018842011690139,Web/CloudOps,7.0 -bizitobs_l2c/H/short,TTM-R1-Pretrained,205.36241149902344,205.36241149902344,9.765654563903809,0.9571609497070312,1.011512279510498,0.8193023204803467,38.28643798828125,14.33047103881836,0.7724491357803345,0.5263938307762146,0.5263938307762146,Web/CloudOps,7.0 -electricity/15T/medium,TTM-R1-Pretrained,651701.3125,651701.3125,105.96629333496094,1.6732470989227295,0.2470926195383072,0.2312248647212982,66.92987823486328,807.2802124023438,1.3910250663757324,0.1825906038284301,0.1825905889272689,Energy,1.0 -electricity/H/medium,TTM-R1-Pretrained,6365218.5,6365218.5,276.15478515625,1.2359539270401,0.3267567455768585,0.1584555804729461,49.43815612792969,2522.9384765625,0.9847859740257264,0.1077923104166984,0.107792317867279,Energy,1.0 -solar/10T/medium,TTM-R1-Pretrained,26.01571273803711,26.01571273803711,2.7237861156463623,1.1924405097961426,3.2334301471710205,1.5169990062713623,47.6976203918457,5.100560188293457,1.1318345069885254,0.6044189929962158,0.6044189929962158,Energy,1.0 -solar/H/medium,TTM-R1-Pretrained,660.3209228515625,660.3209228515625,12.42536163330078,0.9503023624420166,5.636295795440674,1.412003517150879,38.01209259033203,25.69671058654785,0.925655484199524,0.4475905001163482,0.447590559720993,Energy,1.0 -kdd_cup_2018/H/medium,TTM-R1-Pretrained,4856.63623046875,4856.63623046875,25.49256134033203,1.081132888793945,1.1571370363235474,0.5635550618171692,43.24531936645508,69.68956756591797,1.458728909492493,0.5336055159568787,0.5336055159568787,Nature,1.0 -loop_seattle/5T/medium,TTM-R1-Pretrained,283.7026062011719,283.7026062011719,12.758612632751465,1.9841841459274288,0.320991039276123,0.2810423076152801,79.36736297607422,16.843473434448242,0.2996800243854522,0.2270019352436065,0.2270019352436065,Transport,1.0 -loop_seattle/H/medium,TTM-R1-Pretrained,90.72705841064452,90.72705841064452,6.5600266456604,1.3280991315841677,0.1687787175178527,0.143565759062767,53.12396240234375,9.52507495880127,0.1683546304702758,0.1159477308392524,0.1159477308392524,Transport,1.0 -sz_taxi/15T/medium,TTM-R1-Pretrained,18.11094856262207,18.11094856262207,2.979530096054077,0.5815892219543457,12430919860224.0,0.4278164207935333,23.263568878173828,4.255696296691895,0.3960635662078857,0.2772950232028961,0.2772950232028961,Transport,1.0 -m_dense/H/medium,TTM-R1-Pretrained,56997.91015625,56997.91015625,121.361083984375,1.0112087726593018,0.4383679628372192,0.2912840247154236,40.4483528137207,238.7423553466797,0.415381520986557,0.2111529558897018,0.2111529558897018,Transport,1.0 -ett1/15T/medium,TTM-R1-Pretrained,15.956341743469238,15.956341743469238,2.393866777420044,1.4170336723327637,0.7636879682540894,0.5661241412162781,56.68134689331055,3.994539022445679,0.7613630890846252,0.4562734067440033,0.4562733471393585,Energy,7.0 -ett1/H/medium,TTM-R1-Pretrained,147.88861083984375,147.88861083984375,7.473272800445557,1.3740049600601196,4030999822336.0,0.4816672801971435,54.96020126342773,12.160945892333984,0.5824804902076721,0.3579520583152771,0.3579520285129547,Energy,7.0 -ett2/15T/medium,TTM-R1-Pretrained,15.00575351715088,15.00575351715088,2.521174669265747,1.052714705467224,0.1634148955345153,0.1984668672084808,42.10858917236328,3.873726129531861,0.2064378708600998,0.1343579590320587,0.1343579739332199,Energy,7.0 -ett2/H/medium,TTM-R1-Pretrained,305.8819885253906,305.8819885253906,11.393686294555664,1.2107949256896973,0.1966899782419204,0.2058309763669967,48.43179702758789,17.48948097229004,0.2356820106506347,0.15353724360466,0.15353724360466,Energy,7.0 -jena_weather/10T/medium,TTM-R1-Pretrained,1537.4500732421875,1537.4500732421875,11.29836082458496,0.680471658706665,0.9417414665222168,0.6616309285163879,27.218868255615234,39.210330963134766,0.2406331300735473,0.0693378448486328,0.0693378448486328,Nature,21.0 -jena_weather/H/medium,TTM-R1-Pretrained,1495.2928466796875,1495.2928466796875,12.530884742736816,0.8401451706886292,3.385291337966919,0.6819665431976318,33.60580825805664,38.66901779174805,0.2366225719451904,0.0766787081956863,0.0766787081956863,Nature,21.0 -bitbrains_fast_storage/5T/medium,TTM-R1-Pretrained,2465359.25,2465359.25,268.6639404296875,2.242617607116699,3.788220882415772,0.819374680519104,89.70470428466797,1570.146240234375,4.770537853240967,0.8162752389907837,0.8162752389907837,Web/CloudOps,2.0 -bitbrains_rnd/5T/medium,TTM-R1-Pretrained,2117112.5,2117112.5,198.3486785888672,7.39147138595581,2.527876615524292,0.7744432091712952,295.6588439941406,1455.030029296875,6.122053146362305,0.8345540761947632,0.8345540761947632,Web/CloudOps,2.0 -bizitobs_application/10S/medium,TTM-R1-Pretrained,52785060.0,52785060.0,4526.90380859375,13.303730010986328,0.1894544214010238,0.1915669292211532,532.149169921875,7265.33251953125,0.2852882444858551,0.1777581870555877,0.1777581870555877,Web/CloudOps,2.0 -bizitobs_service/10S/medium,TTM-R1-Pretrained,191248.296875,191248.296875,196.9491271972656,7.098899841308594,1.7195463180541992,0.2512445151805877,283.9559936523437,437.3194274902344,0.3291369080543518,0.1482285559177398,0.1482285559177398,Web/CloudOps,2.0 -bizitobs_l2c/5T/medium,TTM-R1-Pretrained,200.64273071289065,200.64273071289065,10.290497779846191,1.0041199922561646,0.6574810743331909,1.018429160118103,40.164798736572266,14.164841651916504,0.7452789545059204,0.5414315462112427,0.5414315462112427,Web/CloudOps,7.0 -bizitobs_l2c/H/medium,TTM-R1-Pretrained,281.8178405761719,281.8178405761719,13.066701889038086,1.2794169187545776,1.4472025632858276,1.070118427276611,51.17667770385742,16.787431716918945,1.016504406929016,0.7912086248397827,0.7912086844444275,Web/CloudOps,7.0 -electricity/15T/long,TTM-R1-Pretrained,1342790.5,1342790.5,123.45643615722656,1.7905689477920532,0.2591364085674286,0.2366401106119156,71.62275695800781,1158.788330078125,1.8281307220458984,0.1947676688432693,0.1947676688432693,Energy,1.0 -electricity/H/long,TTM-R1-Pretrained,9800309.0,9800309.0,320.5233764648437,1.3680378198623655,0.3523074686527252,0.171502023935318,54.72151565551758,3130.544677734375,1.2346858978271484,0.1264143288135528,0.1264143288135528,Energy,1.0 -solar/10T/long,TTM-R1-Pretrained,27.684722900390625,27.684722900390625,2.788307905197144,1.218901753425598,2.6426501274108887,1.5232243537902832,48.75607299804688,5.261627197265625,1.137216567993164,0.602648138999939,0.602648138999939,Energy,1.0 -solar/H/long,TTM-R1-Pretrained,689.2293701171875,689.2293701171875,13.81916046142578,1.0523871183395386,6.962733268737793,1.4221395254135132,42.095481872558594,26.253177642822266,0.9102455377578736,0.4791354835033417,0.4791354835033417,Energy,1.0 -kdd_cup_2018/H/long,TTM-R1-Pretrained,3769.7626953125,3769.7626953125,23.29082298278809,1.0037591457366943,1.1482057571411133,0.5922478437423706,40.15036392211914,61.39839553833008,1.4406543970108032,0.5464968085289001,0.5464968085289001,Nature,1.0 -loop_seattle/5T/long,TTM-R1-Pretrained,265.4754333496094,265.4754333496094,12.66295337677002,1.972639441490173,0.3322114944458008,0.2708892822265625,78.90557861328125,16.29341697692871,0.288070946931839,0.2238836139440536,0.2238836139440536,Transport,1.0 -loop_seattle/H/long,TTM-R1-Pretrained,78.83251953125,78.83251953125,5.85994291305542,1.2037372589111328,0.184073269367218,0.1325975954532623,48.14948654174805,8.878767967224121,0.1586754769086837,0.1047250255942344,0.1047250181436538,Transport,1.0 -sz_taxi/15T/long,TTM-R1-Pretrained,16.81537437438965,16.81537437438965,2.87433123588562,0.5435872673988342,8260150624256.0,0.4148517251014709,21.74349021911621,4.10065507888794,0.3795493543148041,0.2660429775714874,0.2660429775714874,Transport,1.0 -m_dense/H/long,TTM-R1-Pretrained,60970.38671875,60970.38671875,126.48969268798828,1.0458335876464844,0.45860356092453,0.2954391837120056,41.833343505859375,246.9218292236328,0.4274581670761108,0.2189723551273346,0.2189723551273346,Transport,1.0 -ett1/15T/long,TTM-R1-Pretrained,16.58780860900879,16.58780860900879,2.484389305114746,1.488469123840332,0.8089041709899902,0.5957595705986023,59.53876495361328,4.072813510894775,0.7762822508811951,0.4735270738601684,0.4735270738601684,Energy,7.0 -ett1/H/long,TTM-R1-Pretrained,149.5264892578125,149.5264892578125,7.462743759155273,1.3938250541687012,2593034338304.0,0.4885442852973938,55.75300216674805,12.228102684020996,0.5855773687362671,0.3573746383190155,0.3573746681213379,Energy,7.0 -ett2/15T/long,TTM-R1-Pretrained,15.252047538757324,15.252047538757324,2.600973129272461,1.0827611684799194,0.167321115732193,0.2018365561962127,43.31044769287109,3.905386924743652,0.2081251442432403,0.1386105716228485,0.1386105567216873,Energy,7.0 -ett2/H/long,TTM-R1-Pretrained,288.520751953125,288.520751953125,11.162341117858888,1.2191311120986938,0.208345890045166,0.214778259396553,48.76524353027344,16.985898971557617,0.2326088100671768,0.1528596729040146,0.1528596729040146,Energy,7.0 -jena_weather/10T/long,TTM-R1-Pretrained,1562.930419921875,1562.930419921875,11.357297897338867,0.6737896203994751,0.8498238921165466,0.6583142876625061,26.95158576965332,39.53391647338867,0.2419148087501526,0.069497250020504,0.0694972574710846,Nature,21.0 -jena_weather/H/long,TTM-R1-Pretrained,1302.5177001953125,1302.5177001953125,14.308568954467772,1.0584625005722046,3.396106243133545,0.6726809144020081,42.3385009765625,36.09041213989258,0.2173042595386505,0.0861534401774406,0.08615343272686,Nature,21.0 -bitbrains_fast_storage/5T/long,TTM-R1-Pretrained,3354402.25,3354402.25,356.4161071777344,2.0263211727142334,4.350730895996094,0.8388842344284058,81.05284118652344,1831.5028076171875,4.840115070343018,0.9419013857841492,0.9419013261795044,Web/CloudOps,2.0 -bitbrains_rnd/5T/long,TTM-R1-Pretrained,1939892.375,1939892.375,193.2017059326172,6.739154815673828,2.378561496734619,0.7550832629203796,269.5661926269531,1392.8001708984375,5.335503578186035,0.7401121854782104,0.7401122450828552,Web/CloudOps,2.0 -bizitobs_application/10S/long,TTM-R1-Pretrained,63951968.0,63951968.0,5090.4970703125,11.375816345214844,0.1536046713590622,0.169856384396553,455.03265380859375,7996.99755859375,0.3084122240543365,0.1963201314210891,0.1963201314210891,Web/CloudOps,2.0 -bizitobs_service/10S/long,TTM-R1-Pretrained,310560.65625,310560.65625,242.35232543945312,6.615731239318848,0.4960593283176422,0.2466846257448196,264.6292419433594,557.2796630859375,0.4128506183624267,0.1795423477888107,0.1795423477888107,Web/CloudOps,2.0 -bizitobs_l2c/5T/long,TTM-R1-Pretrained,370.515869140625,370.515869140625,14.165494918823242,1.3303046226501465,0.9395979642868042,1.1797410249710083,53.212188720703125,19.248788833618164,1.0807907581329346,0.7953713536262512,0.7953712940216064,Web/CloudOps,7.0 -bizitobs_l2c/H/long,TTM-R1-Pretrained,224.97149658203125,224.97149658203125,11.681208610534668,1.1981686353683472,1.655575394630432,0.9281730651855468,47.92674255371094,14.99905014038086,0.9161852598190308,0.7135218977928162,0.7135219573974609,Web/CloudOps,7.0 diff --git a/results/TTM-R1-Zeroshot/all_results.csv b/results/TTM-R1-Zeroshot/all_results.csv new file mode 100644 index 0000000000000000000000000000000000000000..a23c4a62fb4f57799e17df9ec799b3c01287f25e --- /dev/null +++ b/results/TTM-R1-Zeroshot/all_results.csv @@ -0,0 +1,98 @@ +dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates +m4_yearly/A/short,TTM-R1-Zeroshot,4839935.5,4839935.5,1196.179443359375,5.112272262573242,0.232824757695198,0.2032455503940582,204.4908905029297,2199.9853515625,0.3527778387069702,0.1918129175901413,0.1918129175901413,Econ/Fin,1.0 +m4_quarterly/Q/short,TTM-R1-Zeroshot,2172303.25,2172303.25,762.6624145507812,1.9847232103347776,0.1766680777072906,0.1416244357824325,79.38892364501953,1473.87353515625,0.2467004805803299,0.1276562511920929,0.1276562511920929,Econ/Fin,1.0 +m4_monthly/M/short,TTM-R1-Zeroshot,2287464.25,2287464.25,733.611572265625,1.4358307123184204,0.2279524058103561,0.1735519915819168,57.4332275390625,1512.4365234375,0.3143362402915954,0.152469664812088,0.152469664812088,Econ/Fin,1.0 +m4_weekly/W/short,TTM-R1-Zeroshot,422039.78125,422039.78125,342.1238708496094,2.8267741203308105,0.0932142734527587,0.0928001403808593,113.07096099853516,649.6458740234375,0.1183554157614708,0.0623296685516834,0.0623296722769737,Econ/Fin,1.0 +m4_daily/D/short,TTM-R1-Zeroshot,411048.875,411048.875,216.05897521972656,4.402057647705078,0.0544536747038364,0.040205355733633,176.08229064941406,641.1309204101562,0.0990370139479637,0.0333751402795314,0.0333751402795314,Econ/Fin,1.0 +m4_hourly/H/short,TTM-R1-Zeroshot,1733123.125,1733123.125,274.3367919921875,2.284323215484619,0.1546705365180969,0.1287814080715179,91.3729248046875,1316.4813232421875,0.1797287613153457,0.0374530255794525,0.0374530255794525,Econ/Fin,1.0 +electricity/15T/short,TTM-R1-Zeroshot,257502.15625,257502.15625,88.42657470703125,1.5933691263198853,0.2426293641328811,0.2388309836387634,63.73476409912109,507.44671630859375,1.0002775192260742,0.1743062138557434,0.1743062138557434,Energy,1.0 +electricity/H/short,TTM-R1-Zeroshot,1847637.375,1847637.375,209.80812072753903,1.1588753461837769,0.2957945764064789,0.1582918018102646,46.35501480102539,1359.2783203125,0.6416764259338379,0.099044419825077,0.099044419825077,Energy,1.0 +electricity/D/short,TTM-R1-Zeroshot,2313620736.0,2313620736.0,5347.56494140625,1.626549243927002,0.6676267981529236,0.1234149485826492,65.06197357177734,48100.11328125,0.7904446125030518,0.0878782495856285,0.0878782570362091,Energy,1.0 +electricity/W/short,TTM-R1-Zeroshot,401613684736.0,401613684736.0,71897.9140625,2.820096492767334,0.3193942010402679,0.1503859013319015,112.8038558959961,633729.9375,1.44596266746521,0.1640473157167434,0.1640473008155822,Energy,1.0 +solar/10T/short,TTM-R1-Zeroshot,24.345251083374023,24.345251083374023,2.662130355834961,1.1646230220794678,5.660563945770264,1.5489561557769775,46.584922790527344,4.934090614318848,1.4353264570236206,0.7744134068489075,0.7744134068489075,Energy,1.0 +solar/H/short,TTM-R1-Zeroshot,620.20068359375,620.20068359375,12.117817878723145,0.9356493949890136,4.952815055847168,1.4163239002227783,37.42597579956055,24.903827667236328,0.9179633855819702,0.446666806936264,0.446666806936264,Energy,1.0 +solar/D/short,TTM-R1-Zeroshot,118481.9609375,118481.9609375,253.32144165039065,0.9883174300193788,1.0210517644882202,0.4342885315418243,39.53269577026367,344.21209716796875,0.4972692430019378,0.3659631907939911,0.3659631907939911,Energy,1.0 +solar/W/short,TTM-R1-Zeroshot,7674999.5,7674999.5,2448.83447265625,2.70544981956482,0.5475783348083496,0.4141778945922851,108.2179946899414,2770.37890625,0.5655035972595215,0.4998683333396911,0.4998683035373688,Energy,1.0 +hospital/M/short,TTM-R1-Zeroshot,12836.142578125,12836.142578125,32.64528274536133,0.9913527369499208,0.2491465359926223,0.203258529305458,39.654109954833984,113.29669952392578,0.4112933278083801,0.1185099557042121,0.1185099706053733,Healthcare,1.0 +covid_deaths/D/short,TTM-R1-Zeroshot,1131897.75,1131897.75,222.81866455078125,44.56435394287109,0.1296754330396652,0.4075009226799011,1782.57421875,1063.9068603515625,0.4003001153469085,0.0838366076350212,0.0838366150856018,Healthcare,1.0 +us_births/D/short,TTM-R1-Zeroshot,1401044.5,1401044.5,1088.5601806640625,1.6032034158706665,0.1065103560686111,0.1045429110527038,64.12813568115234,1183.6572265625,0.1109582632780075,0.1020436882972717,0.1020436808466911,Healthcare,1.0 +us_births/M/short,TTM-R1-Zeroshot,368808928.0,368808928.0,14531.2119140625,1.643504619598389,0.0464245378971099,0.0449309945106506,65.74018096923828,19204.3984375,0.0596487298607826,0.0451338477432727,0.0451338477432727,Healthcare,1.0 +us_births/W/short,TTM-R1-Zeroshot,6060797.0,6060797.0,1816.890869140625,1.6510907411575315,0.0242670644074678,0.0244892444461584,66.04362487792969,2461.86865234375,0.0334196239709854,0.0246641151607036,0.0246641170233488,Healthcare,1.0 +saugeen/D/short,TTM-R1-Zeroshot,1217.6488037109375,1217.6488037109375,18.195556640625,4.038330078125,0.736946702003479,0.5322720408439636,161.533203125,34.89482498168945,1.1302517652511597,0.589358389377594,0.589358389377594,Nature,1.0 +saugeen/M/short,TTM-R1-Zeroshot,403.4919128417969,403.4919128417969,12.90295696258545,0.7559629678726196,0.430253803730011,0.3774499595165252,30.2385196685791,20.087108612060547,0.6030752658843994,0.3873854875564575,0.3873854577541351,Nature,1.0 +saugeen/W/short,TTM-R1-Zeroshot,1417.9427490234375,1417.9427490234375,22.10640335083008,1.7919658422470093,0.9105833172798156,0.621161162853241,71.67863464355469,37.65558242797852,1.1366649866104126,0.6673001646995544,0.6673002243041992,Nature,1.0 +temperature_rain/D/short,TTM-R1-Zeroshot,164.8345947265625,164.8345947265625,6.6282057762146,1.636677384376526,95.407470703125,1.397904634475708,65.46709442138672,12.838791847229004,1.511465311050415,0.7803150415420532,0.7803150415420532,Nature,1.0 +kdd_cup_2018/H/short,TTM-R1-Zeroshot,4423.197265625,4423.197265625,24.123720169067383,0.9970476627349854,1.096936583518982,0.5092742443084717,39.88190841674805,66.50711822509766,1.392114520072937,0.504953145980835,0.504953145980835,Nature,1.0 +kdd_cup_2018/D/short,TTM-R1-Zeroshot,2660.199462890625,2660.199462890625,20.4514102935791,1.1857279539108276,0.5912871360778809,0.4502138793468475,47.42911911010742,51.57712173461914,1.15538227558136,0.4581333100795746,0.4581332802772522,Nature,1.0 +car_parts/M/short,TTM-R1-Zeroshot,1.949265718460083,1.949265718460083,0.7429021596908569,1.3475180864334106,0.5989811420440674,1.742508053779602,53.900726318359375,1.3961610794067385,3.347851037979126,1.7814031839370728,1.7814030647277832,Sales,1.0 +restaurant/D/short,TTM-R1-Zeroshot,189.1906890869141,189.1906890869141,8.840666770935059,0.8296910524368286,0.9069666266441344,0.4570354223251343,33.18764114379883,13.754660606384276,0.633021354675293,0.4068679809570312,0.4068679809570312,Sales,1.0 +hierarchical_sales/D/short,TTM-R1-Zeroshot,28.03666305541992,28.03666305541992,2.56404709815979,0.8271127343177795,0.8048070073127747,0.9882603287696838,33.084510803222656,5.294965744018555,1.624436855316162,0.7866212129592896,0.7866212129592896,Sales,1.0 +hierarchical_sales/W/short,TTM-R1-Zeroshot,564.8724365234375,564.8724365234375,12.2916898727417,0.9056759476661682,1.0096296072006226,0.5597119331359863,36.22703552246094,23.76704406738281,1.0911566019058228,0.564317524433136,0.564317524433136,Sales,1.0 +loop_seattle/5T/short,TTM-R1-Zeroshot,82.84699249267578,82.84699249267578,5.735431671142578,0.8927099704742432,0.1390590518712997,0.117562897503376,35.708396911621094,9.102032661437988,0.1560797691345214,0.0983499959111213,0.0983499959111213,Transport,1.0 +loop_seattle/H/short,TTM-R1-Zeroshot,78.17028045654297,78.17028045654297,5.784073352813721,1.1600669622421265,0.1653453558683395,0.1292116194963455,46.402679443359375,8.841395378112793,0.1565016955137252,0.102383978664875,0.102383978664875,Transport,1.0 +loop_seattle/D/short,TTM-R1-Zeroshot,39.735931396484375,39.735931396484375,5.312212944030762,1.637539029121399,0.100871056318283,0.0958108827471733,65.50155639648438,6.303644180297852,0.1126545518636703,0.0949363484978675,0.0949363484978675,Transport,1.0 +sz_taxi/15T/short,TTM-R1-Zeroshot,18.08315086364746,18.08315086364746,2.9162304401397705,0.5838834047317505,1241597870080.0,0.4137653708457947,23.355337142944336,4.252429008483887,0.397620290517807,0.2726800143718719,0.2726800143718719,Transport,1.0 +sz_taxi/H/short,TTM-R1-Zeroshot,7.639774799346924,7.639774799346924,1.977967381477356,0.6016649603843689,1.2737908363342283,0.3176651895046234,24.06659889221192,2.76401424407959,0.2574849724769592,0.1842598617076873,0.1842598617076873,Transport,1.0 +m_dense/H/short,TTM-R1-Zeroshot,61866.4765625,61866.4765625,125.69860076904295,1.0748567581176758,0.4138478338718414,0.2934266030788421,42.99427032470703,248.729721069336,0.4411845505237579,0.2229579836130142,0.2229579985141754,Transport,1.0 +m_dense/D/short,TTM-R1-Zeroshot,18425.615234375,18425.615234375,86.44058990478516,1.195383071899414,0.2065937370061874,0.1755319833755493,47.81532287597656,135.74098205566406,0.2349877208471298,0.1496414542198181,0.1496414393186569,Transport,1.0 +ett1/15T/short,TTM-R1-Zeroshot,7.1254191398620605,7.1254191398620605,1.507292866706848,0.929407835006714,0.4586927890777588,0.3414961099624634,37.17631530761719,2.6693480014801025,0.5046178698539734,0.2849410772323608,0.2849410772323608,Energy,7.0 +ett1/H/short,TTM-R1-Zeroshot,106.6740493774414,106.6740493774414,5.530933856964111,0.9026903510093688,0.49075448513031,0.2867715656757355,36.10761260986328,10.328312873840332,0.4820646345615387,0.2581512928009033,0.2581512928009033,Energy,7.0 +ett1/D/short,TTM-R1-Zeroshot,56256.45703125,56256.45703125,158.85206604003906,1.9342973232269287,1.07516610622406,0.5437031984329224,77.37189483642578,237.18443298339844,0.6229727864265442,0.4172302186489105,0.4172302186489105,Energy,7.0 +ett1/W/short,TTM-R1-Zeroshot,1469617.875,1469617.875,879.71875,1.447729229927063,0.7957108616828918,0.4806247055530548,57.9091682434082,1212.2779541015625,0.4823943376541137,0.3500610888004303,0.3500610888004303,Energy,7.0 +ett2/15T/short,TTM-R1-Zeroshot,9.67288589477539,9.67288589477539,2.0208580493927,0.8757458925247192,0.122892290353775,0.1461191624403,35.02983856201172,3.110126256942749,0.1476487666368484,0.0959373190999031,0.0959373190999031,Energy,7.0 +ett2/H/short,TTM-R1-Zeroshot,120.1141357421875,120.1141357421875,6.845362663269043,0.7833001017570496,0.1272993832826614,0.1162665784358978,31.332002639770508,10.959659576416016,0.1389989703893661,0.0868182405829429,0.0868182480335235,Energy,7.0 +ett2/D/short,TTM-R1-Zeroshot,116203.3046875,116203.3046875,213.6118621826172,1.5472772121429443,0.4215363562107086,0.1568193733692169,61.891090393066406,340.88604736328125,0.1950980871915817,0.1222556978464126,0.1222556978464126,Energy,7.0 +ett2/W/short,TTM-R1-Zeroshot,8909286.0,8909286.0,2076.41357421875,1.2180850505828855,0.2469210624694824,0.2033569812774658,48.72340393066406,2984.8427734375,0.2503825426101684,0.1741792559623718,0.1741792559623718,Energy,7.0 +jena_weather/10T/short,TTM-R1-Zeroshot,782.1746826171875,782.1746826171875,7.494993209838867,0.3400033116340637,0.4026773571968078,0.5611201524734497,13.60013198852539,27.96738624572754,0.1737125217914581,0.0465533025562763,0.046553298830986,Nature,21.0 +jena_weather/H/short,TTM-R1-Zeroshot,1094.249755859375,1094.249755859375,9.976097106933594,0.5863804817199707,1.7231212854385376,0.6161960363388062,23.455219268798828,33.079444885253906,0.2027748823165893,0.0611528344452381,0.0611528381705284,Nature,21.0 +jena_weather/D/short,TTM-R1-Zeroshot,718.0636596679688,718.0636596679688,15.289955139160156,2.276055097579956,1.530358076095581,0.4862256646156311,91.04220581054688,26.796709060668945,0.1613599956035614,0.0920705199241638,0.0920705199241638,Nature,21.0 +bitbrains_fast_storage/5T/short,TTM-R1-Zeroshot,1628531.125,1628531.125,187.12472534179688,1.171704649925232,2.403828859329224,0.7738839983940125,46.86818313598633,1276.13916015625,4.00676155090332,0.5875253677368164,0.5875253677368164,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,TTM-R1-Zeroshot,2192248.25,2192248.25,295.1697998046875,1.3008264303207395,4.625766277313232,0.6319354772567749,52.03305816650391,1480.624267578125,4.220260143280029,0.8413297533988953,0.8413297533988953,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,TTM-R1-Zeroshot,1510926.75,1510926.75,147.6888885498047,2.705741167068481,1.7197303771972656,0.7114022970199585,108.22964477539062,1229.1976318359375,5.029521465301514,0.604300320148468,0.604300320148468,Web/CloudOps,2.0 +bitbrains_rnd/H/short,TTM-R1-Zeroshot,1933281.5,1933281.5,244.4737243652344,5.975783824920654,5.3333740234375,0.6560726165771484,239.0313415527344,1390.4249267578125,6.085179805755615,1.0699366331100464,1.0699366331100464,Web/CloudOps,2.0 +bizitobs_application/10S/short,TTM-R1-Zeroshot,6637856.5,6637856.5,1252.2286376953125,4.147694110870361,0.059621375054121,0.0600129775702953,165.9077606201172,2576.40380859375,0.0993615984916687,0.0482934527099132,0.0482934489846229,Web/CloudOps,2.0 +bizitobs_service/10S/short,TTM-R1-Zeroshot,27536.26171875,27536.26171875,59.36934661865234,2.4597558975219727,0.4024703204631805,0.1219735294580459,98.3902359008789,165.94053649902344,0.1229340583086013,0.0439827106893062,0.0439827106893062,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,TTM-R1-Zeroshot,23.40999794006348,23.40999794006348,2.9665751457214355,0.308409571647644,0.1606444716453552,0.2220607250928878,12.336382865905762,4.838387966156006,0.1661698222160339,0.1018841937184333,0.1018842011690139,Web/CloudOps,7.0 +bizitobs_l2c/H/short,TTM-R1-Zeroshot,205.36241149902344,205.36241149902344,9.765654563903809,0.9571609497070312,1.011512279510498,0.8193023204803467,38.28643798828125,14.33047103881836,0.7724491357803345,0.5263938307762146,0.5263938307762146,Web/CloudOps,7.0 +electricity/15T/medium,TTM-R1-Zeroshot,651701.3125,651701.3125,105.96629333496094,1.6732470989227295,0.2470926195383072,0.2312248647212982,66.92987823486328,807.2802124023438,1.3910250663757324,0.1825906038284301,0.1825905889272689,Energy,1.0 +electricity/H/medium,TTM-R1-Zeroshot,6365218.5,6365218.5,276.15478515625,1.2359539270401,0.3267567455768585,0.1584555804729461,49.43815612792969,2522.9384765625,0.9847859740257264,0.1077923104166984,0.107792317867279,Energy,1.0 +solar/10T/medium,TTM-R1-Zeroshot,26.01571273803711,26.01571273803711,2.7237861156463623,1.1924405097961426,3.2334301471710205,1.5169990062713623,47.6976203918457,5.100560188293457,1.1318345069885254,0.6044189929962158,0.6044189929962158,Energy,1.0 +solar/H/medium,TTM-R1-Zeroshot,660.3209228515625,660.3209228515625,12.42536163330078,0.9503023624420166,5.636295795440674,1.412003517150879,38.01209259033203,25.69671058654785,0.925655484199524,0.4475905001163482,0.447590559720993,Energy,1.0 +kdd_cup_2018/H/medium,TTM-R1-Zeroshot,4856.63623046875,4856.63623046875,25.49256134033203,1.081132888793945,1.1571370363235474,0.5635550618171692,43.24531936645508,69.68956756591797,1.458728909492493,0.5336055159568787,0.5336055159568787,Nature,1.0 +loop_seattle/5T/medium,TTM-R1-Zeroshot,283.7026062011719,283.7026062011719,12.758612632751465,1.9841841459274288,0.320991039276123,0.2810423076152801,79.36736297607422,16.843473434448242,0.2996800243854522,0.2270019352436065,0.2270019352436065,Transport,1.0 +loop_seattle/H/medium,TTM-R1-Zeroshot,90.72705841064452,90.72705841064452,6.5600266456604,1.3280991315841677,0.1687787175178527,0.143565759062767,53.12396240234375,9.52507495880127,0.1683546304702758,0.1159477308392524,0.1159477308392524,Transport,1.0 +sz_taxi/15T/medium,TTM-R1-Zeroshot,18.11094856262207,18.11094856262207,2.979530096054077,0.5815892219543457,12430919860224.0,0.4278164207935333,23.263568878173828,4.255696296691895,0.3960635662078857,0.2772950232028961,0.2772950232028961,Transport,1.0 +m_dense/H/medium,TTM-R1-Zeroshot,56997.91015625,56997.91015625,121.361083984375,1.0112087726593018,0.4383679628372192,0.2912840247154236,40.4483528137207,238.7423553466797,0.415381520986557,0.2111529558897018,0.2111529558897018,Transport,1.0 +ett1/15T/medium,TTM-R1-Zeroshot,15.956341743469238,15.956341743469238,2.393866777420044,1.4170336723327637,0.7636879682540894,0.5661241412162781,56.68134689331055,3.994539022445679,0.7613630890846252,0.4562734067440033,0.4562733471393585,Energy,7.0 +ett1/H/medium,TTM-R1-Zeroshot,147.88861083984375,147.88861083984375,7.473272800445557,1.3740049600601196,4030999822336.0,0.4816672801971435,54.96020126342773,12.160945892333984,0.5824804902076721,0.3579520583152771,0.3579520285129547,Energy,7.0 +ett2/15T/medium,TTM-R1-Zeroshot,15.00575351715088,15.00575351715088,2.521174669265747,1.052714705467224,0.1634148955345153,0.1984668672084808,42.10858917236328,3.873726129531861,0.2064378708600998,0.1343579590320587,0.1343579739332199,Energy,7.0 +ett2/H/medium,TTM-R1-Zeroshot,305.8819885253906,305.8819885253906,11.393686294555664,1.2107949256896973,0.1966899782419204,0.2058309763669967,48.43179702758789,17.48948097229004,0.2356820106506347,0.15353724360466,0.15353724360466,Energy,7.0 +jena_weather/10T/medium,TTM-R1-Zeroshot,1537.4500732421875,1537.4500732421875,11.29836082458496,0.680471658706665,0.9417414665222168,0.6616309285163879,27.218868255615234,39.210330963134766,0.2406331300735473,0.0693378448486328,0.0693378448486328,Nature,21.0 +jena_weather/H/medium,TTM-R1-Zeroshot,1495.2928466796875,1495.2928466796875,12.530884742736816,0.8401451706886292,3.385291337966919,0.6819665431976318,33.60580825805664,38.66901779174805,0.2366225719451904,0.0766787081956863,0.0766787081956863,Nature,21.0 +bitbrains_fast_storage/5T/medium,TTM-R1-Zeroshot,2465359.25,2465359.25,268.6639404296875,2.242617607116699,3.788220882415772,0.819374680519104,89.70470428466797,1570.146240234375,4.770537853240967,0.8162752389907837,0.8162752389907837,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,TTM-R1-Zeroshot,2117112.5,2117112.5,198.3486785888672,7.39147138595581,2.527876615524292,0.7744432091712952,295.6588439941406,1455.030029296875,6.122053146362305,0.8345540761947632,0.8345540761947632,Web/CloudOps,2.0 +bizitobs_application/10S/medium,TTM-R1-Zeroshot,52785060.0,52785060.0,4526.90380859375,13.303730010986328,0.1894544214010238,0.1915669292211532,532.149169921875,7265.33251953125,0.2852882444858551,0.1777581870555877,0.1777581870555877,Web/CloudOps,2.0 +bizitobs_service/10S/medium,TTM-R1-Zeroshot,191248.296875,191248.296875,196.9491271972656,7.098899841308594,1.7195463180541992,0.2512445151805877,283.9559936523437,437.3194274902344,0.3291369080543518,0.1482285559177398,0.1482285559177398,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,TTM-R1-Zeroshot,200.64273071289065,200.64273071289065,10.290497779846191,1.0041199922561646,0.6574810743331909,1.018429160118103,40.164798736572266,14.164841651916504,0.7452789545059204,0.5414315462112427,0.5414315462112427,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,TTM-R1-Zeroshot,281.8178405761719,281.8178405761719,13.066701889038086,1.2794169187545776,1.4472025632858276,1.070118427276611,51.17667770385742,16.787431716918945,1.016504406929016,0.7912086248397827,0.7912086844444275,Web/CloudOps,7.0 +electricity/15T/long,TTM-R1-Zeroshot,1342790.5,1342790.5,123.45643615722656,1.7905689477920532,0.2591364085674286,0.2366401106119156,71.62275695800781,1158.788330078125,1.8281307220458984,0.1947676688432693,0.1947676688432693,Energy,1.0 +electricity/H/long,TTM-R1-Zeroshot,9800309.0,9800309.0,320.5233764648437,1.3680378198623655,0.3523074686527252,0.171502023935318,54.72151565551758,3130.544677734375,1.2346858978271484,0.1264143288135528,0.1264143288135528,Energy,1.0 +solar/10T/long,TTM-R1-Zeroshot,27.684722900390625,27.684722900390625,2.788307905197144,1.218901753425598,2.6426501274108887,1.5232243537902832,48.75607299804688,5.261627197265625,1.137216567993164,0.602648138999939,0.602648138999939,Energy,1.0 +solar/H/long,TTM-R1-Zeroshot,689.2293701171875,689.2293701171875,13.81916046142578,1.0523871183395386,6.962733268737793,1.4221395254135132,42.095481872558594,26.253177642822266,0.9102455377578736,0.4791354835033417,0.4791354835033417,Energy,1.0 +kdd_cup_2018/H/long,TTM-R1-Zeroshot,3769.7626953125,3769.7626953125,23.29082298278809,1.0037591457366943,1.1482057571411133,0.5922478437423706,40.15036392211914,61.39839553833008,1.4406543970108032,0.5464968085289001,0.5464968085289001,Nature,1.0 +loop_seattle/5T/long,TTM-R1-Zeroshot,265.4754333496094,265.4754333496094,12.66295337677002,1.972639441490173,0.3322114944458008,0.2708892822265625,78.90557861328125,16.29341697692871,0.288070946931839,0.2238836139440536,0.2238836139440536,Transport,1.0 +loop_seattle/H/long,TTM-R1-Zeroshot,78.83251953125,78.83251953125,5.85994291305542,1.2037372589111328,0.184073269367218,0.1325975954532623,48.14948654174805,8.878767967224121,0.1586754769086837,0.1047250255942344,0.1047250181436538,Transport,1.0 +sz_taxi/15T/long,TTM-R1-Zeroshot,16.81537437438965,16.81537437438965,2.87433123588562,0.5435872673988342,8260150624256.0,0.4148517251014709,21.74349021911621,4.10065507888794,0.3795493543148041,0.2660429775714874,0.2660429775714874,Transport,1.0 +m_dense/H/long,TTM-R1-Zeroshot,60970.38671875,60970.38671875,126.48969268798828,1.0458335876464844,0.45860356092453,0.2954391837120056,41.833343505859375,246.9218292236328,0.4274581670761108,0.2189723551273346,0.2189723551273346,Transport,1.0 +ett1/15T/long,TTM-R1-Zeroshot,16.58780860900879,16.58780860900879,2.484389305114746,1.488469123840332,0.8089041709899902,0.5957595705986023,59.53876495361328,4.072813510894775,0.7762822508811951,0.4735270738601684,0.4735270738601684,Energy,7.0 +ett1/H/long,TTM-R1-Zeroshot,149.5264892578125,149.5264892578125,7.462743759155273,1.3938250541687012,2593034338304.0,0.4885442852973938,55.75300216674805,12.228102684020996,0.5855773687362671,0.3573746383190155,0.3573746681213379,Energy,7.0 +ett2/15T/long,TTM-R1-Zeroshot,15.252047538757324,15.252047538757324,2.600973129272461,1.0827611684799194,0.167321115732193,0.2018365561962127,43.31044769287109,3.905386924743652,0.2081251442432403,0.1386105716228485,0.1386105567216873,Energy,7.0 +ett2/H/long,TTM-R1-Zeroshot,288.520751953125,288.520751953125,11.162341117858888,1.2191311120986938,0.208345890045166,0.214778259396553,48.76524353027344,16.985898971557617,0.2326088100671768,0.1528596729040146,0.1528596729040146,Energy,7.0 +jena_weather/10T/long,TTM-R1-Zeroshot,1562.930419921875,1562.930419921875,11.357297897338867,0.6737896203994751,0.8498238921165466,0.6583142876625061,26.95158576965332,39.53391647338867,0.2419148087501526,0.069497250020504,0.0694972574710846,Nature,21.0 +jena_weather/H/long,TTM-R1-Zeroshot,1302.5177001953125,1302.5177001953125,14.308568954467772,1.0584625005722046,3.396106243133545,0.6726809144020081,42.3385009765625,36.09041213989258,0.2173042595386505,0.0861534401774406,0.08615343272686,Nature,21.0 +bitbrains_fast_storage/5T/long,TTM-R1-Zeroshot,3354402.25,3354402.25,356.4161071777344,2.0263211727142334,4.350730895996094,0.8388842344284058,81.05284118652344,1831.5028076171875,4.840115070343018,0.9419013857841492,0.9419013261795044,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,TTM-R1-Zeroshot,1939892.375,1939892.375,193.2017059326172,6.739154815673828,2.378561496734619,0.7550832629203796,269.5661926269531,1392.8001708984375,5.335503578186035,0.7401121854782104,0.7401122450828552,Web/CloudOps,2.0 +bizitobs_application/10S/long,TTM-R1-Zeroshot,63951968.0,63951968.0,5090.4970703125,11.375816345214844,0.1536046713590622,0.169856384396553,455.03265380859375,7996.99755859375,0.3084122240543365,0.1963201314210891,0.1963201314210891,Web/CloudOps,2.0 +bizitobs_service/10S/long,TTM-R1-Zeroshot,310560.65625,310560.65625,242.35232543945312,6.615731239318848,0.4960593283176422,0.2466846257448196,264.6292419433594,557.2796630859375,0.4128506183624267,0.1795423477888107,0.1795423477888107,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,TTM-R1-Zeroshot,370.515869140625,370.515869140625,14.165494918823242,1.3303046226501465,0.9395979642868042,1.1797410249710083,53.212188720703125,19.248788833618164,1.0807907581329346,0.7953713536262512,0.7953712940216064,Web/CloudOps,7.0 +bizitobs_l2c/H/long,TTM-R1-Zeroshot,224.97149658203125,224.97149658203125,11.681208610534668,1.1981686353683472,1.655575394630432,0.9281730651855468,47.92674255371094,14.99905014038086,0.9161852598190308,0.7135218977928162,0.7135219573974609,Web/CloudOps,7.0 diff --git a/results/TTM-R2-Pretrained/config.json b/results/TTM-R1-Zeroshot/config.json similarity index 54% rename from results/TTM-R2-Pretrained/config.json rename to results/TTM-R1-Zeroshot/config.json index cd784237c8390f495086065ac1594b270dcfbc42..fda1b1c935fab9a886d905d91089f4eddf00dd93 100644 --- a/results/TTM-R2-Pretrained/config.json +++ b/results/TTM-R1-Zeroshot/config.json @@ -1,8 +1,6 @@ { - "model": "TTM-R2-Pretrained", + "model": "TTM-R1-Zeroshot", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2", - "org": "IBM Research", - "testdata_leakage": "Yes" + "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r1" } \ No newline at end of file diff --git a/results/TTM-R2-Finetuned/config.json b/results/TTM-R2-Finetuned/config.json index c7ed1448ff78a6d72f46677908497c03779d1ae6..4faddb89b51b64a924b4dc0dfc722d4c3e564650 100644 --- a/results/TTM-R2-Finetuned/config.json +++ b/results/TTM-R2-Finetuned/config.json @@ -3,7 +3,5 @@ "model_type": "fine-tuned", "model_dtype": "float32", "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb", - "org": "IBM Research", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/ttm.ipynb" } diff --git a/results/TTM-R2-Pretrained/all_results.csv b/results/TTM-R2-Pretrained/all_results.csv deleted file mode 100644 index ce6371340c6e83622ccf1eb797761770eced3a1c..0000000000000000000000000000000000000000 --- a/results/TTM-R2-Pretrained/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TTM-R2-Pretrained,5041324.0,5041324.0,1227.967529296875,5.130594253540039,0.26075807213783264,0.20997147262096405,205.22377014160156,2245.289306640625,0.360042542219162,0.1969102919101715,0.1969102919101715,Econ/Fin,1 -m4_quarterly/Q/short,TTM-R2-Pretrained,2419099.25,2419099.25,829.3945922851562,2.027404546737671,0.2134242057800293,0.1531578153371811,81.09618377685547,1555.3453369140625,0.2603374123573303,0.13882605731487274,0.13882605731487274,Econ/Fin,1 -m4_monthly/M/short,TTM-R2-Pretrained,2809195.75,2809195.75,853.7319946289062,1.533287525177002,0.2921540439128876,0.20111285150051117,61.331504821777344,1676.0655517578125,0.34834396839141846,0.17743481695652008,0.17743481695652008,Econ/Fin,1 -m4_weekly/W/short,TTM-R2-Pretrained,485578.84375,485578.84375,380.5999450683594,3.483421564102173,0.10566350817680359,0.10109246522188187,139.33685302734375,696.8348999023438,0.12695251405239105,0.06933940947055817,0.06933940947055817,Econ/Fin,1 -m4_daily/D/short,TTM-R2-Pretrained,484801.0625,484801.0625,227.68121337890625,4.401748180389404,0.061012763530015945,0.04191255196928978,176.06993103027344,696.2765502929688,0.10755548626184464,0.035170454531908035,0.035170454531908035,Econ/Fin,1 -m4_hourly/H/short,TTM-R2-Pretrained,2191172.0,2191172.0,294.4319152832031,2.782658100128174,0.1602412611246109,0.13437475264072418,111.30632781982422,1480.2607421875,0.20208826661109924,0.040196456015110016,0.040196459740400314,Econ/Fin,1 -electricity/15T/short,TTM-R2-Pretrained,217371.640625,217371.640625,77.01490783691406,1.4320731163024902,0.2330285906791687,0.22012728452682495,57.282928466796875,466.2313232421875,0.9190338253974915,0.15181156992912292,0.15181156992912292,Energy,1 -electricity/H/short,TTM-R2-Pretrained,1608349.0,1608349.0,205.38311767578125,1.159241795539856,0.32240381836891174,0.1580224633216858,46.36967086791992,1268.20703125,0.5986843109130859,0.09695550054311752,0.09695549309253693,Energy,1 -electricity/D/short,TTM-R2-Pretrained,2721178624.0,2721178624.0,5633.8505859375,1.6594325304031372,0.6992169618606567,0.12481007725000381,66.3772964477539,52164.91796875,0.8572428822517395,0.09258288145065308,0.09258287400007248,Energy,1 -electricity/W/short,TTM-R2-Pretrained,435031867392.0,435031867392.0,69856.296875,2.5212364196777344,0.3161992132663727,0.14358647167682648,100.84944915771484,659569.4375,1.5049197673797607,0.15938901901245117,0.15938901901245117,Energy,1 -solar/10T/short,TTM-R2-Pretrained,25.674766540527344,25.674766540527344,2.6975016593933105,1.1790286302566528,5.4343438148498535,1.5516341924667358,47.16114807128906,5.067027568817139,1.473997712135315,0.7847028970718384,0.7847028970718384,Energy,1 -solar/H/short,TTM-R2-Pretrained,619.8744506835938,619.8744506835938,12.693642616271973,0.9796592593193054,5.864431858062744,1.4270025491714478,39.186370849609375,24.89727783203125,0.9177219867706299,0.4678918719291687,0.4678918719291687,Energy,1 -solar/D/short,TTM-R2-Pretrained,174468.765625,174468.765625,274.03997802734375,1.068386435508728,1.2658942937850952,0.4365691542625427,42.73545837402344,417.694580078125,0.6034263968467712,0.3958944082260132,0.3958944082260132,Energy,1 -solar/W/short,TTM-R2-Pretrained,8813000.0,8813000.0,2602.568359375,2.877580165863037,0.5861544609069824,0.434007465839386,115.10320281982422,2968.669677734375,0.6059797406196594,0.5312492847442627,0.5312492847442627,Energy,1 -hospital/M/short,TTM-R2-Pretrained,12283.0078125,12283.0078125,33.84686279296875,1.050584316253662,0.28865745663642883,0.22228078544139862,42.023372650146484,110.8287353515625,0.4023340344429016,0.12287196516990662,0.12287196516990662,Healthcare,1 -covid_deaths/D/short,TTM-R2-Pretrained,2181495.0,2181495.0,326.9200134277344,53.48585891723633,0.18292130529880524,0.4471057057380676,2139.434326171875,1476.988525390625,0.5557240843772888,0.12300524860620499,0.12300524860620499,Healthcare,1 -us_births/D/short,TTM-R2-Pretrained,1462623.625,1462623.625,1105.77490234375,1.6285947561264038,0.1084594875574112,0.10614823549985886,65.14379119873047,1209.3897705078125,0.11337047815322876,0.10365742444992065,0.10365742444992065,Healthcare,1 -us_births/M/short,TTM-R2-Pretrained,203729680.0,203729680.0,11695.4287109375,1.3240991830825806,0.03641016408801079,0.036412689834833145,52.963966369628906,14273.390625,0.04433305561542511,0.03632592037320137,0.03632592037320137,Healthcare,1 -us_births/W/short,TTM-R2-Pretrained,7337176.0,7337176.0,1957.2852783203125,1.778560757637024,0.026068169623613358,0.026361528784036636,71.1424331665039,2708.72216796875,0.03677063807845116,0.026569955050945282,0.026569955050945282,Healthcare,1 -saugeen/D/short,TTM-R2-Pretrained,1256.2291259765625,1256.2291259765625,18.179595947265625,4.034393787384033,0.7102912664413452,0.54073566198349,161.37574768066406,35.44332504272461,1.1480176448822021,0.588841438293457,0.5888414978981018,Nature,1 -saugeen/M/short,TTM-R2-Pretrained,395.8756408691406,395.8756408691406,13.489892959594727,0.7903389930725098,0.46876707673072815,0.39558157324790955,31.61355972290039,19.896623611450195,0.5973563194274902,0.4050070643424988,0.4050070345401764,Nature,1 -saugeen/W/short,TTM-R2-Pretrained,1452.2073974609375,1452.2073974609375,23.04207420349121,1.867565631866455,0.9696307182312012,0.6474501490592957,74.70262908935547,38.10783767700195,1.150316834449768,0.6955441832542419,0.6955441236495972,Nature,1 -temperature_rain/D/short,TTM-R2-Pretrained,166.6985321044922,166.6985321044922,6.7175984382629395,1.6619073152542114,104.13045501708984,1.4001303911209106,66.4762954711914,12.911178588867188,1.5199871063232422,0.7908389568328857,0.7908389568328857,Nature,1 -kdd_cup_2018/H/short,TTM-R2-Pretrained,4468.35791015625,4468.35791015625,24.576093673706055,1.0117350816726685,1.1454356908798218,0.515670120716095,40.46940231323242,66.84577941894531,1.3992031812667847,0.5144221186637878,0.5144221186637878,Nature,1 -kdd_cup_2018/D/short,TTM-R2-Pretrained,2629.543701171875,2629.543701171875,20.171777725219727,1.172845721244812,0.6205669641494751,0.4447740316390991,46.91382598876953,51.279075622558594,1.1487058401107788,0.45186924934387207,0.4518692195415497,Nature,1 -car_parts/M/short,TTM-R2-Pretrained,2.1810801029205322,2.1810801029205322,0.9530109167098999,1.5749661922454834,0.5873419642448425,1.7117667198181152,62.99864959716797,1.4768480062484741,3.541330099105835,2.2852225303649902,2.2852225303649902,Sales,1 -restaurant/D/short,TTM-R2-Pretrained,202.3482666015625,202.3482666015625,9.526232719421387,0.896973729133606,1.08562433719635,0.4797813296318054,35.87895202636719,14.2249174118042,0.654663622379303,0.43841928243637085,0.43841931223869324,Sales,1 -hierarchical_sales/D/short,TTM-R2-Pretrained,28.789072036743164,28.789072036743164,2.583070755004883,0.8340859413146973,0.8086299300193787,0.9911177754402161,33.36343765258789,5.36554479598999,1.646089792251587,0.7924574613571167,0.7924575209617615,Sales,1 -hierarchical_sales/W/short,TTM-R2-Pretrained,792.2537231445312,792.2537231445312,15.791238784790039,1.086988925933838,1.4281177520751953,0.6353736519813538,43.47955322265625,28.147001266479492,1.2922426462173462,0.7249835133552551,0.7249835133552551,Sales,1 -loop_seattle/5T/short,TTM-R2-Pretrained,45.9228630065918,45.9228630065918,3.994657039642334,0.6305468678474426,0.11244776844978333,0.08397196233272552,25.221874237060547,6.776640892028809,0.11620443314313889,0.06849955022335052,0.06849955022335052,Transport,1 -loop_seattle/H/short,TTM-R2-Pretrained,73.20549011230469,73.20549011230469,5.383979320526123,1.0818450450897217,0.1589573621749878,0.1218806728720665,43.2738037109375,8.556020736694336,0.1514502614736557,0.09530191123485565,0.09530191123485565,Transport,1 -loop_seattle/D/short,TTM-R2-Pretrained,46.15348434448242,46.15348434448242,5.638345241546631,1.7389246225357056,0.10922148823738098,0.10129295289516449,69.5569839477539,6.793635368347168,0.12141135334968567,0.10076476633548737,0.10076476633548737,Transport,1 -sz_taxi/15T/short,TTM-R2-Pretrained,17.630447387695312,17.630447387695312,2.8711466789245605,0.5735141634941101,1263836594176.0,0.41091153025627136,22.940567016601562,4.198862552642822,0.39261162281036377,0.26846447587013245,0.26846450567245483,Transport,1 -sz_taxi/H/short,TTM-R2-Pretrained,7.52171516418457,7.52171516418457,1.9656306505203247,0.5973286032676697,1.2311612367630005,0.31563815474510193,23.893144607543945,2.742574453353882,0.25548774003982544,0.18311062455177307,0.18311062455177307,Transport,1 -m_dense/H/short,TTM-R2-Pretrained,62694.859375,62694.859375,127.04000854492188,1.0879967212677002,0.42501795291900635,0.30154624581336975,43.51987075805664,250.38941955566406,0.4441284239292145,0.22533731162548065,0.22533732652664185,Transport,1 -m_dense/D/short,TTM-R2-Pretrained,18615.037109375,18615.037109375,87.37030792236328,1.20741868019104,0.2061319649219513,0.1770661622285843,48.29674530029297,136.4369354248047,0.2361925095319748,0.1512509286403656,0.1512509435415268,Transport,1 -ett1/15T/short,TTM-R2-Pretrained,5.624592304229736,5.624592304229736,1.2432845830917358,0.8118457794189453,0.4656950831413269,0.2875490188598633,32.47383117675781,2.371622323989868,0.44833534955978394,0.23503252863883972,0.23503251373767853,Energy,7 -ett1/H/short,TTM-R2-Pretrained,104.39521789550781,104.39521789550781,5.366086959838867,0.8821736574172974,0.507716715335846,0.2827994227409363,35.28694534301758,10.217397689819336,0.4768877625465393,0.250457227230072,0.250457227230072,Energy,7 -ett1/D/short,TTM-R2-Pretrained,55083.7578125,55083.7578125,158.2764434814453,1.962099552154541,1.2102364301681519,0.5308582782745361,78.4839859008789,234.6992950439453,0.6164454817771912,0.41571831703186035,0.41571834683418274,Energy,7 -ett1/W/short,TTM-R2-Pretrained,2633276.5,2633276.5,1125.6827392578125,1.5393426418304443,1.0096418857574463,0.45859840512275696,61.573707580566406,1622.7374267578125,0.6457259058952332,0.4479359984397888,0.4479359984397888,Energy,7 -ett2/15T/short,TTM-R2-Pretrained,9.754693031311035,9.754693031311035,1.9594718217849731,0.8321967720985413,0.11617407947778702,0.13838815689086914,33.287872314453125,3.1232504844665527,0.1482718139886856,0.09302309900522232,0.09302308410406113,Energy,7 -ett2/H/short,TTM-R2-Pretrained,124.45040130615234,124.45040130615234,6.933920383453369,0.7901540994644165,0.12842239439487457,0.11721692979335785,31.606163024902344,11.155734062194824,0.14148575067520142,0.08794140070676804,0.08794140070676804,Energy,7 -ett2/D/short,TTM-R2-Pretrained,108511.2109375,108511.2109375,208.45578002929688,1.564162015914917,0.4740043878555298,0.17024555802345276,62.56648254394531,329.410400390625,0.18853026628494263,0.11930473893880844,0.11930473893880844,Energy,7 -ett2/W/short,TTM-R2-Pretrained,12633047.0,12633047.0,2386.069580078125,1.356560468673706,0.2729461193084717,0.21735575795173645,54.262420654296875,3554.2998046875,0.29815125465393066,0.200154647231102,0.20015466213226318,Energy,7 -jena_weather/10T/short,TTM-R2-Pretrained,766.1596069335938,766.1596069335938,7.308579921722412,0.3458213806152344,0.43337202072143555,0.5557089447975159,13.832855224609375,27.679588317871094,0.17192493379116058,0.045395441353321075,0.04539543762803078,Nature,21 -jena_weather/H/short,TTM-R2-Pretrained,1081.8076171875,1081.8076171875,9.729592323303223,0.5753880739212036,1.828482747077942,0.6190334558486938,23.01552391052246,32.890846252441406,0.2016187608242035,0.059641774743795395,0.05964178219437599,Nature,21 -jena_weather/D/short,TTM-R2-Pretrained,1248.5050048828125,1248.5050048828125,20.558490753173828,2.5306453704833984,1.7924801111221313,0.5474997162818909,101.22581481933594,35.334190368652344,0.21276958286762238,0.12379571795463562,0.12379571795463562,Nature,21 -bitbrains_fast_storage/5T/short,TTM-R2-Pretrained,1646846.625,1646846.625,189.82254028320312,0.965898871421814,2.6614601612091064,0.7760849595069885,38.635955810546875,1283.295166015625,4.029229640960693,0.5959958434104919,0.5959958434104919,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TTM-R2-Pretrained,2551485.5,2551485.5,324.970458984375,1.2754968404769897,5.218166351318359,0.635557234287262,51.019874572753906,1597.3369140625,4.552928924560547,0.9262713193893433,0.9262713193893433,Web/CloudOps,2 -bitbrains_rnd/5T/short,TTM-R2-Pretrained,1557151.0,1557151.0,147.8542022705078,2.053879737854004,1.7888224124908447,0.7139753103256226,82.15518951416016,1247.8585205078125,5.105876922607422,0.6049767136573792,0.6049767136573792,Web/CloudOps,2 -bitbrains_rnd/H/short,TTM-R2-Pretrained,1869584.0,1869584.0,242.30137634277344,6.163490295410156,5.455203533172607,0.6784248352050781,246.53961181640625,1367.3272705078125,5.984093189239502,1.060429334640503,1.060429334640503,Web/CloudOps,2 -bizitobs_application/10S/short,TTM-R2-Pretrained,9943514.0,9943514.0,1497.74755859375,4.214677333831787,0.06536141782999039,0.06266415864229202,168.5870819091797,3153.333984375,0.12161148339509964,0.057762134820222855,0.05776213854551315,Web/CloudOps,2 -bizitobs_service/10S/short,TTM-R2-Pretrained,44515.95703125,44515.95703125,71.65859985351562,2.732959747314453,0.35066136717796326,0.13076719641685486,109.31838989257812,210.98805236816406,0.1563066989183426,0.05308698117733002,0.05308698117733002,Web/CloudOps,2 -bizitobs_l2c/5T/short,TTM-R2-Pretrained,24.209394454956055,24.209394454956055,3.1097848415374756,0.32445377111434937,0.17801183462142944,0.22899238765239716,12.978150367736816,4.920304298400879,0.16898316144943237,0.10680259764194489,0.10680260509252548,Web/CloudOps,7 -bizitobs_l2c/H/short,TTM-R2-Pretrained,213.2822723388672,213.2822723388672,10.186047554016113,0.9834961295127869,0.9913681149482727,0.8249644041061401,39.33984375,14.60418701171875,0.7872030735015869,0.5490540862083435,0.5490540862083435,Web/CloudOps,7 -electricity/15T/medium,TTM-R2-Pretrained,479186.78125,479186.78125,82.32447814941406,1.3208063840866089,0.22963841259479523,0.19777949154376984,52.83225631713867,692.233154296875,1.192787528038025,0.14185337722301483,0.14185337722301483,Energy,1 -electricity/H/medium,TTM-R2-Pretrained,6960439.5,6960439.5,280.2503662109375,1.2517058849334717,0.36617472767829895,0.15959197282791138,50.0682373046875,2638.264404296875,1.0298014879226685,0.10939095169305801,0.10939095169305801,Energy,1 -solar/10T/medium,TTM-R2-Pretrained,23.291641235351562,23.291641235351562,2.581629991531372,1.1273674964904785,3.8910577297210693,1.4938726425170898,45.09469985961914,4.826141357421875,1.0709398984909058,0.5728739500045776,0.5728738903999329,Energy,1 -solar/H/medium,TTM-R2-Pretrained,673.9286499023438,673.9286499023438,13.676705360412598,1.0454044342041016,6.807587623596191,1.4293445348739624,41.81617736816406,25.96013641357422,0.9351446628570557,0.4926668405532837,0.4926668703556061,Energy,1 -kdd_cup_2018/H/medium,TTM-R2-Pretrained,4815.142578125,4815.142578125,25.404191970825195,1.0737661123275757,1.1820731163024902,0.559693455696106,42.95064163208008,69.39122772216797,1.452484130859375,0.5317557454109192,0.5317556858062744,Nature,1 -loop_seattle/5T/medium,TTM-R2-Pretrained,116.44446563720703,116.44446563720703,6.789214134216309,1.0654295682907104,0.237552210688591,0.14933940768241882,42.617183685302734,10.790943145751953,0.19199305772781372,0.12079407274723053,0.12079407274723053,Transport,1 -loop_seattle/H/medium,TTM-R2-Pretrained,80.56887817382812,80.56887817382812,5.882454872131348,1.1938070058822632,0.15960149466991425,0.13077092170715332,47.752281188964844,8.976016998291016,0.15865008533000946,0.10397172719240189,0.10397172719240189,Transport,1 -sz_taxi/15T/medium,TTM-R2-Pretrained,17.407360076904297,17.407360076904297,2.9041669368743896,0.5661407113075256,12433696489472.0,0.4201682507991791,22.645627975463867,4.172213077545166,0.3882940411567688,0.27028119564056396,0.27028119564056396,Transport,1 -m_dense/H/medium,TTM-R2-Pretrained,58433.80859375,58433.80859375,122.33402252197266,1.0201516151428223,0.43751060962677,0.29099923372268677,40.80606460571289,241.73086547851562,0.4205811619758606,0.21284574270248413,0.21284572780132294,Transport,1 -ett1/15T/medium,TTM-R2-Pretrained,9.845744132995605,9.845744132995605,1.7487417459487915,1.137412428855896,0.7454333305358887,0.4337901473045349,45.496498107910156,3.1377928256988525,0.5980664491653442,0.33331191539764404,0.33331191539764404,Energy,7 -ett1/H/medium,TTM-R2-Pretrained,138.2218780517578,138.2218780517578,7.0701584815979,1.3201416730880737,3522723315712.0,0.4641319215297699,52.8056640625,11.756780624389648,0.5631219148635864,0.3386438190937042,0.3386438190937042,Energy,7 -ett2/15T/medium,TTM-R2-Pretrained,14.557876586914062,14.557876586914062,2.3989028930664062,0.9867962598800659,0.1484927535057068,0.17977158725261688,39.47185134887695,3.815478563308716,0.2033337652683258,0.12784188985824585,0.12784187495708466,Energy,7 -ett2/H/medium,TTM-R2-Pretrained,269.994140625,269.994140625,10.284690856933594,1.097788691520691,0.17862552404403687,0.1798628270626068,43.91154861450195,16.43149757385254,0.22142501175403595,0.13859282433986664,0.13859283924102783,Energy,7 -jena_weather/10T/medium,TTM-R2-Pretrained,1497.083984375,1497.083984375,11.306266784667969,0.6561654806137085,0.9674649834632874,0.6653227210044861,26.246618270874023,38.692169189453125,0.2374531775712967,0.06938636302947998,0.06938636302947998,Nature,21 -jena_weather/H/medium,TTM-R2-Pretrained,1447.2547607421875,1447.2547607421875,11.986462593078613,0.8147535920143127,3.532595634460449,0.6771256327629089,32.59014129638672,38.04280090332031,0.23279064893722534,0.0733472928404808,0.0733472928404808,Nature,21 -bitbrains_fast_storage/5T/medium,TTM-R2-Pretrained,2803018.75,2803018.75,298.1320495605469,1.2348387241363525,5.41544771194458,0.8186819553375244,49.393550872802734,1674.2218017578125,5.0867486000061035,0.9058074951171875,0.9058074951171875,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TTM-R2-Pretrained,2112217.75,2112217.75,198.39056396484375,4.8130364418029785,2.431964635848999,0.7751559019088745,192.52145385742188,1453.3470458984375,6.114972114562988,0.8347302675247192,0.8347302675247192,Web/CloudOps,2 -bizitobs_application/10S/medium,TTM-R2-Pretrained,34985988.0,34985988.0,3205.0234375,9.018702507019043,0.12491956353187561,0.1292904019355774,360.74810791015625,5914.8955078125,0.2322605401277542,0.1258518397808075,0.1258518248796463,Web/CloudOps,2 -bizitobs_service/10S/medium,TTM-R2-Pretrained,182072.703125,182072.703125,155.6486053466797,5.1217827796936035,0.48198986053466797,0.20569010078907013,204.87130737304688,426.6997985839844,0.32114431262016296,0.11714481562376022,0.11714481562376022,Web/CloudOps,2 -bizitobs_l2c/5T/medium,TTM-R2-Pretrained,195.24891662597656,195.24891662597656,10.268324851989746,0.9921779036521912,0.6358968019485474,1.0157976150512695,39.68711471557617,13.973150253295898,0.7351931929588318,0.540264904499054,0.540264904499054,Web/CloudOps,7 -bizitobs_l2c/H/medium,TTM-R2-Pretrained,289.9211730957031,289.9211730957031,12.913840293884277,1.2396725416183472,1.3855247497558594,1.0475947856903076,49.5869026184082,17.027070999145508,1.0310150384902954,0.7819526791572571,0.7819526791572571,Web/CloudOps,7 -electricity/15T/long,TTM-R2-Pretrained,718197.1875,718197.1875,90.5474853515625,1.3471070528030396,0.2316545844078064,0.19732432067394257,53.88428497314453,847.4651489257812,1.3369802236557007,0.14284975826740265,0.14284975826740265,Energy,1 -electricity/H/long,TTM-R2-Pretrained,10661256.0,10661256.0,325.055908203125,1.3821061849594116,0.39065104722976685,0.1722196787595749,55.28424835205078,3265.157958984375,1.287777304649353,0.12820196151733398,0.1282019466161728,Energy,1 -solar/10T/long,TTM-R2-Pretrained,22.359167098999023,22.359167098999023,2.523888111114502,1.10085928440094,3.154728651046753,1.4919425249099731,44.03437423706055,4.728548049926758,1.0220000743865967,0.5454980731010437,0.5454980134963989,Energy,1 -solar/H/long,TTM-R2-Pretrained,705.2949829101562,705.2949829101562,14.768033027648926,1.1248584985733032,8.415304183959961,1.436413049697876,44.99434280395508,26.557390213012695,0.9207931160926819,0.5120345950126648,0.5120344758033752,Energy,1 -kdd_cup_2018/H/long,TTM-R2-Pretrained,3760.123779296875,3760.123779296875,23.107446670532227,1.0000991821289062,1.153912901878357,0.589838981628418,40.003963470458984,61.319847106933594,1.4388114213943481,0.5421940684318542,0.5421940684318542,Nature,1 -loop_seattle/5T/long,TTM-R2-Pretrained,124.74163818359375,124.74163818359375,7.093927383422852,1.1124976873397827,0.2543075382709503,0.15363603830337524,44.499908447265625,11.168779373168945,0.19746631383895874,0.12542209029197693,0.12542209029197693,Transport,1 -loop_seattle/H/long,TTM-R2-Pretrained,74.29878997802734,74.29878997802734,5.402324199676514,1.114170789718628,0.17675289511680603,0.1240486353635788,44.56683349609375,8.619674682617188,0.15404511988162994,0.09654676914215088,0.09654677659273148,Transport,1 -sz_taxi/15T/long,TTM-R2-Pretrained,16.31110191345215,16.31110191345215,2.8116378784179688,0.5310652256011963,8224755941376.0,0.40968406200408936,21.242610931396484,4.038700580596924,0.3738149106502533,0.26024019718170166,0.26024019718170166,Transport,1 -m_dense/H/long,TTM-R2-Pretrained,62940.5234375,62940.5234375,128.40859985351562,1.0597881078720093,0.470628559589386,0.2987874746322632,42.39152526855469,250.87950134277344,0.4343095123767853,0.22229425609111786,0.22229427099227905,Transport,1 -ett1/15T/long,TTM-R2-Pretrained,10.124308586120605,10.124308586120605,1.8443189859390259,1.1956290006637573,0.7989325523376465,0.45920833945274353,47.82516098022461,3.1818718910217285,0.6064679026603699,0.3515290319919586,0.3515290319919586,Energy,7 -ett1/H/long,TTM-R2-Pretrained,142.861328125,142.861328125,7.135627269744873,1.355435848236084,3003883454464.0,0.4777284562587738,54.217430114746094,11.952461242675781,0.5723775029182434,0.3417097330093384,0.34170976281166077,Energy,7 -ett2/15T/long,TTM-R2-Pretrained,13.01650333404541,13.01650333404541,2.3665223121643066,0.986086905002594,0.14943784475326538,0.1789281815290451,39.443477630615234,3.6078391075134277,0.1922682821750641,0.12611626088619232,0.12611627578735352,Energy,7 -ett2/H/long,TTM-R2-Pretrained,278.0689392089844,278.0689392089844,10.509333610534668,1.131842017173767,0.19035466015338898,0.1890520304441452,45.273681640625,16.675399780273438,0.2283567637205124,0.1439172327518463,0.1439172327518463,Energy,7 -jena_weather/10T/long,TTM-R2-Pretrained,1487.92626953125,1487.92626953125,11.109465599060059,0.6489807367324829,0.8593838214874268,0.6524072885513306,25.959230422973633,38.573646545410156,0.23603875935077667,0.06798072904348373,0.06798072159290314,Nature,21 -jena_weather/H/long,TTM-R2-Pretrained,1266.5963134765625,1266.5963134765625,13.908403396606445,1.0716230869293213,3.8741116523742676,0.6689540147781372,42.864925384521484,35.589271545410156,0.21428684890270233,0.08374399691820145,0.08374399691820145,Nature,21 -bitbrains_fast_storage/5T/long,TTM-R2-Pretrained,3219523.0,3219523.0,355.3838195800781,1.1635642051696777,4.895505428314209,0.8371726274490356,46.54256820678711,1794.302978515625,4.741806983947754,0.9391733407974243,0.9391733407974243,Web/CloudOps,2 -bitbrains_rnd/5T/long,TTM-R2-Pretrained,1991291.25,1991291.25,203.39283752441406,3.782886266708374,2.9021401405334473,0.7515886425971985,151.31544494628906,1411.1312255859375,5.405725955963135,0.7791521549224854,0.7791520953178406,Web/CloudOps,2 -bizitobs_application/10S/long,TTM-R2-Pretrained,41283780.0,41283780.0,3723.089111328125,9.590826034545898,0.13113968074321747,0.13875478506088257,383.633056640625,6425.24560546875,0.24779604375362396,0.14358466863632202,0.14358466863632202,Web/CloudOps,2 -bizitobs_service/10S/long,TTM-R2-Pretrained,267102.25,267102.25,188.78057861328125,5.33817720413208,0.43358832597732544,0.21543104946613312,213.52708435058594,516.8193359375,0.38287630677223206,0.13985468447208405,0.13985469937324524,Web/CloudOps,2 -bizitobs_l2c/5T/long,TTM-R2-Pretrained,358.9239196777344,358.9239196777344,13.98556137084961,1.316333532333374,0.9028239250183105,1.1863360404968262,52.653343200683594,18.945287704467773,1.0637495517730713,0.7852683663368225,0.7852683663368225,Web/CloudOps,7 -bizitobs_l2c/H/long,TTM-R2-Pretrained,248.46641540527344,248.46641540527344,12.290800094604492,1.2374670505523682,1.677781105041504,0.9531240463256836,49.498680114746094,15.7628173828125,0.9628384113311768,0.7507575750350952,0.7507575154304504,Web/CloudOps,7 diff --git a/results/TTM-R2-Zeroshot/all_results.csv b/results/TTM-R2-Zeroshot/all_results.csv new file mode 100644 index 0000000000000000000000000000000000000000..f2846f953d9ce7932881acff448db1c391c7bc0c --- /dev/null +++ b/results/TTM-R2-Zeroshot/all_results.csv @@ -0,0 +1,98 @@ +dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates +m4_yearly/A/short,TTM-R2-Zeroshot,5041324.0,5041324.0,1227.967529296875,5.130594253540039,0.26075807213783264,0.20997147262096405,205.22377014160156,2245.289306640625,0.360042542219162,0.1969102919101715,0.1969102919101715,Econ/Fin,1 +m4_quarterly/Q/short,TTM-R2-Zeroshot,2419099.25,2419099.25,829.3945922851562,2.027404546737671,0.2134242057800293,0.1531578153371811,81.09618377685547,1555.3453369140625,0.2603374123573303,0.13882605731487274,0.13882605731487274,Econ/Fin,1 +m4_monthly/M/short,TTM-R2-Zeroshot,2809195.75,2809195.75,853.7319946289062,1.533287525177002,0.2921540439128876,0.20111285150051117,61.331504821777344,1676.0655517578125,0.34834396839141846,0.17743481695652008,0.17743481695652008,Econ/Fin,1 +m4_weekly/W/short,TTM-R2-Zeroshot,485578.84375,485578.84375,380.5999450683594,3.483421564102173,0.10566350817680359,0.10109246522188187,139.33685302734375,696.8348999023438,0.12695251405239105,0.06933940947055817,0.06933940947055817,Econ/Fin,1 +m4_daily/D/short,TTM-R2-Zeroshot,484801.0625,484801.0625,227.68121337890625,4.401748180389404,0.061012763530015945,0.04191255196928978,176.06993103027344,696.2765502929688,0.10755548626184464,0.035170454531908035,0.035170454531908035,Econ/Fin,1 +m4_hourly/H/short,TTM-R2-Zeroshot,2191172.0,2191172.0,294.4319152832031,2.782658100128174,0.1602412611246109,0.13437475264072418,111.30632781982422,1480.2607421875,0.20208826661109924,0.040196456015110016,0.040196459740400314,Econ/Fin,1 +electricity/15T/short,TTM-R2-Zeroshot,217371.640625,217371.640625,77.01490783691406,1.4320731163024902,0.2330285906791687,0.22012728452682495,57.282928466796875,466.2313232421875,0.9190338253974915,0.15181156992912292,0.15181156992912292,Energy,1 +electricity/H/short,TTM-R2-Zeroshot,1608349.0,1608349.0,205.38311767578125,1.159241795539856,0.32240381836891174,0.1580224633216858,46.36967086791992,1268.20703125,0.5986843109130859,0.09695550054311752,0.09695549309253693,Energy,1 +electricity/D/short,TTM-R2-Zeroshot,2721178624.0,2721178624.0,5633.8505859375,1.6594325304031372,0.6992169618606567,0.12481007725000381,66.3772964477539,52164.91796875,0.8572428822517395,0.09258288145065308,0.09258287400007248,Energy,1 +electricity/W/short,TTM-R2-Zeroshot,435031867392.0,435031867392.0,69856.296875,2.5212364196777344,0.3161992132663727,0.14358647167682648,100.84944915771484,659569.4375,1.5049197673797607,0.15938901901245117,0.15938901901245117,Energy,1 +solar/10T/short,TTM-R2-Zeroshot,25.674766540527344,25.674766540527344,2.6975016593933105,1.1790286302566528,5.4343438148498535,1.5516341924667358,47.16114807128906,5.067027568817139,1.473997712135315,0.7847028970718384,0.7847028970718384,Energy,1 +solar/H/short,TTM-R2-Zeroshot,619.8744506835938,619.8744506835938,12.693642616271973,0.9796592593193054,5.864431858062744,1.4270025491714478,39.186370849609375,24.89727783203125,0.9177219867706299,0.4678918719291687,0.4678918719291687,Energy,1 +solar/D/short,TTM-R2-Zeroshot,174468.765625,174468.765625,274.03997802734375,1.068386435508728,1.2658942937850952,0.4365691542625427,42.73545837402344,417.694580078125,0.6034263968467712,0.3958944082260132,0.3958944082260132,Energy,1 +solar/W/short,TTM-R2-Zeroshot,8813000.0,8813000.0,2602.568359375,2.877580165863037,0.5861544609069824,0.434007465839386,115.10320281982422,2968.669677734375,0.6059797406196594,0.5312492847442627,0.5312492847442627,Energy,1 +hospital/M/short,TTM-R2-Zeroshot,12283.0078125,12283.0078125,33.84686279296875,1.050584316253662,0.28865745663642883,0.22228078544139862,42.023372650146484,110.8287353515625,0.4023340344429016,0.12287196516990662,0.12287196516990662,Healthcare,1 +covid_deaths/D/short,TTM-R2-Zeroshot,2181495.0,2181495.0,326.9200134277344,53.48585891723633,0.18292130529880524,0.4471057057380676,2139.434326171875,1476.988525390625,0.5557240843772888,0.12300524860620499,0.12300524860620499,Healthcare,1 +us_births/D/short,TTM-R2-Zeroshot,1462623.625,1462623.625,1105.77490234375,1.6285947561264038,0.1084594875574112,0.10614823549985886,65.14379119873047,1209.3897705078125,0.11337047815322876,0.10365742444992065,0.10365742444992065,Healthcare,1 +us_births/M/short,TTM-R2-Zeroshot,203729680.0,203729680.0,11695.4287109375,1.3240991830825806,0.03641016408801079,0.036412689834833145,52.963966369628906,14273.390625,0.04433305561542511,0.03632592037320137,0.03632592037320137,Healthcare,1 +us_births/W/short,TTM-R2-Zeroshot,7337176.0,7337176.0,1957.2852783203125,1.778560757637024,0.026068169623613358,0.026361528784036636,71.1424331665039,2708.72216796875,0.03677063807845116,0.026569955050945282,0.026569955050945282,Healthcare,1 +saugeen/D/short,TTM-R2-Zeroshot,1256.2291259765625,1256.2291259765625,18.179595947265625,4.034393787384033,0.7102912664413452,0.54073566198349,161.37574768066406,35.44332504272461,1.1480176448822021,0.588841438293457,0.5888414978981018,Nature,1 +saugeen/M/short,TTM-R2-Zeroshot,395.8756408691406,395.8756408691406,13.489892959594727,0.7903389930725098,0.46876707673072815,0.39558157324790955,31.61355972290039,19.896623611450195,0.5973563194274902,0.4050070643424988,0.4050070345401764,Nature,1 +saugeen/W/short,TTM-R2-Zeroshot,1452.2073974609375,1452.2073974609375,23.04207420349121,1.867565631866455,0.9696307182312012,0.6474501490592957,74.70262908935547,38.10783767700195,1.150316834449768,0.6955441832542419,0.6955441236495972,Nature,1 +temperature_rain/D/short,TTM-R2-Zeroshot,166.6985321044922,166.6985321044922,6.7175984382629395,1.6619073152542114,104.13045501708984,1.4001303911209106,66.4762954711914,12.911178588867188,1.5199871063232422,0.7908389568328857,0.7908389568328857,Nature,1 +kdd_cup_2018/H/short,TTM-R2-Zeroshot,4468.35791015625,4468.35791015625,24.576093673706055,1.0117350816726685,1.1454356908798218,0.515670120716095,40.46940231323242,66.84577941894531,1.3992031812667847,0.5144221186637878,0.5144221186637878,Nature,1 +kdd_cup_2018/D/short,TTM-R2-Zeroshot,2629.543701171875,2629.543701171875,20.171777725219727,1.172845721244812,0.6205669641494751,0.4447740316390991,46.91382598876953,51.279075622558594,1.1487058401107788,0.45186924934387207,0.4518692195415497,Nature,1 +car_parts/M/short,TTM-R2-Zeroshot,2.1810801029205322,2.1810801029205322,0.9530109167098999,1.5749661922454834,0.5873419642448425,1.7117667198181152,62.99864959716797,1.4768480062484741,3.541330099105835,2.2852225303649902,2.2852225303649902,Sales,1 +restaurant/D/short,TTM-R2-Zeroshot,202.3482666015625,202.3482666015625,9.526232719421387,0.896973729133606,1.08562433719635,0.4797813296318054,35.87895202636719,14.2249174118042,0.654663622379303,0.43841928243637085,0.43841931223869324,Sales,1 +hierarchical_sales/D/short,TTM-R2-Zeroshot,28.789072036743164,28.789072036743164,2.583070755004883,0.8340859413146973,0.8086299300193787,0.9911177754402161,33.36343765258789,5.36554479598999,1.646089792251587,0.7924574613571167,0.7924575209617615,Sales,1 +hierarchical_sales/W/short,TTM-R2-Zeroshot,792.2537231445312,792.2537231445312,15.791238784790039,1.086988925933838,1.4281177520751953,0.6353736519813538,43.47955322265625,28.147001266479492,1.2922426462173462,0.7249835133552551,0.7249835133552551,Sales,1 +loop_seattle/5T/short,TTM-R2-Zeroshot,45.9228630065918,45.9228630065918,3.994657039642334,0.6305468678474426,0.11244776844978333,0.08397196233272552,25.221874237060547,6.776640892028809,0.11620443314313889,0.06849955022335052,0.06849955022335052,Transport,1 +loop_seattle/H/short,TTM-R2-Zeroshot,73.20549011230469,73.20549011230469,5.383979320526123,1.0818450450897217,0.1589573621749878,0.1218806728720665,43.2738037109375,8.556020736694336,0.1514502614736557,0.09530191123485565,0.09530191123485565,Transport,1 +loop_seattle/D/short,TTM-R2-Zeroshot,46.15348434448242,46.15348434448242,5.638345241546631,1.7389246225357056,0.10922148823738098,0.10129295289516449,69.5569839477539,6.793635368347168,0.12141135334968567,0.10076476633548737,0.10076476633548737,Transport,1 +sz_taxi/15T/short,TTM-R2-Zeroshot,17.630447387695312,17.630447387695312,2.8711466789245605,0.5735141634941101,1263836594176.0,0.41091153025627136,22.940567016601562,4.198862552642822,0.39261162281036377,0.26846447587013245,0.26846450567245483,Transport,1 +sz_taxi/H/short,TTM-R2-Zeroshot,7.52171516418457,7.52171516418457,1.9656306505203247,0.5973286032676697,1.2311612367630005,0.31563815474510193,23.893144607543945,2.742574453353882,0.25548774003982544,0.18311062455177307,0.18311062455177307,Transport,1 +m_dense/H/short,TTM-R2-Zeroshot,62694.859375,62694.859375,127.04000854492188,1.0879967212677002,0.42501795291900635,0.30154624581336975,43.51987075805664,250.38941955566406,0.4441284239292145,0.22533731162548065,0.22533732652664185,Transport,1 +m_dense/D/short,TTM-R2-Zeroshot,18615.037109375,18615.037109375,87.37030792236328,1.20741868019104,0.2061319649219513,0.1770661622285843,48.29674530029297,136.4369354248047,0.2361925095319748,0.1512509286403656,0.1512509435415268,Transport,1 +ett1/15T/short,TTM-R2-Zeroshot,5.624592304229736,5.624592304229736,1.2432845830917358,0.8118457794189453,0.4656950831413269,0.2875490188598633,32.47383117675781,2.371622323989868,0.44833534955978394,0.23503252863883972,0.23503251373767853,Energy,7 +ett1/H/short,TTM-R2-Zeroshot,104.39521789550781,104.39521789550781,5.366086959838867,0.8821736574172974,0.507716715335846,0.2827994227409363,35.28694534301758,10.217397689819336,0.4768877625465393,0.250457227230072,0.250457227230072,Energy,7 +ett1/D/short,TTM-R2-Zeroshot,55083.7578125,55083.7578125,158.2764434814453,1.962099552154541,1.2102364301681519,0.5308582782745361,78.4839859008789,234.6992950439453,0.6164454817771912,0.41571831703186035,0.41571834683418274,Energy,7 +ett1/W/short,TTM-R2-Zeroshot,2633276.5,2633276.5,1125.6827392578125,1.5393426418304443,1.0096418857574463,0.45859840512275696,61.573707580566406,1622.7374267578125,0.6457259058952332,0.4479359984397888,0.4479359984397888,Energy,7 +ett2/15T/short,TTM-R2-Zeroshot,9.754693031311035,9.754693031311035,1.9594718217849731,0.8321967720985413,0.11617407947778702,0.13838815689086914,33.287872314453125,3.1232504844665527,0.1482718139886856,0.09302309900522232,0.09302308410406113,Energy,7 +ett2/H/short,TTM-R2-Zeroshot,124.45040130615234,124.45040130615234,6.933920383453369,0.7901540994644165,0.12842239439487457,0.11721692979335785,31.606163024902344,11.155734062194824,0.14148575067520142,0.08794140070676804,0.08794140070676804,Energy,7 +ett2/D/short,TTM-R2-Zeroshot,108511.2109375,108511.2109375,208.45578002929688,1.564162015914917,0.4740043878555298,0.17024555802345276,62.56648254394531,329.410400390625,0.18853026628494263,0.11930473893880844,0.11930473893880844,Energy,7 +ett2/W/short,TTM-R2-Zeroshot,12633047.0,12633047.0,2386.069580078125,1.356560468673706,0.2729461193084717,0.21735575795173645,54.262420654296875,3554.2998046875,0.29815125465393066,0.200154647231102,0.20015466213226318,Energy,7 +jena_weather/10T/short,TTM-R2-Zeroshot,766.1596069335938,766.1596069335938,7.308579921722412,0.3458213806152344,0.43337202072143555,0.5557089447975159,13.832855224609375,27.679588317871094,0.17192493379116058,0.045395441353321075,0.04539543762803078,Nature,21 +jena_weather/H/short,TTM-R2-Zeroshot,1081.8076171875,1081.8076171875,9.729592323303223,0.5753880739212036,1.828482747077942,0.6190334558486938,23.01552391052246,32.890846252441406,0.2016187608242035,0.059641774743795395,0.05964178219437599,Nature,21 +jena_weather/D/short,TTM-R2-Zeroshot,1248.5050048828125,1248.5050048828125,20.558490753173828,2.5306453704833984,1.7924801111221313,0.5474997162818909,101.22581481933594,35.334190368652344,0.21276958286762238,0.12379571795463562,0.12379571795463562,Nature,21 +bitbrains_fast_storage/5T/short,TTM-R2-Zeroshot,1646846.625,1646846.625,189.82254028320312,0.965898871421814,2.6614601612091064,0.7760849595069885,38.635955810546875,1283.295166015625,4.029229640960693,0.5959958434104919,0.5959958434104919,Web/CloudOps,2 +bitbrains_fast_storage/H/short,TTM-R2-Zeroshot,2551485.5,2551485.5,324.970458984375,1.2754968404769897,5.218166351318359,0.635557234287262,51.019874572753906,1597.3369140625,4.552928924560547,0.9262713193893433,0.9262713193893433,Web/CloudOps,2 +bitbrains_rnd/5T/short,TTM-R2-Zeroshot,1557151.0,1557151.0,147.8542022705078,2.053879737854004,1.7888224124908447,0.7139753103256226,82.15518951416016,1247.8585205078125,5.105876922607422,0.6049767136573792,0.6049767136573792,Web/CloudOps,2 +bitbrains_rnd/H/short,TTM-R2-Zeroshot,1869584.0,1869584.0,242.30137634277344,6.163490295410156,5.455203533172607,0.6784248352050781,246.53961181640625,1367.3272705078125,5.984093189239502,1.060429334640503,1.060429334640503,Web/CloudOps,2 +bizitobs_application/10S/short,TTM-R2-Zeroshot,9943514.0,9943514.0,1497.74755859375,4.214677333831787,0.06536141782999039,0.06266415864229202,168.5870819091797,3153.333984375,0.12161148339509964,0.057762134820222855,0.05776213854551315,Web/CloudOps,2 +bizitobs_service/10S/short,TTM-R2-Zeroshot,44515.95703125,44515.95703125,71.65859985351562,2.732959747314453,0.35066136717796326,0.13076719641685486,109.31838989257812,210.98805236816406,0.1563066989183426,0.05308698117733002,0.05308698117733002,Web/CloudOps,2 +bizitobs_l2c/5T/short,TTM-R2-Zeroshot,24.209394454956055,24.209394454956055,3.1097848415374756,0.32445377111434937,0.17801183462142944,0.22899238765239716,12.978150367736816,4.920304298400879,0.16898316144943237,0.10680259764194489,0.10680260509252548,Web/CloudOps,7 +bizitobs_l2c/H/short,TTM-R2-Zeroshot,213.2822723388672,213.2822723388672,10.186047554016113,0.9834961295127869,0.9913681149482727,0.8249644041061401,39.33984375,14.60418701171875,0.7872030735015869,0.5490540862083435,0.5490540862083435,Web/CloudOps,7 +electricity/15T/medium,TTM-R2-Zeroshot,479186.78125,479186.78125,82.32447814941406,1.3208063840866089,0.22963841259479523,0.19777949154376984,52.83225631713867,692.233154296875,1.192787528038025,0.14185337722301483,0.14185337722301483,Energy,1 +electricity/H/medium,TTM-R2-Zeroshot,6960439.5,6960439.5,280.2503662109375,1.2517058849334717,0.36617472767829895,0.15959197282791138,50.0682373046875,2638.264404296875,1.0298014879226685,0.10939095169305801,0.10939095169305801,Energy,1 +solar/10T/medium,TTM-R2-Zeroshot,23.291641235351562,23.291641235351562,2.581629991531372,1.1273674964904785,3.8910577297210693,1.4938726425170898,45.09469985961914,4.826141357421875,1.0709398984909058,0.5728739500045776,0.5728738903999329,Energy,1 +solar/H/medium,TTM-R2-Zeroshot,673.9286499023438,673.9286499023438,13.676705360412598,1.0454044342041016,6.807587623596191,1.4293445348739624,41.81617736816406,25.96013641357422,0.9351446628570557,0.4926668405532837,0.4926668703556061,Energy,1 +kdd_cup_2018/H/medium,TTM-R2-Zeroshot,4815.142578125,4815.142578125,25.404191970825195,1.0737661123275757,1.1820731163024902,0.559693455696106,42.95064163208008,69.39122772216797,1.452484130859375,0.5317557454109192,0.5317556858062744,Nature,1 +loop_seattle/5T/medium,TTM-R2-Zeroshot,116.44446563720703,116.44446563720703,6.789214134216309,1.0654295682907104,0.237552210688591,0.14933940768241882,42.617183685302734,10.790943145751953,0.19199305772781372,0.12079407274723053,0.12079407274723053,Transport,1 +loop_seattle/H/medium,TTM-R2-Zeroshot,80.56887817382812,80.56887817382812,5.882454872131348,1.1938070058822632,0.15960149466991425,0.13077092170715332,47.752281188964844,8.976016998291016,0.15865008533000946,0.10397172719240189,0.10397172719240189,Transport,1 +sz_taxi/15T/medium,TTM-R2-Zeroshot,17.407360076904297,17.407360076904297,2.9041669368743896,0.5661407113075256,12433696489472.0,0.4201682507991791,22.645627975463867,4.172213077545166,0.3882940411567688,0.27028119564056396,0.27028119564056396,Transport,1 +m_dense/H/medium,TTM-R2-Zeroshot,58433.80859375,58433.80859375,122.33402252197266,1.0201516151428223,0.43751060962677,0.29099923372268677,40.80606460571289,241.73086547851562,0.4205811619758606,0.21284574270248413,0.21284572780132294,Transport,1 +ett1/15T/medium,TTM-R2-Zeroshot,9.845744132995605,9.845744132995605,1.7487417459487915,1.137412428855896,0.7454333305358887,0.4337901473045349,45.496498107910156,3.1377928256988525,0.5980664491653442,0.33331191539764404,0.33331191539764404,Energy,7 +ett1/H/medium,TTM-R2-Zeroshot,138.2218780517578,138.2218780517578,7.0701584815979,1.3201416730880737,3522723315712.0,0.4641319215297699,52.8056640625,11.756780624389648,0.5631219148635864,0.3386438190937042,0.3386438190937042,Energy,7 +ett2/15T/medium,TTM-R2-Zeroshot,14.557876586914062,14.557876586914062,2.3989028930664062,0.9867962598800659,0.1484927535057068,0.17977158725261688,39.47185134887695,3.815478563308716,0.2033337652683258,0.12784188985824585,0.12784187495708466,Energy,7 +ett2/H/medium,TTM-R2-Zeroshot,269.994140625,269.994140625,10.284690856933594,1.097788691520691,0.17862552404403687,0.1798628270626068,43.91154861450195,16.43149757385254,0.22142501175403595,0.13859282433986664,0.13859283924102783,Energy,7 +jena_weather/10T/medium,TTM-R2-Zeroshot,1497.083984375,1497.083984375,11.306266784667969,0.6561654806137085,0.9674649834632874,0.6653227210044861,26.246618270874023,38.692169189453125,0.2374531775712967,0.06938636302947998,0.06938636302947998,Nature,21 +jena_weather/H/medium,TTM-R2-Zeroshot,1447.2547607421875,1447.2547607421875,11.986462593078613,0.8147535920143127,3.532595634460449,0.6771256327629089,32.59014129638672,38.04280090332031,0.23279064893722534,0.0733472928404808,0.0733472928404808,Nature,21 +bitbrains_fast_storage/5T/medium,TTM-R2-Zeroshot,2803018.75,2803018.75,298.1320495605469,1.2348387241363525,5.41544771194458,0.8186819553375244,49.393550872802734,1674.2218017578125,5.0867486000061035,0.9058074951171875,0.9058074951171875,Web/CloudOps,2 +bitbrains_rnd/5T/medium,TTM-R2-Zeroshot,2112217.75,2112217.75,198.39056396484375,4.8130364418029785,2.431964635848999,0.7751559019088745,192.52145385742188,1453.3470458984375,6.114972114562988,0.8347302675247192,0.8347302675247192,Web/CloudOps,2 +bizitobs_application/10S/medium,TTM-R2-Zeroshot,34985988.0,34985988.0,3205.0234375,9.018702507019043,0.12491956353187561,0.1292904019355774,360.74810791015625,5914.8955078125,0.2322605401277542,0.1258518397808075,0.1258518248796463,Web/CloudOps,2 +bizitobs_service/10S/medium,TTM-R2-Zeroshot,182072.703125,182072.703125,155.6486053466797,5.1217827796936035,0.48198986053466797,0.20569010078907013,204.87130737304688,426.6997985839844,0.32114431262016296,0.11714481562376022,0.11714481562376022,Web/CloudOps,2 +bizitobs_l2c/5T/medium,TTM-R2-Zeroshot,195.24891662597656,195.24891662597656,10.268324851989746,0.9921779036521912,0.6358968019485474,1.0157976150512695,39.68711471557617,13.973150253295898,0.7351931929588318,0.540264904499054,0.540264904499054,Web/CloudOps,7 +bizitobs_l2c/H/medium,TTM-R2-Zeroshot,289.9211730957031,289.9211730957031,12.913840293884277,1.2396725416183472,1.3855247497558594,1.0475947856903076,49.5869026184082,17.027070999145508,1.0310150384902954,0.7819526791572571,0.7819526791572571,Web/CloudOps,7 +electricity/15T/long,TTM-R2-Zeroshot,718197.1875,718197.1875,90.5474853515625,1.3471070528030396,0.2316545844078064,0.19732432067394257,53.88428497314453,847.4651489257812,1.3369802236557007,0.14284975826740265,0.14284975826740265,Energy,1 +electricity/H/long,TTM-R2-Zeroshot,10661256.0,10661256.0,325.055908203125,1.3821061849594116,0.39065104722976685,0.1722196787595749,55.28424835205078,3265.157958984375,1.287777304649353,0.12820196151733398,0.1282019466161728,Energy,1 +solar/10T/long,TTM-R2-Zeroshot,22.359167098999023,22.359167098999023,2.523888111114502,1.10085928440094,3.154728651046753,1.4919425249099731,44.03437423706055,4.728548049926758,1.0220000743865967,0.5454980731010437,0.5454980134963989,Energy,1 +solar/H/long,TTM-R2-Zeroshot,705.2949829101562,705.2949829101562,14.768033027648926,1.1248584985733032,8.415304183959961,1.436413049697876,44.99434280395508,26.557390213012695,0.9207931160926819,0.5120345950126648,0.5120344758033752,Energy,1 +kdd_cup_2018/H/long,TTM-R2-Zeroshot,3760.123779296875,3760.123779296875,23.107446670532227,1.0000991821289062,1.153912901878357,0.589838981628418,40.003963470458984,61.319847106933594,1.4388114213943481,0.5421940684318542,0.5421940684318542,Nature,1 +loop_seattle/5T/long,TTM-R2-Zeroshot,124.74163818359375,124.74163818359375,7.093927383422852,1.1124976873397827,0.2543075382709503,0.15363603830337524,44.499908447265625,11.168779373168945,0.19746631383895874,0.12542209029197693,0.12542209029197693,Transport,1 +loop_seattle/H/long,TTM-R2-Zeroshot,74.29878997802734,74.29878997802734,5.402324199676514,1.114170789718628,0.17675289511680603,0.1240486353635788,44.56683349609375,8.619674682617188,0.15404511988162994,0.09654676914215088,0.09654677659273148,Transport,1 +sz_taxi/15T/long,TTM-R2-Zeroshot,16.31110191345215,16.31110191345215,2.8116378784179688,0.5310652256011963,8224755941376.0,0.40968406200408936,21.242610931396484,4.038700580596924,0.3738149106502533,0.26024019718170166,0.26024019718170166,Transport,1 +m_dense/H/long,TTM-R2-Zeroshot,62940.5234375,62940.5234375,128.40859985351562,1.0597881078720093,0.470628559589386,0.2987874746322632,42.39152526855469,250.87950134277344,0.4343095123767853,0.22229425609111786,0.22229427099227905,Transport,1 +ett1/15T/long,TTM-R2-Zeroshot,10.124308586120605,10.124308586120605,1.8443189859390259,1.1956290006637573,0.7989325523376465,0.45920833945274353,47.82516098022461,3.1818718910217285,0.6064679026603699,0.3515290319919586,0.3515290319919586,Energy,7 +ett1/H/long,TTM-R2-Zeroshot,142.861328125,142.861328125,7.135627269744873,1.355435848236084,3003883454464.0,0.4777284562587738,54.217430114746094,11.952461242675781,0.5723775029182434,0.3417097330093384,0.34170976281166077,Energy,7 +ett2/15T/long,TTM-R2-Zeroshot,13.01650333404541,13.01650333404541,2.3665223121643066,0.986086905002594,0.14943784475326538,0.1789281815290451,39.443477630615234,3.6078391075134277,0.1922682821750641,0.12611626088619232,0.12611627578735352,Energy,7 +ett2/H/long,TTM-R2-Zeroshot,278.0689392089844,278.0689392089844,10.509333610534668,1.131842017173767,0.19035466015338898,0.1890520304441452,45.273681640625,16.675399780273438,0.2283567637205124,0.1439172327518463,0.1439172327518463,Energy,7 +jena_weather/10T/long,TTM-R2-Zeroshot,1487.92626953125,1487.92626953125,11.109465599060059,0.6489807367324829,0.8593838214874268,0.6524072885513306,25.959230422973633,38.573646545410156,0.23603875935077667,0.06798072904348373,0.06798072159290314,Nature,21 +jena_weather/H/long,TTM-R2-Zeroshot,1266.5963134765625,1266.5963134765625,13.908403396606445,1.0716230869293213,3.8741116523742676,0.6689540147781372,42.864925384521484,35.589271545410156,0.21428684890270233,0.08374399691820145,0.08374399691820145,Nature,21 +bitbrains_fast_storage/5T/long,TTM-R2-Zeroshot,3219523.0,3219523.0,355.3838195800781,1.1635642051696777,4.895505428314209,0.8371726274490356,46.54256820678711,1794.302978515625,4.741806983947754,0.9391733407974243,0.9391733407974243,Web/CloudOps,2 +bitbrains_rnd/5T/long,TTM-R2-Zeroshot,1991291.25,1991291.25,203.39283752441406,3.782886266708374,2.9021401405334473,0.7515886425971985,151.31544494628906,1411.1312255859375,5.405725955963135,0.7791521549224854,0.7791520953178406,Web/CloudOps,2 +bizitobs_application/10S/long,TTM-R2-Zeroshot,41283780.0,41283780.0,3723.089111328125,9.590826034545898,0.13113968074321747,0.13875478506088257,383.633056640625,6425.24560546875,0.24779604375362396,0.14358466863632202,0.14358466863632202,Web/CloudOps,2 +bizitobs_service/10S/long,TTM-R2-Zeroshot,267102.25,267102.25,188.78057861328125,5.33817720413208,0.43358832597732544,0.21543104946613312,213.52708435058594,516.8193359375,0.38287630677223206,0.13985468447208405,0.13985469937324524,Web/CloudOps,2 +bizitobs_l2c/5T/long,TTM-R2-Zeroshot,358.9239196777344,358.9239196777344,13.98556137084961,1.316333532333374,0.9028239250183105,1.1863360404968262,52.653343200683594,18.945287704467773,1.0637495517730713,0.7852683663368225,0.7852683663368225,Web/CloudOps,7 +bizitobs_l2c/H/long,TTM-R2-Zeroshot,248.46641540527344,248.46641540527344,12.290800094604492,1.2374670505523682,1.677781105041504,0.9531240463256836,49.498680114746094,15.7628173828125,0.9628384113311768,0.7507575750350952,0.7507575154304504,Web/CloudOps,7 diff --git a/results/TTM-R1-Pretrained/config.json b/results/TTM-R2-Zeroshot/config.json similarity index 54% rename from results/TTM-R1-Pretrained/config.json rename to results/TTM-R2-Zeroshot/config.json index a029ac39c28f4fa5564f7bb9363fb8633268fd53..b7223b7139389853ddb6d52225051d3655915940 100644 --- a/results/TTM-R1-Pretrained/config.json +++ b/results/TTM-R2-Zeroshot/config.json @@ -1,8 +1,6 @@ { - "model": "TTM-R1-Pretrained", + "model": "TTM-R2-Zeroshot", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r1", - "org": "IBM Research", - "testdata_leakage": "Yes" + "model_link": "https://huggingface.co/ibm-granite/granite-timeseries-ttm-r2" } \ No newline at end of file diff --git a/results/TiRex/all_results.csv b/results/TiRex/all_results.csv deleted file mode 100644 index a110ab667f046a65583cf003368f543cfd047202..0000000000000000000000000000000000000000 --- a/results/TiRex/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -ett2/W/short,TiRex,2970755.4285714286,2970755.4285714286,1191.1729910714287,0.7951615508153032,0.12047226088387626,0.12490231915098424,9.391792283581983,1723.5879520846706,0.14458260496610956,0.09992115215592962,0.08619325615838942,Energy,7 -us_births/D/short,TiRex,198695.17333333334,198695.17333333334,277.6393229166667,0.4087953327106353,0.026502461433410646,0.02618951741778194,4.121146512186256,445.752367725998,0.041785667402477786,0.02602643361027811,0.02144543864264523,Healthcare,1 -ett1/W/short,TiRex,2017846.5714285714,2017846.5714285714,1047.0655691964287,1.7735317650085964,0.7534299577985492,0.6073398381716639,9.44359412466487,1420.5092648161685,0.5652545408288483,0.41665238108138475,0.3118608713437383,Energy,7 -electricity/W/short,TiRex,30852364299.07027,30852364299.07027,25863.317117117116,1.4432247256751884,0.11088788617153064,0.09404199740935446,11.231642624958917,175648.41103485756,0.4007717098160171,0.059011554738652834,0.0455652082044092,Energy,1 -electricity/D/short,TiRex,1213819835.438126,1213819835.438126,4221.103567567568,1.4217617878240947,0.32041209530492054,0.0956590571542237,11.667507708831486,34839.91727082781,0.5725355444534372,0.06936674993988069,0.0546806308741351,Energy,1 -m4_daily/D/short,TiRex,352146.5084997803,352147.1400858427,163.873939639731,3.0884185495920518,0.03598843847550224,0.02828467852058975,25.475749423301398,593.4193361357383,0.09166688801712522,0.025313994942767218,0.02014480507468422,Econ/Fin,1 -solar/10T/short,TiRex,28.502920183242093,28.502392696928222,2.4493746198296837,1.0861728909884,5.324996950413727,1.560478734805044,5.655357890626215,5.338812619229307,1.5530600127763068,0.7125228116576577,0.5501821186575035,Energy,1 -solar/10T/medium,TiRex,24.475504589692544,24.47547417606724,2.0546816246405664,0.898124854182947,4.191688743109356,1.4630258088775665,5.471842852484762,4.94727243940462,1.0978193594934198,0.4559419948575269,0.35986911998935855,Energy,1 -solar/10T/long,TiRex,20.45889693582725,20.45862844054136,1.853305788805885,0.808380482237102,3.5351544056009856,1.4545276600227004,6.135346870960489,4.523151217439811,0.9776066748556103,0.40056235632791654,0.3236849399681189,Energy,1 -saugeen/W/short,TiRex,980.3181640625,980.3181640625,14.424923706054688,1.1701196222826737,0.33849353790283204,0.37586861021545986,9.915952281235056,31.31003296169616,0.9451193800342205,0.43542831675667387,0.3504785536560232,Nature,1 -jena_weather/H/short,TiRex,1143.0846908939013,1143.1007727652463,8.320424270311195,0.5178978120879559,1.1269619418791328,0.6042359076182757,4.441072280037575,33.80953550248659,0.20725027608031174,0.05100366513466286,0.041086475887843726,Nature,21 -jena_weather/H/medium,TiRex,1495.4136904761904,1495.4431547619047,10.78253658234127,0.843056650981118,2.1754567314948985,0.681451210878789,5.899888944022455,38.67057913292986,0.23663213206127676,0.06598025366357667,0.052327869559684104,Nature,21 -jena_weather/H/long,TiRex,1196.5870370370371,1196.6232804232804,11.13087177579365,0.9864046950965887,2.342650521253022,0.6542132325312259,7.47200280190376,34.59171919747611,0.2082804746225289,0.06702018026886516,0.056343377351575734,Nature,21 -bitbrains_rnd/H/short,TiRex,2048763.5589275819,2048790.5327138656,159.26088508588722,5.860060925650588,1.5656282883805936,0.5442435066269552,200.24572349988395,1431.3502572492807,6.264289314163356,0.6970035849401268,0.6489405576255898,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,TiRex,2008810.6776013824,2008568.635655564,166.75446872723984,0.6919793318300045,1.8793996079851465,0.7319613124109596,13.867300993590899,1417.3251841413748,4.450050606426177,0.5235677972754552,0.4038768731137463,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TiRex,3476929.3255789443,3476672.290825582,297.01453390974257,0.9948917362202117,4.86948803609243,0.809961664948777,20.91983732708298,1864.652601848115,5.665329928062005,0.9024122382692038,0.6185547825645021,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,TiRex,4256184.889914907,4256092.433378107,361.13987854969605,0.9188876586260262,5.208817204259282,0.8313803980282465,16.17992154383552,2063.05232360086,5.452031349377815,0.954384877610294,0.6569560283439435,Web/CloudOps,2 -jena_weather/10T/short,TiRex,955.7613095238095,955.744246031746,6.360694521949405,0.29892347881070985,0.3630313680394785,0.5639724042139862,2.3487999296931377,30.915389525668434,0.1920233158734188,0.0395078849758147,0.031041611124691042,Nature,21 -jena_weather/10T/medium,TiRex,1572.2976911976912,1572.3064935064936,9.909267902236651,0.6029820936185427,0.7524464410851938,0.6565383306064191,6.196016772731761,39.65220915910854,0.24334492974194863,0.06081300771384657,0.05072731161153592,Nature,21 -jena_weather/10T/long,TiRex,1599.58082010582,1599.6107142857143,10.28968357308201,0.6368085925769856,0.6994138971210838,0.6511399236905666,6.574045717033048,39.994759908090714,0.2447347938377805,0.06296433817083112,0.05224721816048224,Nature,21 -ett2/15T/short,TiRex,8.233495512462797,8.233494931175596,1.7407566615513392,0.7414234884954074,0.10552842309305262,0.12449517553331638,5.744474498139059,2.869406822404728,0.13622094964774278,0.08263991138887421,0.06602884205149234,Energy,7 -ett2/15T/medium,TiRex,12.653265128968254,12.653265128968254,2.2014876302083333,0.9025407012173716,0.13810109007197344,0.16254235312045234,6.733541118904856,3.557142832241665,0.1895665804256405,0.11732126079540803,0.09232424988203031,Energy,7 -ett2/15T/long,TiRex,12.413377976190477,12.413373015873017,2.2431772383432538,0.9189148885047347,0.1406318785867057,0.16500199573199112,7.313302015092101,3.5232624052418346,0.1877610311427991,0.11954297547658138,0.09570045392972987,Energy,7 -restaurant/D/short,TiRex,137.95177220509947,137.95177220509947,7.081024680115812,0.6783902483205515,0.6653670459454095,0.3908695374701871,4.703543674212318,11.745287233826998,0.5405453320578112,0.32588516234826864,0.25526741172525347,Sales,1 -ett2/D/short,TiRex,105545.44761904761,105545.44761904761,193.63712797619047,1.2754186956500064,0.4143649631076389,0.1323978974750113,11.653837258140046,324.87758866848236,0.1859360156010212,0.11082363728406602,0.09102368074327229,Energy,7 -loop_seattle/D/short,TiRex,18.044462719298245,18.044462719298245,2.905175680469556,0.8750021883154138,0.05374959939773607,0.05361570835959038,6.948766493276501,4.247877436944037,0.07591525106368158,0.05191937489741041,0.04228230930538582,Transport,1 -covid_deaths/D/short,TiRex,373414.6245614035,373414.2395989975,108.57943295739348,38.943546228234936,0.10323845337741598,0.3841515645526529,680.3978285893494,611.0766110410409,0.22992054874497342,0.04085354005850716,0.03461859878597194,Healthcare,1 -ett1/D/short,TiRex,41646.368253968256,41646.368253968256,137.80528273809523,1.6921807658166974,0.950717308407738,0.49307234011488266,10.613040717938787,204.07441842124226,0.5360082292010093,0.3619501461595356,0.27706017048595993,Energy,7 -m4_quarterly/Q/short,TiRex,1771182.8773333333,1771182.872,552.947228515625,1.1608850923426102,0.11211042976379394,0.10079044137883154,8.894223332346508,1330.857947841667,0.22276218482177013,0.09255362896922602,0.07387091123638767,Econ/Fin,1 -bizitobs_application/10S/short,TiRex,738717.3688888889,738717.3688888889,423.90506944444445,1.2804317857077658,0.01811051262749566,0.018391086841910036,8.330737403853586,859.4866891865685,0.03314696613544618,0.016348324131485046,0.01312588339289446,Web/CloudOps,2 -bizitobs_application/10S/medium,TiRex,4529530.453333333,4529530.453333333,1178.5428125,2.784767452562252,0.03911426862080892,0.03998799924342233,33.076144206566916,2128.2693563863886,0.08357087375430715,0.04627790758812313,0.04031081739897931,Web/CloudOps,2 -bizitobs_application/10S/long,TiRex,6299779.982222222,6299779.982222222,1505.6433333333334,3.6861970274711395,0.05181477864583333,0.053086275365059755,54.66538418832557,2509.9362506291313,0.09679820868485037,0.058066645138266836,0.05224089460064261,Web/CloudOps,2 -us_births/W/short,TiRex,2324079.8571428573,2324079.8571428573,1180.465122767857,1.073549699546591,0.016011882041181837,0.016022902176482102,6.117694243300344,1524.4933116097484,0.02069484722178624,0.016024698291744036,0.012616317268655659,Healthcare,1 -m4_yearly/A/short,TiRex,3703997.97011114,3704102.759177041,910.7333434897421,3.4264162037569497,0.16704781227513643,0.15304742528910717,28.853171650185807,1924.5773484355311,0.3086148988992169,0.14604031319068989,0.1175523898891408,Econ/Fin,1 -m4_hourly/H/short,TiRex,938324.921095008,938324.921095008,189.94847020933977,0.7025711135679693,0.08703987993863854,0.08130010608814964,5.195918667910062,968.67173030651,0.13224507240404,0.025932158862447968,0.020135420880335434,Econ/Fin,1 -m4_monthly/M/short,TiRex,1832372.7427777778,1832373.308925926,556.7997155671296,0.9219984530483495,0.16112972446724222,0.13202980477432383,7.451806452411567,1353.651632724527,0.281335280794089,0.11572209609784116,0.09217102776527568,Econ/Fin,1 -loop_seattle/5T/short,TiRex,44.22454865478264,44.22454462356489,3.596470366518318,0.5680875434881646,0.1026890474207261,0.07615650715947511,4.6168555779027765,6.650154032410275,0.11403546326295837,0.06167152856289145,0.04900557512478326,Transport,1 -loop_seattle/5T/medium,TiRex,96.03669730392157,96.03488615841073,5.819658253515222,0.9177510763888326,0.18745778023766044,0.13110517982803416,6.670753299702922,9.799831493649346,0.17435913471462647,0.10354367603925826,0.08112071743391197,Transport,1 -loop_seattle/5T/long,TiRex,110.97640752207316,110.97362802430914,6.226973469212246,0.9796835296168676,0.19670561270926498,0.13928461288376484,7.156312289038201,10.534534043899292,0.18625271632814472,0.11009416442255689,0.0864468549405967,Transport,1 -electricity/H/short,TiRex,1367405.391171171,1367405.121981982,156.03959881756757,0.8707681506884576,0.18112333372136932,0.11926849263086182,9.826993254681057,1169.3611038388317,0.5520219496243092,0.07366183403492156,0.06033894832010146,Energy,1 -electricity/H/medium,TiRex,6900100.508828829,6900100.458378378,251.7969552364865,1.0856022248906534,0.23433858757270806,0.13093492386108269,9.070016625741426,2626.804238771673,1.025328258818514,0.09828464941458892,0.07863122671769769,Energy,1 -electricity/H/long,TiRex,10907714.224816816,10907714.224816816,295.3798018018018,1.2219275984803868,0.2934422128216852,0.14457639723100973,10.279550945088932,3302.682882872168,1.3025771521579752,0.11649770646504733,0.0931731605958472,Energy,1 -bizitobs_service/10S/short,TiRex,4127.391746031746,4127.38708994709,22.444016203703704,0.8298255932339903,0.05795688528232473,0.047715458740892495,6.248365249669233,64.24477991270378,0.04759458681332036,0.016627244720866664,0.013071853494199915,Web/CloudOps,2 -bizitobs_service/10S/medium,TiRex,56194.473650793654,56194.5346031746,52.11404761904762,1.249885440308283,0.07660258459666419,0.06985078736539953,11.844491186249495,237.05373578746583,0.17841222775094331,0.039222260311346034,0.030393171961573286,Web/CloudOps,2 -bizitobs_service/10S/long,TiRex,132118.55238095237,132118.55238095237,78.44518518518518,1.5091341967314003,0.08946221891534392,0.08510697181280047,21.778461933795086,363.4811582199996,0.2692784622099258,0.05811470101472029,0.053481070917666546,Web/CloudOps,2 -electricity/15T/short,TiRex,130853.78038288288,130853.75103603603,51.25439963400901,0.9563487965369476,0.16293669922605813,0.16236830647350214,9.86137974391925,361.73717030861354,0.7130552661994293,0.10103252464694247,0.08297902700206361,Energy,1 -electricity/15T/medium,TiRex,235643.2907027027,235643.28317117118,53.323409487612615,0.8351909950386581,0.1254375327689586,0.13226535082303906,7.130713369062645,485.4310359903894,0.8364465570838768,0.0918816041230012,0.07387402719293987,Energy,1 -electricity/15T/long,TiRex,312870.1748048048,312870.1694234234,58.47297250375375,0.8827297695387359,0.13104820071662698,0.13432397019987086,7.216872314805137,559.347990793571,0.8824400060664994,0.09224828024810308,0.07380481100671869,Energy,1 -car_parts/M/short,TiRex,1.3893600226683938,1.3890522763883353,0.45667299787821136,0.8390237711771125,0.8388399471610455,1.9012728228544467,15.784740190145676,1.178711170163579,2.82642847334223,1.0950550031950612,0.9914943977909808,Sales,1 -bizitobs_l2c/H/short,TiRex,70.99114893353175,70.99086216517857,5.090847923642113,0.5104589157273894,0.3990817721799123,0.6755744648839423,3.034635581184298,8.425624542639659,0.4541627412456651,0.2744097409711727,0.2152608281287139,Web/CloudOps,7 -bizitobs_l2c/H/medium,TiRex,84.78590959821429,84.78590959821429,5.361232793898809,0.5400271541988042,0.49330897552890346,0.8263997905076318,3.693762055013273,9.207926454865628,0.5575539328792452,0.3246307889155579,0.25655223090506285,Web/CloudOps,7 -bizitobs_l2c/H/long,TiRex,94.13964533730159,94.13963913690476,5.8163252573164685,0.6064676429114819,0.6829458822884166,0.8737612742536494,4.440192010382922,9.702558700533668,0.5926602887374812,0.3552779336653415,0.27986165295978727,Web/CloudOps,7 -ett1/15T/short,TiRex,4.622242373511905,4.622242373511905,1.0960779825846354,0.6959876476357297,0.45684000166347155,0.24271526432693455,4.693199494468589,2.1499400860284235,0.4064281678522885,0.207204363126246,0.159733585310961,Energy,7 -ett1/15T/medium,TiRex,10.593108878968254,10.593108878968254,1.653461216517857,1.0316980353415084,0.7499513844708642,0.39905972684047286,7.373101838032216,3.254705651663181,0.6203501152732254,0.3151513427158939,0.2497189363448471,Energy,7 -ett1/15T/long,TiRex,9.435853794642858,9.435853794642858,1.6277630518353174,1.0367942068903993,0.7892704750591463,0.39904130003705723,7.445283162141806,3.071783487591998,0.5854849699383163,0.3102532471185309,0.24510896830975445,Energy,7 -loop_seattle/H/short,TiRex,59.64443216691109,59.644446594427244,4.176846127994134,0.848897675888724,0.11319984244171072,0.09573628501359198,6.726118036892866,7.722980782503028,0.13670461345321763,0.07393442395643861,0.05928628411328066,Transport,1 -loop_seattle/H/medium,TiRex,71.77512899896801,71.77401315789474,4.5855629998710015,0.9384201418079954,0.11573108401460913,0.10507816313456297,7.009045827833744,8.472020361104429,0.14974199463387888,0.0810493035725523,0.06475713554605739,Transport,1 -loop_seattle/H/long,TiRex,61.546013931888545,61.54511953904369,4.311520951582387,0.9010199992328466,0.1217122143882224,0.0995874782179967,6.9458186211258175,7.845126763277222,0.14020291839114538,0.07705265171061433,0.06140881481067864,Transport,1 -sz_taxi/15T/short,TiRex,16.698977739383015,16.698977739383015,2.72696779877566,0.5439835291727317,1023556602788.8574,0.3935241049789677,3.87754982260641,4.086438270594946,0.3820994552732309,0.25498315194373905,0.2004169647807729,Transport,1 -sz_taxi/15T/medium,TiRex,16.37261451655983,16.37261451655983,2.7553608690571583,0.5372561300569447,9807495414908.281,0.40515595462526294,3.8386002377658466,4.0463087520059355,0.376576568350883,0.2564323199813817,0.20216258843846632,Transport,1 -sz_taxi/15T/long,TiRex,15.571356392450143,15.571356392450143,2.6923805867165242,0.5082058348907014,6171909608023.813,0.39725447957486887,3.731951076221384,3.9460558019939533,0.36523989585549055,0.2492019511226351,0.19701403584237281,Transport,1 -jena_weather/D/short,TiRex,375.4766617063492,375.47542162698414,9.665390935019841,1.0190211400312152,0.6258229418656144,0.4562943708470215,6.772722846660526,19.377220174894777,0.1166825360507289,0.05820145077780565,0.04600165434577807,Nature,21 -bitbrains_rnd/5T/short,TiRex,1787201.279275027,1787028.3916741398,123.70152230731108,1.66458495188904,0.9086322373516892,0.6527628732067667,53.26968382891597,1336.8624758272733,5.470055298582111,0.5061509166236827,0.40386990397392,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TiRex,2273116.790116314,2272792.112927242,144.3054361976397,4.393507066323417,0.33622587289557354,0.7275127606770119,160.27516041942047,1507.6859056568492,6.343602336808971,0.6071664521389607,0.5848617020384329,Web/CloudOps,2 -bitbrains_rnd/5T/long,TiRex,3004623.0208733412,2999820.239311174,198.0148330075732,3.3410285917779414,2.155750359261086,0.7007474711246955,116.64238815177053,1733.384844999327,6.640207104507196,0.7585502462006376,0.6363780706228606,Web/CloudOps,2 -hospital/M/short,TiRex,2793.6981747066493,2793.6981747066493,17.910358268144286,0.7632120945428317,0.19397523060201408,0.17381936106408727,5.3798056758820785,52.855446026938885,0.19187754286129094,0.06501875955233447,0.05186165892678181,Healthcare,1 -sz_taxi/H/short,TiRex,7.193810096153846,7.193774622729701,1.8476432082999465,0.5634707407815503,1.13784424871461,0.29556207742962726,3.987671086849603,2.6821279045104927,0.24985676541547075,0.17211936645204143,0.1354964977914786,Transport,1 -kdd_cup_2018/D/short,TiRex,3086.2583948400825,3086.2583948400825,22.04650824522907,1.227520474049185,0.5187516363704369,0.48340876978501146,9.124463577853062,55.55410331235743,1.2444708649943474,0.49386517917106465,0.38932207263650737,Nature,1 -saugeen/D/short,TiRex,1431.2284375,1431.2284375,14.327062174479167,3.183650895184356,0.3011397043863932,0.36107309330551013,35.80425145772081,37.8315798969591,1.2253738868197177,0.46405695747484355,0.3915667437974555,Nature,1 -saugeen/M/short,TiRex,443.32835751488096,443.32835751488096,13.166431245349703,0.7714405341577629,0.4060617174421038,0.377133538754412,6.328869702165946,21.055364103118258,0.6321451958763121,0.3952957649091877,0.3193372884079143,Nature,1 -bizitobs_l2c/5T/short,TiRex,23.29142020089286,23.291294642857142,2.8427743094308036,0.2950148348875157,0.16410305787077337,0.21411017480001235,2.665957881676496,4.826118544015766,0.16574844545513134,0.09763237646787504,0.07935712551032867,Web/CloudOps,7 -bizitobs_l2c/5T/medium,TiRex,149.365625,149.36518920068028,8.320692495748299,0.8304317013912187,0.6097517104691572,0.9359692985376182,8.48204455223797,12.22152302292967,0.6430318362855088,0.43779078635859736,0.3561055383893783,Web/CloudOps,7 -bizitobs_l2c/5T/long,TiRex,364.4951984126984,364.50353174603174,12.780913938492063,1.1912677086014891,0.9855804082486288,1.2020434491178862,10.607961835773741,19.09175734218038,1.0719736325349294,0.7176292101456672,0.5944724642883783,Web/CloudOps,7 -bitbrains_fast_storage/H/short,TiRex,3156314.308168179,3156309.7808052506,332.0446880703288,1.0706374264834246,4.060869335398377,0.5765844060439714,18.71855724312543,1776.6018991794922,5.06389238177909,0.9464351958117964,0.6983844225501978,Web/CloudOps,2 -m4_weekly/W/short,TiRex,231988.41225626742,231988.54938932933,244.26357938718664,1.8847096799902023,0.06071092578208699,0.05959688318113693,14.485281925687627,481.65175412975236,0.08774948528187945,0.044501038728826076,0.035874184457216726,Econ/Fin,1 -solar/D/short,TiRex,145153.38588807787,145153.38588807787,249.1293491484185,0.9725997201015677,1.1614453837819343,0.4160351101929897,5.350574793205692,380.99000759610203,0.550400794112742,0.35990705497306863,0.28302893873727863,Energy,1 -kdd_cup_2018/H/short,TiRex,3734.3552116563474,3734.3544393640877,16.067013768522948,0.6573376093992103,0.5341818990021702,0.3612209449004724,5.5945063610035515,61.109370898875625,1.2791298108116052,0.33631169785805065,0.2727573589285206,Nature,1 -kdd_cup_2018/H/medium,TiRex,4371.918119468347,4371.918119468347,19.417421772498496,0.8013862851782936,0.6905952801537807,0.4287402981106577,7.354823968658788,66.1204818454036,1.3840214347956872,0.406441804287525,0.3318153228232925,Nature,1 -kdd_cup_2018/H/long,TiRex,3334.66544020717,3334.66544020717,17.396844112701473,0.7404391263966854,0.6235727405578834,0.4439028364443244,6.282471668212528,57.746562150548584,1.3549676958108359,0.4082002616937342,0.3303145841040082,Nature,1 -solar/H/short,TiRex,616.985585542323,616.9818318606735,9.360162852958124,0.722081080447196,3.744779546107227,1.379546681345698,4.109200815673717,24.839194542946093,0.9155810090020404,0.3450187297546067,0.27041064367413514,Energy,1 -solar/H/medium,TiRex,651.379805352798,651.376399026764,9.64139389446472,0.7404448975946352,4.031936258868576,1.3809051163705899,3.820399028227576,25.52214343178876,0.9193671753873288,0.3473055112020982,0.26547437759873155,Energy,1 -solar/H/long,TiRex,601.8113544201135,601.8090835360908,9.679600060827251,0.7375094210395706,4.478540360308095,1.376801557803167,4.131586995039765,24.531843681633745,0.8505637284569062,0.3356093746787083,0.25521175168383625,Energy,1 -hierarchical_sales/W/short,TiRex,460.8050847457627,460.80495233050846,8.89812715174788,0.7215428881537215,0.5324043064361232,0.45690591798959845,6.686166506803991,21.466371019475154,0.9855315934871568,0.4085173699344657,0.34656412828019534,Sales,1 -solar/W/short,TiRex,1911535.8832116788,1911535.8832116788,1112.8932481751824,1.220791526822605,0.2539241261725878,0.21650740813671626,7.576028967979736,1382.5830474917877,0.28221976627519724,0.22716933565693143,0.1634696132375158,Energy,1 -m_dense/H/short,TiRex,43082.39555555556,43082.08,89.28542534722222,0.7855317186977097,0.29099332361106267,0.21555866677802366,8.158976661388401,207.56299177732902,0.3681650260305369,0.15837009606400382,0.1297492387233034,Transport,1 -m_dense/H/medium,TiRex,37874.40888888889,37874.40888888889,83.60451388888889,0.7278244063518289,0.2678940396684029,0.20557386132251101,7.452901570253836,194.61348588648445,0.33860287864095767,0.1454612918586116,0.12004999713502766,Transport,1 -m_dense/H/long,TiRex,36624.70320987654,36624.70320987654,84.35819444444445,0.7255334906140245,0.26589403164571657,0.20522170827450098,7.165966496068227,191.37581667984213,0.3312998291627926,0.14603650499206397,0.12176832917587532,Transport,1 -temperature_rain/D/short,TiRex,183.41532464323416,183.41443051929193,5.790785418679601,1.3395020097724115,18.177183863449983,1.537019149017625,18.99099487278608,13.543091399057829,1.5943799080297552,0.6817285397554603,0.5498695682400049,Nature,1 -m_dense/D/short,TiRex,8804.717777777778,8804.717777777778,45.50862268518519,0.691536669463316,0.12603228139912667,0.1006767280255517,8.000075278325918,93.83345766717636,0.16243959500844413,0.07878215747515281,0.06625075731674779,Transport,1 -us_births/M/short,TiRex,62625856.0,62625856.0,6349.315104166667,0.7159693729754744,0.019578362504641216,0.01986670892584133,4.345417750162361,7913.649979623815,0.024579742882127633,0.019720929424458168,0.01504508163966473,Healthcare,1 -ett1/H/short,TiRex,105.76156994047619,105.76156994047619,5.020936221168155,0.8297685988720997,0.5058317638578869,0.26251774626967744,5.714507205755161,10.284044434971884,0.47999842754855626,0.23434763494279798,0.17953672203267212,Energy,7 -ett1/H/medium,TiRex,135.87265625,135.87265625,6.665461658296131,1.2306769081846543,4109545449838.5034,0.43185633434681464,8.497057122201332,11.656442692777244,0.5583159734840867,0.31925981301125383,0.25135181626036995,Energy,7 -ett1/H/long,TiRex,145.97402447089948,145.97402447089948,6.9211402529761905,1.3107871863897707,3721692708222.468,0.46172678397956046,8.802213903829887,12.081971050739174,0.5785794258873455,0.33143841656596007,0.2594972280413116,Energy,7 -ett2/H/short,TiRex,113.89380580357142,113.89380580357142,6.539599609375,0.7465717515528231,0.12272715230939782,0.1114306000229953,5.237620289866135,10.672104094487246,0.135351974016252,0.0829403188507172,0.06620842160810164,Energy,7 -ett2/H/medium,TiRex,249.7829799107143,249.7829799107143,9.71401599702381,1.0479167209902442,0.1759455630774431,0.1725140310485191,7.648975041042203,15.804524033032893,0.21297612485420145,0.13090261240982398,0.1057800752345261,Energy,7 -ett2/H/long,TiRex,282.7609126984127,282.7609126984127,10.429313409391535,1.13746330213241,0.1940303823650421,0.19246527035478825,7.845449497189904,16.815496207320578,0.2302752783975551,0.14282142014919874,0.1142258615825124,Energy,7 -hierarchical_sales/D/short,TiRex,28.528165047540305,28.52807978503514,2.2903411146134767,0.7408686373187872,0.6301741816205733,1.0608770896585544,6.907580230191971,5.341176373004387,1.6386137612775828,0.7026512899656305,0.5681087377983529,Sales,1 diff --git a/results/TiRex/config.json b/results/TiRex/config.json deleted file mode 100644 index 12ad54479cca439262900be316fce4ee825ec836..0000000000000000000000000000000000000000 --- a/results/TiRex/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "TiRex", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/NX-AI/TiRex", - "org": "NXAI", - "testdata_leakage": "Yes" -} \ No newline at end of file diff --git a/results/TimeCopilot/all_results.csv b/results/TimeCopilot/all_results.csv deleted file mode 100644 index e4e55303d2ad86a06599d8b55a1d2b8ae2043d8e..0000000000000000000000000000000000000000 --- a/results/TimeCopilot/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,TimeCopilot,4208871.815741128,4208871.815741128,1010.910936873178,3.888188868922098,0.1835835463803872,0.1687864003554149,32.74533867803543,2051.553512765662,0.3289761158104224,0.1621042548358817,0.1294591256953563,Econ/Fin,1 -m4_quarterly/Q/short,TimeCopilot,1861908.9716377887,1861908.9716377887,595.6596766636371,1.3212731592043376,0.1188092252840994,0.1072062181473107,9.58320770007134,1364.517853176641,0.2283962598449484,0.0997029404735933,0.0787399603025537,Econ/Fin,1 -m4_monthly/M/short,TimeCopilot,1941666.4759626216,1941666.4759626216,587.4193242920703,0.992934410565318,0.1705416317550141,0.1387560906795214,8.120019219963858,1393.436929309189,0.28960402974021,0.1220859013307427,0.0967459800559417,Econ/Fin,1 -m4_weekly/W/short,TimeCopilot,304646.4926513002,304646.4926513002,272.4440725607307,2.1001147377185,0.0655132776512277,0.0653989090465896,14.953147197758588,551.9479075522438,0.1005563550321304,0.0496350878623871,0.0388426840774093,Econ/Fin,1 -m4_daily/D/short,TimeCopilot,339660.79501941416,339660.79501941416,180.97843748165687,3.410489917042385,0.0395085010486282,0.0310040592829509,29.307193347701965,582.8042510306648,0.0900271502511467,0.0279561670227446,0.0223116797981767,Econ/Fin,1 -m4_hourly/H/short,TimeCopilot,524954.10817021,524954.10817021,171.82326690154375,0.7706233412968566,0.0912051413921246,0.0818318907111256,6.3723306183586015,724.5371682461915,0.0989153159696496,0.0234576685384462,0.0193328475966307,Econ/Fin,1 -electricity/15T/short,TimeCopilot,151009.2185274338,151009.2185274338,56.61546663541818,1.0584644196925137,0.1673964987414938,0.1757201307820008,11.40825023563569,388.5990459682496,0.766005321169062,0.111600244913476,0.0924337203744175,Energy,1 -electricity/H/short,TimeCopilot,1625470.9663607306,1625470.9663607306,173.63875595318578,0.9445283559815032,0.1850095202789404,0.1266785670399844,10.965349810240992,1274.939593220295,0.6018625391814524,0.0819698934083031,0.0668268918397401,Energy,1 -electricity/D/short,TimeCopilot,1524727850.2334847,1524727850.2334847,4499.245042665888,1.440678272205953,0.4645618045851292,0.0966410120059837,12.834644251743407,39047.76370336059,0.6416844660376455,0.0739375415890234,0.0591240047703936,Energy,1 -electricity/W/short,TimeCopilot,107538242197.20187,107538242197.20187,37600.97314072858,1.731413766670026,0.2579633973676443,0.1083506011356826,14.50780880240442,327930.23983341624,0.7482285899661988,0.0857930123453573,0.0667798023145311,Energy,1 -solar/10T/short,TimeCopilot,27.24892377764872,27.24892377764872,2.378006966259878,1.0505715113645349,3.4210851134341405,1.5918319064210218,4.508536801063111,5.2200501700317705,1.5185120291659178,0.6917619689628381,0.5077658379230758,Energy,1 -solar/H/short,TimeCopilot,659.9567727325225,659.9567727325225,10.368725871971089,0.7996956660754997,3.3920387408800736,1.398975364381254,7.548931352585399,25.689623834001974,0.9469280785839052,0.382194684155454,0.3117904456590773,Energy,1 -solar/D/short,TimeCopilot,125866.4929895851,125866.4929895851,253.13198455998497,0.9872057092459324,1.0797759311803388,0.4284607224409668,6.238664324840403,354.7766804478348,0.5125314647574201,0.3656894998276473,0.2872018924750267,Energy,1 -solar/W/short,TimeCopilot,1348124.5480815065,1348124.5480815065,934.246138134142,1.0178505807998526,0.2077722813767061,0.1845538236489491,5.979179324857676,1161.0876573633475,0.2370070194919315,0.1907029940993756,0.1484664387553441,Energy,1 -hospital/M/short,TimeCopilot,2908.3396358532555,2908.3396358532555,18.55949724881041,0.7798974698104213,0.1918066912745793,0.1761693015233185,5.251147372408516,53.92902405804555,0.195774880414239,0.0673752847914217,0.0529398612868036,Healthcare,1 -covid_deaths/D/short,TimeCopilot,683203.3235435126,683203.3235435126,123.05835295103329,34.38262143232607,0.0891866497388241,0.3651155130761072,585.3302636000165,826.5611432577222,0.3109976526598306,0.046301304168642,0.0394855918185296,Healthcare,1 -us_births/D/short,TimeCopilot,195114.15537626744,195114.15537626744,274.3993636067708,0.4038139053701383,0.0262962687231841,0.025846322311671,4.150256571072779,441.7172799158614,0.0414074106631264,0.0257227137157287,0.0213319129099562,Healthcare,1 -us_births/M/short,TimeCopilot,81372043.76497395,81372043.76497395,7473.096354166667,0.8427784924069014,0.02294957067664,0.0233201502709241,4.542482907186801,9020.64541842622,0.0280180631676483,0.0232113863251144,0.0171916281354551,Healthcare,1 -us_births/W/short,TimeCopilot,3309825.666670663,3309825.666670663,1350.247907366071,1.2273411952942834,0.0180909590033902,0.0182124756432147,9.375108623321603,1819.292628103204,0.0246967190368072,0.0183294829447113,0.0146125351543498,Healthcare,1 -saugeen/D/short,TimeCopilot,1350.1767921390351,1350.1767921390351,13.578996993700663,3.017412246735102,0.2945186519735365,0.3381408161271264,44.3771975879136,36.74475189927176,1.190171269021097,0.4398269480313645,0.3758806057524924,Nature,1 -saugeen/M/short,TimeCopilot,402.1315480732868,402.1315480732868,12.827814573333375,0.7516726001427814,0.3835155888529804,0.3696029572855505,6.452525792510636,20.053217898214907,0.6020577603946569,0.3851294765747559,0.3053685922627424,Nature,1 -saugeen/W/short,TimeCopilot,1188.771119779659,1188.771119779659,16.004309606552123,1.2985654533191504,0.3771862082685798,0.4056451972601779,14.143459388159318,34.47856029157336,1.0407640121972903,0.4831033934625676,0.4017461788808135,Nature,1 -temperature_rain/D/short,TimeCopilot,172.84589088882612,172.84589088882612,5.690968262597378,1.329354786151759,28.23893063539611,1.4921973717873052,20.160673384342687,13.147086783345811,1.5477597169526005,0.6699774309287958,0.5439475176876,Nature,1 -kdd_cup_2018/H/short,TimeCopilot,4062.4779847188897,4062.4779847188897,20.480178734790616,0.8616459031733725,0.7792248697868767,0.4569235767972537,7.456057861756592,63.73757121760202,1.334142792044846,0.4286872297899118,0.3476365368258818,Nature,1 -kdd_cup_2018/D/short,TimeCopilot,2806.392915042132,2806.392915042132,20.651221516354063,1.1720349651780184,0.5342432330567164,0.454346968795786,9.671998096586076,52.97539914943663,1.186705210611965,0.4626092973802523,0.3721230913159204,Nature,1 -car_parts/M/short,TimeCopilot,1.3944985371303251,1.3944985371303251,0.4601047288939888,0.8277407819169432,0.8074556971925223,1.895373104789929,15.759363077256392,1.1808888758601823,2.831650388212677,1.1032839421424192,0.9497286228477376,Sales,1 -restaurant/D/short,TimeCopilot,146.2636063982368,146.2636063982368,7.367976427367725,0.7043412881788105,0.7247356264999066,0.402722738788297,4.9338596056814135,12.09394916469541,0.5565915619579526,0.3390913466173199,0.2671417966057537,Sales,1 -hierarchical_sales/D/short,TimeCopilot,27.93154911370736,27.93154911370736,2.2822688005103853,0.7417091758407094,0.6222577983869355,1.0205722742182668,7.988893691905033,5.285030663459519,1.6213888793654423,0.7001747933942881,0.5712055217154575,Sales,1 -hierarchical_sales/W/short,TimeCopilot,461.69125742091273,461.69125742091273,9.041678339432355,0.7303589909292997,0.5696615853201108,0.459559482310429,7.15437288782513,21.48700205754429,0.9864787745363024,0.4151078751771685,0.3493664205223431,Sales,1 -loop_seattle/5T/short,TimeCopilot,35.11582550573566,35.11582550573566,3.1766584739115924,0.5043912097603322,0.0911021588354409,0.0676089741244228,4.28356480301807,5.925860739650879,0.1016154291613943,0.0544726797182081,0.0436016046427134,Transport,1 -loop_seattle/H/short,TimeCopilot,62.242283569235695,62.242283569235695,4.265975248197058,0.8660501767633345,0.123675206764086,0.097661989261773,7.890145339704824,7.889377894944296,0.1396500126038301,0.0755121005878186,0.0611125144330223,Transport,1 -loop_seattle/D/short,TimeCopilot,18.448784817802004,18.448784817802004,2.9454716708022866,0.8893667597973234,0.0544913823365161,0.0542830942696274,9.469108156661244,4.295204863310015,0.0767610532582133,0.0526395166210478,0.0437976451719324,Transport,1 -sz_taxi/15T/short,TimeCopilot,16.970653448392113,16.970653448392113,2.768767421497555,0.5523431523968114,1039448800203.1052,0.3955646394848205,4.344994837662684,4.11954529631513,0.3851951037709525,0.2588915954378182,0.2062483022006385,Transport,1 -sz_taxi/H/short,TimeCopilot,7.265841640229874,7.265841640229874,1.8697022724468728,0.5699377641751315,1.123909766413656,0.2984359175663999,4.38008848435448,2.695522517106818,0.2511045562354697,0.174174304401337,0.1384303641463761,Transport,1 -m_dense/H/short,TimeCopilot,43089.78857211956,43089.78857211956,89.26722765106294,0.7921908695312612,0.2863753692819957,0.2105748544654512,8.59052092151455,207.5808001047293,0.3681966135657942,0.1583378178855925,0.1305263828021132,Transport,1 -m_dense/D/short,TimeCopilot,8167.028572960703,8167.028572960703,45.84068879727964,0.6825423804230932,0.1226612034991929,0.0986211480672491,8.179580734166276,90.37161375653696,0.1564466311253342,0.0793570130342006,0.0657543630403932,Transport,1 -ett1/15T/short,TimeCopilot,4.57781770733472,4.57781770733472,1.1049286956787996,0.7011452908525583,0.4687633367503997,0.2446696987448156,5.476351101158151,2.139583535956173,0.4044703487956138,0.2088775163133622,0.165339278673051,Energy,7 -ett1/H/short,TimeCopilot,101.1330136191198,101.1330136191198,4.961560403976944,0.8270870321136256,0.500319278383962,0.2620214363922102,5.835271817402984,10.056491118631778,0.4693775833157873,0.2315763226379548,0.1804979758372916,Energy,7 -ett1/D/short,TimeCopilot,49677.91394363521,49677.91394363521,147.56372262291492,1.774252403142336,1.3441692775236855,0.4872234815226898,10.623516253669065,222.88542784048312,0.5854159694082027,0.3875810122077724,0.2956041137794847,Energy,7 -ett1/W/short,TimeCopilot,1011957.1728767653,1011957.1728767653,812.9205210549491,1.52983148890384,0.6616017363557185,0.5386038150186447,8.846642452735148,1005.9608207463972,0.4002958205953012,0.3234804779106662,0.2541024452808174,Energy,7 -ett2/15T/short,TimeCopilot,7.702216342646174,7.702216342646174,1.801961813884832,0.7948216853625055,0.115007599762545,0.1358810774745831,6.191634075356738,2.775286713593061,0.1317527332543178,0.0855455377047745,0.068010871082508,Energy,7 -ett2/H/short,TimeCopilot,111.08349459162535,111.08349459162535,6.602935576013157,0.7581288520706242,0.1240775591525643,0.1131448359612007,5.520679300893479,10.539615485947548,0.1336716498232208,0.0837435951338946,0.0666364159578577,Energy,7 -ett2/D/short,TimeCopilot,107394.3400500349,107394.3400500349,199.7623817375728,1.3566302605487015,0.4383444393541361,0.1382760401563498,10.962740740785602,327.7107566895461,0.1875575124100927,0.1143292816210743,0.0930115935089939,Energy,7 -ett2/W/short,TimeCopilot,4393268.626847372,4393268.626847372,1470.3404562813896,1.0012167025904044,0.1426203775302994,0.1534762620113483,7.95602334210496,2096.0125540767576,0.1758233194561104,0.1233390225889547,0.0961875259247906,Energy,7 -jena_weather/10T/short,TimeCopilot,828.619751644976,828.619751644976,5.681305431614895,0.3033015347286767,0.3190194943656728,0.5562812420202121,2.386514715249575,28.78575605477431,0.1787956229039812,0.0352880272319573,0.0282470367123362,Nature,21 -jena_weather/H/short,TimeCopilot,1114.2671516666535,1114.2671516666535,8.239287884239024,0.5215663102724813,1.4170110986586484,0.5992748477517932,5.037852500049764,33.3806403723274,0.2046211765433347,0.0505063043113419,0.0420540956391067,Nature,21 -jena_weather/D/short,TimeCopilot,366.6637699521878,366.6637699521878,9.47778113030058,1.000378140254235,0.6344829675545672,0.4403002028370409,6.598024324559487,19.148466516987405,0.1153050651496088,0.0570717331193879,0.0448270766953056,Nature,21 -bitbrains_fast_storage/5T/short,TimeCopilot,1886235.9982757785,1886235.9982757785,159.91455783573943,0.7070124590577969,1.985398128544987,0.7268134543963763,14.151383176995845,1373.4030720352196,4.312145981561116,0.5020921621666673,0.3930489602672484,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TimeCopilot,2180596.7738726847,2180596.7738726847,244.70522848757423,1.0603996704920255,2.545562792561041,0.576196309633268,20.687137138924285,1476.68438532839,4.209030040412352,0.6974893673851099,0.557749278298169,Web/CloudOps,2 -bitbrains_rnd/5T/short,TimeCopilot,1733240.319796704,1733240.319796704,120.4828040522798,1.6858308386473713,1.0072696102492449,0.6441691774017078,54.381235308620106,1316.525852308531,5.386843722332412,0.4929808522331466,0.3980883834548485,Web/CloudOps,2 -bitbrains_rnd/H/short,TimeCopilot,1844106.5278123424,1844106.5278123424,140.91685249202595,5.852422890114724,0.9910948342263344,0.5371950862930858,198.33763043037143,1357.9788392358485,5.94317979941055,0.6167210908812049,0.5713244162831072,Web/CloudOps,2 -bizitobs_application/10S/short,TimeCopilot,756604.9621277261,756604.9621277261,444.8492035908169,1.4228439608426573,0.0198724553981974,0.0202845111979339,10.254230931460237,869.8304214775004,0.0335458825448278,0.0171560556694139,0.0136133576144484,Web/CloudOps,2 -bizitobs_service/10S/short,TimeCopilot,5254.437530127733,5254.437530127733,22.953738129045917,0.8410992505054966,0.0590632527498088,0.0471612085982144,8.960408771953439,72.48749913004127,0.0537010583013426,0.0170048635184711,0.0137872465011043,Web/CloudOps,2 -bizitobs_l2c/5T/short,TimeCopilot,20.32080670057079,20.32080670057079,2.637088461300092,0.2736371811669997,0.1583539871525464,0.2086028748171467,2.804417770489809,4.507860545821131,0.1548181775859693,0.0905682918895841,0.0733395549985485,Web/CloudOps,7 -bizitobs_l2c/H/short,TimeCopilot,86.38183412618716,86.38183412618716,5.496603702405645,0.5456377091995832,0.4186559845174549,0.6898036735068019,4.033211022331053,9.294182811102177,0.5009802563319643,0.296281016605036,0.2379357584545769,Web/CloudOps,7 -electricity/15T/medium,TimeCopilot,270351.74737311015,270351.74737311015,59.07229263138479,0.9379705137279136,0.1540899053362026,0.1501940965091556,8.562701574520556,519.953601173326,0.8959324315113344,0.1017875107562592,0.0829051407312251,Energy,1 -electricity/15T/long,TimeCopilot,350754.79833892337,350754.79833892337,63.68678300407552,0.9701533725723728,0.1538012347711226,0.150148343762289,8.60454450165919,592.2455557781108,0.9343399947114034,0.1004737104646307,0.0814141452988151,Energy,1 -electricity/H/medium,TimeCopilot,6416190.090371392,6416190.090371392,243.6408192827964,1.0483407271121,0.2227124077583914,0.1291339734182908,10.040587447375373,2533.0199545940004,0.988721118650361,0.0951010365920263,0.0767664954882301,Energy,1 -electricity/H/long,TimeCopilot,9939024.767879302,9939024.767879302,283.4081134433646,1.1538882844696006,0.2823846439229823,0.1400982265073691,11.401534499230491,3152.6218878703644,1.2433931577852149,0.1117760777060194,0.0903506800598112,Energy,1 -solar/10T/medium,TimeCopilot,22.019168289905668,22.019168289905668,2.1670351192539576,0.947507266525788,2.267577131790845,1.5238331844463489,6.533838178257635,4.692458661502056,1.0412752123224644,0.4808736989894127,0.3654965748217511,Energy,1 -solar/10T/long,TimeCopilot,22.625751067938847,22.625751067938847,2.212217256905629,0.9671587071177912,2.277119444532844,1.5325572626893469,6.191737729498273,4.756653347463828,1.0280745251539265,0.4781353695120983,0.3611726608331724,Energy,1 -solar/H/medium,TimeCopilot,660.6757207529897,660.6757207529897,10.32349824264134,0.7884339037797935,3.4372374784898354,1.3971808793552492,8.60678230418119,25.70361299025858,0.925904132435434,0.37187650186276,0.3056480122368017,Energy,1 -solar/H/long,TimeCopilot,603.7713699914224,603.7713699914224,10.574238680127047,0.8031220443323172,3.3848381887836827,1.4022686196231828,7.66178282160624,24.57175960307732,0.8519476861980545,0.3666281260423847,0.2919721600130173,Energy,1 -kdd_cup_2018/H/medium,TimeCopilot,4501.496239798726,4501.496239798726,21.267457002191247,0.8809323711096007,0.8523815087719337,0.4762398641605064,8.014151088322132,67.09319071112004,1.4043819809200055,0.4451663883816173,0.3634352254434075,Nature,1 -kdd_cup_2018/H/long,TimeCopilot,3476.279481839817,3476.279481839817,19.940171223992767,0.846932989143261,0.8171217384323863,0.5069393709400944,7.379851777933759,58.959982037309146,1.3834393999393408,0.4678769829908051,0.3729545163616967,Nature,1 -loop_seattle/5T/medium,TimeCopilot,53.09786497968351,53.09786497968351,4.2271080844403,0.6750152284090478,0.1411380644379274,0.0975960577179127,4.733693288938917,7.28682818376305,0.1296476484100133,0.0752089398709585,0.0581236058564988,Transport,1 -loop_seattle/5T/long,TimeCopilot,65.77390008978465,65.77390008978465,4.678719174527285,0.741335837948898,0.161000568672698,0.1063086322592711,5.458349474297115,8.110110978882142,0.1433884214432887,0.0827207122761697,0.0643679642964059,Transport,1 -loop_seattle/H/medium,TimeCopilot,67.05025518941878,67.05025518941878,4.574280084455069,0.9385925235329952,0.1291700466289554,0.1038880948135676,8.364170945220577,8.1884220207204,0.1447294386535952,0.0808498862408728,0.0651454144718348,Transport,1 -loop_seattle/H/long,TimeCopilot,66.77893425129199,66.77893425129199,4.528082261100478,0.9470132077266724,0.1427466849514623,0.1040273097569529,8.695303930807215,8.171837874755713,0.1460416861629384,0.0809228938007257,0.0652841225947393,Transport,1 -sz_taxi/15T/medium,TimeCopilot,16.746692647366423,16.746692647366423,2.80319449785868,0.546313980390436,12260400705409.943,0.4086868235270063,4.15504472063875,4.092272308555043,0.3808542444886428,0.2608840375565462,0.206937670340773,Transport,1 -sz_taxi/15T/long,TimeCopilot,15.88749669690106,15.88749669690106,2.7324282289642734,0.5160497951836313,6899154928645.429,0.4008666353985959,3.9047299348177553,3.985912279127711,0.3689289403819629,0.2529086895515328,0.1998971444247135,Transport,1 -m_dense/H/medium,TimeCopilot,33620.6758636312,33620.6758636312,80.97299096213446,0.6994088185298244,0.2611041852375969,0.1935244968643103,7.419700992410988,183.3594171664799,0.3190222208686621,0.1408827744236561,0.1150419135776963,Transport,1 -m_dense/H/long,TimeCopilot,35131.00096581889,35131.00096581889,84.76255368632299,0.7212917507999137,0.2611897313593505,0.1976007999709247,7.238621727643479,187.43265714869136,0.3244736371093977,0.146736510614921,0.1191795538083717,Transport,1 -ett1/15T/medium,TimeCopilot,9.650517354548349,9.650517354548349,1.634607786774018,1.0469833055521922,0.7259542038227474,0.4093312196634479,7.423650658102283,3.1065281834466507,0.5921073433217752,0.3115578603655285,0.2432714779273141,Energy,7 -ett1/15T/long,TimeCopilot,8.249822969890293,8.249822969890293,1.5941603270479447,1.0582463352692113,0.7452691376585209,0.4145760095963351,7.904996533423086,2.8722505061171617,0.5474537863824479,0.3038485345496112,0.2390871499557088,Energy,7 -ett1/H/medium,TimeCopilot,131.36659254489592,131.36659254489592,6.723023053242026,1.2854796996107485,3449389115141.716,0.4519796988339821,9.722145582667212,11.461526623661262,0.5489799558203433,0.3220168673803568,0.2512439885196175,Energy,7 -ett1/H/long,TimeCopilot,145.42598346736074,145.42598346736074,7.067940720999071,1.3787753932428612,4333740885596.531,0.4873355925106218,10.491053386405422,12.059269607540944,0.5774923029404946,0.3384683730318362,0.266623667537051,Energy,7 -ett2/15T/medium,TimeCopilot,11.654150753345416,11.654150753345416,2.164660533491581,0.9173180469176152,0.1407238647415174,0.166368457672974,6.956952946467317,3.413817621570522,0.1819285205115292,0.1153586781494928,0.0912573450345453,Energy,7 -ett2/15T/long,TimeCopilot,11.83917756320148,11.83917756320148,2.2042152614773265,0.9346659806338128,0.1440029299374223,0.1701361833127588,7.591942824199662,3.4408105968218416,0.1833670250235007,0.1174666212031033,0.0941863260552289,Energy,7 -ett2/H/medium,TimeCopilot,244.61828849213703,244.61828849213703,9.713107990908126,1.0717401895462983,0.1805422241171783,0.1809646816155467,7.611095036531764,15.640277762627395,0.2107628007376633,0.1308903764434982,0.1051952148012162,Energy,7 -ett2/H/long,TimeCopilot,267.647744319421,267.647744319421,10.34468404672764,1.1446436946420455,0.1949381236785513,0.1993334897572211,7.941608680412425,16.359943285947573,0.2240368317587744,0.1416624861630846,0.1121997537911523,Energy,7 -jena_weather/10T/medium,TimeCopilot,1432.124056361338,1432.124056361338,9.415487818263491,0.6076936640898429,0.6648073173510344,0.657911977219078,5.769151149084737,37.84341496695743,0.2322443907066463,0.0577826877798355,0.0476556763226186,Nature,21 -jena_weather/10T/long,TimeCopilot,1461.9712240412546,1461.9712240412546,9.656132026235216,0.6271024171144943,0.6880532193880432,0.6554077436138248,6.178353600980851,38.23573229377534,0.2339710022427383,0.0590875276196621,0.0493391718038347,Nature,21 -jena_weather/H/medium,TimeCopilot,1374.7494097731546,1374.7494097731546,10.41510045264059,0.8262192834746671,2.556791650149903,0.6689623999322188,6.409873839105692,37.0776133235832,0.2268844917564687,0.0637318468200035,0.0529729670141724,Nature,21 -jena_weather/H/long,TimeCopilot,1294.7696650536095,1294.7696650536095,11.430013058836357,0.9956425809445488,2.514330090168181,0.6534341790118225,8.026939150897899,35.98290795716224,0.2166569722895802,0.0688213422190891,0.0564411235445146,Nature,21 -bitbrains_fast_storage/5T/medium,TimeCopilot,3040509.589662659,3040509.589662659,283.21892608931984,1.0087087717530026,4.287881643868664,0.810720179602496,22.07875482157477,1743.7057061507423,5.297859557653899,0.8604973242893447,0.6118706188243893,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,TimeCopilot,4030523.8806433696,4030523.8806433696,371.4619560289608,0.9134105304687804,4.239415674241728,0.825838708432873,16.990356306754105,2007.616467516485,5.305530996817342,0.9816630585263264,0.6737192028941127,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TimeCopilot,2317870.158848923,2317870.158848923,153.50325611456742,4.433770453902802,0.594711554396072,0.7129055265784028,162.0072193283584,1522.455306026723,6.405745871560616,0.6458664797807487,0.6111270144334976,Web/CloudOps,2 -bitbrains_rnd/5T/long,TimeCopilot,2280941.4486649195,2280941.4486649195,167.9207402001444,3.355373219889245,1.4696731836119947,0.6935498764481771,117.07688210774565,1510.2785996844818,5.785537053993971,0.6432665236500025,0.6102253099979892,Web/CloudOps,2 -bizitobs_application/10S/medium,TimeCopilot,4570776.178464267,4570776.178464267,1182.757140528361,2.6990382044848347,0.0384832876936519,0.0393211753062574,33.14222071025751,2137.937365421229,0.0839505080144545,0.0464433918462883,0.039958439029638,Web/CloudOps,2 -bizitobs_application/10S/long,TimeCopilot,6617673.59866569,6617673.59866569,1552.8124705314635,3.4816992433063123,0.0497868597441512,0.0509601432809278,77.05664865666928,2572.4839355505587,0.0992104228820281,0.0598857702195689,0.055663982472999,Web/CloudOps,2 -bizitobs_service/10S/medium,TimeCopilot,60797.75518485954,60797.75518485954,52.89482176999606,1.1797597839799314,0.0704116585906638,0.0658206812028272,12.969505753217875,246.5720081129639,0.185575903802245,0.0398098893363783,0.0306251405257861,Web/CloudOps,2 -bizitobs_service/10S/long,TimeCopilot,131170.94048455538,131170.94048455538,77.31256830274113,1.378960440029548,0.0798102979298142,0.0774093466099856,24.48723397931568,362.1752897210899,0.2683110330783669,0.0572756222193541,0.0535407422224889,Web/CloudOps,2 -bizitobs_l2c/5T/medium,TimeCopilot,126.08155468883244,126.08155468883244,7.413843337915914,0.7114207052594923,0.4971504127500155,0.8069137239630945,6.695381593254608,11.228604307251745,0.5907897103387891,0.3900771848622152,0.3072603977907794,Web/CloudOps,7 -bizitobs_l2c/5T/long,TimeCopilot,324.36213081406584,324.36213081406584,12.03023948555428,1.1287304672383875,0.8257551978752894,1.005355722027217,8.340879600261944,18.010056380091257,1.0112377406594684,0.6754799618735338,0.5238501660372177,Web/CloudOps,7 -bizitobs_l2c/H/medium,TimeCopilot,170.49214821827232,170.49214821827232,7.960913137206807,0.7967643166622312,0.6335708803287097,0.9380559367882922,5.633552917767068,13.057264193477604,0.79063717976047,0.4820453823905438,0.3677423620221114,Web/CloudOps,7 -bizitobs_l2c/H/long,TimeCopilot,141.57640361192122,141.57640361192122,7.304780622766841,0.7629465060919579,0.7216246090200941,0.8693538484765067,7.455372083944178,11.898588303320745,0.7268001150074588,0.4461970833337841,0.3603500176749528,Web/CloudOps,7 \ No newline at end of file diff --git a/results/TimeCopilot/config.json b/results/TimeCopilot/config.json deleted file mode 100644 index 972b51b710543ef5ae802d7f6607432befdc1213..0000000000000000000000000000000000000000 --- a/results/TimeCopilot/config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "model": "TimeCopilot", - "model_type": "agentic", - "model_dtype": "float32", - "model_link": "https://github.com/AzulGarza/TimeCopilot", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/Timer/config.json b/results/Timer/config.json index 44b1ef1075bab4aefd9839ee17e201b1277fbf34..cf6cfff8cd47e0eacd90d3408216df7e12ca548b 100644 --- a/results/Timer/config.json +++ b/results/Timer/config.json @@ -2,7 +2,5 @@ "model": "Timer", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://drive.google.com/drive/folders/15oaiAl4OO5gFqZMJD2lOtX2fxHbpgcU8", - "org": "THUML @ Tsinghua University", - "testdata_leakage": "Yes" + "model_link": "https://drive.google.com/drive/folders/15oaiAl4OO5gFqZMJD2lOtX2fxHbpgcU8" } \ No newline at end of file diff --git a/results/Toto_Open_Base_1.0/all_results.csv b/results/Toto_Open_Base_1.0/all_results.csv deleted file mode 100644 index 7b0dd4434c49e205691b2a1be056781f27d5a000..0000000000000000000000000000000000000000 --- a/results/Toto_Open_Base_1.0/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/MAAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bizitobs_l2c/H/medium,Toto_Open_Base_1.0,37398.41667,157.5728981,7.473105004,0.7566620283,0.5795229645,0.6969685872,0.9660157703,6.472265453,193.3867024,11.70985857,0.452507859,0.3562059739,Web/CloudOps,7 -m_dense/D/short,Toto_Open_Base_1.0,20314.09185,11139,51.54292824,0.7633603786,0.1379826619,0.111935843,0.1106136294,8.515403944,142.5275126,0.2467362069,0.08922843298,0.0745206581,Transport,1 -ett2/H/long,Toto_Open_Base_1.0,21834922.12,250.4522817,9.895243262,1.077451519,0.1819867424,0.1717069757,0.1776895674,8.036886582,4672.785264,63.99019775,0.1355077405,0.1082210883,Energy,7 -bizitobs_application/10S/medium,Toto_Open_Base_1.0,135623775.6,3939721.813,1075.867917,2.303904467,0.03309280396,0.03302663167,0.03399109523,17.39248079,11645.76213,0.4572948033,0.04224616662,0.03422610427,Web/CloudOps,2 -ett2/W/short,Toto_Open_Base_1.0,99790116.57,5574654.857,1591.609794,0.9866367346,0.1466089998,0.143275312,0.1567967449,11.79440226,9989.500316,0.8379657713,0.1335116358,0.1058714494,Energy,7 -m4_weekly/W/short,Toto_Open_Base_1.0,1183982.392,477587.8294,339.1406685,2.399577752,0.08559447641,0.08309790376,0.09231963647,20.03856176,1088.10955,0.1982366778,0.061786168,0.049082368,Econ/Fin,1 -jena_weather/10T/short,Toto_Open_Base_1.0,3224.957937,766.3789187,5.19020066,0.2656077383,0.2951250528,0.4302952358,0.5413362049,2.447266509,56.78871311,0.3527291101,0.03223765357,0.02662086378,Nature,21 -bitbrains_rnd/H/short,Toto_Open_Base_1.0,12556663.97,1934760.008,139.8294697,5.637954952,0.748071646,0.3988558514,0.5126118129,194.5472316,3543.538341,15.50825784,0.6119621858,0.5927710423,Web/CloudOps,2 -ett1/W/short,Toto_Open_Base_1.0,4061652.857,1412445.286,901.6033761,1.578953942,0.7711945942,0.4131914548,0.5430430004,8.184546837,2015.354276,0.8019575683,0.3587695026,0.2629936609,Energy,7 -solar/H/long,Toto_Open_Base_1.0,229425639.3,826.3135645,12.58941638,0.9574328268,3.103650801,1.13842416,1.450042967,5.574609656,15146.80294,525.1673257,0.436498062,0.3313992776,Energy,1 -sz_taxi/15T/medium,Toto_Open_Base_1.0,40822.83077,16.65023872,2.799652152,0.5453405167,9227365555594,0.3907462878,0.4130393525,4.340224252,202.0466054,18.8038091,0.2605543631,0.2054407513,Transport,1 -loop_seattle/H/medium,Toto_Open_Base_1.0,320976.3534,72.4425882,4.540610488,0.9291967897,0.1228339669,0.09776583928,0.1037258278,7.571202991,566.5477503,10.01366808,0.08025478219,0.06448085654,Transport,1 -saugeen/M/short,Toto_Open_Base_1.0,762.5173456,428.9092262,12.91658093,0.7569064687,0.3590344247,0.3242794446,0.3679617927,5.364999916,27.61371662,0.8290466133,0.3877945088,0.2987905772,Nature,1 -ett2/15T/short,Toto_Open_Base_1.0,9.490230887,8.237386068,1.822001721,0.7858460986,0.112535502,0.1269984836,0.1317325047,5.123620462,3.080621835,0.1462480777,0.08649690337,0.06808393659,Energy,7 -ett1/D/short,Toto_Open_Base_1.0,58832.92698,44959.96825,137.6339658,1.658963218,1.487493703,0.3903144231,0.4549627879,10.99019236,242.5549979,0.6370787475,0.3615001765,0.2838008056,Energy,7 -bizitobs_application/10S/short,Toto_Open_Base_1.0,11946828.23,562513.0667,372.4225347,1.24709962,0.01725755056,0.01722629865,0.01764700784,10.70973968,3456.418411,0.1333002424,0.01436284855,0.01159086669,Web/CloudOps,2 -kdd_cup_2018/D/short,Toto_Open_Base_1.0,101953.7318,2979.821774,21.49683797,1.211969669,0.5038438613,0.3928861529,0.4758361871,9.826820957,319.3019446,7.152702717,0.4815520042,0.3873948742,Nature,1 -restaurant/D/short,Toto_Open_Base_1.0,550.5961258,181.4894111,8.217001659,0.7834700744,0.6456841315,0.3971230152,0.467267707,5.308714575,23.4647848,1.079903764,0.378165455,0.2973646443,Sales,1 -hospital/M/short,Toto_Open_Base_1.0,5870.95904,3106.503857,18.43248095,0.7832408926,0.194963272,0.1766570687,0.1788603291,5.280802512,76.62218373,0.2781563197,0.06691418614,0.05238767787,Healthcare,1 -sz_taxi/H/short,Toto_Open_Base_1.0,25.21437947,7.372503297,1.861796518,0.5675622397,1.311333885,0.2796560923,0.2980483862,4.380735836,5.021392184,0.4677736798,0.1734378388,0.1373674751,Transport,1 -bizitobs_service/10S/long,Toto_Open_Base_1.0,7877186.208,130185.9285,75.13797619,1.302467646,0.07940020849,0.06686133329,0.07720037228,20.63749334,2806.632539,2.079243111,0.05566461486,0.05102757171,Web/CloudOps,2 -us_births/W/short,Toto_Open_Base_1.0,3784342.286,3362242.286,1358.152344,1.234635428,0.01820384392,0.01819532684,0.01831153248,9.091229699,1945.338604,0.02640778081,0.01843678489,0.01447765155,Healthcare,1 -ett1/H/long,Toto_Open_Base_1.0,64097.39259,160.3568287,7.115493242,1.369602803,4903367495009,0.3615915773,0.481282649,9.37411769,253.1746287,12.12398463,0.3407455603,0.2668288663,Energy,7 -ett1/15T/short,Toto_Open_Base_1.0,9.704888044,4.876736305,1.09491272,0.6927710736,0.5079014172,0.2077161335,0.2401852562,4.874973738,3.11526693,0.5889150585,0.2069840572,0.1616278199,Energy,7 -hierarchical_sales/W/short,Toto_Open_Base_1.0,2810.427436,488.3472425,9.143537101,0.7442683038,0.5216713315,0.4083515587,0.4689143229,6.695418071,53.01346467,2.433874094,0.4197842607,0.3558391793,Sales,1 -hierarchical_sales/D/short,Toto_Open_Base_1.0,123.3105751,29.73635348,2.291699299,0.7354232305,0.5850923014,0.7704961101,1.053829294,6.894183285,11.10452949,3.406746671,0.703067966,0.5704088874,Sales,1 -m4_hourly/H/short,Toto_Open_Base_1.0,3400969.634,3374528.178,325.7894297,0.8608470877,0.1000763959,0.08706018813,0.09206759258,6.491836545,1844.171802,0.2517701712,0.04447745075,0.03460384789,Econ/Fin,1 -covid_deaths/D/short,Toto_Open_Base_1.0,9959467.475,321507.2149,89.58891602,32.61906192,0.08334196633,0.2733040927,0.3592514707,580.5163294,3155.862398,1.187408529,0.03370826405,0.02727374002,Healthcare,1 -bizitobs_l2c/5T/short,Toto_Open_Base_1.0,45.78455636,19.32186454,2.504416039,0.2585959492,0.1496430341,0.1714225042,0.2033655439,2.81569694,6.766428627,0.2323865475,0.08601178389,0.06947410461,Web/CloudOps,7 -solar/D/short,Toto_Open_Base_1.0,444993.8453,154174.6287,259.1266138,1.011716133,1.185910688,0.4337246415,0.4276860555,5.586282249,667.07859,0.9637013844,0.3743497097,0.2897205121,Energy,1 -solar/W/short,Toto_Open_Base_1.0,4998447.65,2598564.438,1285.94594,1.426511618,0.2974762186,0.2783865546,0.2451364002,7.223323168,2235.720834,0.4563665181,0.2624937165,0.1860354557,Energy,1 -ett2/15T/medium,Toto_Open_Base_1.0,67342.93841,12.99678819,2.234408172,0.9225758172,0.1396207755,0.1561704218,0.165291283,6.545983746,259.5051799,13.82949986,0.1190756482,0.09284270687,Energy,7 -saugeen/D/short,Toto_Open_Base_1.0,2401.326667,1292.026354,13.34403158,2.964942115,0.2790883382,0.2586500549,0.3283487956,26.48296733,49.00333322,1.587229639,0.4322163622,0.3532752348,Nature,1 -m4_daily/D/short,Toto_Open_Base_1.0,792623.8843,387850.5572,174.9001043,3.312199429,0.03974167746,0.03059911665,0.03097884076,30.36738222,890.2942684,0.1375258593,0.02701723239,0.02186022254,Econ/Fin,1 -ett2/15T/long,Toto_Open_Base_1.0,17880.44825,10.6662748,2.076824932,0.8713644491,0.1326568412,0.1502894035,0.1596898251,7.092517536,133.7177933,7.126062783,0.1106777526,0.08833617314,Energy,7 -ett2/H/short,Toto_Open_Base_1.0,153.0302176,103.0952195,6.32607945,0.7349521832,0.1200016492,0.1090898786,0.1105930964,6.540849498,12.37053829,0.1568928454,0.08023228913,0.06450730164,Energy,7 -sz_taxi/15T/long,Toto_Open_Base_1.0,1767026.507,16.06504184,2.745417112,0.5175745084,8452365932963,0.3840168227,0.4043976613,4.809020317,1329.295493,123.0372229,0.2541109174,0.2018226322,Transport,1 -jena_weather/H/short,Toto_Open_Base_1.0,2050.049916,1141.295948,8.399357769,0.5438768279,1.19870911,0.494068474,0.6225096432,4.050156961,45.27747692,0.2775480409,0.05148752652,0.04224081567,Nature,21 -bizitobs_l2c/H/short,Toto_Open_Base_1.0,159.4930866,60.77701048,4.745320638,0.4696071924,0.4324744428,0.5125146291,0.6477213057,3.146160506,12.62905723,0.6807385995,0.2557849621,0.1989570271,Web/CloudOps,7 -bitbrains_rnd/5T/medium,Toto_Open_Base_1.0,66664251431,2268161.072,152.7860747,4.416975059,0.7249773977,0.543368251,0.7244324769,172.9594116,258194.2126,1086.3548,0.6428489778,0.6281348146,Web/CloudOps,2 -jena_weather/D/short,Toto_Open_Base_1.0,1598.264187,400.0442708,10.0961325,1.195979531,0.5814537397,0.3924275716,0.4858788869,8.356709248,39.97829644,0.2407346861,0.06079521903,0.05050018266,Nature,21 -bizitobs_service/10S/short,Toto_Open_Base_1.0,17440.30815,2418.000741,19.37142733,0.7885676144,0.05277840751,0.04206951,0.0429771496,6.834426153,132.0617588,0.09783557861,0.0143509735,0.01143907001,Web/CloudOps,2 -bizitobs_l2c/H/long,Toto_Open_Base_1.0,43524.07619,138.8084077,7.564872427,0.7971592662,0.751385241,0.6967385428,0.9400415814,8.978071088,208.6242464,12.7433711,0.4620842414,0.3685722769,Web/CloudOps,7 -m_dense/H/short,Toto_Open_Base_1.0,76576.03111,56413.84,101.2946419,0.8785644308,0.3165610233,0.2169596693,0.227740682,10.54627774,276.7237451,0.4908389989,0.1796714648,0.1483854659,Transport,1 -sz_taxi/15T/short,Toto_Open_Base_1.0,35.3490883,17.17536928,2.758330405,0.5498899178,918322775258,0.3595493862,0.3941651881,3.879414269,5.945509928,0.5559306294,0.2579157005,0.2026779445,Transport,1 -saugeen/W/short,Toto_Open_Base_1.0,1621.368164,1042.626758,16.11312256,1.307169824,0.433044672,0.3785656452,0.4306035042,10.04483078,40.26621616,1.215469217,0.4863880029,0.3900837548,Nature,1 -bizitobs_l2c/5T/long,Toto_Open_Base_1.0,1942267.124,304.8233929,12.6343192,1.177213459,0.8224734879,0.7236397104,1.065886579,6.787914198,1393.65244,78.251501,0.7093981345,0.5326296421,Web/CloudOps,7 -bitbrains_fast_storage/H/short,Toto_Open_Base_1.0,24009418.08,3322312.737,278.0267631,0.9447498096,2.774899699,0.444406509,0.5689816659,19.45060895,4899.94062,13.96642267,0.7924666008,0.6227602305,Web/CloudOps,2 -kdd_cup_2018/H/short,Toto_Open_Base_1.0,9510.549401,5040.024176,24.1799674,0.9901861829,0.812027319,0.440681097,0.5372339059,7.707072322,97.52204572,2.041313042,0.5061305107,0.4029938215,Nature,1 -electricity/W/short,Toto_Open_Base_1.0,132357951466,70021748932,36191.03851,1.789913428,0.2654867862,0.1058043033,0.1129543184,12.99505113,363810.3235,0.8300951004,0.08257600681,0.06366477644,Energy,1 -jena_weather/10T/medium,Toto_Open_Base_1.0,2884813.096,1499.197547,9.720549242,0.5977261209,0.6488183008,0.5138719787,0.6587338226,6.209115747,1698.473755,10.42350324,0.05965483792,0.04890392733,Nature,21 -solar/H/medium,Toto_Open_Base_1.0,8428203.529,814.2207953,11.34814768,0.8654755987,3.420207596,1.133391799,1.428819249,6.837183454,2903.136843,104.5777606,0.4087867486,0.3312314098,Energy,1 -jena_weather/H/long,Toto_Open_Base_1.0,690451.9111,1390.475926,11.6833664,1.013920841,1.949344184,0.5521645198,0.6782610212,9.512091035,830.9343603,5.003145961,0.07034681705,0.05715514827,Nature,21 -jena_weather/10T/long,Toto_Open_Base_1.0,16709049.02,1481.672751,9.802677538,0.6347010837,0.6718184922,0.5140006575,0.6538680142,7.150488703,4087.670366,25.01315589,0.05998426473,0.04982757937,Nature,21 -m4_quarterly/Q/short,Toto_Open_Base_1.0,8191969.069,1884532.298,576.4840781,1.226756767,0.1140628526,0.09880934656,0.1050152807,10.04283167,2862.161608,0.4790754509,0.09649328287,0.07759517241,Econ/Fin,1 -jena_weather/H/medium,Toto_Open_Base_1.0,22609.89048,1427.170437,10.30184539,0.7525596047,1.445821565,0.5293127151,0.6805553134,6.384433833,150.3658554,0.9201152673,0.06303881424,0.05268077696,Nature,21 -m4_yearly/A/short,Toto_Open_Base_1.0,14551917.71,3767044.312,918.9365125,3.39694196,0.16269661,0.1370154016,0.1588259838,34.50765278,3814.697591,0.6117044402,0.1473557291,0.1216313197,Econ/Fin,1 -bizitobs_application/10S/long,Toto_Open_Base_1.0,102915131.2,6357697.422,1513.660972,3.274733168,0.04762808482,0.04752889845,0.04876194424,51.24138813,10144.70952,0.3912408965,0.05837585342,0.05250523855,Web/CloudOps,2 -solar/H/short,Toto_Open_Base_1.0,892.3889222,729.6633282,10.73672368,0.8282911464,3.436526479,1.129916488,1.41346713,9.66753028,29.87287938,1.1011243,0.3957592166,0.3281712475,Energy,1 -ett2/H/medium,Toto_Open_Base_1.0,301437.2,229.789881,9.397626023,1.017258383,0.1697165203,0.1604266212,0.1658503941,7.095124107,549.032968,7.398571851,0.1266390461,0.1017818934,Energy,7 -solar/10T/short,Toto_Open_Base_1.0,38.99912798,28.8004132,2.328592015,1.032864175,4.504511879,1.233119881,1.578546369,7.924051156,6.244928181,1.816648932,0.6773871651,0.5406264183,Energy,1 -us_births/D/short,Toto_Open_Base_1.0,298692.0533,298147.7333,338.3152604,0.4979574637,0.03238560359,0.03221077601,0.0322724088,4.706409047,546.5272668,0.05123249645,0.03171431039,0.02593239217,Healthcare,1 -bizitobs_service/10S/medium,Toto_Open_Base_1.0,151640546.4,63206.46095,50.15047619,1.083177612,0.07057723757,0.06131290496,0.0660597011,10.93458683,12314.24161,9.267987315,0.03774442567,0.02729092177,Web/CloudOps,2 -loop_seattle/H/long,Toto_Open_Base_1.0,122602991.5,70.56007267,4.516094838,0.9429943618,0.1409278158,0.1038558997,0.1031635044,7.743303101,11072.62351,197.8825949,0.0807086563,0.06461756368,Transport,1 -loop_seattle/H/short,Toto_Open_Base_1.0,117.8778003,70.50443307,4.440241586,0.89936996,0.1235948057,0.09710036909,0.1015866682,8.034821411,10.85715434,0.1921826755,0.0785967926,0.06349726817,Transport,1 -ett1/H/medium,Toto_Open_Base_1.0,249021.1429,142.4777995,6.782524182,1.271840857,5101261675861,0.3463809059,0.4439577375,8.757355818,499.0201828,23.90188383,0.3248668304,0.2538733743,Energy,7 -ett1/H/short,Toto_Open_Base_1.0,151.5017857,123.0018322,5.310755557,0.8649675899,0.5511348906,0.2243749709,0.2674878438,6.242362432,12.30860616,0.5744930063,0.2478746891,0.1937806943,Energy,7 -solar/10T/long,Toto_Open_Base_1.0,248394281.7,24.29987644,2.012680234,0.8802104975,2.653774763,1.173673626,1.502075159,10.68856756,15760.52923,3406.386319,0.4350086417,0.352392243,Energy,1 -bitbrains_rnd/5T/long,Toto_Open_Base_1.0,185133487404,2150290.438,167.7686039,3.336747658,1.638555064,0.5464242455,0.7114870421,129.292375,430271.4113,1648.272946,0.642683766,0.5889880229,Web/CloudOps,2 -m_dense/H/long,Toto_Open_Base_1.0,999569700.3,42277.33728,90.20064043,0.7801472265,0.2769145027,0.2013021135,0.2136976906,7.749645307,31615.97223,54.73192163,0.1561506427,0.1284824893,Transport,1 -ett1/15T/long,Toto_Open_Base_1.0,5819.542857,10.78491195,1.689712767,1.069131671,0.8389982592,0.315474776,0.4087703063,8.044604274,76.2859283,14.54017336,0.322060924,0.2511173411,Energy,7 -electricity/15T/short,Toto_Open_Base_1.0,264941.1271,164515.0974,60.84776745,1.144713264,0.1923987992,0.1668248233,0.1841301134,11.95686549,514.7243215,1.014623107,0.1199429448,0.09944710545,Energy,1 -loop_seattle/5T/short,Toto_Open_Base_1.0,106.8326729,45.32785115,3.543208074,0.5618224956,0.09517780755,0.0738451314,0.07534042072,4.340436915,10.33598921,0.177239396,0.06075819607,0.04834597093,Transport,1 -kdd_cup_2018/H/long,Toto_Open_Base_1.0,724480550.1,4217.507247,24.57593103,1.041030982,0.8698310088,0.4958640978,0.6338830421,10.62248093,26916.17636,631.5622494,0.5766506384,0.4566059798,Nature,1 -car_parts/M/short,Toto_Open_Base_1.0,13.16870377,1.412413313,0.4252767785,0.810434487,0.9362636455,1.386385595,1.963242548,14.08657555,3.628870867,8.701660087,1.01977009,0.8989729648,Sales,1 -electricity/H/medium,Toto_Open_Base_1.0,7554248694,5027753.095,238.8937961,1.102564665,0.2017909919,0.1235894356,0.1342696326,9.839924909,86915.18104,33.9258586,0.09324811904,0.07507115984,Energy,1 -bitbrains_fast_storage/5T/medium,Toto_Open_Base_1.0,52323102590,2994436.864,279.761322,0.9847437587,4.050228287,0.6095507418,0.8091713329,24.59009573,228742.4372,694.9827225,0.8499921903,0.629301822,Web/CloudOps,2 -bizitobs_l2c/5T/medium,Toto_Open_Base_1.0,221127.0531,134.4320047,7.904009885,0.7541455962,0.4971016052,0.5809160505,0.8020871532,5.693576376,470.2414838,24.74161725,0.4158671535,0.3155920059,Web/CloudOps,7 -m4_monthly/M/short,Toto_Open_Base_1.0,14562640.19,1990627.53,581.9357991,0.9827794763,0.1631085373,0.1355008769,0.1373329189,8.427619296,3816.102749,0.7931171544,0.1209462363,0.09687896759,Econ/Fin,1 -electricity/H/long,Toto_Open_Base_1.0,5922055962,6609210.048,261.1795038,1.235409238,0.2984009959,0.1365030847,0.1484970644,11.27322499,76954.89564,30.35098714,0.1030091159,0.08349723518,Energy,1 -solar/10T/medium,Toto_Open_Base_1.0,3413751.302,24.3248469,2.008026238,0.8811358769,3.107376946,1.176572155,1.494887214,7.98364065,1847.633974,409.997344,0.4455890266,0.3530037168,Energy,1 -electricity/H/short,Toto_Open_Base_1.0,8549918.024,1703296.617,175.6078327,0.9756653182,0.1660374339,0.1175242395,0.1308332802,11.92837172,2924.024286,1.380348293,0.08289943877,0.0689230012,Energy,1 -bitbrains_fast_storage/5T/long,Toto_Open_Base_1.0,60371635062396,3498794.655,356.8014712,0.8969665165,4.132958995,0.625213384,0.8308686532,20.3471869,7769918.601,20533.57434,0.9429197281,0.6688295234,Web/CloudOps,2 -ett1/15T/medium,Toto_Open_Base_1.0,6064.968889,11.92386781,1.753412078,1.080607726,0.821602986,0.3157942321,0.409499473,7.527531484,77.87791015,14.84360492,0.3342020363,0.2604685887,Energy,7 -temperature_rain/D/short,Toto_Open_Base_1.0,2721.590325,190.2997162,5.890219042,1.365017681,23.98484399,1.084659305,1.546957797,18.23690176,52.16886356,6.141654542,0.6934345137,0.559701936,Nature,1 -us_births/M/short,Toto_Open_Base_1.0,43483594.67,45510197.33,5144.822917,0.5814147368,0.01584235082,0.01583728567,0.01602387925,4.620249499,6594.209177,0.02048156874,0.0159797849,0.01269073845,Healthcare,1 -bitbrains_rnd/5T/short,Toto_Open_Base_1.0,5755247.409,1885353.991,121.8783778,1.649742225,0.9224266805,0.4824698187,0.6465897673,53.0055946,2399.009673,9.816055112,0.4986911419,0.3985907533,Web/CloudOps,2 -kdd_cup_2018/H/medium,Toto_Open_Base_1.0,688932790.1,5444.038506,26.52247464,1.077645508,0.8338213196,0.4752643862,0.6078889982,10.70520091,26247.52922,549.4082931,0.5551633983,0.4414215174,Nature,1 -m_dense/H/medium,Toto_Open_Base_1.0,20470620.16,37971.29778,84.76822049,0.7282179456,0.2750284714,0.1948773024,0.20424096,7.242183538,4524.446945,7.871966082,0.1474859943,0.121148238,Transport,1 -loop_seattle/5T/medium,Toto_Open_Base_1.0,7447219.553,81.10869925,5.071234732,0.8042105073,0.1584395226,0.1121562314,0.115953968,6.636613702,2728.959427,48.55379626,0.09022768735,0.07234440444,Transport,1 -loop_seattle/5T/long,Toto_Open_Base_1.0,86819307.2,94.12713751,5.375400578,0.8478243958,0.1684977292,0.1162421453,0.1224421713,7.401495938,9317.687868,164.7386371,0.09503818732,0.07661619596,Transport,1 -loop_seattle/D/short,Toto_Open_Base_1.0,28.73048971,19.93816394,3.069733063,0.9245607514,0.05696543328,0.05646385295,0.05660744698,7.53428119,5.360082995,0.09579184361,0.05486023066,0.04439360883,Transport,1 -electricity/15T/medium,Toto_Open_Base_1.0,474772741.9,286474.7632,62.27742545,0.9876350728,0.171121329,0.1424783133,0.1554430993,8.213132241,21789.28044,37.54512416,0.1073102748,0.08595130674,Energy,1 -electricity/15T/long,Toto_Open_Base_1.0,1205277865547,372753.3002,67.9122961,1.043512109,0.1755482863,0.1452407467,0.1576501538,8.931468734,1097851.477,1731.995287,0.1071399723,0.08562482807,Energy,1 -ett2/D/short,Toto_Open_Base_1.0,629423.0857,148927.2,240.0302827,1.614564587,0.516818092,0.1579685514,0.1562183683,14.4767306,793.3618882,0.4540618892,0.1373756482,0.1114829481,Energy,7 -bitbrains_fast_storage/5T/short,Toto_Open_Base_1.0,7262583.103,2039738.605,152.8956294,0.6717791317,1.543328694,0.5455201532,0.726179967,13.40476802,2694.918014,8.461376119,0.4800544656,0.3706565882,Web/CloudOps,2 -electricity/D/short,Toto_Open_Base_1.0,2239868034,1483995494,4526.021261,1.484873289,0.5564405235,0.09443621331,0.1015192079,12.33176477,47327.2441,0.7777437967,0.07437756046,0.05886452498,Energy,1 \ No newline at end of file diff --git a/results/Toto_Open_Base_1.0/config.json b/results/Toto_Open_Base_1.0/config.json deleted file mode 100644 index 182f2d58b52a17f46da50886604ceb61fc65903e..0000000000000000000000000000000000000000 --- a/results/Toto_Open_Base_1.0/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "Toto_Open_Base_1.0", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Datadog/Toto-Open-Base-1.0", - "org": "Datadog", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/YingLong_110m/all_results.csv b/results/YingLong_110m/all_results.csv deleted file mode 100644 index e106cc19a2b198a2f4c3f46127c6d529ee71d4cc..0000000000000000000000000000000000000000 --- a/results/YingLong_110m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -car_parts/M/short,YingLong_110m,1.843900305255248,1.6918696857977946,0.5801768418848395,1.042972189100596,0.7036974853004412,1.8118402604620656,15.124621236334674,1.35790290715325,3.2561118770763025,1.3912045520443417,1.1609190089229378,Sales,1 -solar/D/short,YingLong_110m,120124.25498783456,131737.36642335766,249.5613290754258,0.9732760927275498,1.0896855041058393,0.4227962893001064,5.250237033751217,346.5894617379971,0.5007037223242208,0.3605311188333409,0.2842868608780193,Energy,1 -bizitobs_application/10S/short,YingLong_110m,1279220.0533333332,1388033.5644444444,529.7608333333334,1.6094207599473829,0.0231995264689127,0.0237628661261664,10.486369538670612,1131.0261063889432,0.043619156082872,0.0204307578270988,0.0160324211305041,Web/CloudOps,2 -bizitobs_application/10S/medium,YingLong_110m,6811315.626666667,6560801.706666667,1474.25625,3.6654339255789594,0.0516912905375162,0.0526861190795898,46.45101624411667,2609.849732583596,0.1024811177518351,0.0578897039420983,0.0493313648740927,Web/CloudOps,2 -bizitobs_application/10S/long,YingLong_110m,9162671.786666667,7306552.32,1622.5309722222223,4.3767350951622985,0.0592618391248914,0.0611039733886718,73.29213639082548,3026.990549484201,0.1167389262665924,0.062574534156966,0.0575771418541374,Web/CloudOps,2 -ett1/D/short,YingLong_110m,42552.828571428574,42676.48888888889,139.67012648809524,1.7278003705101963,1.2535656156994048,0.4905046735491071,10.87539346641944,206.283369594906,0.5418101127302961,0.3668482201264097,0.2799443289488829,Energy,7 -ett1/W/short,YingLong_110m,1051475.5714285714,1083364.857142857,812.9561941964286,1.5250727690609622,0.7231317247663226,0.5172080993652344,8.476241409068537,1025.4148289490315,0.4080370347825129,0.3234946731050985,0.2528383972571947,Energy,7 -hospital/M/short,YingLong_110m,2993.247501086484,2879.5432420686657,18.77489746305954,0.780682014214452,0.1940906828043513,0.175651129200789,5.249674961654481,54.71057942561461,0.1986121078867453,0.0681572375881279,0.054137476375947,Healthcare,1 -electricity/W/short,YingLong_110m,47025732067.4018,49203407619.22883,30903.73063063063,1.6906766671378068,0.2551323863093288,0.1062141109157253,13.11570548512226,216854.17235414637,0.4947896592008528,0.0705121150345822,0.0568398686961493,Energy,1 -hierarchical_sales/W/short,YingLong_110m,498.3057468220339,466.9772245762712,9.377093816207626,0.7420677178441749,0.5937939546080364,0.4801955142263638,6.707491989947868,22.32276297464169,1.024848967041315,0.4305069637798339,0.3687032562737851,Sales,1 -ett2/W/short,YingLong_110m,6688521.142857143,5669832.0,1705.195591517857,1.1859454986372289,0.176666396004813,0.1952486889702933,10.109138397730945,2586.217535873025,0.2169440021284078,0.1430397678866268,0.1128394040690246,Energy,7 -bizitobs_service/10S/short,YingLong_110m,6259.745185185186,6483.464973544974,27.44454365079365,1.0746929494411337,0.0941598333126653,0.0679244171626984,7.816723963908653,79.11855146035717,0.0586135522783464,0.0203317953165151,0.0158236069161317,Web/CloudOps,2 -bizitobs_service/10S/medium,YingLong_110m,79450.71746031746,85633.0361904762,70.39871527777778,1.8281844017773472,0.1630574738033234,0.1044094122023809,17.778647770566124,281.8700364712742,0.2121420317466209,0.0529837320715061,0.0448172043138526,Web/CloudOps,2 -bizitobs_service/10S/long,YingLong_110m,140205.97164021165,137503.37015873016,88.94500661375662,1.89335579626426,0.2043532856936177,0.1174223012772817,26.90652190611916,374.4408786981086,0.2773977735135658,0.0658933043998598,0.0601586077501251,Web/CloudOps,2 -us_births/W/short,YingLong_110m,3334228.571428572,3300954.571428572,1429.017857142857,1.2990108718894304,0.0192109942436218,0.0193279428141457,8.854691384000024,1825.987012940829,0.0247875946545639,0.019398777289189,0.0152737308948069,Healthcare,1 -us_births/M/short,YingLong_110m,50607674.66666666,46719130.66666666,5761.875,0.6502323111027395,0.017891256759564,0.0180254578590393,3.857575419670789,7113.907130871661,0.0220957470464813,0.0178963444660325,0.0135742571718928,Healthcare,1 -solar/H/short,YingLong_110m,628.0685987322321,690.0911048149571,11.383386507155205,0.8804431656373053,4.218507305138539,1.4016672154542604,6.0146237700365095,25.061296828620662,0.9237677734507826,0.4195954295563567,0.3430711896492029,Energy,1 -solar/H/medium,YingLong_110m,652.3963503649635,678.9728710462288,12.2689762013382,0.9391865615780406,4.646920398663614,1.406865948932927,6.913374973875051,25.54205062959831,0.920084279116056,0.4419571587027988,0.3672737353222672,Energy,1 -solar/H/long,YingLong_110m,677.4370235198702,645.0960259529603,13.07360604217356,0.9975936672777356,5.069801592934555,1.408540045807264,7.285826330718738,26.02762039679905,0.9024250331472882,0.4532857474521184,0.3695688623863586,Energy,1 -loop_seattle/H/short,YingLong_110m,60.858273416707405,61.76369935093151,4.378860739673293,0.884904762340182,0.1304836147839394,0.1008803440288966,7.167057891058617,7.801171284923015,0.1380886648590238,0.0775102880145323,0.0625666537078707,Transport,1 -loop_seattle/H/medium,YingLong_110m,66.9749613003096,63.77637383900929,4.805631208075336,0.9752768181998628,0.1354313680138996,0.1095736004619775,7.272618869116842,8.183823146935032,0.1446481417088102,0.0849389840793782,0.0675196517741423,Transport,1 -loop_seattle/H/long,YingLong_110m,66.06846835225318,62.81858875128999,4.76953528121775,0.9799117689366712,0.1578273147252322,0.111065071245055,7.561400268775225,8.12825124810086,0.1452627319847349,0.0852379809752005,0.0682400740590624,Transport,1 -bitbrains_rnd/5T/short,YingLong_110m,1429166.7504790674,1626556.2823995843,128.40966224866665,1.7586382192009924,1.232069256738781,0.6738116679194635,54.80158042631478,1195.4776244158932,4.891549304445742,0.5254152660226297,0.4293571966948413,Web/CloudOps,2 -bitbrains_rnd/5T/medium,YingLong_110m,2181518.301122295,2375060.339156109,184.90390293294303,4.513589574300016,1.7640844087507292,0.7407258189766511,162.0242568357637,1476.9963781683066,6.214475867189912,0.7779848749196192,0.6734731764383273,Web/CloudOps,2 -m4_quarterly/Q/short,YingLong_110m,2149980.7386666667,2117257.156,654.1768684895833,1.4733073524974245,0.1304903262456258,0.1188058980305989,12.473758867339992,1466.2812617866557,0.2454296628490938,0.1094976881048318,0.0882082277748772,Econ/Fin,1 -hierarchical_sales/D/short,YingLong_110m,28.503154712691195,28.95848232740802,2.333928986668045,0.7557288351703992,0.626868412230361,1.01677911197809,6.907991797036802,5.338834583754323,1.6378953262694147,0.7160235664054078,0.5847970303492365,Sales,1 -solar/W/short,YingLong_110m,5132242.686131387,5374631.7080291975,1965.0248631386864,2.1729912508262035,0.4503507126856894,0.3491745552007299,12.500431608881795,2265.445361541829,0.4624340372200045,0.4011107026127813,0.3172612244118224,Energy,1 -bitbrains_rnd/5T/long,YingLong_110m,2271092.1071532364,2337607.7655438865,192.9968373725689,3.4640875677908043,2.1271747879066623,0.7269518750794107,118.47955108327018,1507.0143022391117,5.773032517961116,0.7393274346235844,0.6479758135832705,Web/CloudOps,2 -jena_weather/10T/short,YingLong_110m,740.4624007936508,815.5128968253969,6.045567491319445,0.3244455062148568,0.340106691267744,0.5535157509893068,2.910167493160289,27.2114387858057,0.169017139538176,0.0375505511601541,0.0306395209585856,Nature,21 -jena_weather/10T/medium,YingLong_110m,1612.6080808080808,1556.7261183261182,10.586028814935064,0.6405198409986587,1.0609466420006453,0.6560839085345505,6.401859123674272,40.15729175141273,0.2464446129763671,0.0649662778656273,0.0557072030209336,Nature,21 -jena_weather/10T/long,YingLong_110m,1754.6976190476191,1692.8407407407408,11.351511863425928,0.647390345084517,1.0158162019373005,0.6491443095564889,7.031053205526741,41.88911098421187,0.2563266528997248,0.0694618475527007,0.059911383129092,Nature,21 -ett2/H/short,YingLong_110m,107.60275297619047,105.81012834821428,6.420584542410714,0.7430531378811844,0.1210455605198008,0.1112654731387183,5.256734516054753,10.373174681657996,0.1315607173193783,0.0814308766536893,0.0646473544645334,Energy,7 -ett2/H/medium,YingLong_110m,247.20969122023808,236.4450892857143,9.41319115048363,1.0296640197015383,0.1720335914613535,0.1707638331821986,7.271572223720213,15.72290339664523,0.2118762342906105,0.1268488041494753,0.1041337052597714,Energy,7 -ett2/H/long,YingLong_110m,273.4015211640212,259.3898148148148,10.023366195436507,1.0983183735154745,0.186180762808678,0.1862538332661623,7.549927260006394,16.534857760622593,0.2264321508653267,0.1372622854941352,0.1108865766785165,Energy,7 -sz_taxi/H/short,YingLong_110m,7.214532835870727,7.199728941306089,1.851638402694311,0.5647840552609904,1.2481388189853766,0.2953322973006811,4.051169520996518,2.6859882419457324,0.2502163796692724,0.1724915434637727,0.1361426119807096,Transport,1 -bitbrains_rnd/H/short,YingLong_110m,2108299.8555901283,2196922.7105408977,185.93737479220837,5.915073307974757,2.532121997766209,0.5655497275287612,200.3061379767857,1451.998572860913,6.35465644980072,0.813752960964861,0.6922823514290165,Web/CloudOps,2 -temperature_rain/D/short,YingLong_110m,168.42888534974455,179.74203293980327,5.994417313635806,1.4092067762158809,43.05256994026332,1.4657871271208651,18.447379060434574,12.978015462687065,1.5278555309201594,0.7057013973834304,0.5695113851444837,Nature,1 -saugeen/W/short,YingLong_110m,788.195947265625,846.0529296875,13.776004028320312,1.1173300535021184,0.3653525829315185,0.3688099384307861,10.117556462639788,28.07482764445091,0.8474620173780107,0.415840136691114,0.3386746927817969,Nature,1 -jena_weather/D/short,YingLong_110m,383.0036458333333,372.2569940476191,9.961922588045637,1.119835715322047,0.7614433688762426,0.4450140574621776,7.276908702603315,19.57047893724968,0.1178462696669062,0.0599870559875349,0.047642228426339,Nature,21 -bitbrains_fast_storage/5T/short,YingLong_110m,1646127.6025572307,1837058.534358384,173.4253635675557,0.7746741240390985,2.140447317106568,0.6970448224612723,14.6627237639006,1283.0150437766624,4.02834997747612,0.5445127575158513,0.4190319350630545,Web/CloudOps,2 -m4_yearly/A/short,YingLong_110m,5335253.9982008645,5441574.807463509,1210.3423761643596,4.716552389805132,0.2106766300993446,0.2040163579638429,46.482874293828445,2309.81687546889,0.370389842776649,0.1940840103709212,0.1588989513840055,Econ/Fin,1 -covid_deaths/D/short,YingLong_110m,2225042.157393484,2112747.340350877,242.95526315789476,46.26887410004859,0.1288096924356339,0.1849375042032819,1107.9592552876968,1491.6575201410958,0.5612434011933948,0.0914131001194397,0.0787579692151736,Healthcare,1 -saugeen/D/short,YingLong_110m,1083.0783333333334,1276.1467708333332,13.531979166666666,3.006842335499278,0.3131029764811198,0.3444833628336589,28.575796289368288,32.91015547415924,1.0659677771599811,0.4383040294110111,0.3600684836259978,Nature,1 -bizitobs_l2c/H/short,YingLong_110m,76.98164682539682,80.9236576140873,5.459219796316964,0.544079942342998,0.4506522345600082,0.6493920110887097,3.1917779476069663,8.77391855589034,0.4729366805562608,0.2942659283250179,0.2286716253990275,Web/CloudOps,7 -bizitobs_l2c/H/medium,YingLong_110m,101.10079985119047,100.93075706845238,6.324358258928571,0.6549118588142214,0.5744248679293306,0.8125947780217326,4.330943289945748,10.0548893505195,0.60883882267385,0.3829494987266835,0.3056147204016453,Web/CloudOps,7 -bizitobs_l2c/H/long,YingLong_110m,127.51450892857144,111.91805555555555,7.303777591765873,0.7759753162582762,0.8824020441286401,0.8202975802951389,5.61243380948337,11.292232238515618,0.6897621365177885,0.4461358153053192,0.3580713160303097,Web/CloudOps,7 -kdd_cup_2018/D/short,YingLong_110m,2641.803311390385,2844.666533679101,20.860599940155595,1.175281319525241,0.5278960250224417,0.4562750780262983,8.945472781917406,51.39847576913527,1.1513803983148565,0.4672995747201962,0.3749615411328965,Nature,1 -bizitobs_l2c/5T/short,YingLong_110m,21.63178013392857,21.427143787202382,2.7550249372209823,0.2880674767032388,0.1536247137713749,0.2131534785914942,2.7375406337990342,4.65099775681827,0.1597340887873169,0.0946187078435359,0.077275721016552,Web/CloudOps,7 -bizitobs_l2c/5T/medium,YingLong_110m,160.88086734693877,173.8452168367347,9.424848533163264,0.9018982754305164,0.5122450865934992,0.8432613865858843,5.950172826255396,12.683882187522036,0.6673587276290389,0.4958856312443436,0.3862613771152057,Web/CloudOps,7 -bizitobs_l2c/5T/long,YingLong_110m,298.20722222222224,352.9578968253968,13.497110615079364,1.2567234613394984,0.8347074233885171,1.088086232112377,9.350956603048376,17.268677489090535,0.9696104242932112,0.7578425828200893,0.5918333986329958,Web/CloudOps,7 -restaurant/D/short,YingLong_110m,147.13038199308863,149.52120108340338,7.436090291398151,0.7097982312540012,0.7350719648214953,0.405928857143691,4.923523323043857,12.12973132402728,0.5582383398368355,0.3422261044582373,0.2703437743348678,Sales,1 -bitbrains_fast_storage/5T/medium,YingLong_110m,2932373.6551262,3267816.9807411344,302.2288904408331,1.059909032669241,5.002733475089009,0.7522894046457049,22.88002887017856,1712.4174885600182,5.202797581522949,0.9182548944733104,0.6557298339121019,Web/CloudOps,2 -electricity/H/short,YingLong_110m,1944081.4857657657,2079194.669189189,207.620185106982,1.1389196928115577,0.1812231216702186,0.1434392298420292,10.57890264352921,1394.303225903808,0.6582106952284659,0.0980115543332733,0.079940864424704,Energy,1 -bitbrains_fast_storage/5T/long,YingLong_110m,3831983.867698639,4409853.582701519,400.0782705183567,0.995757746542128,5.283809294674965,0.7659855844544042,18.343332336100374,1957.545367979664,5.173207965674115,1.057287422193827,0.7196851456747015,Web/CloudOps,2 -ett1/H/short,YingLong_110m,99.30339471726192,104.8042224702381,4.941267903645834,0.8253969858404319,0.4627948579334077,0.2599838075183687,5.633980002590112,9.96510886630256,0.4651123997393038,0.2306291886274444,0.179656885094139,Energy,7 -ett1/H/medium,YingLong_110m,136.56222098214286,135.01561569940475,6.831343005952381,1.2686854868617772,3853813528516.11,0.4550558907645089,8.41479905221366,11.685983954384964,0.5597309298877828,0.3272051363436405,0.2535399540538676,Energy,7 -ett1/H/long,YingLong_110m,151.84737103174604,154.01362433862434,7.408285796957672,1.4119925613288111,3872398931859.4976,0.5106130125661376,8.918774871107171,12.322636529239432,0.5901043744074848,0.3547667615832361,0.2729411016142002,Energy,7 -electricity/H/medium,YingLong_110m,7619830.620540541,7453080.60972973,294.38128096846845,1.3000870172048071,0.2470112886589681,0.1519741369682371,10.45759363785584,2760.4040683458898,1.077476674986685,0.1149067150833084,0.0921588430756736,Energy,1 -solar/10T/short,YingLong_110m,25.458110933698297,27.031522772201946,2.5718477940617395,1.1395126606310668,4.472551976386037,1.558937234038504,6.638171239118119,5.045603128833886,1.4677654037728944,0.7481502447787418,0.5700685263101939,Energy,1 -electricity/H/long,YingLong_110m,10095027.529033031,9583277.077525526,329.76702702702704,1.444133612638563,0.3441131514403158,0.1687742574398107,11.59887223743284,3177.2673052535306,1.2531132854398166,0.1300600179230385,0.1037060848487748,Energy,1 -m4_weekly/W/short,YingLong_110m,337790.21727019496,322943.9451467752,287.17623741161344,2.312578906208,0.0718715578630544,0.0693259851207614,18.299805806386555,581.1972275142019,0.1058851278425693,0.0523190599888635,0.0416169408903756,Econ/Fin,1 -ett2/15T/short,YingLong_110m,7.498291596912202,7.571107700892857,1.729876999627976,0.7462979204359157,0.1085578173003524,0.1276543571835472,5.397103975973882,2.73830085945869,0.1299968831830147,0.0821234151334548,0.0647997474034762,Energy,7 -ett2/15T/medium,YingLong_110m,12.128210565476191,12.200902777777776,2.169119853670635,0.8945756200628378,0.1355653714114049,0.1622458418588789,6.65633130351552,3.4825580491179458,0.1855918223247291,0.1155963233937895,0.0912416193690419,Energy,7 -ett2/15T/long,YingLong_110m,12.159797867063492,11.908239087301586,2.1875968812003967,0.9064558057674303,0.139153641765881,0.1659342060391865,7.0699985720637,3.487090171914614,0.1858333473523007,0.1165809976366959,0.0927031899970379,Energy,7 -jena_weather/H/short,YingLong_110m,1083.419695071011,1138.2622180451128,8.647916177161655,0.5382559556516687,1.6086323952505086,0.6101880199083264,4.506082416898583,32.91534133304728,0.2017689233870139,0.053011169440773,0.0437685577017571,Nature,21 -jena_weather/H/medium,YingLong_110m,1434.5523809523809,1486.8956349206348,11.310623604910717,0.8849009643070249,3.0977005882841278,0.6892198372227872,6.145043432664278,37.87548522398599,0.2317668114200839,0.0692117118125467,0.0566642850114182,Nature,21 -jena_weather/H/long,YingLong_110m,1199.3948412698412,1184.2037037037037,11.636966765873016,1.0097983231584071,3.803772474315069,0.6546696309702974,9.875330096630517,34.63228033597905,0.2085246976150432,0.0700674328247747,0.0603954391211289,Nature,21 -m4_monthly/M/short,YingLong_110m,2124889.5944444444,2104369.640407407,624.3422008101852,1.0851097148969189,0.1816056700812445,0.147112749311659,10.271842581025457,1457.7001044262995,0.302960125247973,0.1297597432974679,0.1054375180146469,Econ/Fin,1 -m_dense/D/short,YingLong_110m,12796.386666666667,12255.59037037037,60.13559606481481,0.8671633461373934,0.1517253222511466,0.1262981499565972,7.978575124196496,113.12111503457994,0.1958293829296734,0.1041036120080754,0.0833182725876339,Transport,1 -solar/10T/medium,YingLong_110m,26.257030385976552,26.577802200840523,2.627392912934085,1.1379832379438883,2.623091047284714,1.4972573596180625,7.652865015652232,5.124161432466443,1.1370717280344853,0.5830288992861634,0.4569334412231867,Energy,1 -m4_daily/D/short,YingLong_110m,364378.0656324986,360539.007874548,183.8920455025516,3.543748673772196,0.0428049835985834,0.0340059473715273,31.538818489347257,603.6373626876476,0.0932452907395394,0.0284062390890253,0.0226415174904807,Econ/Fin,1 -bitbrains_fast_storage/H/short,YingLong_110m,2891610.768801035,2610107.800015423,300.8787582791387,1.1046303884623228,3.795777175365274,0.5500887496021343,20.674034815140484,1700.4736895350763,4.846902260900079,0.8576021744615674,0.6399073021233013,Web/CloudOps,2 -electricity/15T/short,YingLong_110m,146139.65227477477,148886.21432432433,55.86144320101352,1.0489393678544376,0.1730247960245589,0.1694501277593406,10.66752529448951,382.2821631658673,0.7535534968302381,0.1101139156310653,0.0909779068139793,Energy,1 -solar/10T/long,YingLong_110m,30.30442977620641,32.5395356536395,2.8617998926018857,1.2375559443485704,2.179261924975194,1.5324585701628723,8.354899939932718,5.504945937627944,1.1898058752920673,0.6185322008076112,0.4813229342949809,Energy,1 -m4_hourly/H/short,YingLong_110m,749210.383252818,717480.5539452496,211.8672252415459,0.9638390525024426,0.12359827996836,0.1075143123018568,7.537097839151907,865.5693982880969,0.1181693283349051,0.0289245527308261,0.022922751640237,Econ/Fin,1 -loop_seattle/D/short,YingLong_110m,18.943187242002065,19.119933565531475,3.118676631836945,0.9304386245067072,0.0582435757506127,0.0575321791956914,7.541500154312252,4.352377194361957,0.0777828014905089,0.0557349224422685,0.0443372585836093,Transport,1 -saugeen/M/short,YingLong_110m,404.8509579613095,425.13257998511904,13.560282389322916,0.7947187874178349,0.4134275345575242,0.4136600040254138,5.383273345142056,20.120908477534247,0.6040900346556595,0.4071203577936292,0.3223218449697684,Nature,1 -ett2/D/short,YingLong_110m,110078.70476190477,109161.5746031746,194.42872023809525,1.265202085176369,0.460411871047247,0.1295007857065352,11.129440348945582,331.78110971226914,0.1898870828376392,0.1112766864210114,0.0905975178997903,Energy,7 -m_dense/H/short,YingLong_110m,57181.484444444446,58352.39111111111,110.0059375,0.9481986678278248,0.3765221804507182,0.2631729125976562,8.327033240738814,239.1265030155471,0.4241508298442539,0.1951231214023421,0.1589397861698921,Transport,1 -m_dense/H/medium,YingLong_110m,56845.386666666665,46238.07111111111,112.59459201388889,0.9497984247475948,0.5246484551925824,0.2914479573567708,8.042056551789994,238.42270585383991,0.4148253866772913,0.1959003652888921,0.1616473374171873,Transport,1 -m_dense/H/long,YingLong_110m,72637.4162962963,58406.763456790126,127.50513117283953,1.0276977408385104,0.6006675419517632,0.3136951135706018,8.334321690062021,269.51329521249284,0.4665673553225138,0.2207302307459761,0.1803886234454,Transport,1 -loop_seattle/5T/short,YingLong_110m,47.50567424132482,49.14025836074561,3.851548163981876,0.6053939536032894,0.113768793308821,0.0814232156003591,4.735004991270783,6.892436016483926,0.1181900645187271,0.0660455497750451,0.0524655617089425,Transport,1 -electricity/15T/medium,YingLong_110m,282733.474027027,275577.52007207205,56.71742623873874,0.8804414343468178,0.1462577081588512,0.1354073623051717,7.434214337292104,531.726879165448,0.9162189567864776,0.097729836756855,0.0790904614363773,Energy,1 -loop_seattle/5T/medium,YingLong_110m,113.91570554373064,115.6213668246904,6.600064963074045,1.0335103121146776,0.240231029719145,0.1453582803039236,7.5971683231511635,10.673130072463778,0.1898969105068196,0.1174287146434775,0.0934372085902928,Transport,1 -loop_seattle/5T/long,YingLong_110m,124.5887650498796,124.61946451095058,6.909670192638459,1.0803133892896442,0.2572309386108244,0.1504683244825708,8.099115069579383,11.161933750469924,0.1973452714506879,0.1221643821119755,0.0975790263086008,Transport,1 -sz_taxi/15T/short,YingLong_110m,16.64501446361798,16.796248253157433,2.741521441747272,0.5468035215974539,1079318087152.523,0.3946966769785526,3.918143195430888,4.079830200341428,0.3814815724429811,0.2563439798049376,0.2018717640127759,Transport,1 -sz_taxi/15T/medium,YingLong_110m,16.41774172008547,16.29446781517094,2.7590465578258545,0.5375678696241388,10190647795473.94,0.4063753568209134,3.7108302648962153,4.051881256908384,0.3770951829455409,0.2567753348409959,0.2018226805281084,Transport,1 -sz_taxi/15T/long,YingLong_110m,15.671237313034188,15.4512775997151,2.6950200876958688,0.5080189682874582,6628394884256.456,0.3973311117232015,3.5659556213184054,3.9586913637001535,0.3664094209390731,0.2494462586314947,0.1962359118574886,Transport,1 -ett1/15T/short,YingLong_110m,5.136664109002976,5.460441080729167,1.140609232584635,0.7041810358768417,0.5033803079865247,0.2458350590297154,4.867918564445051,2.2664209911230038,0.4284479074510879,0.215622622996504,0.1676887519959624,Energy,7 -ett1/15T/medium,YingLong_110m,9.287917906746031,9.688941592261903,1.6029811507936509,1.009852841367202,0.7418127876548242,0.3889829799107143,7.195348188018159,3.0476085553669834,0.5808772039534426,0.3055297922770673,0.240882488272878,Energy,7 -ett1/15T/long,YingLong_110m,8.228581969246031,8.670805431547619,1.569231460813492,1.0137601444292446,0.8001605865673274,0.3896575055803571,6.783171848764059,2.8685504996855173,0.5467485614982224,0.2990970679970918,0.2329755020422297,Energy,7 -electricity/15T/long,YingLong_110m,342645.7764804805,337793.8654414414,61.17964283033033,0.9112326708689272,0.148884438010155,0.1351548392835597,7.609666871529001,585.3595275388285,0.9234763931119212,0.0965183844027915,0.0775482475996398,Energy,1 -us_births/D/short,YingLong_110m,358157.8933333333,347439.73333333334,388.5833333333333,0.5717365850482113,0.0364318148295084,0.036666940053304,4.616306166578116,598.4629423225246,0.0561010445960638,0.0364265343281222,0.0298180647071071,Healthcare,1 -electricity/D/short,YingLong_110m,1297677372.4436758,1266918193.723964,4237.877837837837,1.4186643866874746,0.5282984296377244,0.0975097330705704,11.631629748682268,36023.28930627624,0.5919822769855417,0.0696424069079309,0.055195447196494,Energy,1 -kdd_cup_2018/H/short,YingLong_110m,4418.228545304156,4548.484061050581,21.99192171600269,0.9172755726602436,0.8375542966641358,0.4853699780605646,7.743883423661001,66.46975662136997,1.3913323924162797,0.460330752062966,0.3736459314080033,Nature,1 -kdd_cup_2018/H/medium,YingLong_110m,4890.508440364557,5138.728227720904,25.029952653218835,1.0340430992906833,0.9328160726903276,0.5543136609066536,8.841430063525657,69.93217028209948,1.4638069770650932,0.5239222403879714,0.421088051598333,Nature,1 -kdd_cup_2018/H/long,YingLong_110m,3825.816441197321,4073.1029229967016,23.835689092517622,1.0066094026393546,0.998323822848604,0.6002705705380212,7.802349250918457,61.8531845679535,1.4513256521506386,0.5592815836127703,0.4437795527641178,Nature,1 diff --git a/results/YingLong_110m/config.json b/results/YingLong_110m/config.json deleted file mode 100644 index 88a8a9418b8dfe67847c46db8cf414cd9fb926d2..0000000000000000000000000000000000000000 --- a/results/YingLong_110m/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "YingLong_110m", - "model_type": "pretrained", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_110m", - "org": "Alibaba", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/YingLong_300m/all_results.csv b/results/YingLong_300m/all_results.csv deleted file mode 100644 index f13f330c8dd017ddf526cd5f1953fd3c90199ab9..0000000000000000000000000000000000000000 --- a/results/YingLong_300m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -car_parts/M/short,YingLong_300m,2.140451681887912,1.8299107668772627,0.580173079388161,1.0651554318735337,0.7310368808628804,1.8206315619134603,15.415473223745524,1.46302825737848,3.508191683111761,1.3911955299632646,1.1909752401446398,Sales,1 -ett1/W/short,YingLong_300m,1266404.2857142857,1252840.5714285714,876.1472516741071,1.5891326412551992,0.7785827772957938,0.5494017260415214,8.218942503845803,1125.346295908191,0.4478021506247268,0.3486399031037617,0.2702477427520409,Energy,7 -hospital/M/short,YingLong_300m,3526.4856584093877,3309.5495436766623,19.362625964254672,0.7925509856820293,0.1971561012450429,0.1785367976474223,5.817898089598588,59.38422061801761,0.2155785107008762,0.0702908285263521,0.0571739399990139,Healthcare,1 -electricity/W/short,YingLong_300m,56024946654.78919,55650156780.16576,32040.56216216216,1.5985525104586955,0.2532723454411594,0.1020619658736495,12.953576291218274,236695.89488368656,0.5400619222233788,0.0731059894339027,0.0571041755428816,Energy,1 -us_births/W/short,YingLong_300m,2994969.1428571427,2996735.1428571427,1360.232142857143,1.2366136774693737,0.0183419585227966,0.0184230485132762,7.65173526097592,1730.5979148424808,0.023492697001202,0.0184650179625041,0.0146625855770976,Healthcare,1 -bitbrains_rnd/H/short,YingLong_300m,2016634.5495929413,2029023.1120583096,181.98897105835212,5.891558599317641,2.444099645239502,0.5708674190770602,199.5416414615088,1420.0825854833026,6.214976467442832,0.796472813640522,0.6726041862317288,Web/CloudOps,2 -sz_taxi/H/short,YingLong_300m,7.258240789429754,7.306476821247329,1.865656143579728,0.5678432184047901,1.2825355203742654,0.2966730818789229,3.9790141391573983,2.6941122451430557,0.2509731806994151,0.1737973825291552,0.136745560885467,Transport,1 -bitbrains_fast_storage/H/short,YingLong_300m,2828396.814641544,2454481.489242475,296.4950003855744,1.116041561674029,4.006003154639603,0.5545716818021557,20.78528469074592,1681.783819235262,4.793630060824541,0.8451070407959799,0.6314508563229755,Web/CloudOps,2 -loop_seattle/H/short,YingLong_300m,61.74249684291999,62.71841392374124,4.418428308823529,0.8946830528220908,0.1302929391788961,0.1014617551940649,7.371551018983021,7.8576393937950595,0.1390882078092375,0.0782106742252841,0.0630410510266255,Transport,1 -loop_seattle/H/medium,YingLong_300m,66.5091847265222,64.02369711042311,4.718248919633643,0.9658017991621825,0.1293049752281669,0.1070802418932211,7.47193216776216,8.155316347421612,0.1441442872752958,0.0833945120869589,0.0668446691847725,Transport,1 -loop_seattle/H/long,YingLong_300m,65.67417870657034,64.11391038871689,4.7273988433092535,0.980583866721561,0.1591987853940918,0.1101696873925008,7.602251006171908,8.103960680221144,0.1448286270162778,0.0844849464171062,0.0680189711717979,Transport,1 -solar/H/short,YingLong_300m,672.2013221923422,746.2710494301447,11.799409885628762,0.9113878939471538,4.744568416277159,1.4033726929429309,7.281014323939222,25.926845588932377,0.9556721899139048,0.4349301902698205,0.3553887270331152,Energy,1 -solar/H/medium,YingLong_300m,723.4724452554744,826.7110097323601,12.653674346107056,0.9667742024331428,5.460635722061639,1.4120378747476288,6.491292193523192,26.897443098842583,0.96890867935974,0.4558148837672371,0.3741601759997349,Energy,1 -solar/H/long,YingLong_300m,688.6034874290349,731.6656934306569,12.617222982562854,0.9605376905649284,6.969098705162071,1.4051437800137203,6.16670622014489,26.24125544689192,0.9098321496728228,0.4374621150409241,0.3519056717370039,Energy,1 -hierarchical_sales/D/short,YingLong_300m,28.81281779661017,28.70021444811906,2.3483247855518807,0.7628497507907794,0.6420465861655035,1.0103917974079126,6.993795589397193,5.367757240841855,1.646768477164764,0.7204400380791133,0.5886648287710353,Sales,1 -electricity/H/short,YingLong_300m,1984023.0149549549,2107041.9585585585,201.48073902027028,1.092141316358599,0.1818618676648489,0.1407964399552443,11.213333626226326,1408.5535186690474,0.6649378510823422,0.0951132973386854,0.0783453463677904,Energy,1 -m4_quarterly/Q/short,YingLong_300m,2063543.7866666664,2028244.752,629.7319420572917,1.3902340567998526,0.1276051349639892,0.1145791289011637,13.006706666329526,1436.504015541435,0.2404454761879113,0.1054060378812329,0.0858312457511847,Econ/Fin,1 -solar/D/short,YingLong_300m,114811.94355231144,128232.76690997567,249.15471411192212,0.9718423636674904,1.0682757375304135,0.4241992662712895,5.156159962169716,338.839111603592,0.4895071061831731,0.3599436987059138,0.2780932732986481,Energy,1 -jena_weather/10T/short,YingLong_300m,736.9854662698413,785.972123015873,5.79593990265377,0.3190417201037026,0.3542642892024339,0.5516509426646484,2.7715034516383183,27.1474762412612,0.1686198519709293,0.0360000509709435,0.0297604143954231,Nature,21 -jena_weather/10T/medium,YingLong_300m,1453.4891774891776,1474.2339105339106,9.960123556998555,0.6168747497369166,0.8123377555588925,0.6476998426857323,6.01765140831233,38.12465314582124,0.2339703446024747,0.0611251079977263,0.0505212199444347,Nature,21 -jena_weather/10T/long,YingLong_300m,1428.3079365079366,1394.7974867724868,10.071142319775133,0.6389360993754846,0.857084003881742,0.6403959273913475,6.651505448945648,37.79296146781748,0.2312616116361389,0.0616270467682569,0.0517809597026227,Nature,21 -electricity/H/medium,YingLong_300m,6697165.174774774,6783113.28,262.5517004504504,1.1394256712480204,0.238575305449455,0.1358960838391234,9.37409242522388,2587.888168908149,1.0101380342999615,0.1024825808864167,0.0823824332110503,Energy,1 -electricity/H/long,YingLong_300m,10788752.264072072,10874608.261765763,310.58578978978977,1.2904492927112967,0.3256845672558769,0.1510404453444876,10.5750235895188,3284.623610715857,1.2954545805609838,0.1224949436299781,0.0973975950166174,Energy,1 -bizitobs_service/10S/short,YingLong_300m,7097.78708994709,7754.496507936508,29.26458333333333,1.1375627888817756,0.1028141211454199,0.0737919624772652,9.36021751103007,84.24836550311875,0.062413882517257,0.0216801388985467,0.0168141396060885,Web/CloudOps,2 -bizitobs_service/10S/medium,YingLong_300m,79036.43936507936,83061.78539682539,71.17019345238096,2.024475608191864,0.199401138547867,0.1157342432415674,20.704316738600188,281.1342016992585,0.2115882251571748,0.0535643647256803,0.0454953792638194,Web/CloudOps,2 -bizitobs_service/10S/long,YingLong_300m,143507.8162962963,138727.52423280422,91.6943716931217,2.209092702254538,0.2256087239583333,0.1289170231894841,30.394555370464637,378.82425515837326,0.280645118928353,0.0679301219456458,0.0621107889211743,Web/CloudOps,2 -temperature_rain/D/short,YingLong_300m,169.8573192622022,179.59816848414107,5.988265373235231,1.4169223545586886,43.17501018061756,1.4562404630752914,18.35573106066468,13.03293210533233,1.5343206716380424,0.704977151354141,0.570969453004899,Nature,1 -jena_weather/H/short,YingLong_300m,1097.892439431913,1140.749582289056,9.048075918964075,0.5433205131930411,2.0868194362760684,0.611689395497154,4.561840662804369,33.13445999909932,0.2031121067037529,0.0554641228970155,0.0452479824277923,Nature,21 -jena_weather/H/medium,YingLong_300m,1460.507738095238,1471.3926587301587,11.443912760416667,0.8864643831135963,3.157411993222846,0.6841133754278699,5.952747439205969,38.21658982817852,0.2338540909890952,0.0700273317943319,0.0573387430363795,Nature,21 -jena_weather/H/long,YingLong_300m,1266.8092592592593,1151.557671957672,12.092056258267196,1.0297684460143597,3.487787066881547,0.6550576079496593,9.710695016466998,35.59226403671533,0.2143048630840619,0.0728075757743191,0.060212749301546,Nature,21 -ett2/W/short,YingLong_300m,5192896.0,3621361.142857143,1325.118024553571,0.920251381772724,0.1422298039708818,0.1493746212550572,9.130618127945622,2278.792662793173,0.1911557683875646,0.1111570869625623,0.0913238713292226,Energy,7 -m4_weekly/W/short,YingLong_300m,322603.114634669,310905.3627598029,284.32563745446754,2.2478891646814683,0.0702888353084991,0.0683257121561629,15.625143070590372,567.9816146977549,0.1034774479942459,0.0517997248533857,0.0409224454402673,Econ/Fin,1 -us_births/M/short,YingLong_300m,61678576.0,59888624.0,6444.708333333333,0.7297496794067668,0.0198117395242055,0.0200638261934121,4.488259551838323,7853.570907555365,0.0243931376938969,0.020017220143867,0.0147512790400836,Healthcare,1 -ett2/15T/short,YingLong_300m,7.8107073102678575,7.9362060546875,1.755917794363839,0.7526828731917684,0.1089503491888287,0.1278966540382022,5.480821304810825,2.7947642673878343,0.1326774020234941,0.0833596642985447,0.0656481454070023,Energy,7 -ett2/15T/medium,YingLong_300m,11.774804067460318,11.666077628968257,2.1428738839285715,0.887583082146435,0.1351627821972178,0.1633011881510416,6.635062649703288,3.431443437893202,0.1828678321684062,0.1141976281575843,0.0902405995156334,Energy,7 -ett2/15T/long,YingLong_300m,12.091335565476191,11.55820560515873,2.1818497333829363,0.9056944228890282,0.1384972828893669,0.1660144430493551,6.927418405885268,3.4772597782558887,0.1853094678799243,0.1162747217264114,0.0923770822217173,Energy,7 -bitbrains_fast_storage/5T/short,YingLong_300m,1623703.492605833,1833989.1113537804,174.27381623355913,0.8032211889881609,2.1813455291714123,0.7021580837758448,14.560358990451094,1274.2462448859064,4.000818117280552,0.5471766890843569,0.4236067899107265,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,YingLong_300m,2884643.2224165485,3210277.6705504307,299.1882810123437,1.071828860739069,5.004896097549624,0.7550169182353349,22.36133822426951,1698.423746423886,5.160280725540856,0.909016682713278,0.6451886415427616,Web/CloudOps,2 -m4_monthly/M/short,YingLong_300m,2115643.5334444446,2077160.3182962963,613.2118324652778,1.0481616578806128,0.1776379590564304,0.1442686033602114,10.328364197653602,1454.525191753118,0.3023002693982013,0.1274464706444784,0.1040263144155038,Econ/Fin,1 -m_dense/D/short,YingLong_300m,10192.354814814817,9848.373333333331,50.06791666666667,0.7452390651863391,0.1358169544352877,0.1083497789171007,7.522153103474118,100.95719298204963,0.1747718345769766,0.0866749697650207,0.0727910211679623,Transport,1 -bitbrains_rnd/5T/short,YingLong_300m,1399005.2873932128,1607149.0893850003,127.8545537579058,1.785717368673824,1.287260952103256,0.6904630002774508,54.44360319841742,1182.795539133122,4.839657872789569,0.5231439223383848,0.4254771941136492,Web/CloudOps,2 -bitbrains_rnd/5T/medium,YingLong_300m,2154692.144613578,2237569.5868353653,170.8998017476835,4.498291984870307,1.302720488480961,0.742535158557506,161.37266202016482,1467.8869658844917,6.176147931090124,0.7190624901772755,0.6523731293826035,Web/CloudOps,2 -electricity/15T/short,YingLong_300m,148046.51135135136,151065.45126126127,56.60763900619369,1.0720575790780826,0.1753974046930317,0.1731773346699035,11.37966539568751,384.76812673524734,0.7584538210441255,0.1115848146488403,0.0928686267327801,Energy,1 -bitbrains_rnd/5T/long,YingLong_300m,2495840.3812932014,2664985.362653644,219.90548587644668,3.4700286239149376,3.13265080653652,0.7299715676745755,117.8975914945948,1579.8228955465868,6.051945847533729,0.8424084091017047,0.6887456579557683,Web/CloudOps,2 -hierarchical_sales/W/short,YingLong_300m,526.165717690678,466.37486758474574,9.348125910354874,0.7470547667388,0.5960424784768493,0.474093744310282,6.830222255351081,22.938302415189277,1.053108683839589,0.4291770330528773,0.3712959863530989,Sales,1 -bitbrains_fast_storage/5T/long,YingLong_300m,3764342.004972065,4058291.128677089,387.3317370508165,1.009582292054786,5.223704907595844,0.7684563330215575,17.863852992642816,1940.191228969986,5.127346157498022,1.0236021398256,0.7091054073249635,Web/CloudOps,2 -saugeen/M/short,YingLong_300m,435.9256417410714,451.20191592261904,13.39521716889881,0.7850987702047343,0.3970942270188105,0.3903743198939732,6.467658630148655,20.87883238452456,0.6268451840939336,0.4021646046854742,0.3277683276184382,Nature,1 -ett1/D/short,YingLong_300m,44104.08253968254,45032.7492063492,142.73630952380952,1.7279010891714286,1.193524654327877,0.5022412496899802,10.298170440655811,210.0097201076239,0.5515974959562591,0.3749016515760436,0.284186477558998,Energy,7 -m_dense/H/short,YingLong_300m,56680.333333333336,58421.52,108.44579861111112,0.9290678683994812,0.3376354017763754,0.2654865180121528,8.735681521715637,238.0763182959056,0.4222880638408689,0.1923558237751465,0.1564930420470122,Transport,1 -kdd_cup_2018/H/short,YingLong_300m,4379.112130473512,4493.657248686884,22.089731872523508,0.927031062482016,0.8771597718772081,0.4876331401105727,7.713783069151611,66.17486026032478,1.3851596774796953,0.4623780957872693,0.3740945141201524,Nature,1 -m_dense/H/medium,YingLong_300m,41800.30666666666,36464.53333333333,91.74852430555556,0.7876295415627308,0.3532931748005365,0.2336693996853298,7.203348506611248,204.4512329790815,0.3557193157147895,0.1596308410972175,0.1340288035217407,Transport,1 -m_dense/H/long,YingLong_300m,48228.20345679012,40142.02074074074,99.05324074074076,0.8425839120449523,0.3943751161026657,0.2482007137345679,7.669552484776432,219.60920622048184,0.3801760001115421,0.1714758024538058,0.1450698737138724,Transport,1 -kdd_cup_2018/H/medium,YingLong_300m,4827.251245101868,5073.050808932262,24.9392565809393,1.0340730004504828,0.970128537306293,0.552104804941354,8.735375001149587,69.47842287431305,1.454309222045284,0.5220238073369216,0.4169610440086544,Nature,1 -kdd_cup_2018/H/long,YingLong_300m,3780.132152963716,4039.107903047205,23.72086670455411,1.0039256698315402,1.0027367671188725,0.5979061908619002,7.661119924565232,61.48277932042204,1.4426344482750642,0.556587386406003,0.4394137146588515,Nature,1 -m4_daily/D/short,YingLong_300m,371833.55571327184,372676.6711953767,182.73388389688736,3.51317346219464,0.0425897182039283,0.0342663411028246,30.821644341592496,609.7815639335711,0.0941944000342118,0.028227335127278,0.0225400668982757,Econ/Fin,1 -solar/10T/short,YingLong_300m,24.90353938564477,27.648199893552317,2.5009282095878955,1.10731808027984,5.283718417722162,1.5479865117989604,6.1093779056853625,4.990344615920304,1.4516907083504589,0.7275197453354145,0.553421584303218,Energy,1 -m4_yearly/A/short,YingLong_300m,5000833.433736688,4916918.619424857,1118.2171567133862,4.312498587797275,0.197400464768342,0.1904156140444542,51.570179669169846,2236.2543311834384,0.3585937478127971,0.1793113043998976,0.1515941545796118,Econ/Fin,1 -saugeen/W/short,YingLong_300m,825.80419921875,866.23046875,14.6233642578125,1.1859630934214436,0.4206714630126953,0.4009381294250488,10.372063312467995,28.736809134257584,0.8674444791022005,0.4414184098198283,0.360389737554416,Nature,1 -solar/10T/medium,YingLong_300m,19.27412076974121,21.211836429993365,2.034577224687569,0.8896344776528704,3.385049980445471,1.45993566937038,5.788836723467309,4.390230149974054,0.9742094680060194,0.4514807488376786,0.3482281957351236,Energy,1 -solar/10T/long,YingLong_300m,20.30869037662206,21.51395533949209,2.027292605212135,0.8863020744614358,2.819984049958087,1.4675204210037165,6.560452865681614,4.506516434744476,0.9740113330649068,0.4381668194287328,0.3512486118243954,Energy,1 -loop_seattle/D/short,YingLong_300m,17.843253353973168,18.015608875129,3.013710373129515,0.9072788780417246,0.0559316786696175,0.0555139689253579,7.063688757702453,4.224127525770638,0.0754908083870195,0.0538590350134818,0.0428155122248671,Transport,1 -ett2/D/short,YingLong_300m,110412.06349206348,110744.29206349206,198.483878968254,1.299525306770167,0.481592765687004,0.133729989188058,13.158700287625312,332.2831074431312,0.1901743893837749,0.1135975607540354,0.092369399197667,Energy,7 -ett1/H/short,YingLong_300m,101.64408482142858,107.17801339285714,5.043887183779762,0.8324057044261892,0.5000860305059524,0.259050550914946,5.5807663009249255,10.081869113484291,0.4705620781612315,0.2354188502641587,0.1820929093721376,Energy,7 -ett1/H/medium,YingLong_300m,134.91138392857144,134.96585751488095,6.794502766927083,1.2602670254898185,4084204535571.4873,0.4403334844680059,8.777061283906667,11.615135984075756,0.5563374800544929,0.3254405762238074,0.251884158597903,Energy,7 -ett1/H/long,YingLong_300m,139.63209325396826,138.84641203703703,7.119143208498677,1.3700696073732894,3946868136624.5366,0.4871406353339947,9.286321544286984,11.816602441225154,0.5658714979262243,0.3409203492613036,0.2635700437027675,Energy,7 -bizitobs_application/10S/short,YingLong_300m,1445730.1333333333,1696495.36,572.6089930555555,1.817812998221645,0.0260726356506347,0.0266502592298719,13.865050401736086,1202.385185093917,0.0463711905181336,0.0220832400785958,0.0167920703778449,Web/CloudOps,2 -bizitobs_application/10S/medium,YingLong_300m,7385415.68,6472782.933333334,1427.790625,3.8681272482165134,0.0535245768229166,0.0546028264363606,46.89840908821573,2717.6121283214798,0.1067126298687804,0.056065135604854,0.0482781477326485,Web/CloudOps,2 -bizitobs_application/10S/long,YingLong_300m,11097452.08888889,7829531.875555555,1706.585,4.599798576803993,0.0632481596204969,0.0646431435479058,80.28070935257594,3331.283849942675,0.128474302570193,0.065816162034804,0.0606065105006177,Web/CloudOps,2 -m4_hourly/H/short,YingLong_300m,680276.8180354268,711011.2979066023,221.3985507246377,0.9250319562832946,0.1259653825498817,0.1112879140365527,6.632736911403227,824.7889536332472,0.1126018975042944,0.0302257890415205,0.024848441847268,Econ/Fin,1 -electricity/15T/medium,YingLong_300m,278774.65377477475,275488.4064864865,56.76663063063063,0.8830048993986506,0.1490606893164232,0.1366740812645642,7.348494437699491,527.9911493337504,0.9097819181048782,0.0978146208788825,0.0788211182303169,Energy,1 -loop_seattle/5T/short,YingLong_300m,46.9253664376935,48.42047601627,3.857237383800148,0.6071451751382285,0.112301051136879,0.081556795354475,4.697483555905233,6.850209225833434,0.1174659682631413,0.0661431073375346,0.0524441375006667,Transport,1 -loop_seattle/5T/medium,YingLong_300m,110.54684412087204,110.4436410764964,6.519525495436661,1.022574347155637,0.2322959196013932,0.1443163168073561,7.40182114791904,10.514125932328946,0.1870679002010895,0.1159957520566482,0.0920612168122998,Transport,1 -electricity/15T/long,YingLong_300m,354041.6156996997,339702.3521441441,62.13003922672673,0.918390176709199,0.1527356066870213,0.1376696520456582,7.614968616067751,595.0139626090296,0.9387074476292172,0.0980177511934217,0.0781690330929683,Energy,1 -ett2/H/short,YingLong_300m,106.4244605654762,104.88314732142857,6.394745163690477,0.7407379492686809,0.1214849699767802,0.1111269905453636,5.261647960582421,10.316223173500862,0.1308384137338789,0.0811031614359429,0.0644934721112446,Energy,7 -ett2/H/medium,YingLong_300m,250.7732328869048,239.66867559523808,9.44057384672619,1.0184489875158365,0.1694063243810091,0.1668871924990699,7.150306133250036,15.835821193954695,0.2133978742240738,0.1272178035904968,0.1043022206096907,Energy,7 -ett2/H/long,YingLong_300m,261.07665343915346,244.61593915343917,9.685083912037037,1.0566042765999606,0.1798375119565762,0.177695154260706,7.226095618540751,16.157866611627707,0.2212695472336587,0.1326297699842532,0.1074672897321776,Energy,7 -solar/W/short,YingLong_300m,3862026.510948905,3956599.124087591,1628.1543111313867,1.7994194368063492,0.3761719195512089,0.2988925014969206,6.805938593758485,1965.203936223644,0.4011472559064694,0.3323470007686349,0.2550112813871233,Energy,1 -us_births/D/short,YingLong_300m,296733.0133333333,282991.52,343.6533333333333,0.5056032079818389,0.0324939346313476,0.0324668312072753,4.374781613832464,544.732056458341,0.0510642100472872,0.0322147114140432,0.026229993355089,Healthcare,1 -loop_seattle/5T/long,YingLong_300m,121.08967205595688,119.8049902534113,6.827265436876505,1.0697318088504102,0.2490078142916523,0.1493035153114608,7.929456457836664,11.004075247650611,0.1945543008369836,0.1207074491773945,0.0963152927793633,Transport,1 -ett1/15T/short,YingLong_300m,4.989907691592262,5.426839192708333,1.141652352469308,0.7168367160193663,0.5069723305352889,0.2472874959309896,4.930563499085433,2.233810128814054,0.4222830970424437,0.2158198160747358,0.1664695648975054,Energy,7 -ett1/15T/medium,YingLong_300m,9.615633060515872,10.269906374007936,1.6350288318452382,1.0285318121682634,0.745706922602835,0.395733894469246,6.911382373698324,3.10090842504513,0.5910362118139507,0.3116381119724082,0.2433109284140673,Energy,7 -ett1/15T/long,YingLong_300m,8.188919270833333,8.516012524801587,1.5882097904265873,1.0330601058192537,0.7845586517624605,0.3966448102678571,6.774528152155858,2.861628779355095,0.545429274759444,0.3027143563860313,0.234386699323145,Energy,7 -bizitobs_l2c/H/short,YingLong_300m,75.45882936507937,79.87349640376983,5.463115389384921,0.5544664321973307,0.438568115234375,0.6435810235464935,3.342718499167766,8.686704171610737,0.4682356018836918,0.294475910768167,0.2293816088144503,Web/CloudOps,7 -bizitobs_l2c/H/medium,YingLong_300m,113.69686569940475,110.27954799107142,6.904127139136905,0.7066118148803214,0.6698913210481228,0.827325504925879,4.58752951107538,10.662873238457108,0.645653170562967,0.4180553850416573,0.3300024212056354,Web/CloudOps,7 -bizitobs_l2c/H/long,YingLong_300m,160.58421378968254,147.89649057539685,8.480064174107143,0.8682625214808085,1.075153924125715,0.8582013811383928,5.829721432727708,12.672182676622151,0.7740534920595641,0.5179867947268668,0.4064934051278263,Web/CloudOps,7 -electricity/D/short,YingLong_300m,1282925632.1337657,1243548164.7602162,4178.923099099099,1.3960895808857603,0.4919655604049015,0.0964651131501923,11.42025889604222,35817.95125539379,0.5886078908799401,0.0686735847612102,0.0544394748229051,Energy,1 -restaurant/D/short,YingLong_300m,144.66475203138134,145.78415989539553,7.351071436910432,0.7028993806063948,0.7281346041870972,0.4021850939367937,4.760680038577257,12.027666109074584,0.5535410621619854,0.3383133397880059,0.2660543066940184,Sales,1 -covid_deaths/D/short,YingLong_300m,2211933.930827068,2010009.6,239.6091478696742,45.40439854073493,0.1270707583128245,0.179707526749991,1027.51135296984,1487.257183820965,0.5595877532384465,0.090154107958177,0.0782561074650183,Healthcare,1 -kdd_cup_2018/D/short,YingLong_300m,2642.2679699448104,2860.213046080192,20.99158853647184,1.1828928942779466,0.5280815777416218,0.4594672016818771,8.995735851027813,51.40299572928421,1.1514816502185126,0.4702338582751935,0.3737529937229737,Nature,1 -jena_weather/D/short,YingLong_300m,443.6748511904762,421.8808531746032,10.51873992435516,1.1223821038179145,0.7764330249147366,0.4529628208705357,7.463217046476573,21.063590652841604,0.1268372425727124,0.0633400064278558,0.0498921349319168,Nature,21 -saugeen/D/short,YingLong_300m,1158.5739583333334,1324.6688541666667,14.110291341145832,3.1354633168907866,0.3316891988118489,0.3666178385416667,31.62129070008756,34.037831281286614,1.1024934652389848,0.4570356985341836,0.3812078377644103,Nature,1 -sz_taxi/15T/short,YingLong_300m,16.783106541895606,16.917020506620116,2.7601179372114437,0.5506225498685873,1435920906467.7017,0.3976650820255862,3.961607921482223,4.096718997184894,0.3830607471782375,0.2580828316647546,0.2029748081706777,Transport,1 -sz_taxi/15T/medium,YingLong_300m,16.644449452457266,16.39372329059829,2.7757130158253207,0.5409075762721375,10721217586650.363,0.4071870983156384,3.82111000336121,4.079760955308199,0.3796898542357274,0.2583264269460883,0.2033638725101678,Transport,1 -sz_taxi/15T/long,YingLong_300m,15.92368011039886,15.55902666488604,2.7135422231125355,0.5112924792403588,7093442563239.458,0.3995969593015491,3.699541891371406,3.9904486101688943,0.3693488151024904,0.251160634491863,0.1984183486358434,Transport,1 -bizitobs_l2c/5T/short,YingLong_300m,21.18319847470238,21.35645461309524,2.7568990071614583,0.2863604562118306,0.154044101423466,0.2115164802187965,2.582952677078044,4.602520882592754,0.1580691967068341,0.0946830709909545,0.0769831298202451,Web/CloudOps,7 -bizitobs_l2c/5T/medium,YingLong_300m,156.47931547619046,165.9232674319728,9.161406515731292,0.8765213191442975,0.5612127561711044,0.8780798256802721,5.991860301799829,12.509169256037367,0.6581662581680209,0.4820247070447808,0.378771384090228,Web/CloudOps,7 -bizitobs_l2c/5T/long,YingLong_300m,281.7161507936508,324.4997619047619,13.082249503968256,1.2142850755593633,0.8502009243068485,1.1255081288206574,8.598363535277072,16.784402008819104,0.9424190800691136,0.7345487516496757,0.5755636810205136,Web/CloudOps,7 diff --git a/results/YingLong_300m/config.json b/results/YingLong_300m/config.json deleted file mode 100644 index 9f0b068dce5f8fb4ff39f08633b7bac6adb80180..0000000000000000000000000000000000000000 --- a/results/YingLong_300m/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "YingLong_300m", - "model_type": "pretrained", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_300m", - "org": "Alibaba", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/YingLong_50m/all_results.csv b/results/YingLong_50m/all_results.csv deleted file mode 100644 index 45fccda0749752760a83b092903e9a1827899fde..0000000000000000000000000000000000000000 --- a/results/YingLong_50m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -,dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -0,car_parts/M/short,YingLong_50m,2.345947492672213,2.159477054478876,0.7046011231312068,1.2694769664624517,0.7896742665264872,1.8166425885386976,15.541603892619609,1.5316486191918215,3.672736271633272,1.6895612149756591,1.4066018375789853,Sales,1 -1,hierarchical_sales/W/short,YingLong_50m,506.3414327330509,469.1426112288136,9.835895085738876,0.7762142640077655,0.6525183068950571,0.4975210610082594,6.810471240877617,22.502031746779014,1.0330792840570182,0.4515707544804065,0.3884757395322085,Sales,1 -2,m4_weekly/W/short,YingLong_50m,341134.72809085064,329510.0955646025,285.9197289479323,2.2658492182747123,0.0715250489745989,0.0683844194234452,17.1217089367556,584.0674002979885,0.1064080288437343,0.0520901436193186,0.0414693148038312,Econ/Fin,1 -3,solar/H/short,YingLong_50m,644.0886156998336,693.0945703675246,11.36824447352414,0.8779149150095859,4.844241733675308,1.4027870115210337,7.285883170317518,25.378900994720667,0.9354747691167784,0.4190372891381436,0.3410760500052428,Energy,1 -4,solar/H/medium,YingLong_50m,699.8939172749392,783.696897810219,12.152376064476886,0.9282774260861631,6.186506391005668,1.4087745690411022,6.9437785740605165,26.455508259622214,0.9529891549700594,0.437756949627003,0.3593152234850635,Energy,1 -5,bitbrains_fast_storage/H/short,YingLong_50m,2779094.920323197,2396860.934049645,285.2965966634964,1.11194384477403,3.75212288793506,0.5462979873385828,21.42075720979008,1667.0617626000535,4.751667418280039,0.8131879533951912,0.623302913456145,Web/CloudOps,2 -6,solar/H/long,YingLong_50m,683.3649635036496,695.7158150851582,12.816001368613138,0.9753705831403884,7.868069870639642,1.4131793830220591,6.751444721076464,26.14125022839668,0.9063647864932614,0.4443541239486017,0.3515286755533442,Energy,1 -7,bizitobs_service/10S/short,YingLong_50m,8828.645925925926,8728.380105820106,30.63292989417989,1.1677389729333216,0.11160320302166,0.0703267738301918,10.776195731661758,93.96087444210984,0.0696092196403678,0.0226938537757618,0.0177720397177592,Web/CloudOps,2 -8,bizitobs_service/10S/medium,YingLong_50m,84027.80952380953,84558.35428571429,71.37823412698413,2.11970590850764,0.2086375984312996,0.1171220179966517,21.88575470696702,289.875506940151,0.2181671374712877,0.053720941039888,0.0466532159788119,Web/CloudOps,2 -9,bizitobs_service/10S/long,YingLong_50m,150605.20634920636,143200.4673015873,96.26845238095238,2.3440938635306883,0.2365248584242724,0.1303954819775132,29.702485428959225,388.0788661460533,0.2875012306105031,0.0713187471488748,0.0652027971221242,Web/CloudOps,2 -10,jena_weather/H/short,YingLong_50m,1058.9135338345864,1098.6013993316624,8.542908377715122,0.5349701924956606,1.5319804349726602,0.6067653079627356,4.510380017575527,32.54095164303875,0.1994739386892743,0.0523674783902325,0.0437365212110991,Nature,21 -11,jena_weather/H/medium,YingLong_50m,1433.5343253968254,1443.396626984127,11.270207093253967,0.8804535156623439,3.437548347597054,0.6806450719007342,5.971317705796436,37.862043333618765,0.231684558108966,0.0689643960097257,0.056707822484238,Nature,21 -12,jena_weather/H/long,YingLong_50m,1265.4759259259258,1147.4583333333333,11.866300843253969,1.0546677344811275,3.406120945809831,0.6630678835119225,10.351846591977424,35.573528443576215,0.2141920540557199,0.0714482780557197,0.060924276451579,Nature,21 -13,jena_weather/10T/short,YingLong_50m,929.2649801587302,1088.3414682539683,7.2913194444444445,0.3355857765130435,0.36007344982751,0.5732276958533923,2.867170582323744,30.48384785683609,0.189342901249565,0.0452882321166316,0.0369492828071498,Nature,21 -14,jena_weather/10T/medium,YingLong_50m,1749.5961038961038,1838.22886002886,10.89862012987013,0.6360092629119818,0.8894896365168352,0.6662731209017687,6.367873579534676,41.82817356634286,0.2566987861103216,0.0668846454215329,0.0577173812096614,Nature,21 -15,jena_weather/10T/long,YingLong_50m,1826.5833333333333,1982.88082010582,11.474210482804232,0.6418221135358174,0.8611013964834784,0.6545774939021869,6.865585264621856,42.73854622391048,0.2615244927853477,0.0702126614439889,0.0602355313198452,Nature,21 -16,ett2/H/short,YingLong_50m,107.4489490327381,106.79140625,6.4644891648065474,0.7520336049129952,0.1244616642814223,0.1131137393769763,5.303594440600407,10.365758488057596,0.1314666593496833,0.0819877094260368,0.0649365320886791,Energy,7 -17,ett2/H/medium,YingLong_50m,240.5033110119048,232.72336309523808,9.28452380952381,1.010321635022601,0.1702085426368962,0.1661948976062593,7.061515230128147,15.508169170211705,0.2089825524990052,0.125114928987171,0.102082007057541,Energy,7 -18,ett2/H/long,YingLong_50m,256.7726851851852,248.11107804232805,9.654962384259258,1.060695091675036,0.1827469019936677,0.1782821736007771,7.263948572087742,16.02412821919449,0.2194381028819546,0.1322172788445764,0.1068073671577059,Energy,7 -19,hierarchical_sales/D/short,YingLong_50m,29.1740104381976,29.092078338156263,2.364517233360893,0.7687472673995499,0.6495397235299631,1.006649990957007,7.098579794570063,5.401297107010278,1.6570581366736246,0.725407701746376,0.591013011304968,Sales,1 -20,m4_daily/D/short,YingLong_50m,360999.3035249586,360344.2695596336,193.09807170739128,3.708535578366285,0.0435554471905981,0.0339308457044369,31.566226549101973,600.8321758402745,0.0928119668942469,0.0298283157249116,0.023740030190624,Econ/Fin,1 -21,ett2/15T/short,YingLong_50m,8.028053501674107,8.161034574962798,1.7920915876116073,0.7730018202384242,0.1120487116501903,0.1315044766380673,5.499145219882313,2.8333819900737187,0.1345107227002379,0.08507696295069,0.0670705947508995,Energy,7 -22,ett2/15T/medium,YingLong_50m,12.2265091765873,12.068477182539684,2.1940415736607144,0.909095213257458,0.1392216305434914,0.1669336906312004,6.4364931291520415,3.4966425577383946,0.1863424112839223,0.1169244469636455,0.0919309167813532,Energy,7 -23,ett2/15T/long,YingLong_50m,12.33646949404762,11.64891121031746,2.199874441964285,0.9193657819633304,0.1421542997484582,0.1693870132688492,6.754968759117255,3.512331062705738,0.1871784801118855,0.1172352910738002,0.0931966929661242,Energy,7 -24,us_births/W/short,YingLong_50m,2937836.285714286,3116259.714285714,1357.3392857142858,1.2339246271767508,0.0182938064847673,0.0183947086334228,7.413372415602534,1714.0117519183718,0.0232675414658539,0.0184257477104473,0.0143340729500147,Healthcare,1 -25,ett2/W/short,YingLong_50m,6623990.285714285,4113258.5714285714,1388.510602678571,0.9505465144564608,0.1422410437038966,0.1490610837936401,12.31331514823101,2573.711383530462,0.2158949276778677,0.1164747523997943,0.109217580130966,Energy,7 -26,bitbrains_fast_storage/5T/short,YingLong_50m,1625323.5851269332,1807582.7612112092,171.9134607317288,0.8170880115615706,2.188540019955796,0.6879133037170319,14.894242836302633,1274.881792609391,4.002813579976022,0.5397657564700009,0.4157287325802631,Web/CloudOps,2 -27,temperature_rain/D/short,YingLong_50m,172.66202881747603,177.03384917584646,6.184557662395261,1.4583460504277337,55.47725823619599,1.4426676192473649,18.659867846409888,13.140092420431298,1.5469362738146486,0.7280859433364272,0.5855959025173892,Nature,1 -28,saugeen/D/short,YingLong_50m,1106.04375,1263.1034375,13.639124348958331,3.030752481941585,0.3296083831787109,0.3492827606201172,29.81457571490539,33.2572360547295,1.0772097998619516,0.4417744873944299,0.3670472305631288,Nature,1 -29,saugeen/W/short,YingLong_50m,912.7845703125,931.8578125,15.276841735839843,1.2393039893169084,0.4460285186767578,0.4208316326141357,11.57681965079929,30.212324808139144,0.9119841466469698,0.4611441708771578,0.3761910558095485,Nature,1 -30,bitbrains_fast_storage/5T/medium,YingLong_50m,2870817.369644257,3254084.6308887354,298.25332927733194,1.0843732914669508,5.019108437274568,0.7493626578769188,22.878649997043716,1694.3486564589523,5.147899476017486,0.906176040954912,0.643268657087845,Web/CloudOps,2 -31,kdd_cup_2018/H/short,YingLong_50m,4438.921620014656,4562.074270052173,22.455049457146558,0.933982231959928,0.8712875178771664,0.4929287654149919,7.8220884471991505,66.62523260758385,1.3945867863972063,0.4700248544763467,0.3798612742472347,Nature,1 -32,kdd_cup_2018/H/medium,YingLong_50m,4856.280447402948,5070.645750417959,24.90104127814369,1.0303783760583116,0.9588919119228289,0.5491351752796269,8.970134709056229,69.68701778238862,1.458675491254284,0.5212238918382718,0.4193356351147657,Nature,1 -33,electricity/H/short,YingLong_50m,1963977.596036036,2063356.6875675677,202.36813837274772,1.0953413361090494,0.1902020470908761,0.1394756007306852,10.844345889803993,1401.4198500221253,0.6615702500380454,0.0955322132056848,0.0786484151989655,Energy,1 -34,kdd_cup_2018/H/long,YingLong_50m,3813.569594741067,4078.6333335286295,23.86171278584025,1.0096902984019391,0.9942768613202848,0.601537233567298,7.891046642279318,61.75410589378707,1.4490008660882356,0.5598922046171155,0.4445782346906876,Nature,1 -35,bizitobs_l2c/H/short,YingLong_50m,95.82296316964286,101.37220982142856,6.389646499875992,0.6454355930506707,0.527434936816172,0.7312637518416564,4.273780109539138,9.788920429222156,0.5276478809935692,0.3444183105108198,0.268816795141293,Web/CloudOps,7 -36,bizitobs_l2c/H/medium,YingLong_50m,129.63949032738094,134.55741257440476,7.505065336681548,0.7737421733293213,0.6573928147968812,0.8902070027584474,4.899897526602889,11.3859338803359,0.6894355906947746,0.4544431056756912,0.3582639308630035,Web/CloudOps,7 -37,bizitobs_l2c/H/long,YingLong_50m,143.0390625,121.79383680555556,7.483891369047619,0.7900304220953858,0.8887128190205733,0.8322667681981647,6.230490059681806,11.95989391675361,0.7305448388148346,0.4571376846620619,0.3770265652827749,Web/CloudOps,7 -38,restaurant/D/short,YingLong_50m,149.56512094891193,150.78612122910246,7.516571372700103,0.7169512887675535,0.747977100991174,0.4093964885705613,4.910284339555336,12.229681964340362,0.5628382998873502,0.345930030292545,0.2713240697110609,Sales,1 -39,sz_taxi/15T/short,YingLong_50m,16.772697029532967,16.85576893267323,2.7634824722532243,0.5513067916951179,1255242130478.4165,0.3971661275414174,3.967935671947546,4.095448330712154,0.3829419344286209,0.2583974300806815,0.2032656653026313,Transport,1 -40,sz_taxi/15T/medium,YingLong_50m,16.9963374732906,16.79073684561966,2.8291869073851497,0.5522790594283759,11016456208215.96,0.4117452768179087,3.994914936733256,4.122661455090704,0.3836824618143321,0.263303065115379,0.2082080492337384,Transport,1 -41,sz_taxi/15T/long,YingLong_50m,16.179738693019942,15.89880698005698,2.7589086983618234,0.5209434231991072,7478219016552.588,0.4030867791243768,3.936516157070564,4.02240459091573,0.3723066037566794,0.2553596746288545,0.2043178718052039,Transport,1 -42,jena_weather/D/short,YingLong_50m,435.4188988095238,373.4659226190476,9.950619264632936,1.0802152985573867,0.8180907067106863,0.4451001364087301,7.25892292119835,20.866693528432428,0.1256516000703413,0.059918991506165,0.0489986381024393,Nature,21 -43,kdd_cup_2018/D/short,YingLong_50m,2654.8831704235654,2825.608085643992,20.737810858434734,1.1712600846547196,0.5410068262621351,0.453783898842177,9.20092703933402,51.52555841932784,1.1542271845319196,0.4645489689929856,0.3718150430487579,Nature,1 -44,m_dense/D/short,YingLong_50m,11828.093333333334,11316.230370370373,56.79396990740741,0.8310552920904485,0.1478149323573017,0.1208352322048611,7.51145069906171,108.75703808643068,0.1882745202008855,0.0983187628383452,0.0802924636732568,Transport,1 -45,bitbrains_fast_storage/5T/long,YingLong_50m,3655905.308869216,3892526.2015919313,378.6719158481997,1.007303638308903,5.08547967450603,0.7616484369449431,18.311073355116317,1912.042182816377,5.052956632652286,1.0007168178506989,0.6903949173762876,Web/CloudOps,2 -46,m_dense/H/short,YingLong_50m,54753.804444444446,55595.96444444444,108.64254340277778,0.9371542769194656,0.3416542823024893,0.260730726453993,8.538897815674979,233.9953085949469,0.4150493695538056,0.1927048000099034,0.1575588611417751,Transport,1 -47,m_dense/H/medium,YingLong_50m,52862.44,46354.77333333333,107.02614583333332,0.8893852585453887,0.4622745488663682,0.2679356384277344,8.161983479242796,229.91833332729252,0.4000288528944082,0.1862119724331516,0.156467421056435,Transport,1 -48,m_dense/H/long,YingLong_50m,64903.02814814815,55913.53283950617,122.08861882716047,0.9842398728974996,0.5558742264659896,0.2963102213541667,8.394663164590755,254.76072724842845,0.4410284793506166,0.2113534471694824,0.1747923377750923,Transport,1 -49,bizitobs_application/10S/short,YingLong_50m,1780789.9022222222,1970887.68,599.4136111111111,1.885777229108565,0.0269266404045952,0.0274116579691569,15.091735876583115,1334.4624019515209,0.0514648808446078,0.0231169870558781,0.0176288755103418,Web/CloudOps,2 -50,bizitobs_application/10S/medium,YingLong_50m,8732044.8,6254582.613333333,1417.6616666666666,4.080549996140289,0.0554265276590983,0.056578197479248,44.76144069936248,2955.003350251908,0.1160342844698754,0.0556674012224096,0.0489637633139335,Web/CloudOps,2 -51,bizitobs_application/10S/long,YingLong_50m,12305676.515555557,8048074.524444444,1721.628611111111,4.865825362311909,0.0655826059977213,0.0675031195746527,67.58463998206184,3507.944770881599,0.1352874081569255,0.0663963339843274,0.0601136315919493,Web/CloudOps,2 -52,ett1/H/short,YingLong_50m,97.85457589285714,100.60135788690476,4.928938802083334,0.8304398933425667,0.4636341276622954,0.2600812457856677,5.630882199233684,9.8921471831376,0.4617069794873857,0.2300537390170789,0.1793112094888531,Energy,7 -53,ett1/H/medium,YingLong_50m,133.38772321428573,132.90990513392856,6.761934988839286,1.2665184884028238,4272706268616.796,0.4422677902948289,8.465435611100284,11.54936029459146,0.5531869804489171,0.3238806568550475,0.2512376460359611,Energy,7 -54,ett1/H/long,YingLong_50m,135.28277116402117,136.3495701058201,6.983536396329365,1.3444550410010925,3239410691356.6665,0.4766467543506117,9.208496499410815,11.631112206664554,0.5569887723370733,0.3344264327304795,0.2592799199082443,Energy,7 -55,ett1/W/short,YingLong_50m,1364905.7142857143,1400135.7142857143,923.1282087053572,1.733683991631802,0.850813456944057,0.5598954132625035,9.790559825487488,1168.2917932972543,0.4648911890481847,0.3673347472361833,0.2902033017798772,Energy,7 -56,electricity/H/medium,YingLong_50m,7920814.653693694,7897946.254414414,272.0144481981982,1.1566413945307807,0.2453377881846304,0.1373175527187671,9.572521634062108,2814.394189464883,1.0985507984646588,0.1061762031703422,0.086222377009911,Energy,1 -57,m4_yearly/A/short,YingLong_50m,5003386.283624968,5004212.268245263,1134.3502510083863,4.356100991573442,0.2020312156753626,0.1948036310521314,44.14944905738486,2236.82504537681,0.3586852644791607,0.1818983208525387,0.1503592848334618,Econ/Fin,1 -58,ett1/15T/short,YingLong_50m,5.336710611979167,5.779307919456845,1.1641288394019718,0.7179857460793408,0.5370524839818158,0.247701172601609,4.932365771034554,2.3101321633142917,0.4367111384796321,0.2200688076922992,0.1699979607237043,Energy,7 -59,ett1/15T/medium,YingLong_50m,9.55883308531746,9.986096230158733,1.6359509858630952,1.0279384672562828,0.7153626003996103,0.4004739040798611,7.047193897866914,3.091736257399305,0.5892879875917432,0.3118138754399848,0.2452464456609152,Energy,7 -60,ett1/15T/long,YingLong_50m,8.809058159722222,8.916981026785715,1.6117877294146823,1.0351120073786877,0.7736472612527088,0.3960380239335317,7.09317100956946,2.9680057546646066,0.5657048314328871,0.3072083348696756,0.2422142447896858,Energy,7 -61,electricity/H/long,YingLong_50m,12860407.7587988,12703120.227171171,330.4777117117117,1.3246994223916422,0.3215943027644249,0.1537320356114761,10.851398147439134,3586.1410678888246,1.414372976489062,0.1303403117524763,0.1046904349320689,Energy,1 -62,electricity/D/short,YingLong_50m,1468617804.8276756,1457061033.079928,4508.499171171171,1.4785628623154317,0.5245328362423753,0.1005155816066071,11.861664471410982,38322.54956063956,0.6297667588297139,0.0740896141505971,0.058313981731172,Energy,1 -63,solar/10T/short,YingLong_50m,26.09515710538321,27.7849409785584,2.5307373046875,1.1219539696809864,3.741759049011934,1.5730944926769048,7.540801049475076,5.1083419135158925,1.4860161094427018,0.736191207871765,0.5696661614103602,Energy,1 -64,solar/10T/medium,YingLong_50m,35.16942324706923,32.85600116124751,2.838931773252599,1.2271129636096665,2.4403789970388856,1.5303478816768263,9.623389839344616,5.930381374504445,1.315975128083644,0.6299702106829513,0.5363962280170552,Energy,1 -65,bitbrains_rnd/5T/short,YingLong_50m,1413526.150639543,1617651.6682116392,128.62127531770662,1.8033149761164065,1.1654302492101956,0.6559372417056644,55.5143854083498,1188.9180588415431,4.864709455863586,0.5262811256083998,0.4295090572542891,Web/CloudOps,2 -66,electricity/15T/short,YingLong_50m,147501.08966216215,149329.09209459458,56.5797974732545,1.0781102571023975,0.1780705432119419,0.1734740378651663,10.355844945135749,384.05870601011264,0.7570554129580094,0.1115299334994566,0.0931792249724961,Energy,1 -67,solar/10T/long,YingLong_50m,39.365362524077455,41.476229692568936,3.161267119544556,1.362179537472376,1.9718213520460528,1.5801417159006357,10.068966022347482,6.27418221954682,1.3560639744786465,0.683257244452153,0.5594268831323339,Energy,1 -68,hospital/M/short,YingLong_50m,3306.233811386354,3063.4567579313343,19.830595053780964,0.8277984503138229,0.2061077119577765,0.186392229155632,5.845252568523061,57.49985922927424,0.2087378412843228,0.071989664990386,0.0578865357962606,Healthcare,1 -69,loop_seattle/D/short,YingLong_50m,18.65639834881321,18.91279831011352,3.120340113841589,0.9369963193505348,0.0582251644232939,0.057477707365841,6.617972802598223,4.319305308589937,0.0771917626602223,0.0557646510904927,0.0438613585539295,Transport,1 -70,saugeen/M/short,YingLong_50m,352.7041248139881,365.5207868303572,12.7148684547061,0.7450967247924493,0.402371951511928,0.3969873700823103,6.10345306099761,18.780418653852955,0.5638445087169407,0.3817384952583827,0.3148663600775754,Nature,1 -71,ett1/D/short,YingLong_50m,40004.263492063496,39826.63492063492,133.8670386904762,1.6953082577971474,1.4413104344928076,0.4696768624441964,11.177447030439016,200.01065844615252,0.5253346288301407,0.3516062175355745,0.2713516288121217,Energy,7 -72,bitbrains_rnd/5T/medium,YingLong_50m,2108276.8289839784,2271763.29469824,162.83312770682485,4.562232492964939,0.9072726409906402,0.7263479851234265,162.53609101742742,1451.990643559379,6.109264008470763,0.6851218848404009,0.6379403883415798,Web/CloudOps,2 -73,m4_hourly/H/short,YingLong_50m,610484.0450885668,701484.5732689211,214.16810084541063,0.9642894045509044,0.1272972094647934,0.1155495021654211,7.699171827419456,781.3347842561259,0.1066694442327237,0.0292386730373303,0.0240692645292731,Econ/Fin,1 -74,ett2/D/short,YingLong_50m,115713.23174603174,111845.38412698412,200.70328621031743,1.310885593087438,0.4282265314980158,0.1362759786938864,12.50183286594022,340.1664765170603,0.1946862495004617,0.1148677860757533,0.0940236788229717,Energy,7 -75,bitbrains_rnd/5T/long,YingLong_50m,2403903.1805204945,2571618.586995048,204.93930604546205,3.5096062818253326,2.5363544050581006,0.7185924567258016,118.96630136303492,1550.452572805919,5.939434753250842,0.7850763435031533,0.6614450212712936,Web/CloudOps,2 -76,us_births/M/short,YingLong_50m,49769536.0,40883354.66666666,5374.541666666667,0.6073190193919541,0.0167430688937505,0.0168678288658459,4.00501332323018,7054.752724227831,0.0219120138627552,0.0166932897735045,0.0128850775476625,Healthcare,1 -77,electricity/W/short,YingLong_50m,78800425941.56396,79297323436.05045,35512.92972972973,1.7377926176399563,0.2567041640255809,0.1087707746427741,13.061693300587018,280714.1356283363,0.6404970215356374,0.0810287863380408,0.0631888170242435,Energy,1 -78,loop_seattle/5T/short,YingLong_50m,55.46593369049923,56.872313596491225,4.197359930300245,0.6588380813048039,0.12175173725014,0.0874300621119323,5.350901955219515,7.4475454809285475,0.1277089665819101,0.0719754582826837,0.0574285432218288,Transport,1 -79,electricity/15T/medium,YingLong_50m,293482.7428468469,291231.64663063065,60.58548001126126,0.9499317309341716,0.1648060511921295,0.1467763945386848,8.37699985883902,541.7404755478834,0.9334733917053883,0.1043948829132895,0.0849946371244059,Energy,1 -80,loop_seattle/5T/medium,YingLong_50m,126.56000669182148,129.6099725877193,7.018233570772058,1.0945748166882907,0.2608085923642689,0.1530180621860689,7.923993573446473,11.249889185757407,0.2001586400067423,0.1248687932458795,0.0994782709867671,Transport,1 -81,loop_seattle/5T/long,YingLong_50m,131.4119894507511,133.67282536406375,7.0721103514505215,1.1035848649746882,0.2718466672579119,0.1531503306046468,8.383921989740612,11.463506856575394,0.2026771456417622,0.1250363573406324,0.1004954530569604,Transport,1 -82,bizitobs_l2c/5T/short,YingLong_50m,21.605775669642856,21.62736235119048,2.78084222702753,0.2877068481078589,0.1540279500589987,0.2104753351041046,2.612555731886186,4.648201337038108,0.1596380484130131,0.0955053780759954,0.0779665012791904,Web/CloudOps,7 -83,bizitobs_l2c/5T/medium,YingLong_50m,142.88671343537416,149.1109056122449,8.69811330782313,0.8418890347772198,0.5265403205971789,0.8457501422146139,5.890603479443451,11.953523055374683,0.6289310968819162,0.4576486712871379,0.3589505826234094,Web/CloudOps,7 -84,bizitobs_l2c/5T/long,YingLong_50m,266.64021825396827,306.94638888888886,12.69935763888889,1.187276933414684,0.8327570502377598,1.064004944329774,7.577885517520446,16.329121784528656,0.9168557761203632,0.7130499458498359,0.5499149616032214,Web/CloudOps,7 -85,electricity/15T/long,YingLong_50m,361712.035003003,339027.2284084084,65.0468125,1.004045441403098,0.1701865799081604,0.1494019458655707,8.851890312349786,601.4250036396916,0.948821650556743,0.1026193184955768,0.083771915015824,Energy,1 -86,solar/D/short,YingLong_50m,121799.3498783455,129854.88661800486,253.6771593673966,0.989843435216182,1.0813892373783456,0.4291314118099908,5.553985313786724,348.99763592085475,0.5041827137837863,0.3664770917354322,0.2884009884155804,Energy,1 -87,bitbrains_rnd/H/short,YingLong_50m,2060177.1368654363,2117281.9518349604,184.44738075955843,5.867890053533356,2.520784680684783,0.5681762230904476,198.2342075690009,1435.3317166653276,6.28171412933239,0.8072320177857726,0.6792077052027463,Web/CloudOps,2 -88,m4_monthly/M/short,YingLong_50m,2155715.8595555555,2111149.437962963,619.4549046585648,1.0533479404780126,0.1806442969640096,0.1456027859935054,9.55228028430166,1468.235628077304,0.3051497687522639,0.1287439953739253,0.1047997367605427,Econ/Fin,1 -89,covid_deaths/D/short,YingLong_50m,1793368.2847117796,1472458.105263158,194.51929824561404,43.57945993519842,0.1186095936102697,0.1703691975257426,890.1879936255372,1339.167011508191,0.5038681052161073,0.0731888326046813,0.0593550539782383,Healthcare,1 -90,loop_seattle/H/short,YingLong_50m,61.95739851515398,62.46728178806149,4.4340315614646135,0.8965250597533395,0.13190690814934,0.1018883100798372,7.382973603801189,7.871302212159941,0.1393300535881994,0.0784868676641886,0.0634633846793447,Transport,1 -91,loop_seattle/H/medium,YingLong_50m,69.45846233230134,66.83237229102167,4.966542505159959,1.0132323030102617,0.1342672909007353,0.1121721612164602,7.526380705270291,8.334174364164776,0.1473055823432752,0.0877830729221292,0.0696447769590137,Transport,1 -92,loop_seattle/H/long,YingLong_50m,65.69606553147575,62.03056415548676,4.751472738218094,0.9835414056424908,0.1565596815069229,0.110881079748237,7.709076965029921,8.105310946007917,0.1448527580736363,0.0849151791494899,0.0681958697173843,Transport,1 -93,solar/W/short,YingLong_50m,5121434.627737226,5059590.072992701,1883.7143020072992,2.080787091174768,0.4297032043011519,0.3356885144310276,11.17355031713386,2263.0586885313483,0.4619468575887953,0.3845131842215099,0.3045864028182746,Energy,1 -94,sz_taxi/H/short,YingLong_50m,7.290910248063568,7.258826622596154,1.862273387419872,0.5666622235072498,1.1346744635166266,0.2970678867437901,4.108246478491044,2.7001685591946973,0.25153736372616,0.173482257918256,0.13685702572313,Transport,1 -95,m4_quarterly/Q/short,YingLong_50m,2202503.0186666665,2156719.5866666664,647.456951171875,1.4015457984951658,0.1313249570528666,0.1182681825955708,11.334767466026577,1484.083224979875,0.2484093980052535,0.1083728922797461,0.0875069168020507,Econ/Fin,1 -96,us_births/D/short,YingLong_50m,390455.28,379573.06666666665,410.8233333333333,0.6041797786177266,0.038495709101359,0.0388207817077636,4.469942328335667,624.8642092486975,0.0585759491364682,0.0385113538609314,0.0307806063870929,Healthcare,1 diff --git a/results/YingLong_50m/config.json b/results/YingLong_50m/config.json deleted file mode 100644 index 048ca4fbee283236053483a3837930f79e3265c4..0000000000000000000000000000000000000000 --- a/results/YingLong_50m/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "YingLong_50m", - "model_type": "pretrained", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_50m", - "org": "Alibaba", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/YingLong_6m/all_results.csv b/results/YingLong_6m/all_results.csv deleted file mode 100644 index 394491323a6cd9a079dc498160bb5db3741e074a..0000000000000000000000000000000000000000 --- a/results/YingLong_6m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -,dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -0,car_parts/M/short,YingLong_6m,1.4256717159372094,1.3781951381277402,0.6313851595843505,1.1116554622469257,0.6382586963869232,1.7779277178192112,16.78015743895224,1.1940149563289435,2.8631253826976617,1.5139968449160262,1.2105464409696856,Sales,1 -1,hospital/M/short,YingLong_6m,4015.826162538027,3838.0612777053457,21.013732140916996,0.8359615058789307,0.2085148737980769,0.1878369135319901,5.688476650672462,63.370546490763566,0.2300497993002959,0.0762847273578859,0.0619081969301056,Healthcare,1 -2,bizitobs_application/10S/short,YingLong_6m,2320305.635555556,1858212.6933333331,613.4881944444444,2.149902125424402,0.0298106257120768,0.0304501024881998,25.748018490394287,1523.254947655039,0.0587458547070207,0.0236597874773274,0.0198596027188103,Web/CloudOps,2 -3,bizitobs_application/10S/medium,YingLong_6m,11444406.613333331,8742759.253333334,1521.3734375,4.596128402307508,0.0628451538085937,0.0632885360717773,46.49517702247203,3382.9582636109085,0.1328388143709146,0.0597398572210546,0.0523605555669505,Web/CloudOps,2 -4,bizitobs_application/10S/long,YingLong_6m,13244051.91111111,10289149.724444443,1901.4120833333332,5.507461199402938,0.0754376898871527,0.0766571892632378,69.45892650247822,3639.23782008144,0.1403508562712475,0.0733298638928626,0.064830209163392,Web/CloudOps,2 -5,ett2/15T/short,YingLong_6m,8.20115269252232,8.28952404203869,1.8478413899739583,0.7985663190479825,0.1153554605776208,0.1349930536179315,5.37556537228766,2.8637654744273875,0.1359531347904118,0.0877236043962933,0.0688702376315566,Energy,7 -6,ett2/15T/medium,YingLong_6m,12.236853918650796,12.131070188492064,2.207648499503968,0.921881067240266,0.1424043299788135,0.1685027785528274,6.493926040291256,3.498121484261345,0.1864212259553815,0.1176495846721545,0.0928659678493093,Energy,7 -7,ett2/15T/long,YingLong_6m,12.351812996031748,11.778629712301589,2.236570560515873,0.94641495562293,0.1467957712771317,0.1739019290984623,6.895564035671887,3.514514617416144,0.1872948456949237,0.1191908936562061,0.0946459018278231,Energy,7 -8,jena_weather/H/short,YingLong_6m,1093.234857978279,1123.939640768588,8.614744869987469,0.5534368630739412,1.9122423732753688,0.6048327551283558,4.840350400390098,33.06410225574375,0.2026808182663901,0.052807831463258,0.0463518219474755,Nature,21 -9,jena_weather/H/medium,YingLong_6m,1481.1103174603174,1477.3252976190477,11.698043774801588,0.9287345089526038,4.576984468393407,0.6886579995473037,6.47764261574141,38.48519608187436,0.2354977402936103,0.071582404542275,0.0615594287683964,Nature,21 -10,jena_weather/H/long,YingLong_6m,1238.1851851851852,1178.522222222222,11.791174768518518,1.0847386146010818,3.86556487963336,0.6651762179768127,11.902302009548809,35.1878556491467,0.2118698765364644,0.0709959358517055,0.0640282769833356,Nature,21 -11,bizitobs_service/10S/short,YingLong_6m,14452.285291005292,12271.09417989418,34.80434854497354,1.3457522148879075,0.1474375310019841,0.0980934490224041,15.064629365476948,120.21765798336487,0.0890610842960005,0.025784174069172,0.0217088115020862,Web/CloudOps,2 -12,bizitobs_service/10S/medium,YingLong_6m,99495.53777777778,100139.45904761904,80.52168650793651,2.56449512959571,0.1497442626953125,0.1331268310546875,25.507747086169825,315.42913273472027,0.2373993984182764,0.0606025187682515,0.0528472399110938,Web/CloudOps,2 -13,bizitobs_service/10S/long,YingLong_6m,158795.78412698413,156422.89439153438,106.1684126984127,2.8186576443050546,0.1849508876901455,0.1412263609871031,32.81866787443415,398.4918871532821,0.2952155294685723,0.0786529542458256,0.0703410124367075,Web/CloudOps,2 -14,m4_weekly/W/short,YingLong_6m,386603.2243411185,352563.78144418256,314.4123098350118,3.007684597800195,0.0753146887387508,0.0726404674359197,26.088581357810188,621.7742551289161,0.1132776334380136,0.0572810502977567,0.0460643184049049,Econ/Fin,1 -15,loop_seattle/H/short,YingLong_6m,66.51445255214274,67.7222079361523,4.7398130066604205,0.9552631427715462,0.1485241628937651,0.1089159061257218,8.08626601773549,8.155639309836031,0.1443631093632809,0.0838995101974141,0.0683763311341952,Transport,1 -16,loop_seattle/H/medium,YingLong_6m,75.21790505675955,70.65750773993808,5.423755563080495,1.0987527117909544,0.1504415695143189,0.1207601793287216,8.391361001844356,8.672825667379666,0.1532912055438018,0.0958642616288985,0.0767024007197392,Transport,1 -17,bitbrains_fast_storage/H/short,YingLong_6m,2707360.0128867524,2258208.1280449666,283.7603740928292,1.2550857824673614,3.7548346359841647,0.5633419689119171,22.842427385021125,1645.4057289576797,4.689940689387193,0.8088092201652473,0.626181025860868,Web/CloudOps,2 -18,us_births/W/short,YingLong_6m,3451126.285714286,3316488.285714286,1418.0,1.2886437559112136,0.0190166447843824,0.0192207012857709,8.768189706544595,1857.7207232827773,0.0252183766608251,0.0192492109588244,0.015425635124951,Healthcare,1 -19,loop_seattle/H/long,YingLong_6m,77.03095115239078,77.21209150326797,5.328770532335741,1.0925809377147078,0.1867445497936016,0.122045201372764,8.66671102574153,8.776727815785948,0.1568518764359512,0.0952322636222274,0.0776206161634525,Transport,1 -20,us_births/D/short,YingLong_6m,291700.4266666667,278277.17333333334,360.48333333333335,0.5302691111278264,0.0340310351053873,0.0339402770996093,4.642845159482257,540.0929796494921,0.0506293342403947,0.0337923873464703,0.0276366354815189,Healthcare,1 -21,m4_daily/D/short,YingLong_6m,487054.95204298897,453533.78647470346,259.19196534185,4.848722708823648,0.0514984758713499,0.0418586928208943,44.35645812622036,697.8932239554908,0.1078052165014137,0.0400379957563352,0.0323670658642097,Econ/Fin,1 -22,electricity/W/short,YingLong_6m,121555547433.05225,148941582800.95135,38160.5963963964,1.789992584226223,0.2676569298852848,0.1128648620468002,13.950345574134747,348648.17141790985,0.7955000729021929,0.0870698879385118,0.0705048804977863,Energy,1 -23,ett2/W/short,YingLong_6m,6308226.285714285,5570140.571428572,1630.1685267857142,1.0456053839619526,0.1521472930908203,0.160482747214181,12.255732843150977,2511.6182603481534,0.210686266588469,0.1367461473906063,0.1208295737805687,Energy,7 -24,bitbrains_rnd/5T/short,YingLong_6m,1411730.3493651769,1572914.8245244725,133.84738430405437,1.8703412026772617,1.2843956277336963,0.6748114196869792,56.965790015981455,1188.1625938250947,4.861618311119202,0.5476648548016729,0.4486004945457798,Web/CloudOps,2 -25,bitbrains_rnd/5T/medium,YingLong_6m,2120893.5838062745,2242790.2817224977,184.4575629429833,4.716103863100836,1.737886768178811,0.763055867398377,163.4471494050614,1456.3288034665368,6.127516856243132,0.7761068953001928,0.67858376583898,Web/CloudOps,2 -26,bitbrains_rnd/5T/long,YingLong_6m,2554275.375110407,2890143.4071811587,235.6223082051662,3.66585199150068,3.708038489884353,0.7385314958387462,119.27105109574332,1598.210053500605,6.1223829101573015,0.9026160171170392,0.7269553805930001,Web/CloudOps,2 -27,bitbrains_fast_storage/5T/short,YingLong_6m,1660855.3978340656,1746959.7239333368,182.2329702517244,0.839242730401285,2.3897731391605705,0.6932475103115835,15.5882633909942,1288.74178865825,4.046330540312583,0.5721664645864641,0.4380864950051189,Web/CloudOps,2 -28,bitbrains_fast_storage/5T/medium,YingLong_6m,2883905.47595491,3269161.232629103,320.7089593357298,1.1248485031990805,5.561914312475275,0.7682795851990999,23.09546434583923,1698.20654690615,5.159620813379753,0.9744024510096536,0.6903823536588576,Web/CloudOps,2 -29,temperature_rain/D/short,YingLong_6m,173.95016437325032,183.78863917195116,6.196907171296386,1.4494874376699838,44.49994781273643,1.4786377054508604,19.25694551667524,13.18901680843763,1.5526959677391374,0.7295398070286827,0.5965084148536008,Nature,1 -30,covid_deaths/D/short,YingLong_6m,2521502.73283208,1429387.8696741855,190.8111058897243,43.99552852963286,0.1187168827274223,0.1744953259505021,990.3348606461624,1587.9240324499406,0.5974641449363226,0.0717936071846385,0.0645705954238884,Healthcare,1 -31,bitbrains_fast_storage/5T/long,YingLong_6m,3557613.766486999,3612921.949718246,375.43088978436447,1.0492137198460367,5.260199865673104,0.7790442698512434,17.98245515654289,1886.163769794924,4.984567713257915,0.992151753600007,0.7163534024934399,Web/CloudOps,2 -32,m_dense/H/short,YingLong_6m,58425.37333333334,58457.79555555555,115.704453125,0.9912151708333264,0.3735685572615229,0.2771017711118441,8.980336145676718,241.71341157108623,0.4287393610054854,0.2052308681420126,0.1681412299803744,Transport,1 -33,m_dense/H/medium,YingLong_6m,63860.333333333336,58787.55111111111,130.19722222222222,1.0937945539709597,0.5707514801886628,0.3245477973090278,8.963314518927264,252.7060215612864,0.4396765515030423,0.2265267179953562,0.1823752116213765,Transport,1 -34,m_dense/H/long,YingLong_6m,76637.2661728395,70293.0962962963,145.43271604938272,1.1986693071260404,0.6752887811929162,0.3559894627700617,9.28459470404453,276.8343659534334,0.4792412110261316,0.2517655303462199,0.1996784596665274,Transport,1 -35,electricity/H/short,YingLong_6m,2086959.0506306307,2143283.6264864863,218.35821860923423,1.1828402007772778,0.2308749496262992,0.1515115087414799,11.424128323613807,1444.631112301902,0.681969051717956,0.1030806729909616,0.0844180273551706,Energy,1 -36,solar/10T/short,YingLong_6m,30.63052530033455,32.533699247262774,2.694103083181265,1.184814199998189,2.774939421239352,1.6279012530401566,9.084060634000648,5.534485098031664,1.6099811156698989,0.7837142951441862,0.6442934399924607,Energy,1 -37,electricity/H/medium,YingLong_6m,8272421.735495496,7590927.824144145,300.43863738738736,1.2885369002786058,0.2831753755766807,0.1536465776676589,11.089191636103468,2876.181798060668,1.1226685382653605,0.117271100909394,0.0949288107500603,Energy,1 -38,solar/10T/medium,YingLong_6m,54.561721687679714,60.905640345056405,4.133981955734351,1.7762782629536509,1.6602498323897497,1.6801763122034226,11.411678287324456,7.38659066739722,1.6391137408834482,0.9173469782367212,0.7501030995299636,Energy,1 -39,m4_monthly/M/short,YingLong_6m,2213308.539185185,2133837.343148148,614.0911432291666,1.050113930620086,0.175333191341824,0.143268569946289,10.907367839620989,1487.7192407121663,0.3091991323396145,0.1276292216083775,0.1049130441022329,Econ/Fin,1 -40,jena_weather/D/short,YingLong_6m,456.9145833333333,381.391121031746,10.172071087549604,1.2265688762946565,0.8045389398100412,0.4825322711278522,8.080186964146192,21.375560421503184,0.1287158104709992,0.061252493426345,0.0515118234851049,Nature,21 -41,kdd_cup_2018/D/short,YingLong_6m,2804.9000598444045,3063.910100405612,22.16047858900193,1.235084621197566,0.529840773364253,0.48696614713204,9.390973827828384,52.9613071953894,1.1863894806491615,0.496418235810306,0.3931704265712783,Nature,1 -42,saugeen/D/short,YingLong_6m,1239.9059375,1419.1428125,14.692249348958333,3.264783412067605,0.3566678873697916,0.3851711018880208,34.43539381044104,35.212298100237646,1.1405347253396367,0.4758854570676986,0.3974965662985171,Nature,1 -43,ett1/15T/short,YingLong_6m,4.8676272437686015,4.990239606584821,1.1404894147600446,0.7283943219594146,0.4423740879988088,0.2559108007521856,5.036526940554215,2.2062699843329696,0.41707686337269,0.2155999724402206,0.1682085004302174,Energy,7 -44,ett1/15T/medium,YingLong_6m,9.993009052579366,10.509025297619049,1.7060029141865078,1.077458748159638,0.718713344201674,0.4138988095238095,7.440758873505662,3.1611721010693747,0.6025225280494372,0.325165842241755,0.2547738538940717,Energy,7 -45,electricity/H/long,YingLong_6m,12435422.78918919,11308260.072168168,349.5649129129129,1.447709422068271,0.3546584281847244,0.1710208984494093,12.161615080735844,3526.3894834787025,1.3908069692706584,0.1378682982607375,0.1097024346345031,Energy,1 -46,ett1/15T/long,YingLong_6m,8.801176215277778,8.903399057539682,1.655717540922619,1.0939685705011315,0.7376511119753871,0.4196805245535714,7.341803746068504,2.966677639258734,0.5654516913233636,0.3155814003783656,0.2463456125350392,Energy,7 -47,hierarchical_sales/W/short,YingLong_6m,593.9550450211865,536.1059984110169,10.393634343551376,0.7924142448297052,0.6735013868940373,0.5134435104111493,6.942178945762895,24.371192933896086,1.118893388432303,0.477176836617141,0.4238735763580704,Sales,1 -48,us_births/M/short,YingLong_6m,70465296.0,59346416.0,6786.041666666667,0.7683487106292306,0.0210015053550402,0.0212315022945404,4.335564397122846,8394.360964361731,0.0260728279232775,0.021077399150019,0.0166777022892018,Healthcare,1 -49,sz_taxi/15T/short,YingLong_6m,16.908707205338064,17.10427457741911,2.7716585748507137,0.553238366544088,1046035122388.7012,0.3983898279285547,4.012979207869956,4.112019844959174,0.3844914418841482,0.2591619306412892,0.2040290328973894,Transport,1 -50,sz_taxi/15T/medium,YingLong_6m,16.695684762286326,16.662596821581197,2.7991895365918804,0.5460415234734954,11066967960346.691,0.4087407038762019,3.838801948937468,4.086035335418225,0.3802737900338029,0.2605113090618455,0.2046726696245767,Transport,1 -51,sz_taxi/15T/long,YingLong_6m,15.90224915420228,15.79834290420228,2.7299501424501424,0.5143364474420461,8393565261042.143,0.4010567602608618,3.6545857891035154,3.987762424493501,0.3691001865413246,0.252679322278043,0.1992851416979396,Transport,1 -52,saugeen/W/short,YingLong_6m,911.599609375,960.18984375,15.580357360839844,1.2639585670474955,0.4376567840576172,0.4278181076049804,11.711795701281426,30.19270788410672,0.911391993483082,0.4703060423987124,0.3854927856982198,Nature,1 -53,m_dense/D/short,YingLong_6m,13341.748148148148,12546.767407407408,64.25738425925925,0.9290418115872155,0.1582925851650817,0.1377927879050925,7.827893388393261,115.5064853077443,0.1999588117150047,0.1112390370649989,0.0872931534659928,Transport,1 -54,bizitobs_l2c/H/short,YingLong_6m,114.42757936507935,126.6770755828373,6.601813422309028,0.6749339592467777,0.5073041298406588,0.7045015169432383,4.831414432867954,10.697082750221172,0.5766001559393172,0.3558546509988396,0.2866723190263739,Web/CloudOps,7 -55,bizitobs_l2c/H/medium,YingLong_6m,134.3912388392857,144.24291294642856,7.931118629092261,0.8366655400172881,0.7647520299463647,0.9057084583100818,4.982667404934468,11.592723529839123,0.7019570180676354,0.480241279668949,0.3697913207905763,Web/CloudOps,7 -56,bizitobs_l2c/H/long,YingLong_6m,163.82595486111111,163.56238839285714,8.791390749007936,0.9157663167342724,0.98063049475104,0.890716068328373,6.1518163495244025,12.799451350003684,0.7818274299497194,0.5370035204656358,0.4142741912553321,Web/CloudOps,7 -57,electricity/D/short,YingLong_6m,1533642808.236973,1485540612.298955,4602.094126126126,1.5200583704182054,0.4938796391668024,0.102420258687438,12.38869562345204,39161.751853523776,0.6435576395005285,0.0756276900902348,0.0594639002034952,Energy,1 -58,electricity/15T/short,YingLong_6m,169974.29711711712,168176.98011261263,62.85628994228603,1.166133278671916,0.1918052702106809,0.1839524637051659,10.652215801644042,412.2793920597016,0.8126839478587841,0.1239021373415051,0.1025828727259494,Energy,1 -59,solar/10T/long,YingLong_6m,62.130653449412,74.06619874036902,4.560471019141575,1.9646781716857096,1.5017374238717625,1.7485805326707158,12.36408185977238,7.882300010112023,1.7036328728937742,0.9856727521309536,0.7950374809879271,Energy,1 -60,ett1/H/short,YingLong_6m,96.08734188988096,97.44706101190476,4.944199335007441,0.8355688178660703,0.4380381992885044,0.2681375231061663,5.863344281192608,9.802415104956582,0.4575188162895408,0.230766010522073,0.1821882587743685,Energy,7 -61,ett1/H/medium,YingLong_6m,144.43982514880952,144.67147507440475,7.006554594494047,1.3020510597474362,4901045378767.825,0.4599154881068638,8.750688141923838,12.018312075695553,0.5756486591175269,0.3355973560971806,0.2617993112689566,Energy,7 -62,ett1/H/long,YingLong_6m,150.16276455026454,154.31660052910053,7.399727182539682,1.4042094601479322,3797929727094.4585,0.4959285675533234,8.927667085037163,12.254091747259956,0.5868219132561785,0.3543569080754304,0.2741528388126436,Energy,7 -63,ett1/D/short,YingLong_6m,40297.57777777778,40178.444444444445,135.17145337301588,1.7178217424849256,1.3497804671999007,0.4811476934523809,10.745280435240572,200.7425659340285,0.5272570081246664,0.355032305967142,0.2731873254716103,Energy,7 -64,ett2/D/short,YingLong_6m,111597.38412698412,104680.0507936508,196.2031498015873,1.3196671681082288,0.4579360235305059,0.1361082410055493,13.11068159392628,334.06194654133253,0.1911924659327621,0.1122922392769428,0.0934227546884637,Energy,7 -65,loop_seattle/D/short,YingLong_6m,19.58421536377709,19.838204656862747,3.15921617002064,0.9511760644378504,0.0582751578221749,0.0583228395696271,8.306610462454897,4.425405672226795,0.079087918060891,0.0564594182086612,0.0455482690902607,Transport,1 -66,saugeen/M/short,YingLong_6m,422.8811383928572,431.0413876488095,13.497557140531994,0.7908826313922438,0.4121134621756417,0.4043764840988886,6.196140045153902,20.564073973628307,0.6173951923324396,0.4052371576509418,0.330658762144861,Nature,1 -67,m4_hourly/H/short,YingLong_6m,1257200.1288244766,1117143.085346216,255.2020933977456,1.162580452863786,0.1515378230243873,0.1439845012965793,9.289869046444084,1121.249360679629,0.153075286752863,0.0348407187524388,0.0274400106343165,Econ/Fin,1 -68,ett1/W/short,YingLong_6m,1530810.5714285714,1635119.4285714286,958.4317801339286,1.616038709138826,0.8329011372157505,0.5475828647613525,10.051207219177323,1237.2592983803238,0.4923350054198341,0.3813828809243914,0.3001318502614921,Energy,7 -69,kdd_cup_2018/H/short,YingLong_6m,4400.338209159561,4512.022298842,22.4222406458294,0.9355944265482335,0.8910455518531085,0.4936754558677208,7.899186291474847,66.33504510558171,1.388512636408817,0.469338106633982,0.3778594455561658,Nature,1 -70,kdd_cup_2018/H/medium,YingLong_6m,4861.196719513105,5068.561403816703,25.01046982575156,1.0362150684167428,0.9895599268400797,0.5509716319720219,9.123059247672622,69.72228280480427,1.4594136520413743,0.5235144295240369,0.4199486741754273,Nature,1 -71,kdd_cup_2018/H/long,YingLong_6m,3808.0005624527626,4054.502235163844,23.896696175907103,1.0145360063015778,1.045777418031586,0.6001156763201528,8.031144110772448,61.70899903946557,1.4479424770138187,0.5607130563122684,0.443385552750686,Nature,1 -72,restaurant/D/short,YingLong_6m,155.43575464649294,158.31785280657513,7.730971852526385,0.7359646030201444,0.7514430906766602,0.4180397097109368,5.113213956181143,12.467387643227148,0.5737780659882443,0.355797237135071,0.2792917209108109,Sales,1 -73,bizitobs_l2c/5T/short,YingLong_6m,25.520079985119047,23.281775483630952,3.0645804268973214,0.3140058967085348,0.1640628778096951,0.2186434474080827,2.8339294917401223,5.0517402927228,0.1734972095536489,0.1052500963450835,0.0862905443981476,Web/CloudOps,7 -74,bizitobs_l2c/5T/medium,YingLong_6m,151.3609906462585,147.68826530612245,8.864945392219388,0.855587909665183,0.5131953472364185,0.7810369997608418,8.138101425179208,12.30288546017797,0.6473126969712253,0.4664264922984302,0.3777525911006123,Web/CloudOps,7 -75,bizitobs_l2c/5T/long,YingLong_6m,255.129126984127,270.9380357142857,12.081734871031744,1.1310586877828337,0.7863878990967644,0.9703994605654762,12.098654228981454,15.972762033666156,0.8968467088682183,0.6783713507823539,0.5570813152362109,Web/CloudOps,7 -76,m4_yearly/A/short,YingLong_6m,5946464.286933055,6271486.670264937,1350.404881605293,5.270358387025677,0.2338295769959213,0.2302942916654481,50.81537036373012,2438.5373253106163,0.3910307635723022,0.2165436823562338,0.177432191591014,Econ/Fin,1 -77,electricity/15T/medium,YingLong_6m,323643.0010810811,297403.4505765766,65.74022888513514,1.029878923600594,0.1769668669185374,0.1593413279700174,8.818135834764806,568.8963008150793,0.980265613185778,0.1132770342973414,0.0919811912232511,Energy,1 -78,loop_seattle/5T/short,YingLong_6m,59.32077467911507,61.91805824303405,4.440177182098168,0.6930516664203245,0.136395591405154,0.092089331974309,5.74872904092521,7.701998096540603,0.1320722673052282,0.0761392381984669,0.0606622336274395,Transport,1 -79,loop_seattle/5T/medium,YingLong_6m,143.55213412667698,152.0149394511094,7.543936413586816,1.1698861469748054,0.294770224052563,0.1614691643965871,8.945093428100222,11.981324389510409,0.2131723749172696,0.1342221268057066,0.1073335596907305,Transport,1 -80,electricity/15T/long,YingLong_6m,427130.7330570571,377739.4658858859,71.10913194444444,1.0776169606145911,0.1803762871579164,0.1602488576761639,9.394873488037192,653.5523950358204,1.0310589989283163,0.1121833703834594,0.0910301376974757,Energy,1 -81,solar/D/short,YingLong_6m,121620.9907542579,124691.05206812652,252.89779501216543,0.9876464983558,1.03813322118309,0.4334201190883516,5.563061242944722,348.7420117425744,0.5038134239128848,0.3653511756970328,0.2886610173572294,Energy,1 -82,sz_taxi/H/short,YingLong_6m,7.401238857171474,7.458362025073451,1.872837686131143,0.5705636704225204,1.1252373589409723,0.2969712803506443,4.165126290372363,2.7205217986944112,0.2534333935831153,0.1744663875343128,0.1380375211958415,Transport,1 -83,hierarchical_sales/D/short,YingLong_6m,28.1761290822654,29.048646134766432,2.31473781779661,0.7547697733321663,0.6108387172504037,1.0257519410138487,7.139694087114035,5.308119166170386,1.628472176307393,0.7101359283250264,0.5806447906103447,Sales,1 -84,jena_weather/10T/short,YingLong_6m,1294.21875,1514.3083333333334,8.747539992559524,0.3614230021215637,0.408061852448748,0.6081725513985363,3.290174630413293,35.97525191016736,0.2234514029149214,0.0543331868327899,0.04751797037363,Nature,21 -85,jena_weather/10T/medium,YingLong_6m,2153.5473304473303,2340.593217893218,12.827224251443,0.6620934052029728,0.9885680672350696,0.6976142815589148,6.9173921247698535,46.40632856030878,0.2847948450432316,0.0787204559455066,0.0684501072410684,Nature,21 -86,jena_weather/10T/long,YingLong_6m,2293.6833333333334,2528.382804232804,13.373459201388888,0.6617058281251397,0.905035600700856,0.681582378917909,7.5457536312034845,47.89241415227816,0.2930618943803265,0.0818344900199733,0.0709459708477972,Nature,21 -87,loop_seattle/5T/long,YingLong_6m,149.38093337919963,156.49258112601763,7.589826568054122,1.1776338798942303,0.3043673048532278,0.1606137316266053,9.674586819463173,12.22214929458807,0.2160900990968317,0.1341896859290376,0.1086805463172179,Transport,1 -88,m4_quarterly/Q/short,YingLong_6m,2223853.2213333333,2197359.973333333,665.29750390625,1.4687421750313596,0.1341299676895141,0.1230659607251485,13.812310096535851,1491.2589383917648,0.2496104860701967,0.1113590866455555,0.0906396630489106,Econ/Fin,1 -89,bitbrains_rnd/H/short,YingLong_6m,2075879.546822386,1939742.5855675377,183.50577021439835,6.06972423368074,2.442471223785261,0.6024472826633097,206.0600628480097,1440.791291902608,6.305607902813446,0.8031110691596235,0.6920231138529406,Web/CloudOps,2 -90,ett2/H/short,YingLong_6m,116.23372395833331,113.91056547619048,6.712948753720238,0.7770926335150614,0.1264926730156087,0.115883799961635,5.602928147235157,10.78117451664397,0.1367352904471069,0.0851388683282542,0.068046699928005,Energy,7 -91,ett2/H/medium,YingLong_6m,283.94916294642854,275.0946986607143,10.345156715029765,1.128187364748504,0.1837476101249718,0.1845985775902157,7.962617794351183,16.85079116677993,0.2270752473106589,0.1394076394563621,0.1124304754724707,Energy,7 -92,ett2/H/long,YingLong_6m,318.9388558201058,306.72460317460315,10.977796378968254,1.2004794387609048,0.1982187573096972,0.1981620990410053,8.088656623438526,17.858859309040593,0.2445633330440612,0.1503324722738818,0.1201268788616959,Energy,7 -93,solar/W/short,YingLong_6m,5541395.620437956,6155899.328467153,2133.059306569343,2.3621086683270733,0.4856537867636576,0.3722002349630759,7.349534061933132,2354.016911672037,0.4805137049996196,0.4354107335853979,0.3259369901003652,Energy,1 -94,solar/H/short,YingLong_6m,660.2696087847355,673.4941573825074,12.277839972307596,0.9476317131566676,4.625272248395754,1.4195587045275764,8.772532354802598,25.695711875422628,0.9471525240179856,0.4525652830962397,0.380009302778114,Energy,1 -95,solar/H/medium,YingLong_6m,715.0728710462287,716.3605231143553,13.413013229927008,1.0211985848631435,5.368780852321784,1.42377526542029,9.668975323556722,26.74084649083175,0.9632677040409178,0.4831680426680602,0.416752720446112,Energy,1 -96,solar/H/long,YingLong_6m,721.4557177615571,720.9772100567722,14.540393096107056,1.1052415852486046,5.611037135556817,1.4333167342057556,8.586769921994845,26.85992773187517,0.9312826452913922,0.5041419277554371,0.4169607668135643,Energy,1 diff --git a/results/YingLong_6m/config.json b/results/YingLong_6m/config.json deleted file mode 100644 index ba74bc444af3f2806f1125a5264c173a111be2a1..0000000000000000000000000000000000000000 --- a/results/YingLong_6m/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "YingLong_6m", - "model_type": "pretrained", - "model_dtype": "bf16", - "model_link": "https://huggingface.co/qcw2333/YingLong_6m", - "org": "Alibaba", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/auto_arima/config.json b/results/auto_arima/config.json index d89838c6d14a62f990e5637a07384f7eeefc52a0..5eb6a9bb09477f57ec930d1609814035d30d997b 100644 --- a/results/auto_arima/config.json +++ b/results/auto_arima/config.json @@ -1,6 +1,5 @@ { "model": "Auto_Arima", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/auto_ets/config.json b/results/auto_ets/config.json index 9016c289853e156b0756324f444c4bbff273da50..c68c6cc9eae4738e51ce65d90579625ee58ddeca 100644 --- a/results/auto_ets/config.json +++ b/results/auto_ets/config.json @@ -1,6 +1,5 @@ { "model": "Auto_ETS", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/auto_theta/config.json b/results/auto_theta/config.json index 548524a37aae2797a4d0feb5d79e130e5dddf0d2..28984714533528ea26486371d9120787158ae553 100644 --- a/results/auto_theta/config.json +++ b/results/auto_theta/config.json @@ -1,6 +1,5 @@ { "model": "Auto_Theta", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/chronos_base/config.json b/results/chronos_base/config.json index 0d5638057f751db0fc81a0b485b8c6574f8eef78..ba962e3824e1e0deff839c439eb5ee6a97b1db84 100644 --- a/results/chronos_base/config.json +++ b/results/chronos_base/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-t5-base", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/chronos_bolt_base/config.json b/results/chronos_bolt_base/config.json index ddd82219863d241678a9d86ebdb2c26a546869f8..e0c0950188b890aac3103a0260ae4c446a67ca0c 100644 --- a/results/chronos_bolt_base/config.json +++ b/results/chronos_bolt_base/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-bolt-base", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/chronos_bolt_small/config.json b/results/chronos_bolt_small/config.json index 73bf4aae2f0c0468d6d74fb834fd102f3b82768b..68813b26c5e322842ccacb197a93d18367617e3c 100644 --- a/results/chronos_bolt_small/config.json +++ b/results/chronos_bolt_small/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-bolt-small", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/chronos_large/config.json b/results/chronos_large/config.json index a1ef17d870570b777bfc0b5c180c5576041343e4..9ac2a7a48c60de66f27c94067ff75c2f69cd771a 100644 --- a/results/chronos_large/config.json +++ b/results/chronos_large/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/amazon/chronos-t5-large", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb", - "org": "AWS AI Labs", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/chronos.ipynb" } \ No newline at end of file diff --git a/results/crossformer/config.json b/results/crossformer/config.json index 4965940552a4969353e7c34601afbe9669032ddf..4841c673f38238dca63f1399a3bb85fa9cc14b21 100644 --- a/results/crossformer/config.json +++ b/results/crossformer/config.json @@ -1,7 +1,5 @@ { "model": "Crossformer", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Shanghai Jiao Tong University", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/deepar/config.json b/results/deepar/config.json index 7ca0ae15dd11978965542bdef5cd70f3117eb8a7..3037ea9d5bb2bda278dea88d26279c6fd0274c0a 100644 --- a/results/deepar/config.json +++ b/results/deepar/config.json @@ -1,7 +1,5 @@ { "model": "DeepAR", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Amazon Research", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/iTransformer/config.json b/results/iTransformer/config.json index 8fa358b00583feda708a77558b6cb4aeec19ec3e..234e955ce7c114a40c89fd6f3d35b54c4c2a68e1 100644 --- a/results/iTransformer/config.json +++ b/results/iTransformer/config.json @@ -1,7 +1,5 @@ { "model": "i_transformer", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Tsinghua University", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/naive/all_results.csv b/results/naive/all_results.csv index 9c0a8a37448fd3c8119c87f371a267074a8b75ca..135fbcae0566f049b6967b2e6bc0625684c40645 100644 --- a/results/naive/all_results.csv +++ b/results/naive/all_results.csv @@ -1,98 +1,98 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -m4_yearly/A/short,Naive,4070174.893125562,4070174.893125562,1007.6444123429384,3.965954931783283,0.17520083696448324,0.16353434447586868,52.037766411775884,2017.4674453694568,0.323510256896443,0.1615804524853844,0.13714565740983498,Econ/Fin,1 -ett1/W/short,Naive,1571455.142857143,1571455.142857143,970.0256696428571,1.7689207877683926,0.6150428227015904,0.6251754760742188,12.799615221235317,1253.5769393448265,0.4988281842249676,0.38599636628003803,0.3118251468740881,Energy,7 -bizitobs_l2c/H/short,Naive,263.3054935515873,263.3054935515873,11.038767981150794,1.0860940042784581,1.2889937073778477,1.012692318579292,9.95825254038738,16.2266907763594,0.8746601901190734,0.5950178649574238,0.48632480596516664,Web/CloudOps,7 -bizitobs_l2c/H/medium,Naive,538.6467633928571,538.6467633928571,15.811579241071428,1.4850898274863054,1.1477882042734406,1.7614684262279354,15.525790979764977,23.20876479679298,1.4053259605351764,0.9574151394565906,0.8639523074274909,Web/CloudOps,7 -bizitobs_l2c/H/long,Naive,326.6440476190476,326.6440476190476,13.937010168650794,1.4278952141944932,2.5975948022295894,0.9060760013640873,17.133983554777487,18.07329653436383,1.103969115064981,0.8513128057896183,0.8178194151135174,Web/CloudOps,7 -ett2/D/short,Naive,122878.94603174603,122878.94603174603,211.56051587301587,1.3901143157307532,0.4713712661985367,0.14168287004743305,32.70861808020462,350.5409334610525,0.20062382493032205,0.12108166507005774,0.15331100213050436,Energy,7 -solar/W/short,Naive,2813150.8321167883,2813150.8321167883,1341.8134124087592,1.4703506784640206,0.3086572076282362,0.2534797557079009,11.583228430599936,1677.245012547895,0.3423676402558515,0.2738976644635694,0.20974762813186765,Energy,1 -jena_weather/D/short,Naive,633.5864087301587,633.5864087301587,13.136656746031745,1.5734140297468562,1.7707135350571066,0.4854136849221913,31.00640730511845,25.17114238031637,0.15157141745411598,0.07910414448099358,0.2105908566035542,Nature,21 -saugeen/M/short,Naive,1026.8541666666667,1026.8541666666667,21.004798525855655,1.2302953456028718,0.7310131617954799,0.6058716092790876,19.163894291336852,32.04456532185554,0.9620739837576904,0.6306270655515779,0.7678932982488066,Nature,1 -kdd_cup_2018/H/short,Naive,5594.245732646461,5594.245732646461,31.058507069381246,1.2800082967549335,1.348935586862045,0.6193935963094522,14.046664157776483,74.79469053780797,1.565588323464133,0.6501107988069305,0.5618630829214255,Nature,1 -kdd_cup_2018/H/medium,Naive,5746.392704032155,5746.392704032155,32.48218551304779,1.4458736272627521,1.5174890734649364,0.7064537834436008,35.987336722040006,75.80496490357446,1.5867351909559555,0.6799109649120403,0.9996707248005586,Nature,1 -kdd_cup_2018/H/long,Naive,4181.4676662897455,4181.4676662897455,25.868315472284554,1.180184135046071,1.0975976860084167,0.7152840645066674,43.77688578853556,64.66426885297433,1.5172850161804052,0.6069751990727064,1.2460511734345185,Nature,1 -loop_seattle/H/short,Naive,193.11455617565585,193.11455617565585,7.998571198519228,1.591891706430253,0.30128818640632,0.16068070405310833,28.110449663719383,13.896566344808196,0.24598335775575475,0.14158284513176175,0.1820636511546452,Transport,1 -loop_seattle/H/medium,Naive,216.44939692982456,216.44939692982456,8.814659825690145,1.776697703849278,0.32173781891971426,0.17238567340484715,87.84401294486503,14.712219306747183,0.2600368222627069,0.15579812145323396,0.45764837206742065,Transport,1 -loop_seattle/H/long,Naive,210.4300653594771,210.4300653594771,8.603209924320605,1.7512023647663921,0.32551775113142845,0.17357919100514382,108.10293428245187,14.506207821463097,0.25924535980099905,0.15375088236182935,0.5564661074073203,Transport,1 -ett1/H/short,Naive,345.0893973214286,345.0893973214286,10.155936104910714,1.8250258188592174,0.9246851603190104,0.5119235084170387,15.402812126847282,18.57658196013003,0.8670450800238758,0.47401908767171513,0.4067901701225723,Energy,7 -ett1/H/medium,Naive,471.78404017857144,471.78404017857144,11.986143275669642,2.0328290422336406,9728700156311.344,0.649679928734189,46.87255743203719,21.720590235501692,1.0403647838029844,0.5741078498521575,0.8414627329278955,Energy,7 -ett1/H/long,Naive,443.52450396825395,443.52450396825395,12.352705439814814,2.2728552413599847,5652510468487.882,0.749044170076885,57.27242142453815,21.06002146172349,1.008518814962206,0.5915443094102009,0.9868526334206741,Energy,7 -ett1/D/short,Naive,57976.62857142857,57976.62857142857,154.97894345238095,1.7783697877928102,1.1623038155691965,0.5164825923859127,23.47582377746543,240.7833643992636,0.6324254934604606,0.4070573356817528,0.40843357791503654,Energy,7 -covid_deaths/D/short,Naive,4435194.482205514,4435194.482205514,353.7093984962406,46.91239825526407,0.132724402667158,0.17781104711702989,1472.7784117419058,2105.990142950701,0.792389040211866,0.13308488253209907,0.12672327831223676,Healthcare,1 -m4_quarterly/Q/short,Naive,2269820.328,2269820.328,655.560453125,1.4770341883914722,0.13182427295049032,0.11610267241795857,13.675610580733625,1506.5922899046045,0.2521770186622125,0.10972927561734654,0.0943391723614522,Econ/Fin,1 -ett2/15T/short,Naive,19.819986979166668,19.819986979166668,3.024351283482143,1.2443102434383637,0.16288884327408776,0.1946806408110119,13.571980290549968,4.451964395541216,0.2113505875232857,0.14357671442317277,0.12107340753866416,Energy,7 -ett2/15T/medium,Naive,18.141490575396826,18.141490575396826,2.8998893229166667,1.216851599992199,0.18442374644793325,0.21048138330853175,34.48033209514033,4.259282871023809,0.226984894057983,0.15454035119858,0.2420306513801469,Energy,7 -ett2/15T/long,Naive,16.585460069444444,16.585460069444444,2.8831569320436508,1.300946032338565,0.2003668146589512,0.24919358803902203,42.00999676184563,4.07252502379598,0.2170322303229923,0.153648651801133,0.28312587034305786,Energy,7 -bizitobs_l2c/5T/short,Naive,21.968577938988094,21.968577938988094,2.758282761346726,0.2830657721671069,0.16774414213623878,0.15846396092072984,3.0909789998825157,4.687064959971015,0.16097278253260314,0.09473059471069084,0.08008460690704661,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Naive,157.30076530612246,157.30076530612246,8.568459024234693,0.8255719817212336,0.6241842431613175,0.8702898646468468,6.3292605433256774,12.541960185956677,0.6598915432932841,0.4508269493215748,0.35883413736823033,Web/CloudOps,7 -bizitobs_l2c/5T/long,Naive,352.5859523809524,352.5859523809524,13.23037326388889,1.2484065615509454,0.986276036425049,1.2111292430391314,9.782056549262665,18.77727222950534,1.054315763614314,0.7428656793238004,0.5919795411117497,Web/CloudOps,7 -solar/10T/short,Naive,36.5841692328163,36.5841692328163,3.299997772439553,1.4492625864683233,2.6560337692441602,1.410252805214504,17.739652658708565,6.04848487084297,1.7595035939179897,0.9599689946351024,0.9431423497567353,Energy,1 -solar/10T/medium,Naive,80.77481613581067,80.77481613581067,5.939547217153285,2.564335860697831,2.8615413042870976,1.6386301084254773,33.25251391362155,8.987481078467463,1.994357743628945,1.3180091154305162,1.373706540018747,Energy,1 -solar/10T/long,Naive,94.56340575577859,94.56340575577859,4.626759277739761,2.017857283966539,1.0,2.0,40.636930571439166,9.724371740929005,2.10176738342858,1.0,1.5490163371145615,Energy,1 -temperature_rain/D/short,Naive,310.64911445557556,310.64911445557556,7.890313322375261,2.0115357653222166,97.02872693676676,1.3339967533230714,47.47823470729249,17.625240833973745,2.0749568489172328,0.9288984940737532,1.2679455200160903,Nature,1 -m4_hourly/H/short,Naive,57543048.45088567,57543048.45088567,1218.0648148148148,11.607687602745548,0.37716947546327745,0.43002987675812704,75.79187144378025,7585.713443762931,1.0356173223815475,0.16629273322245952,0.13648849999564483,Econ/Fin,1 -m4_daily/D/short,Naive,497624.29605596676,497624.29605596676,180.83018393659805,3.278424296610481,0.04217648847004251,0.03045251709590828,32.19152086932198,705.4249046184624,0.10896865245350737,0.02793326594720746,0.024356147648337902,Econ/Fin,1 -ett2/H/short,Naive,171.92237723214285,171.92237723214285,8.963274274553571,1.0872035620086857,0.18046948507834626,0.15999321710495723,14.11974224626444,13.111917374363783,0.1662955949497229,0.11367925724568231,0.1127608016960007,Energy,7 -ett2/H/medium,Naive,411.36316964285714,411.36316964285714,13.140455264136905,1.4275550147417344,0.2418973113685871,0.21611560639880953,43.67294503723677,20.282089873651017,0.27331420397137396,0.17707608499481128,0.29476933710790065,Energy,7 -ett2/H/long,Naive,294.4490740740741,294.4490740740741,11.495035548941798,1.3228699351602262,0.21793927625643253,0.22216192982184194,53.48074459500154,17.159518468595618,0.23498639848662972,0.1574156646100037,0.3493770970075422,Energy,7 -car_parts/M/short,Naive,2.9952172180151453,2.9952172180151453,0.6895841636774279,1.2125433213650758,0.8496029665824577,1.6621891716563262,25.79123698119671,1.7306695866095139,4.149968135842565,1.6535520866517999,2.4999765507853837,Sales,1 -loop_seattle/5T/short,Naive,101.24308777170408,101.24308777170408,5.645426243328335,0.8927905562283007,0.13816935201670486,0.12006684510946519,14.599096828070424,10.061962421501288,0.17254044173284333,0.09680659666474113,0.11567081973538601,Transport,1 -loop_seattle/5T/medium,Naive,392.4867645446336,392.4867645446336,12.876183142253613,1.9871339455955137,0.36263277508526026,0.29879087623304873,44.143153465916726,19.81127872058322,0.3524833622181743,0.22909375969775447,0.3268672871228811,Transport,1 -loop_seattle/5T/long,Naive,213.5793191721133,213.5793191721133,9.076179516397202,1.408835030757464,0.3416879652137262,0.18775294267697942,54.0115563888563,14.614353190343843,0.2583847545162609,0.16046871084512437,0.3605186433614601,Transport,1 -loop_seattle/D/short,Naive,58.65786974329205,58.65786974329205,5.940060831075852,1.7323957056549062,0.11095823557630288,0.10706747427075271,21.844122635807963,7.658842585096788,0.13687375626545378,0.10615682844516516,0.1032547596878987,Transport,1 -sz_taxi/15T/short,Naive,33.12582751449939,33.12582751449939,3.9447246546855923,0.7855968930284677,1313272123878.9246,0.5283226274017595,22.646180528246813,5.755504106027498,0.5381642491850911,0.36884862610167457,0.6611611222089492,Transport,1 -sz_taxi/15T/medium,Naive,30.582425213675215,30.582425213675215,3.9741160857371796,0.7792049328037046,9997834048909.348,0.5282687717013889,70.06940102207474,5.530137901867838,0.5146716380849939,0.3698578357504329,1.9355403237762074,Transport,1 -sz_taxi/15T/long,Naive,28.46968482905983,28.46968482905983,3.853752114494302,0.742344566525504,9279995729984.182,0.5353563966234864,85.04158178037052,5.335699094688514,0.49386280363180085,0.35669643096266895,2.4019918478793696,Transport,1 -m_dense/H/short,Naive,358130.1511111111,358130.1511111111,343.1718142361111,2.7139864906441935,1.04015513186624,0.7294788180763577,28.457280672803915,598.4397639788913,1.0614830196673901,0.608701285519745,0.5387397922383297,Transport,1 -m_dense/H/medium,Naive,360975.25333333336,360975.25333333336,348.6905208333333,2.7352300662596214,1.184018235880775,0.7301190185546875,88.57658717624187,600.812161439275,1.0453372563528116,0.6066774538833799,1.2024808459166658,Transport,1 -m_dense/H/long,Naive,395605.0488888889,395605.0488888889,362.1829012345679,2.855573013916035,1.0900783741369702,0.7696336082175926,108.34227578348323,628.9714213610098,1.088842509259061,0.6269921424055026,1.4425473283951151,Transport,1 -electricity/15T/short,Naive,603056.6183220721,603056.6183220721,127.26604527378942,2.456280689463761,0.3533350538350351,0.2991550499122379,20.028534644922413,776.567201420503,1.5307670327684602,0.2508664622193399,0.22064410727117006,Energy,1 -electricity/15T/medium,Naive,2925108.410720721,2925108.410720721,223.189305884009,3.4575185664171464,0.4035447466155227,0.41189880728913403,54.65956633387852,1710.294831519034,2.9470102785062213,0.38457765665388643,0.4298202929628236,Energy,1 -electricity/15T/long,Naive,6142069.7442042045,6142069.7442042045,249.06663475975975,3.412327584432877,0.5159208992945213,0.384422579940197,66.52365216918754,2478.3199438741167,3.9098536420593453,0.3929331608043482,0.4779696439404405,Energy,1 -saugeen/D/short,Naive,994.7777083333333,994.7777083333333,15.3725,3.413049305768806,0.47135218302408854,0.4242374928792318,58.71112805300772,31.540096834558597,1.0215912513857213,0.49791893773514423,0.5850141736335629,Nature,1 -bitbrains_rnd/H/short,Naive,4185060.895273006,4185060.895273006,250.85095744000682,5.871568507205478,3.2411984858834244,0.3079789245627823,242.30099047281863,2045.742138020578,8.953168487020333,1.0978465200232173,1.4590277821900182,Web/CloudOps,2 -bitbrains_rnd/5T/short,Naive,1851989.8842618705,1851989.8842618705,130.16222649958368,2.0564165206768776,1.6385894817054072,0.3454797128234705,132.3593826217692,1360.878350280388,5.568321339995986,0.53258625455368,0.8683346368401388,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Naive,2446023.529571928,2446023.529571928,157.91522160865162,4.774544594207593,0.575097733397328,0.3963432651932816,389.568567896354,1563.9768315329763,6.580448103895253,0.6644298685628833,2.0082074202220164,Web/CloudOps,2 -bitbrains_rnd/5T/long,Naive,4798684.04501758,4798684.04501758,287.5256742664699,3.9868856753563806,5.680248975320888,0.42959668137427853,405.57989889313706,2190.589885171932,8.391656316532812,1.101446079412625,2.3614946872229625,Web/CloudOps,2 -m_dense/D/short,Naive,39929.671111111114,39929.671111111114,118.38299768518519,1.6693368812171987,0.2532523956185716,0.2557375985604745,23.291872240434824,199.82410042612756,0.3459250756939904,0.20493848013227253,0.22686565739428607,Transport,1 -saugeen/W/short,Naive,1800.4892578125,1800.4892578125,24.552409362792968,1.990658004828632,0.824161148071289,0.557866096496582,23.291068065474015,42.43217243805106,1.2808504084686672,0.7411348925661494,0.7340352210434381,Nature,1 -jena_weather/H/short,Naive,2411.4258563074354,2411.4258563074354,12.323681110327904,0.6477717456282629,1.0957840912703458,0.4789195589483279,38.34991731121663,49.10627104869026,0.3010182802235531,0.0755433718470736,0.3077983594115,Nature,21 -jena_weather/H/medium,Naive,2461.295238095238,2461.295238095238,13.411072358630953,0.8727345201332312,2.037994607709956,0.5234983085857982,118.95415019975488,49.611442612518715,0.3035812160888766,0.08206473025764742,0.9274165841787126,Nature,21 -jena_weather/H/long,Naive,3726.665343915344,3726.665343915344,17.702639302248677,1.4657949793485425,2.372808282131346,0.5753933143958292,144.34046179305136,61.046419583095485,0.36756708078564854,0.1065895018080746,1.1409491476417122,Nature,21 -bitbrains_fast_storage/H/short,Naive,3129276.3703227686,3129276.3703227686,280.24068195254864,1.4078564206723017,4.944441147552041,0.34317742724255007,60.99672161140896,1768.9760796355524,5.042156288732694,0.7987769496334254,1.2058405212636514,Web/CloudOps,2 -kdd_cup_2018/D/short,Naive,3183.23772462431,3183.23772462431,23.884613481510574,1.4970343738270926,0.5893064466306873,0.5721879846611051,24.580612052672397,56.42018898075679,1.263872161836989,0.5350407118596068,0.674506129145155,Nature,1 -sz_taxi/H/short,Naive,14.13685542701656,14.13685542701656,2.687046148838141,0.8346056944917577,0.9688676361344818,0.39366466163570046,25.403498760775808,3.7599009863314965,0.3502579788038407,0.250314930224525,0.4858205976799266,Transport,1 -ett1/15T/short,Naive,18.248468889508928,18.248468889508928,2.6184166317894344,1.9694015637556788,0.9318440512861078,0.5606311979747954,12.695925468598913,4.271822665971626,0.8075522991563836,0.4949897354107341,0.3848114505420264,Energy,7 -ett1/15T/medium,Naive,25.974303075396826,25.974303075396826,2.7894202628968254,1.8231059347573026,1.1089171131632836,0.6168787629867231,32.362039726312574,5.096499099911314,0.9713977675690157,0.5316662601268594,0.6174724527459637,Energy,7 -ett1/15T/long,Naive,28.417566964285715,28.417566964285715,3.0355567956349208,2.0446378289055525,1.1703466395952207,0.6797395967709925,39.70402516501359,5.330812974048678,1.0160582236557134,0.5785801266324228,0.7155861341837335,Energy,7 -m4_monthly/M/short,Naive,2501230.9708518516,2501230.9708518516,671.266107421875,1.205128987689887,0.1901403370256777,0.15256483493027864,12.668739500059264,1581.528049340843,0.3286958215348495,0.13951214127343117,0.1293608518757588,Econ/Fin,1 -electricity/W/short,Naive,317380739662.4144,317380739662.4144,56604.72342342342,2.0896995064573947,0.285636731125347,0.1196364761812733,15.938528356933686,563365.5471027798,1.2854142672489681,0.12915329923232946,0.0993032024444414,Energy,1 -bitbrains_fast_storage/5T/short,Naive,2299344.812804004,2299344.812804004,173.28187846869332,1.0948146284753992,2.4327068429200516,0.37465512436513404,70.19326241539363,1516.3590646031053,4.760992442912181,0.5440622430011399,0.8929464680662709,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Naive,3704568.4175853366,3704568.4175853366,280.36127147074325,1.271487396616441,4.117625747562423,0.4499728768046956,196.7313435217375,1924.7255434438794,5.847847822156081,0.8518149802561188,2.0491239073790615,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Naive,3499892.2950397297,3499892.2950397297,289.35216809290046,1.1903598806207039,1.795788841111804,0.5099463825492213,231.56717491055915,1870.7999078040734,4.9439656728737384,0.7646714009634807,2.122607238439539,Web/CloudOps,2 -hospital/M/short,Naive,5851.3624511082135,5851.3624511082135,24.06573229030856,0.9676000297092351,0.24346126737101395,0.21603344482527978,11.846825178636562,76.49419880689132,0.27769170470586313,0.08736419661785731,0.08634250116644743,Healthcare,1 -us_births/W/short,Naive,5288246.857142857,5288246.857142857,1720.3214285714287,1.5634204337929731,0.023174324205943515,0.023229679891041348,13.279190377688378,2299.6188504060533,0.031217100406456557,0.023353194707727513,0.01929679613291714,Healthcare,1 -us_births/M/short,Naive,261017514.66666666,261017514.66666666,13253.958333333334,1.49946000119159,0.040852939089139305,0.04129129648208618,18.13215937344219,16156.036477634812,0.05018053914906859,0.041166704218987014,0.039028587852434786,Healthcare,1 -jena_weather/10T/short,Naive,1652.7815476190476,1652.7815476190476,10.021488250248016,0.3638272449122593,0.4180473438261545,0.4286353601518544,51.18596442950847,40.65441609000242,0.2525148769682579,0.06224600217963847,0.28772334069999134,Nature,21 -jena_weather/10T/medium,Naive,3627.5171717171715,3627.5171717171715,15.930331213924964,0.7603204747977048,0.9488791837688123,0.5798423298817329,161.65362364190366,60.22887323964455,0.36962356543938746,0.09776417032562872,0.8818873317406091,Nature,21 -jena_weather/10T/long,Naive,2737.9084656084656,2737.9084656084656,13.276913855820105,0.7531930861787588,0.8439116329840955,0.4988705728488919,197.68405969526736,52.325027143886565,0.3201858133426328,0.08124371249566584,1.070325463540521,Nature,21 -electricity/D/short,Naive,4108036968.189694,4108036968.189694,6964.321117117117,1.987041399842847,0.6052061423957089,0.13582226488305205,24.595303547998157,64093.9698270414,1.0532768308357283,0.11444692995228782,0.10405987098536745,Energy,1 -bizitobs_application/10S/short,Naive,5848946.915555555,5848946.915555555,1048.0411805555555,3.763641918540148,0.05442916870117188,0.05418765597873264,47.88283566865831,2418.4596162755242,0.09327032054038906,0.04041875919370658,0.03821885679668207,Web/CloudOps,2 -bizitobs_application/10S/medium,Naive,21648810.666666668,21648810.666666668,1831.66875,6.51945902607138,0.08660387674967449,0.08887063344319661,105.36258094385984,4652.828243839082,0.18270287104380797,0.07192424088925747,0.08853450662390008,Web/CloudOps,2 -bizitobs_application/10S/long,Naive,20331799.893333334,20331799.893333334,2235.4125,7.207407834346513,0.09553615146213108,0.09904331631130642,110.06515727907836,4509.079716897156,0.17389718137400212,0.08621092492587615,0.09735421334116763,Web/CloudOps,2 -us_births/D/short,Naive,2855120.2133333334,2855120.2133333334,1266.88,1.8648391912491094,0.12472813924153646,0.1233361307779948,22.730582398368462,1689.7100974230264,0.15839661042681866,0.11875971986174946,0.11952686561898423,Healthcare,1 -restaurant/D/short,Naive,269.8283833006444,269.8283833006444,10.66129634818343,1.0060757450046092,0.9434227416570351,0.5561225187815214,20.74904381007386,16.426453765211907,0.7559834619860213,0.49065755991884735,0.6770164746085788,Sales,1 -hierarchical_sales/D/short,Naive,58.96940884663084,58.96940884663084,3.4495245969408845,1.1348033996861053,1.219942090035255,1.1912773542022284,28.8672586949405,7.679154175209066,2.3558794594517027,1.058275945161004,1.7364572065613715,Sales,1 -solar/D/short,Naive,181869.82773722627,181869.82773722627,295.1889902676399,1.155861190610263,1.1535581565541362,0.49778203291324513,22.3811319807538,426.46198861941525,0.6160923187357285,0.4264475482750424,0.5590651164917229,Energy,1 -solar/H/short,Naive,3225.9262389550518,3225.9262389550518,27.129435080666138,2.0928584989912538,1.0,2.0,33.34841823228013,56.79723795181463,2.093565080988042,1.0,1.329536973647586,Energy,1 -solar/H/medium,Naive,3336.7024330900244,3336.7024330900244,27.760555428832117,2.120411643361188,1.0,2.0,103.26192446836347,57.76419680987544,2.080801191386882,1.0,2.9763472520215295,Energy,1 -solar/H/long,Naive,3580.4882400648826,3580.4882400648826,28.841864355231145,2.203647240050593,1.0,2.0,126.30882847556744,59.83718108387863,2.074664118341773,1.0,3.4587368807623573,Energy,1 -hierarchical_sales/W/short,Naive,1045.4721927966102,1045.4721927966102,13.569120762711865,1.0250133777288337,0.9503837463030324,0.5798107884238987,15.297190661198302,32.33376242871544,1.4844588487218926,0.6229649713667368,0.8322267526416594,Sales,1 -ett2/W/short,Naive,4688738.857142857,4688738.857142857,1323.111607142857,0.7785209533197451,0.12217171703066144,0.1269862311226981,16.565488704288857,2165.3495923621335,0.1816396339666693,0.11098877930205708,0.13367758870767996,Energy,7 -bizitobs_service/10S/short,Naive,24411.641058201058,24411.641058201058,51.05920717592593,2.18868692523134,0.29019553694144756,0.09446033861271289,26.76123260084578,156.2422511941026,0.11574925802661973,0.03782629410977125,0.036596222506683165,Web/CloudOps,2 -bizitobs_service/10S/medium,Naive,120879.21777777778,120879.21777777778,96.6510615079365,3.7066999349383667,0.1335696265811012,0.13655030265687004,61.109367709322896,347.67688703417974,0.2616698182259184,0.0727418664837433,0.09526178896511943,Web/CloudOps,2 -bizitobs_service/10S/long,Naive,209141.57037037038,209141.57037037038,126.71164021164022,3.859549432045509,0.15100740947420635,0.15907528831845238,66.98322651223562,457.31998684768894,0.33879726640926433,0.09387203393810047,0.11879424998333914,Web/CloudOps,2 -m4_weekly/W/short,Naive,453525.1459181487,453525.1459181487,347.99148275123207,2.777295047362158,0.08937289522218837,0.09161286714732429,26.631225199626535,673.442756229621,0.1226908336142798,0.06339865521526268,0.06087039452311706,Econ/Fin,1 -electricity/H/short,Naive,14602393.667747747,14602393.667747747,705.7568060247747,3.922376436778821,0.5214915893589874,0.39891619637702774,34.02521326369052,3821.3078478117604,1.8039302069745988,0.3331676148246089,0.2943305500709037,Energy,1 -electricity/H/medium,Naive,70374987.66846848,70374987.66846848,969.0920326576577,4.089287698069245,0.5908175828889569,0.3929812789135593,100.07077757479067,8388.98013279734,3.2744952546433783,0.378268539443073,0.5591833066588809,Energy,1 -electricity/H/long,Naive,62374735.49069069,62374735.49069069,944.6613333333333,4.015252993124353,0.5441184704539119,0.3889218944666957,122.42396003880515,7897.767753656136,3.1148773103603298,0.3725741557559165,0.6680188049702744,Energy,1 +bitbrains_fast_storage/5T/short,Naive,2300000.0,2300000.0,173.0,1.09,2.43,0.375,70.2,1520.0,4.76,0.544,0.893,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,Naive,3130000.0,3130000.0,280.0,1.41,4.94,0.343,61.0,1770.0,5.04,0.799,1.21,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,Naive,1850000.0,1850000.0,130.0,2.06,1.64,0.345,132.0,1360.0,5.57,0.533,0.868,Web/CloudOps,2.0 +bitbrains_rnd/H/short,Naive,4190000.0,4190000.0,251.0,5.87,3.24,0.308,242.0,2050.0,8.95,1.1,1.46,Web/CloudOps,2.0 +bizitobs_application/10S/short,Naive,5850000.0,5850000.0,1050.0,3.76,0.0544,0.0542,47.9,2420.0,0.0933,0.0404,0.0382,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,Naive,22.0,22.0,2.76,0.283,0.168,0.158,3.09,4.69,0.161,0.0947,0.0801,Web/CloudOps,7.0 +bizitobs_l2c/H/short,Naive,263.0,263.0,11.0,1.09,1.29,1.01,9.96,16.2,0.875,0.595,0.486,Web/CloudOps,7.0 +bizitobs_service/10S/short,Naive,24400.0,24400.0,51.1,2.19,0.29,0.0945,26.8,156.0,0.116,0.0378,0.0366,Web/CloudOps,2.0 +car_parts/M/short,Naive,3.0,3.0,0.69,1.21,0.85,1.66,25.8,1.73,4.15,1.65,2.5,Sales,1.0 +covid_deaths/D/short,Naive,4440000.0,4440000.0,354.0,46.9,0.133,0.178,1470.0,2110.0,0.792,0.133,0.127,Healthcare,1.0 +electricity/15T/short,Naive,603000.0,603000.0,127.0,2.46,0.353,0.299,20.0,777.0,1.53,0.251,0.221,Energy,1.0 +electricity/D/short,Naive,4110000000.0,4110000000.0,6960.0,1.99,0.605,0.136,24.6,64100.0,1.05,0.114,0.104,Energy,1.0 +electricity/H/short,Naive,14600000.0,14600000.0,706.0,3.92,0.521,0.399,34.0,3820.0,1.8,0.333,0.294,Energy,1.0 +electricity/W/short,Naive,317000000000.0,317000000000.0,56600.0,2.09,0.286,0.12,15.9,563000.0,1.29,0.129,0.0993,Energy,1.0 +ett1/15T/short,Naive,18.2,18.2,2.62,1.97,0.932,0.561,12.7,4.27,0.808,0.495,0.385,Energy,7.0 +ett1/D/short,Naive,58000.0,58000.0,155.0,1.78,1.16,0.516,23.5,241.0,0.632,0.407,0.408,Energy,7.0 +ett1/H/short,Naive,345.0,345.0,10.2,1.83,0.925,0.512,15.4,18.6,0.867,0.474,0.407,Energy,7.0 +ett1/W/short,Naive,1570000.0,1570000.0,970.0,1.77,0.615,0.625,12.8,1250.0,0.499,0.386,0.312,Energy,7.0 +ett2/15T/short,Naive,19.8,19.8,3.02,1.24,0.163,0.195,13.6,4.45,0.211,0.144,0.121,Energy,7.0 +ett2/D/short,Naive,123000.0,123000.0,212.0,1.39,0.471,0.142,32.7,351.0,0.201,0.121,0.153,Energy,7.0 +ett2/H/short,Naive,172.0,172.0,8.96,1.09,0.18,0.16,14.1,13.1,0.166,0.114,0.113,Energy,7.0 +ett2/W/short,Naive,4690000.0,4690000.0,1320.0,0.779,0.122,0.127,16.6,2170.0,0.182,0.111,0.134,Energy,7.0 +hierarchical_sales/D/short,Naive,59.0,59.0,3.45,1.13,1.22,1.19,28.9,7.68,2.36,1.06,1.74,Sales,1.0 +hierarchical_sales/W/short,Naive,1050.0,1050.0,13.6,1.03,0.95,0.58,15.3,32.3,1.48,0.623,0.832,Sales,1.0 +hospital/M/short,Naive,5850.0,5850.0,24.1,0.968,0.243,0.216,11.8,76.5,0.278,0.0874,0.0863,Healthcare,1.0 +jena_weather/10T/short,Naive,1650.0,1650.0,10.0,0.364,0.418,0.429,51.2,40.7,0.253,0.0622,0.288,Nature,21.0 +jena_weather/D/short,Naive,634.0,634.0,13.1,1.57,1.77,0.485,31.0,25.2,0.152,0.0791,0.211,Nature,21.0 +jena_weather/H/short,Naive,2410.0,2410.0,12.3,0.648,1.1,0.479,38.3,49.1,0.301,0.0755,0.308,Nature,21.0 +kdd_cup_2018/D/short,Naive,3180.0,3180.0,23.9,1.5,0.589,0.572,24.6,56.4,1.26,0.535,0.675,Nature,1.0 +kdd_cup_2018/H/short,Naive,5590.0,5590.0,31.1,1.28,1.35,0.619,14.0,74.8,1.57,0.65,0.562,Nature,1.0 +loop_seattle/5T/short,Naive,101.0,101.0,5.65,0.893,0.138,0.12,14.6,10.1,0.173,0.0968,0.116,Transport,1.0 +loop_seattle/D/short,Naive,58.7,58.7,5.94,1.73,0.111,0.107,21.8,7.66,0.137,0.106,0.103,Transport,1.0 +loop_seattle/H/short,Naive,193.0,193.0,8.0,1.59,0.301,0.161,28.1,13.9,0.246,0.142,0.182,Transport,1.0 +m_dense/D/short,Naive,39900.0,39900.0,118.0,1.67,0.253,0.256,23.3,200.0,0.346,0.205,0.227,Transport,1.0 +m_dense/H/short,Naive,358000.0,358000.0,343.0,2.71,1.04,0.729,28.5,598.0,1.06,0.609,0.539,Transport,1.0 +m4_daily/D/short,Naive,498000.0,498000.0,181.0,3.28,0.0422,0.0305,32.2,705.0,0.109,0.0279,0.0244,Econ/Fin,1.0 +m4_hourly/H/short,Naive,57500000.0,57500000.0,1220.0,11.6,0.377,0.43,75.8,7590.0,1.04,0.166,0.136,Econ/Fin,1.0 +m4_monthly/M/short,Naive,2500000.0,2500000.0,671.0,1.21,0.19,0.153,12.7,1580.0,0.329,0.14,0.129,Econ/Fin,1.0 +m4_quarterly/Q/short,Naive,2270000.0,2270000.0,656.0,1.48,0.132,0.116,13.7,1510.0,0.252,0.11,0.0943,Econ/Fin,1.0 +m4_weekly/W/short,Naive,454000.0,454000.0,348.0,2.78,0.0894,0.0916,26.6,673.0,0.123,0.0634,0.0609,Econ/Fin,1.0 +m4_yearly/A/short,Naive,4070000.0,4070000.0,1010.0,3.97,0.175,0.164,52.0,2020.0,0.324,0.162,0.137,Econ/Fin,1.0 +restaurant/D/short,Naive,274.0,274.0,10.7,1.01,0.944,0.556,20.7,16.6,0.757,0.49,0.674,Sales,1.0 +saugeen/D/short,Naive,995.0,995.0,15.4,3.41,0.471,0.424,58.7,31.5,1.02,0.498,0.585,Nature,1.0 +saugeen/M/short,Naive,1030.0,1030.0,21.0,1.23,0.731,0.606,19.2,32.0,0.962,0.631,0.768,Nature,1.0 +saugeen/W/short,Naive,1800.0,1800.0,24.6,1.99,0.824,0.558,23.3,42.4,1.28,0.741,0.734,Nature,1.0 +solar/10T/short,Naive,36.6,36.6,3.3,1.45,2.66,1.41,17.7,6.05,1.76,0.96,0.943,Energy,1.0 +solar/D/short,Naive,182000.0,182000.0,295.0,1.16,1.15,0.498,22.4,426.0,0.616,0.426,0.559,Energy,1.0 +solar/H/short,Naive,3230.0,3230.0,27.1,2.09,1.0,2.0,33.3,56.8,2.09,1.0,1.33,Energy,1.0 +solar/W/short,Naive,2810000.0,2810000.0,1340.0,1.47,0.309,0.253,11.6,1680.0,0.342,0.274,0.21,Energy,1.0 +sz_taxi/15T/short,Naive,33.1,33.1,3.94,0.786,1310000000000.0,0.528,22.6,5.76,0.538,0.369,0.661,Transport,1.0 +sz_taxi/H/short,Naive,14.1,14.1,2.69,0.835,0.969,0.394,25.4,3.76,0.35,0.25,0.486,Transport,1.0 +temperature_rain/D/short,Naive,311.0,311.0,7.89,2.01,97.0,1.33,47.5,17.6,2.07,0.929,1.27,Nature,1.0 +us_births/D/short,Naive,2860000.0,2860000.0,1270.0,1.86,0.125,0.123,22.7,1690.0,0.158,0.119,0.12,Healthcare,1.0 +us_births/M/short,Naive,261000000.0,261000000.0,13300.0,1.5,0.0409,0.0413,18.1,16200.0,0.0502,0.0412,0.039,Healthcare,1.0 +us_births/W/short,Naive,5290000.0,5290000.0,1720.0,1.56,0.0232,0.0232,13.3,2300.0,0.0312,0.0234,0.0193,Healthcare,1.0 +bitbrains_fast_storage/5T/medium,Naive,3700000.0,3700000.0,280.0,1.27,4.12,0.45,197.0,1920.0,5.85,0.852,2.05,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,Naive,2450000.0,2450000.0,158.0,4.77,0.575,0.396,390.0,1560.0,6.58,0.664,2.01,Web/CloudOps,2.0 +bizitobs_application/10S/medium,Naive,21600000.0,21600000.0,1830.0,6.52,0.0866,0.0889,105.0,4650.0,0.183,0.0719,0.0885,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,Naive,157.0,157.0,8.57,0.826,0.624,0.87,6.33,12.5,0.66,0.451,0.359,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,Naive,559.0,559.0,16.5,1.58,1.0,2.0,17.9,23.6,1.43,1.0,0.877,Web/CloudOps,7.0 +bizitobs_service/10S/medium,Naive,121000.0,121000.0,96.7,3.71,0.134,0.137,61.1,348.0,0.262,0.0727,0.0953,Web/CloudOps,2.0 +electricity/15T/medium,Naive,2930000.0,2930000.0,223.0,3.46,0.404,0.412,54.7,1710.0,2.95,0.385,0.43,Energy,1.0 +electricity/H/medium,Naive,70400000.0,70400000.0,969.0,4.09,0.591,0.393,100.0,8390.0,3.27,0.378,0.559,Energy,1.0 +ett1/15T/medium,Naive,26.0,26.0,2.79,1.82,1.11,0.617,32.4,5.1,0.971,0.532,0.617,Energy,7.0 +ett1/H/medium,Naive,472.0,472.0,12.0,2.03,9730000000000.0,0.65,46.9,21.7,1.04,0.574,0.841,Energy,7.0 +ett2/15T/medium,Naive,18.1,18.1,2.9,1.22,0.184,0.21,34.5,4.26,0.227,0.155,0.242,Energy,7.0 +ett2/H/medium,Naive,411.0,411.0,13.1,1.43,0.242,0.216,43.7,20.3,0.273,0.177,0.295,Energy,7.0 +jena_weather/10T/medium,Naive,3630.0,3630.0,15.9,0.76,0.949,0.58,162.0,60.2,0.37,0.0978,0.882,Nature,21.0 +jena_weather/H/medium,Naive,2460.0,2460.0,13.4,0.873,2.04,0.523,119.0,49.6,0.304,0.0821,0.927,Nature,21.0 +kdd_cup_2018/H/medium,Naive,5750.0,5750.0,32.5,1.45,1.52,0.706,36.0,75.8,1.59,0.68,1.0,Nature,1.0 +loop_seattle/5T/medium,Naive,392.0,392.0,12.9,1.99,0.363,0.299,44.1,19.8,0.352,0.229,0.327,Transport,1.0 +loop_seattle/H/medium,Naive,216.0,216.0,8.81,1.78,0.322,0.172,87.8,14.7,0.26,0.156,0.458,Transport,1.0 +m_dense/H/medium,Naive,361000.0,361000.0,349.0,2.74,1.18,0.73,88.6,601.0,1.05,0.607,1.2,Transport,1.0 +solar/10T/medium,Naive,80.8,80.8,5.94,2.56,2.86,1.64,33.3,8.99,1.99,1.32,1.37,Energy,1.0 +solar/H/medium,Naive,3340.0,3340.0,27.8,2.12,1.0,2.0,103.0,57.8,2.08,1.0,2.98,Energy,1.0 +sz_taxi/15T/medium,Naive,30.6,30.6,3.97,0.779,10000000000000.0,0.528,70.1,5.53,0.515,0.37,1.94,Transport,1.0 +bitbrains_fast_storage/5T/long,Naive,3500000.0,3500000.0,289.0,1.19,1.8,0.51,232.0,1870.0,4.94,0.765,2.12,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,Naive,4800000.0,4800000.0,288.0,3.99,5.68,0.43,406.0,2190.0,8.39,1.1,2.36,Web/CloudOps,2.0 +bizitobs_application/10S/long,Naive,1370000000.0,1370000000.0,28300.0,38600.0,559.0,1.04,605000.0,37100.0,1.43,1.09,0.983,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,Naive,353.0,353.0,13.2,1.25,0.986,1.21,9.78,18.8,1.05,0.743,0.592,Web/CloudOps,7.0 +bizitobs_l2c/H/long,Naive,944.0,944.0,27.7,4.15,6.13,1.12,25.6,30.7,1.88,1.69,1.27,Web/CloudOps,7.0 +bizitobs_service/10S/long,Naive,209000.0,209000.0,127.0,3.86,0.151,0.159,67.0,457.0,0.339,0.0939,0.119,Web/CloudOps,2.0 +electricity/15T/long,Naive,6140000.0,6140000.0,249.0,3.41,0.516,0.384,66.5,2480.0,3.91,0.393,0.478,Energy,1.0 +electricity/H/long,Naive,62400000.0,62400000.0,945.0,4.02,0.544,0.389,122.0,7900.0,3.11,0.373,0.668,Energy,1.0 +ett1/15T/long,Naive,28.4,28.4,3.04,2.04,1.17,0.68,39.7,5.33,1.02,0.579,0.716,Energy,7.0 +ett1/H/long,Naive,444.0,444.0,12.4,2.27,5650000000000.0,0.749,57.3,21.1,1.01,0.592,0.987,Energy,7.0 +ett2/15T/long,Naive,16.6,16.6,2.88,1.3,0.2,0.249,42.0,4.07,0.217,0.154,0.283,Energy,7.0 +ett2/H/long,Naive,294.0,294.0,11.5,1.32,0.218,0.222,53.5,17.2,0.235,0.157,0.349,Energy,7.0 +jena_weather/10T/long,Naive,2740.0,2740.0,13.3,0.753,0.844,0.499,198.0,52.3,0.32,0.0812,1.07,Nature,21.0 +jena_weather/H/long,Naive,3730.0,3730.0,17.7,1.47,2.37,0.575,144.0,61.0,0.368,0.107,1.14,Nature,21.0 +kdd_cup_2018/H/long,Naive,4180.0,4180.0,25.9,1.18,1.1,0.715,43.8,64.7,1.52,0.607,1.25,Nature,1.0 +loop_seattle/5T/long,Naive,214.0,214.0,9.08,1.41,0.342,0.188,54.0,14.6,0.258,0.16,0.361,Transport,1.0 +loop_seattle/H/long,Naive,210.0,210.0,8.6,1.75,0.326,0.174,108.0,14.5,0.259,0.154,0.556,Transport,1.0 +m_dense/H/long,Naive,396000.0,396000.0,362.0,2.86,1.09,0.77,108.0,629.0,1.09,0.627,1.44,Transport,1.0 +solar/10T/long,Naive,94.6,94.6,4.63,2.02,1.0,2.0,40.6,9.72,2.1,1.0,1.55,Energy,1.0 +solar/H/long,Naive,3580.0,3580.0,28.8,2.2,1.0,2.0,126.0,59.8,2.07,1.0,3.46,Energy,1.0 +sz_taxi/15T/long,Naive,28.5,28.5,3.85,0.742,9280000000000.0,0.535,85.0,5.34,0.494,0.357,2.4,Transport,1.0 diff --git a/results/naive/config.json b/results/naive/config.json index 105bf1f715922d2506cb918021d91ffe590107ee..e0cc6841f66fcb202023d04e4deeeb2fef50b12a 100644 --- a/results/naive/config.json +++ b/results/naive/config.json @@ -1,6 +1,5 @@ { "model": "Naive", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/seasonal_naive/all_results.csv b/results/seasonal_naive/all_results.csv index 7f8f9cc8f419a62b733cabf20886baeccdf5d23e..e1e69e07f06592482f6b27958c2a21b0837a8622 100644 --- a/results/seasonal_naive/all_results.csv +++ b/results/seasonal_naive/all_results.csv @@ -1,98 +1,98 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/H/short,Seasonal_Naive,4198734.112364942,4198734.112364942,323.91137144521844,1.2985124569638047,4.122732886534151,0.31268171558395724,34.57780677446709,2049.0812849579547,5.840547085975178,0.9232525964178199,1.0222395520114795,Web/CloudOps,2 -solar/H/short,Seasonal_Naive,1009.7531694199001,1009.7531694199001,12.29965959124688,0.9519362458316395,3.8670855712527974,0.6200499536903793,16.15202026601857,31.77661356123242,1.171296544389828,0.4533695432534923,0.5916559631947232,Energy,1 -solar/H/medium,Seasonal_Naive,993.4109489051095,993.4109489051095,12.224457306873479,0.9349685908749716,5.243116882202785,0.568037363492515,29.963349764914504,31.518422373353484,1.1353671382460326,0.4403534842165714,0.9459874672135314,Energy,1 -solar/H/long,Seasonal_Naive,1160.9764801297647,1160.9764801297647,13.995331508515815,1.0711966477927781,5.3301362264546865,0.7115105305381197,36.28762062617156,34.073104938202576,1.1813766446766685,0.48524364916713286,1.0779764182077312,Energy,1 -m_dense/D/short,Seasonal_Naive,39929.671111111114,39929.671111111114,118.38299768518519,1.6693368812171987,0.2532523956185716,0.2557375985604745,23.291872240434824,199.82410042612756,0.3459250756939904,0.20493848013227253,0.22686565739428607,Transport,1 -ett2/H/short,Seasonal_Naive,151.78024553571427,151.78024553571427,7.9655081612723215,0.9231678277377344,0.1445715792364967,0.13716111864362443,9.954323499249957,12.319912562015782,0.1562507702523094,0.10102480674150278,0.08895289185095483,Energy,7 -ett2/H/medium,Seasonal_Naive,338.06856398809526,338.06856398809526,11.509515671502976,1.2384249730532626,0.20117125149055434,0.19754707699730284,25.271323860489556,18.3866409109466,0.24777181028131637,0.15509812516606425,0.1862050814081793,Energy,7 -ett2/H/long,Seasonal_Naive,258.848330026455,258.848330026455,10.011796254960318,1.1284016697207502,0.19021284402568,0.18827246610449735,30.708493118791996,16.08876409257265,0.2203232414204263,0.1371038440641976,0.20790633225768584,Energy,7 -ett2/D/short,Seasonal_Naive,122878.94603174603,122878.94603174603,211.56051587301587,1.3901143157307532,0.4713712661985367,0.14168287004743305,32.70861808020462,350.5409334610525,0.20062382493032205,0.12108166507005774,0.15331100213050436,Energy,7 -electricity/15T/short,Seasonal_Naive,366722.87168918917,366722.87168918917,94.21436127533784,1.7170601412372184,0.30235384997180187,0.23004003312303262,26.97516311288429,605.5764788110492,1.193710612408421,0.18571507783204827,0.16489261500071847,Energy,1 -electricity/15T/medium,Seasonal_Naive,373592.9609594595,373592.9609594595,70.78214646677928,1.1507882041044137,0.21527551573127818,0.162019429539667,12.499327683031675,611.2225134592635,1.0531979612070823,0.12196476848793535,0.11275485280084273,Energy,1 -electricity/15T/long,Seasonal_Naive,432425.0750750751,432425.0750750751,73.83208042417418,1.1635317075319347,0.19999027586553747,0.1567721679648986,14.199582505532762,657.5903550654275,1.0374294291948993,0.11647916132088339,0.11255332426055248,Energy,1 -bitbrains_rnd/H/short,Seasonal_Naive,3710112.576957504,3710112.576957504,254.47180959890883,6.0373618687533925,4.792930625938438,0.36119250186151963,222.24992066836998,1926.1652517262125,8.429841528920612,1.1136931405932016,1.2432170953390527,Web/CloudOps,2 -loop_seattle/H/short,Seasonal_Naive,125.56935646488512,125.56935646488512,6.351615330169464,1.2928412613913995,0.1710389719335216,0.14238265652237242,14.224911524098493,11.205773354163698,0.1983535851600468,0.11243005123144872,0.10423508438727377,Transport,1 -loop_seattle/H/medium,Seasonal_Naive,153.75436661506708,153.75436661506708,7.21920260497291,1.4805126192799751,0.20086050437084785,0.16072294158463138,25.49530583293511,12.399772845301122,0.21916459102759323,0.1275985944423001,0.16208090743299983,Transport,1 -loop_seattle/H/long,Seasonal_Naive,167.0609584623323,167.0609584623323,7.511452824002408,1.5460268753677606,0.23476947145360338,0.16369716459260514,30.628061299733826,12.925206321847721,0.23099074580000042,0.13423972094936726,0.18709450197084374,Transport,1 -solar/W/short,Seasonal_Naive,2813150.8321167883,2813150.8321167883,1341.8134124087592,1.4703506784640206,0.3086572076282362,0.2534797557079009,11.583228430599936,1677.245012547895,0.3423676402558515,0.2738976644635694,0.20974762813186765,Energy,1 -bizitobs_application/10S/short,Seasonal_Naive,3938171.448888889,3938171.448888889,1054.6502083333332,2.2423302386755415,0.0340565554300944,0.03481890360514323,26.67150527524178,1984.4826653031992,0.07653356419683467,0.04067364297805646,0.03483868247005456,Web/CloudOps,2 -bizitobs_application/10S/medium,Seasonal_Naive,5143417.173333333,5143417.173333333,1276.9247916666666,2.6914183385868085,0.03906738917032877,0.03992526690165202,28.10250665325578,2267.9103098079813,0.08905416300729038,0.05014107835453231,0.04268912394422428,Web/CloudOps,2 -bizitobs_application/10S/long,Seasonal_Naive,5953087.1466666665,5953087.1466666665,1470.5443055555556,3.2063450561622804,0.04643516540527344,0.04741138034396701,21.30987046900455,2439.894904840507,0.09409699394104339,0.05671301593170149,0.04573090228155046,Web/CloudOps,2 -bizitobs_service/10S/short,Seasonal_Naive,73535.24306878306,73535.24306878306,55.138688409391534,1.225305042620374,0.08240041742879878,0.07553190362516535,16.350613928029045,271.1738244535838,0.2008942442703866,0.04084850431410673,0.03998233993351736,Web/CloudOps,2 -bizitobs_service/10S/medium,Seasonal_Naive,96480.73142857142,96480.73142857142,63.760550595238094,1.3205790727650968,0.0796314687577505,0.07705034528459821,17.448776648119132,310.6134759288003,0.23377502162466005,0.04798769290234763,0.04755831292581585,Web/CloudOps,2 -bizitobs_service/10S/long,Seasonal_Naive,130947.25079365079,130947.25079365079,75.32497354497355,1.367193469742303,0.08491984049479166,0.08144445348668981,17.936180702599792,361.8663438255218,0.26808215608217806,0.055803148480992645,0.05345299584740217,Web/CloudOps,2 -bizitobs_l2c/5T/short,Seasonal_Naive,174.3096912202381,174.3096912202381,9.685235305059523,0.9860210805885202,0.42617953698061745,0.5068802606491816,7.245192912077175,13.202639555037399,0.45343208257188783,0.3326301832497065,0.26206838623738704,Web/CloudOps,7 -bizitobs_l2c/5T/medium,Seasonal_Naive,294.66615646258504,294.66615646258504,12.864580676020408,1.2435997989335994,0.9360730119589105,1.1668089007914848,7.637297246345627,17.16584272509174,0.9031757619892758,0.6768661254104057,0.5203917750796179,Web/CloudOps,7 -bizitobs_l2c/5T/long,Seasonal_Naive,416.33936507936505,416.33936507936505,15.353609871031747,1.4542830442631411,1.0210265852224123,1.298652753589897,8.166517613570452,20.404395729336485,1.1456763155758307,0.862082240570442,0.6484921267253538,Web/CloudOps,7 -ett2/W/short,Seasonal_Naive,4688738.857142857,4688738.857142857,1323.111607142857,0.7785209533197451,0.12217171703066144,0.1269862311226981,16.565488704288857,2165.3495923621335,0.1816396339666693,0.11098877930205708,0.13367758870767996,Energy,7 -loop_seattle/5T/short,Seasonal_Naive,76.27992181453173,76.27992181453173,4.806023281794214,0.7623397118337274,0.13014939133965933,0.10246611478035911,9.641956549599044,8.733837748351622,0.14976603568936095,0.08241268902447142,0.08083202568463041,Transport,1 -loop_seattle/5T/medium,Seasonal_Naive,149.91421294504644,149.91421294504644,7.272513948013416,1.1532851467900647,0.22437167928437823,0.16228546394529234,12.161708217874148,12.243945971174751,0.217844961125775,0.12939296873911593,0.11727937998585615,Transport,1 -loop_seattle/5T/long,Seasonal_Naive,172.4441060371517,172.4441060371517,7.891544207229676,1.2506857542807588,0.23036973349395495,0.1761461930635284,12.853493858763652,13.131797517367975,0.2321728668857043,0.13952411620149754,0.12716656087996273,Transport,1 -loop_seattle/D/short,Seasonal_Naive,58.65786974329205,58.65786974329205,5.940060831075852,1.7323957056549062,0.11095823557630288,0.10706747427075271,21.844122635807963,7.658842585096788,0.13687375626545378,0.10615682844516516,0.1032547596878987,Transport,1 -bitbrains_rnd/5T/short,Seasonal_Naive,3407833.9580161185,3407833.9580161185,210.5518366560826,1.9708126376716957,3.046095137251105,0.38408369415249266,74.37164472655921,1846.0319493486884,7.553429809324837,0.8615173317922609,1.1015904440824371,Web/CloudOps,2 -bitbrains_rnd/5T/medium,Seasonal_Naive,3267017.897300195,3267017.897300195,204.8128727866788,4.542392314855541,2.750306364851926,0.3918429569214854,178.58186300111112,1807.489390646649,7.605029622998441,0.8617522032352579,1.169291300089368,Web/CloudOps,2 -bitbrains_rnd/5T/long,Seasonal_Naive,3851065.095876048,3851065.095876048,237.4491928177016,3.501257904408163,3.179405138911208,0.40348581728614835,137.13909859136817,1962.413079826989,7.517562382830161,0.909614361068727,1.175220506770106,Web/CloudOps,2 -electricity/W/short,Seasonal_Naive,317380739662.4144,317380739662.4144,56604.72342342342,2.0896995064573947,0.285636731125347,0.1196364761812733,15.938528356933686,563365.5471027798,1.2854142672489681,0.12915329923232946,0.0993032024444414,Energy,1 -solar/D/short,Seasonal_Naive,181869.82773722627,181869.82773722627,295.1889902676399,1.155861190610263,1.1535581565541362,0.49778203291324513,22.3811319807538,426.46198861941525,0.6160923187357285,0.4264475482750424,0.5590651164917229,Energy,1 -ett1/H/short,Seasonal_Naive,141.21642485119048,141.21642485119048,5.822230747767857,0.9773176243434534,0.5700293404715402,0.28737351553780693,9.542335004679739,11.883451722929264,0.5546493091237623,0.271747328730894,0.24036844107833774,Energy,7 -ett1/H/medium,Seasonal_Naive,204.51871279761906,204.51871279761906,8.163152785528274,1.5678007461285395,2345799664153.64,0.45697086879185267,22.421278808802057,14.301003908733787,0.6849841868181539,0.3909956677413786,0.4349118298516912,Energy,7 -ett1/H/long,Seasonal_Naive,199.49484126984126,199.49484126984126,7.749353091931217,1.4787012870437326,2646635889605.55,0.47721551158440806,25.35082572229849,14.124264273576916,0.6763804250290198,0.37109973563904386,0.47106248989542887,Energy,7 -temperature_rain/D/short,Seasonal_Naive,310.64911445557556,310.64911445557556,7.890313322375261,2.0115357653222166,97.02872693676676,1.3339967533230714,47.47823470729249,17.625240833973745,2.0749568489172328,0.9288984940737532,1.2679455200160903,Nature,1 -saugeen/W/short,Seasonal_Naive,1800.4892578125,1800.4892578125,24.552409362792968,1.990658004828632,0.824161148071289,0.557866096496582,23.291068065474015,42.43217243805106,1.2808504084686672,0.7411348925661494,0.7340352210434381,Nature,1 -covid_deaths/D/short,Seasonal_Naive,4435194.482205514,4435194.482205514,353.7093984962406,46.91239825526407,0.132724402667158,0.17781104711702989,1472.7784117419058,2105.990142950701,0.792389040211866,0.13308488253209907,0.12672327831223676,Healthcare,1 -us_births/D/short,Seasonal_Naive,2855120.2133333334,2855120.2133333334,1266.88,1.8648391912491094,0.12472813924153646,0.1233361307779948,22.730582398368462,1689.7100974230264,0.15839661042681866,0.11875971986174946,0.11952686561898423,Healthcare,1 -m4_weekly/W/short,Seasonal_Naive,453525.1459181487,453525.1459181487,347.99148275123207,2.777295047362158,0.08937289522218837,0.09161286714732429,26.631225199626535,673.442756229621,0.1226908336142798,0.06339865521526268,0.06087039452311706,Econ/Fin,1 -car_parts/M/short,Seasonal_Naive,2.5046831406935035,2.5046831406935035,0.6672313006509898,1.2014638390969912,0.8541762945076465,1.6763314144736843,20.368070639282948,1.5826190763078472,3.7949581992256025,1.5999522140809175,1.7217438941460104,Sales,1 -sz_taxi/H/short,Seasonal_Naive,12.029984433426817,12.029984433426817,2.4300792726696048,0.7381672994945998,1.552363729884482,0.38466275859082866,8.057108467274903,3.468426795166191,0.3231053592422672,0.22637687999566908,0.21382494112013678,Transport,1 -ett1/15T/short,Seasonal_Naive,8.776201520647321,8.776201520647321,1.4945350283668155,0.9341549587802876,0.6328330752819759,0.30603092738560267,8.439802226867357,2.962465446321243,0.5600292824430536,0.2825293306160358,0.24136394938567346,Energy,7 -ett1/15T/medium,Seasonal_Naive,12.883658234126985,12.883658234126985,1.8389095052083333,1.188305978388322,0.9080556087099148,0.42800251405065676,12.40964763458674,3.5893813163450585,0.6841396278696877,0.3504979699009272,0.32155116949786394,Energy,7 -ett1/15T/long,Seasonal_Naive,13.023148561507936,13.023148561507936,1.8557530381944445,1.190885705867336,0.9840622608053838,0.4204419926758006,13.893107103740524,3.608759975602137,0.6878332194845934,0.35370836394199906,0.3401793468576481,Energy,7 -saugeen/M/short,Seasonal_Naive,713.7325613839286,713.7325613839286,16.662892659505207,0.9763741730697157,0.4971456073579334,0.4422340847197033,10.586458554994822,26.71577364374703,0.8020876713555654,0.500270025848136,0.4450846286544009,Nature,1 -us_births/M/short,Seasonal_Naive,74172117.33333333,74172117.33333333,6704.75,0.7605310792868888,0.020606640726327896,0.020953829089800518,5.646371868808973,8612.323573422756,0.02674981830097076,0.020824907787592,0.016825596814792485,Healthcare,1 -m4_quarterly/Q/short,Seasonal_Naive,2487884.428,2487884.428,708.8467662760416,1.602247174616002,0.14192582162221273,0.12521366802851358,12.540135359281015,1577.302896719587,0.26401272904907985,0.11864846608179633,0.09805397548466407,Econ/Fin,1 -ett1/W/short,Seasonal_Naive,1571455.142857143,1571455.142857143,970.0256696428571,1.7689207877683926,0.6150428227015904,0.6251754760742188,12.799615221235317,1253.5769393448265,0.4988281842249676,0.38599636628003803,0.3118251468740881,Energy,7 -saugeen/D/short,Seasonal_Naive,994.7777083333333,994.7777083333333,15.3725,3.413049305768806,0.47135218302408854,0.4242374928792318,58.71112805300772,31.540096834558597,1.0215912513857213,0.49791893773514423,0.5850141736335629,Nature,1 -electricity/H/short,Seasonal_Naive,2887718.36009009,2887718.36009009,247.69792300112613,1.3577726051465433,0.3631860614314555,0.16452154722835458,18.287059022037134,1699.3287969342748,0.8022045515456071,0.11693110926995137,0.10566499518474065,Energy,1 -electricity/H/medium,Seasonal_Naive,8353587.978378379,8353587.978378379,308.2015061936937,1.3924537851702354,0.3866618745283321,0.15667597765115268,20.69150990156008,2890.2574242406813,1.128162669425473,0.1203011991361897,0.12738701179892217,Energy,1 -electricity/H/long,Seasonal_Naive,13526606.943327328,13526606.943327328,357.2064234234234,1.5240786163496851,0.36051114996741657,0.16771684918985819,24.628137038027603,3677.8535782882013,1.4505443840804357,0.14088210974822626,0.15341839756227935,Energy,1 -ett1/D/short,Seasonal_Naive,57976.62857142857,57976.62857142857,154.97894345238095,1.7783697877928102,1.1623038155691965,0.5164825923859127,23.47582377746543,240.7833643992636,0.6324254934604606,0.4070573356817528,0.40843357791503654,Energy,7 -bizitobs_l2c/H/short,Seasonal_Naive,281.8430679563492,281.8430679563492,12.53165302579365,1.2140641267600039,1.3605904339028776,1.138373051002047,7.486930567002142,16.788182389894065,0.9049260260934667,0.675488192208351,0.5211675771895117,Web/CloudOps,7 -bizitobs_l2c/H/medium,Seasonal_Naive,456.3732886904762,456.3732886904762,15.667392113095238,1.5102861226255864,1.6912910179529201,1.4024095456148358,18.533653530822832,21.362895138311103,1.293555748999092,0.9486843898499615,0.9042051354742671,Web/CloudOps,7 -bizitobs_l2c/H/long,Seasonal_Naive,309.27222222222224,309.27222222222224,13.635487971230159,1.4260541525613812,2.4383105011700468,0.9168538411458333,22.036198194504,17.586137217201003,1.0742120179443015,0.8328949597245134,0.9410651242377539,Web/CloudOps,7 -hospital/M/short,Seasonal_Naive,3464.0771403737504,3464.0771403737504,20.005975662755322,0.9205278266364826,0.23307140471364352,0.21025354682958836,6.599875528051222,58.85641120875236,0.21366243998041978,0.07262633732682543,0.06248817651596496,Healthcare,1 -m4_daily/D/short,Seasonal_Naive,497624.29605596676,497624.29605596676,180.83018393659805,3.278424296610481,0.04217648847004251,0.03045251709590828,32.19152086932198,705.4249046184624,0.10896865245350737,0.02793326594720746,0.024356147648337902,Econ/Fin,1 -bitbrains_fast_storage/5T/short,Seasonal_Naive,5404398.959598898,5404398.959598898,346.43736615760264,1.1360241737534567,5.932190463200192,0.4761820977820179,31.06038796757978,2324.736320445589,7.299097101583847,1.087727650212227,1.210386425305063,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,Seasonal_Naive,5191990.375993042,5191990.375993042,332.75293038959035,1.2202713595882653,5.768419487236171,0.4516151765437165,36.13464652226377,2278.593947150971,6.92299777327362,1.0109953109538277,1.1979128978088514,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,Seasonal_Naive,6473012.871295428,6473012.871295428,404.5292260549034,1.1366422318753115,6.462525817893773,0.5033018991430218,32.16149564789198,2544.2116404291974,6.723591850916995,1.0690499817466734,1.1773077032683998,Web/CloudOps,2 -ett2/15T/short,Seasonal_Naive,14.617721121651785,14.617721121651785,2.534710984002976,1.067314800011479,0.15340707960692476,0.17031889997991706,8.528224043296179,3.8233128464267456,0.181506262086668,0.12033174753313121,0.09638438775638664,Energy,7 -ett2/15T/medium,Seasonal_Naive,14.948670634920635,14.948670634920635,2.4787872023809525,1.0512391501712754,0.1650384327009311,0.1899925877490716,13.312453415228259,3.8663510749698657,0.20604484738813625,0.1320990569451144,0.12412877798161984,Energy,7 -ett2/15T/long,Seasonal_Naive,13.674988839285714,13.674988839285714,2.362206101190476,1.0126577751121104,0.1609817984920091,0.1894470412273871,15.633896791936913,3.697970908388236,0.19707156352079597,0.12588623903557683,0.13282444765695306,Energy,7 -jena_weather/10T/short,Seasonal_Naive,1986.203373015873,1986.203373015873,10.556470114087302,0.7428874675800381,0.8774809708937492,0.39362146904346257,20.65580527060839,44.566841631597285,0.2768159431081858,0.06556891005829443,0.15520092375681044,Nature,21 -jena_weather/10T/medium,Seasonal_Naive,2572.8219336219336,2572.8219336219336,12.447531114718615,0.7160100391723763,0.8990536291309329,0.4023956803535614,27.512189615220795,50.72299215959104,0.311286135757249,0.07639028565640756,0.21183609008512827,Nature,21 -jena_weather/10T/long,Seasonal_Naive,2044.5376984126983,2044.5376984126983,11.441627397486773,0.7614923703210493,0.7718160693153647,0.4043349656668781,32.10701219708694,45.21656442513847,0.2766879110680792,0.0700132799578619,0.2371949131202996,Nature,21 -jena_weather/D/short,Seasonal_Naive,633.5864087301587,633.5864087301587,13.136656746031745,1.5734140297468562,1.7707135350571066,0.4854136849221913,31.00640730511845,25.17114238031637,0.15157141745411598,0.07910414448099358,0.2105908566035542,Nature,21 -us_births/W/short,Seasonal_Naive,5288246.857142857,5288246.857142857,1720.3214285714287,1.5634204337929731,0.023174324205943515,0.023229679891041348,13.279190377688378,2299.6188504060533,0.031217100406456557,0.023353194707727513,0.01929679613291714,Healthcare,1 -m_dense/H/short,Seasonal_Naive,145691.3111111111,145691.3111111111,180.18139756944444,1.4875444172240992,0.5399732053067908,0.3908195287297072,14.354127446705878,381.69531187992226,0.6770323708995337,0.31959690096169935,0.27479786933969136,Transport,1 -m_dense/H/medium,Seasonal_Naive,158402.70222222223,158402.70222222223,189.1006423611111,1.569946511898904,0.5428334697930458,0.397995368109809,22.89829990230752,397.99836962256796,0.6924668813918432,0.32901122738058514,0.3773835994986822,Transport,1 -m_dense/H/long,Seasonal_Naive,137584.05530864198,137584.05530864198,175.8238888888889,1.4779178744488217,0.4978060829089136,0.38833718102796383,27.64819460133164,370.92324719359664,0.6421229733186591,0.3043771431636812,0.4189507904607682,Transport,1 -kdd_cup_2018/D/short,Seasonal_Naive,3183.23772462431,3183.23772462431,23.884613481510574,1.4970343738270926,0.5893064466306873,0.5721879846611051,24.580612052672397,56.42018898075679,1.263872161836989,0.5350407118596068,0.674506129145155,Nature,1 -electricity/D/short,Seasonal_Naive,4108036968.189694,4108036968.189694,6964.321117117117,1.987041399842847,0.6052061423957089,0.13582226488305205,24.595303547998157,64093.9698270414,1.0532768308357283,0.11444692995228782,0.10405987098536745,Energy,1 -restaurant/D/short,Seasonal_Naive,269.8283833006444,269.8283833006444,10.66129634818343,1.0060757450046092,0.9434227416570351,0.5561225187815214,20.74904381007386,16.426453765211907,0.7559834619860213,0.49065755991884735,0.6770164746085788,Sales,1 -jena_weather/H/short,Seasonal_Naive,1941.9557226399331,1941.9557226399331,11.035693889672096,0.7229102185557323,1.1349936220434893,0.40077710759656054,17.382581140637207,44.06762669624872,0.2701317147149199,0.06764809310095782,0.15420788787522166,Nature,21 -jena_weather/H/medium,Seasonal_Naive,1951.8111111111111,1951.8111111111111,12.670400080605159,0.8886133410507664,1.7017065286708326,0.45474790719102887,42.286346185217376,44.1793063674738,0.27034101099927343,0.077532425227887,0.3428055217231077,Nature,21 -jena_weather/H/long,Seasonal_Naive,2714.1007936507935,2714.1007936507935,15.67983630952381,1.2677178473920059,1.8661278391921836,0.5020780815368567,51.127769997760254,52.097032484113655,0.31368185519406616,0.09440998667650671,0.41914340130808575,Nature,21 -m4_monthly/M/short,Seasonal_Naive,2652970.8577407408,2652970.8577407408,700.2370131655092,1.259717038974429,0.19222692431343927,0.15988281532570167,10.588031340884587,1628.794295711015,0.33851937015163663,0.14553329004027185,0.12192063664829357,Econ/Fin,1 -hierarchical_sales/D/short,Seasonal_Naive,58.96940884663084,58.96940884663084,3.4495245969408845,1.1348033996861053,1.219942090035255,1.1912773542022284,28.8672586949405,7.679154175209066,2.3558794594517027,1.058275945161004,1.7364572065613715,Sales,1 -m4_hourly/H/short,Seasonal_Naive,3614355.993558776,3614355.993558776,353.8562298711755,1.1932101877276544,0.1561203125786282,0.1391227341312525,9.229226853352637,1901.1459685039379,0.25954838815818215,0.04830918594592099,0.03757255147212195,Econ/Fin,1 -m4_yearly/A/short,Seasonal_Naive,4070174.893125562,4070174.893125562,1007.6444123429384,3.965954931783283,0.17520083696448324,0.16353434447586868,52.037766411775884,2017.4674453694568,0.323510256896443,0.1615804524853844,0.13714565740983498,Econ/Fin,1 -hierarchical_sales/W/short,Seasonal_Naive,1045.4721927966102,1045.4721927966102,13.569120762711865,1.0250133777288337,0.9503837463030324,0.5798107884238987,15.297190661198302,32.33376242871544,1.4844588487218926,0.6229649713667368,0.8322267526416594,Sales,1 -kdd_cup_2018/H/short,Seasonal_Naive,5615.976023835748,5615.976023835748,32.23957254610716,1.340425278558367,1.4218929148819215,0.6653979848821596,11.487358350395635,74.9398160114885,1.5686260624453057,0.6748326381021144,0.547536580422314,Nature,1 -kdd_cup_2018/H/medium,Seasonal_Naive,5785.5976861948375,5785.5976861948375,31.3213270934715,1.4289997072062277,1.6378374272295781,0.6550327560864638,21.848163370875977,76.06311646386071,1.592138770600474,0.6556120959870002,0.7587892026846513,Nature,1 -kdd_cup_2018/H/long,Seasonal_Naive,4863.8870949314805,4863.8870949314805,29.603438898808303,1.3355336658089667,1.3504101339311876,0.8098963463700346,25.694140506460055,69.74157364822995,1.6364191009705393,0.694616286015704,0.9362833347539716,Nature,1 -sz_taxi/15T/short,Seasonal_Naive,31.23822759271978,31.23822759271978,3.8183243114578564,0.7644167735543712,1312692316403.7466,0.538321787329966,7.055930253561324,5.58911688844667,0.5226063327326282,0.3570296534180041,0.3087068505119876,Transport,1 -sz_taxi/15T/medium,Seasonal_Naive,28.930445379273504,28.930445379273504,3.6711446647970085,0.7134840540400068,18917890105834.12,0.5172677325387286,10.816848306969383,5.378702945810774,0.5005780877467839,0.34166128798341444,0.379172158612487,Transport,1 -sz_taxi/15T/long,Seasonal_Naive,28.48211582977208,28.48211582977208,3.6789685719373217,0.6909431344941248,3284113398767.079,0.5217545990251068,12.685376303466118,5.336863857151696,0.4939706119705816,0.3405187776085254,0.4279704063043717,Transport,1 -solar/10T/short,Seasonal_Naive,37.917036857512166,37.917036857512166,2.5271684706936206,1.1057028063509833,4.40491207343912,0.925039670208932,18.785745549933484,6.157681126650857,1.7912687729072838,0.7351530344494263,0.8595386998005952,Energy,1 -solar/10T/medium,Seasonal_Naive,29.228987641008626,29.228987641008626,2.118083017100752,0.9270005404761702,3.219568147534267,0.6587355148950821,15.807425762449464,5.406383970918883,1.1996980737201242,0.4700110331163171,0.6550635536405177,Energy,1 -solar/10T/long,Seasonal_Naive,26.90307906275345,26.90307906275345,1.995840144718167,0.8709317964102903,2.310040346896705,0.6597688060895561,18.202191205836492,5.18681781661487,1.1210476934837867,0.4313689182665159,0.6738494837655387,Energy,1 +bitbrains_fast_storage/5T/short,Seasonal_Naive,5400000.0,5400000.0,346.0,1.14,5.93,0.476,31.1,2320.0,7.3,1.09,1.21,Web/CloudOps,2.0 +bitbrains_fast_storage/H/short,Seasonal_Naive,4200000.0,4200000.0,324.0,1.3,4.12,0.313,35.8,2050.0,5.84,0.923,1.08,Web/CloudOps,2.0 +bitbrains_rnd/5T/short,Seasonal_Naive,3410000.0,3410000.0,211.0,1.97,3.05,0.384,74.4,1850.0,7.55,0.862,1.1,Web/CloudOps,2.0 +bitbrains_rnd/H/short,Seasonal_Naive,3710000.0,3710000.0,254.0,6.04,4.79,0.361,223.0,1930.0,8.43,1.11,1.3,Web/CloudOps,2.0 +bizitobs_application/10S/short,Seasonal_Naive,3940000.0,3940000.0,1050.0,2.24,0.0341,0.0348,26.7,1980.0,0.0765,0.0407,0.0348,Web/CloudOps,2.0 +bizitobs_l2c/5T/short,Seasonal_Naive,174.0,174.0,9.69,0.986,0.426,0.507,7.25,13.2,0.453,0.333,0.262,Web/CloudOps,7.0 +bizitobs_l2c/H/short,Seasonal_Naive,282.0,282.0,12.5,1.21,1.36,1.14,8.5,16.8,0.905,0.675,0.536,Web/CloudOps,7.0 +bizitobs_service/10S/short,Seasonal_Naive,73500.0,73500.0,55.1,1.23,0.0824,0.0755,16.4,271.0,0.201,0.0408,0.04,Web/CloudOps,2.0 +car_parts/M/short,Seasonal_Naive,2.5,2.5,0.667,1.2,0.854,1.68,20.4,1.58,3.79,1.6,1.72,Sales,1.0 +covid_deaths/D/short,Seasonal_Naive,4440000.0,4440000.0,354.0,46.9,0.133,0.178,1390.0,2110.0,0.792,0.133,0.125,Healthcare,1.0 +electricity/15T/short,Seasonal_Naive,367000.0,367000.0,94.2,1.72,0.302,0.23,27.0,606.0,1.19,0.186,0.165,Energy,1.0 +electricity/D/short,Seasonal_Naive,4110000000.0,4110000000.0,6960.0,1.99,0.605,0.136,33.8,64100.0,1.05,0.114,0.122,Energy,1.0 +electricity/H/short,Seasonal_Naive,2890000.0,2890000.0,248.0,1.36,0.363,0.165,18.0,1700.0,0.802,0.117,0.109,Energy,1.0 +electricity/W/short,Seasonal_Naive,317000000000.0,317000000000.0,56600.0,2.09,0.286,0.12,18.0,563000.0,1.29,0.129,0.0989,Energy,1.0 +ett1/15T/short,Seasonal_Naive,8.78,8.78,1.49,0.934,0.633,0.306,8.44,2.96,0.56,0.283,0.241,Energy,7.0 +ett1/D/short,Seasonal_Naive,58000.0,58000.0,155.0,1.78,1.16,0.516,34.2,241.0,0.632,0.407,0.515,Energy,7.0 +ett1/H/short,Seasonal_Naive,141.0,141.0,5.82,0.977,0.57,0.287,10.2,11.9,0.555,0.272,0.25,Energy,7.0 +ett1/W/short,Seasonal_Naive,1570000.0,1570000.0,970.0,1.77,0.615,0.625,17.8,1250.0,0.499,0.386,0.338,Energy,7.0 +ett2/15T/short,Seasonal_Naive,14.6,14.6,2.53,1.07,0.153,0.17,8.53,3.82,0.182,0.12,0.0964,Energy,7.0 +ett2/D/short,Seasonal_Naive,123000.0,123000.0,212.0,1.39,0.471,0.142,48.0,351.0,0.201,0.121,0.205,Energy,7.0 +ett2/H/short,Seasonal_Naive,152.0,152.0,7.97,0.923,0.145,0.137,11.5,12.3,0.156,0.101,0.0941,Energy,7.0 +ett2/W/short,Seasonal_Naive,4690000.0,4690000.0,1320.0,0.779,0.122,0.127,23.0,2170.0,0.182,0.111,0.169,Energy,7.0 +hierarchical_sales/D/short,Seasonal_Naive,59.0,59.0,3.45,1.13,1.22,1.19,40.8,7.68,2.36,1.06,2.36,Sales,1.0 +hierarchical_sales/W/short,Seasonal_Naive,1050.0,1050.0,13.6,1.03,0.95,0.58,20.6,32.3,1.48,0.623,1.03,Sales,1.0 +hospital/M/short,Seasonal_Naive,3460.0,3460.0,20.0,0.921,0.233,0.21,6.6,58.9,0.214,0.0726,0.0625,Healthcare,1.0 +jena_weather/10T/short,Seasonal_Naive,1990.0,1990.0,10.6,0.743,0.877,0.394,20.7,44.6,0.277,0.0656,0.155,Nature,21.0 +jena_weather/D/short,Seasonal_Naive,634.0,634.0,13.1,1.57,1.77,0.485,45.4,25.2,0.152,0.0791,0.297,Nature,21.0 +jena_weather/H/short,Seasonal_Naive,1940.0,1940.0,11.0,0.723,1.13,0.401,19.8,44.1,0.27,0.0676,0.173,Nature,21.0 +kdd_cup_2018/D/short,Seasonal_Naive,3180.0,3180.0,23.9,1.5,0.589,0.572,35.5,56.4,1.26,0.535,0.888,Nature,1.0 +kdd_cup_2018/H/short,Seasonal_Naive,5620.0,5620.0,32.2,1.34,1.42,0.665,12.1,74.9,1.57,0.675,0.559,Nature,1.0 +loop_seattle/5T/short,Seasonal_Naive,76.3,76.3,4.81,0.762,0.13,0.102,9.64,8.73,0.15,0.0824,0.0808,Transport,1.0 +loop_seattle/D/short,Seasonal_Naive,58.7,58.7,5.94,1.73,0.111,0.107,32.0,7.66,0.137,0.106,0.131,Transport,1.0 +loop_seattle/H/short,Seasonal_Naive,126.0,126.0,6.35,1.29,0.171,0.142,14.4,11.2,0.198,0.112,0.108,Transport,1.0 +m_dense/D/short,Seasonal_Naive,39900.0,39900.0,118.0,1.67,0.253,0.256,34.1,200.0,0.346,0.205,0.294,Transport,1.0 +m_dense/H/short,Seasonal_Naive,146000.0,146000.0,180.0,1.49,0.54,0.391,14.1,382.0,0.677,0.32,0.281,Transport,1.0 +m4_daily/D/short,Seasonal_Naive,498000.0,498000.0,181.0,3.28,0.0422,0.0305,36.5,705.0,0.109,0.0279,0.0265,Econ/Fin,1.0 +m4_hourly/H/short,Seasonal_Naive,3610000.0,3610000.0,354.0,1.19,0.156,0.139,10.0,1900.0,0.26,0.0483,0.0395,Econ/Fin,1.0 +m4_monthly/M/short,Seasonal_Naive,2650000.0,2650000.0,700.0,1.26,0.192,0.16,11.1,1630.0,0.339,0.146,0.126,Econ/Fin,1.0 +m4_quarterly/Q/short,Seasonal_Naive,2490000.0,2490000.0,709.0,1.6,0.142,0.125,12.6,1580.0,0.264,0.119,0.0992,Econ/Fin,1.0 +m4_weekly/W/short,Seasonal_Naive,454000.0,454000.0,348.0,2.78,0.0894,0.0916,31.2,673.0,0.123,0.0634,0.0727,Econ/Fin,1.0 +m4_yearly/A/short,Seasonal_Naive,4070000.0,4070000.0,1010.0,3.97,0.175,0.164,47.0,2020.0,0.324,0.162,0.138,Econ/Fin,1.0 +restaurant/D/short,Seasonal_Naive,274.0,274.0,10.7,1.01,0.944,0.556,30.2,16.6,0.757,0.49,0.907,Sales,1.0 +saugeen/D/short,Seasonal_Naive,995.0,995.0,15.4,3.41,0.471,0.424,77.2,31.5,1.02,0.498,0.754,Nature,1.0 +saugeen/M/short,Seasonal_Naive,714.0,714.0,16.7,0.976,0.497,0.442,10.6,26.7,0.802,0.5,0.445,Nature,1.0 +saugeen/W/short,Seasonal_Naive,1800.0,1800.0,24.6,1.99,0.824,0.558,30.4,42.4,1.28,0.741,0.855,Nature,1.0 +solar/10T/short,Seasonal_Naive,37.9,37.9,2.53,1.11,4.4,0.925,18.8,6.16,1.79,0.735,0.86,Energy,1.0 +solar/D/short,Seasonal_Naive,182000.0,182000.0,295.0,1.16,1.15,0.498,32.8,426.0,0.616,0.426,0.757,Energy,1.0 +solar/H/short,Seasonal_Naive,1010.0,1010.0,12.3,0.952,3.87,0.62,16.6,31.8,1.17,0.453,0.628,Energy,1.0 +solar/W/short,Seasonal_Naive,2810000.0,2810000.0,1340.0,1.47,0.309,0.253,16.1,1680.0,0.342,0.274,0.236,Energy,1.0 +sz_taxi/15T/short,Seasonal_Naive,31.2,31.2,3.82,0.764,1310000000000.0,0.538,7.06,5.59,0.523,0.357,0.309,Transport,1.0 +sz_taxi/H/short,Seasonal_Naive,12.0,12.0,2.43,0.738,1.55,0.385,9.27,3.47,0.323,0.226,0.229,Transport,1.0 +temperature_rain/D/short,Seasonal_Naive,311.0,311.0,7.89,2.01,97.0,1.33,61.8,17.6,2.07,0.929,1.63,Nature,1.0 +us_births/D/short,Seasonal_Naive,2860000.0,2860000.0,1270.0,1.86,0.125,0.123,32.8,1690.0,0.158,0.119,0.144,Healthcare,1.0 +us_births/M/short,Seasonal_Naive,74200000.0,74200000.0,6700.0,0.761,0.0206,0.021,5.65,8610.0,0.0267,0.0208,0.0168,Healthcare,1.0 +us_births/W/short,Seasonal_Naive,5290000.0,5290000.0,1720.0,1.56,0.0232,0.0232,17.0,2300.0,0.0312,0.0234,0.0216,Healthcare,1.0 +bitbrains_fast_storage/5T/medium,Seasonal_Naive,5190000.0,5190000.0,333.0,1.22,5.77,0.452,39.1,2280.0,6.92,1.01,1.27,Web/CloudOps,2.0 +bitbrains_rnd/5T/medium,Seasonal_Naive,3270000.0,3270000.0,205.0,4.54,2.75,0.392,183.0,1810.0,7.61,0.862,1.26,Web/CloudOps,2.0 +bizitobs_application/10S/medium,Seasonal_Naive,5140000.0,5140000.0,1280.0,2.69,0.0391,0.0399,21.6,2270.0,0.0891,0.0501,0.0419,Web/CloudOps,2.0 +bizitobs_l2c/5T/medium,Seasonal_Naive,295.0,295.0,12.9,1.24,0.936,1.17,8.51,17.2,0.903,0.677,0.53,Web/CloudOps,7.0 +bizitobs_l2c/H/medium,Seasonal_Naive,447.0,447.0,15.6,1.65,1.6,1.48,47.1,21.1,1.28,0.946,1.42,Web/CloudOps,7.0 +bizitobs_service/10S/medium,Seasonal_Naive,96500.0,96500.0,63.8,1.32,0.0796,0.0771,17.4,311.0,0.234,0.048,0.0486,Web/CloudOps,2.0 +electricity/15T/medium,Seasonal_Naive,374000.0,374000.0,70.8,1.15,0.215,0.162,14.7,611.0,1.05,0.122,0.124,Energy,1.0 +electricity/H/medium,Seasonal_Naive,8350000.0,8350000.0,308.0,1.39,0.387,0.157,28.7,2890.0,1.13,0.12,0.156,Energy,1.0 +ett1/15T/medium,Seasonal_Naive,12.9,12.9,1.84,1.19,0.908,0.428,14.9,3.59,0.684,0.35,0.352,Energy,7.0 +ett1/H/medium,Seasonal_Naive,205.0,205.0,8.16,1.57,2350000000000.0,0.457,29.9,14.3,0.685,0.391,0.54,Energy,7.0 +ett2/15T/medium,Seasonal_Naive,14.9,14.9,2.48,1.05,0.165,0.19,17.5,3.87,0.206,0.132,0.143,Energy,7.0 +ett2/H/medium,Seasonal_Naive,338.0,338.0,11.5,1.24,0.201,0.198,36.6,18.4,0.248,0.155,0.241,Energy,7.0 +jena_weather/10T/medium,Seasonal_Naive,2570.0,2570.0,12.4,0.716,0.899,0.402,37.5,50.7,0.311,0.0764,0.277,Nature,21.0 +jena_weather/H/medium,Seasonal_Naive,1950.0,1950.0,12.7,0.889,1.7,0.455,61.3,44.2,0.27,0.0775,0.486,Nature,21.0 +kdd_cup_2018/H/medium,Seasonal_Naive,5790.0,5790.0,31.3,1.43,1.64,0.655,30.7,76.1,1.59,0.656,0.949,Nature,1.0 +loop_seattle/5T/medium,Seasonal_Naive,150.0,150.0,7.27,1.15,0.224,0.162,12.3,12.2,0.218,0.129,0.123,Transport,1.0 +loop_seattle/H/medium,Seasonal_Naive,154.0,154.0,7.22,1.48,0.201,0.161,35.8,12.4,0.219,0.128,0.206,Transport,1.0 +m_dense/H/medium,Seasonal_Naive,158000.0,158000.0,189.0,1.57,0.543,0.398,32.6,398.0,0.692,0.329,0.479,Transport,1.0 +solar/10T/medium,Seasonal_Naive,29.2,29.2,2.12,0.927,3.22,0.659,19.5,5.41,1.2,0.47,0.771,Energy,1.0 +solar/H/medium,Seasonal_Naive,993.0,993.0,12.2,0.935,5.24,0.568,43.1,31.5,1.14,0.44,1.27,Energy,1.0 +sz_taxi/15T/medium,Seasonal_Naive,28.9,28.9,3.67,0.713,18900000000000.0,0.517,14.3,5.38,0.501,0.342,0.454,Transport,1.0 +bitbrains_fast_storage/5T/long,Seasonal_Naive,6470000.0,6470000.0,405.0,1.14,6.46,0.503,37.2,2540.0,6.72,1.07,1.29,Web/CloudOps,2.0 +bitbrains_rnd/5T/long,Seasonal_Naive,3850000.0,3850000.0,237.0,3.5,3.18,0.403,145.0,1960.0,7.52,0.91,1.29,Web/CloudOps,2.0 +bizitobs_application/10S/long,Seasonal_Naive,1230000000.0,1230000000.0,26000.0,36400.0,538.0,1.03,1310000.0,35000.0,1.35,1.0,0.973,Web/CloudOps,2.0 +bizitobs_l2c/5T/long,Seasonal_Naive,416.0,416.0,15.4,1.45,1.02,1.3,10.4,20.4,1.15,0.862,0.674,Web/CloudOps,7.0 +bizitobs_l2c/H/long,Seasonal_Naive,907.0,907.0,27.0,4.04,5.97,1.11,59.4,30.1,1.84,1.65,1.82,Web/CloudOps,7.0 +bizitobs_service/10S/long,Seasonal_Naive,131000.0,131000.0,75.3,1.37,0.0849,0.0814,19.3,362.0,0.268,0.0558,0.0559,Web/CloudOps,2.0 +electricity/15T/long,Seasonal_Naive,432000.0,432000.0,73.8,1.16,0.2,0.157,17.8,658.0,1.04,0.116,0.129,Energy,1.0 +electricity/H/long,Seasonal_Naive,13500000.0,13500000.0,357.0,1.52,0.361,0.168,34.9,3680.0,1.45,0.141,0.19,Energy,1.0 +ett1/15T/long,Seasonal_Naive,13.0,13.0,1.86,1.19,0.984,0.42,18.7,3.61,0.688,0.354,0.396,Energy,7.0 +ett1/H/long,Seasonal_Naive,199.0,199.0,7.75,1.48,2650000000000.0,0.477,36.6,14.1,0.676,0.371,0.616,Energy,7.0 +ett2/15T/long,Seasonal_Naive,13.7,13.7,2.36,1.01,0.161,0.189,22.2,3.7,0.197,0.126,0.165,Energy,7.0 +ett2/H/long,Seasonal_Naive,259.0,259.0,10.0,1.13,0.19,0.188,45.0,16.1,0.22,0.137,0.287,Energy,7.0 +jena_weather/10T/long,Seasonal_Naive,2040.0,2040.0,11.4,0.761,0.772,0.404,42.1,45.2,0.277,0.07,0.304,Nature,21.0 +jena_weather/H/long,Seasonal_Naive,2710.0,2710.0,15.7,1.27,1.87,0.502,74.9,52.1,0.314,0.0944,0.598,Nature,21.0 +kdd_cup_2018/H/long,Seasonal_Naive,4860.0,4860.0,29.6,1.34,1.35,0.81,37.3,69.7,1.64,0.695,1.25,Nature,1.0 +loop_seattle/5T/long,Seasonal_Naive,172.0,172.0,7.89,1.25,0.23,0.176,13.7,13.1,0.232,0.14,0.137,Transport,1.0 +loop_seattle/H/long,Seasonal_Naive,167.0,167.0,7.51,1.55,0.235,0.164,44.0,12.9,0.231,0.134,0.245,Transport,1.0 +m_dense/H/long,Seasonal_Naive,138000.0,138000.0,176.0,1.48,0.498,0.388,39.9,371.0,0.642,0.304,0.552,Transport,1.0 +solar/10T/long,Seasonal_Naive,26.9,26.9,2.0,0.871,2.31,0.66,21.6,5.19,1.12,0.431,0.786,Energy,1.0 +solar/H/long,Seasonal_Naive,1160.0,1160.0,14.0,1.07,5.33,0.712,52.7,34.1,1.18,0.485,1.47,Energy,1.0 +sz_taxi/15T/long,Seasonal_Naive,28.5,28.5,3.68,0.691,3280000000000.0,0.522,18.0,5.34,0.494,0.341,0.554,Transport,1.0 diff --git a/results/seasonal_naive/config.json b/results/seasonal_naive/config.json index 8f73dc6672723e7b63ae7b1ea1686760f197da63..b1bbd11a136f62b9b129ec1ad6e61bb5af33681e 100644 --- a/results/seasonal_naive/config.json +++ b/results/seasonal_naive/config.json @@ -1,6 +1,5 @@ { "model": "Seasonal_Naive", "model_type": "statistical", - "model_dtype": "float32", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/sundial_base_128m/all_results.csv b/results/sundial_base_128m/all_results.csv deleted file mode 100644 index 874fce4bc2b976be3ecceaabe372fa01a47e4e0e..0000000000000000000000000000000000000000 --- a/results/sundial_base_128m/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -loop_seattle/5T/short,sundial_base_128m,37.38912407685113,37.79311926559275,3.4166639455946854,0.5418677171663021,0.09302349149627213,0.07274604222599813,7.18608638094916,6.114664674113466,0.10485300271420192,0.0585882453174582,0.04965148377952988,Transport,1 -loop_seattle/5T/medium,sundial_base_128m,73.50833688080495,74.1760066756966,5.1912426631836945,0.8196109744635476,0.16633652882925373,0.11945720078160273,10.00461001995478,8.57370030271673,0.1525437419054575,0.09236287169151089,0.07743672868191195,Transport,1 -loop_seattle/5T/long,sundial_base_128m,89.1548085081986,89.1674085540649,5.676028121775026,0.8928347181609574,0.1759994486189944,0.12873686830817854,11.336979324492034,9.44218240176489,0.16693971589687603,0.10035333800527821,0.08449760538340932,Transport,1 -loop_seattle/D/short,sundial_base_128m,18.429789086687308,18.52360197368421,2.9816349812951497,0.8995514672181137,0.05507054127160733,0.05483099245557598,17.340802301170243,4.292993021970489,0.07672152690732546,0.05328580486260833,0.047140832678401315,Transport,1 -loop_seattle/H/short,sundial_base_128m,60.18822434617891,59.60882590570854,4.346736920183586,0.8804201037957281,0.11585792996093326,0.0996091810639487,14.29746495628067,7.758107007909785,0.13732638336324776,0.07694166374242092,0.06656832037283406,Transport,1 -loop_seattle/H/medium,sundial_base_128m,68.30358617131063,68.25909442724458,4.956817998581012,1.0140278217947962,0.13040064761513157,0.11203929479892286,15.993568057047899,8.264598367211235,0.1460758345241723,0.0876111933762954,0.07529871485222338,Transport,1 -loop_seattle/H/long,sundial_base_128m,61.90195218438252,61.99824991400069,4.742837332301342,0.9871012881124152,0.1341950817127838,0.10877739716363519,14.785401125459641,7.867779368054402,0.14060775076603935,0.08476085288459285,0.07223909651791788,Transport,1 -m_dense/D/short,sundial_base_128m,7680.636296296297,7703.6059259259255,46.08437789351852,0.681301001844435,0.11990517064144737,0.09835581461588541,9.91669473889226,87.63923947808023,0.1517164870782257,0.07977887490612931,0.0667818156593893,Transport,1 -m_dense/H/short,sundial_base_128m,37860.537777777776,38285.991111111114,88.01905381944445,0.7911527641202473,0.2899245252902517,0.21493893093532987,12.412157782362964,194.5778450332354,0.34513261139769036,0.15612386853328483,0.1332013739663772,Transport,1 -m_dense/H/medium,sundial_base_128m,35793.24888888889,35781.76222222222,88.92399305555556,0.7587427424857901,0.31726641572233644,0.2202114529079861,10.309195756556583,189.19103807762377,0.32916850450703183,0.15471651356382551,0.12820447024260126,Transport,1 -m_dense/H/long,sundial_base_128m,36111.89728395062,36283.50419753086,91.45063271604938,0.7708725770495878,0.31554077114423046,0.22397265022183643,9.930669545560853,190.03130606284486,0.32897228252995997,0.15831456409323705,0.13037685659946585,Transport,1 -sz_taxi/15T/short,sundial_base_128m,16.603792407137135,16.572001362894916,2.7728853889437386,0.5537644512663443,1009814854269.8967,0.40059981852660687,8.565742344193046,4.074775135775854,0.3810089022864843,0.2592766430058536,0.22330872874467816,Transport,1 -sz_taxi/15T/medium,sundial_base_128m,17.097115384615385,16.801088408119657,2.8827486478365385,0.5632305341689838,10381046999526.62,0.4233590150490785,7.808679476853401,4.1348658242578304,0.384818282074604,0.26828787909035945,0.22793119415486818,Transport,1 -sz_taxi/15T/long,sundial_base_128m,16.49652221331909,16.136390446937323,2.842568108974359,0.5372017514900559,6770085977715.236,0.42102996739227205,6.8094228015750105,4.061591093810293,0.3759336366610085,0.26310304065120377,0.22107241287489293,Transport,1 -sz_taxi/H/short,sundial_base_128m,7.454074435763889,7.3442643646501065,1.9091939029530582,0.581415842067849,1.1498924320579593,0.30563041491386217,9.275420074651208,2.7302150896520754,0.25433638345203247,0.17785319348140846,0.1539936485059681,Transport,1 -bitbrains_fast_storage/5T/short,sundial_base_128m,1667311.4097309958,1711803.8958942452,181.00176684311495,0.7400817080132996,2.7738987359024674,0.7436391409035517,15.030792252691075,1291.2441325059315,4.054187281222474,0.5683007903315925,0.4616509497480503,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,sundial_base_128m,2907838.320472667,2877131.708869361,312.31870573567153,1.107871728301013,5.429366768086938,0.8071652781288694,23.570398005670967,1705.238493722408,5.1809858111874965,0.9489105418050499,0.7275862010337261,Web/CloudOps,2 -bitbrains_fast_storage/5T/long,sundial_base_128m,3992984.6565324483,4074699.162123445,402.33512084912155,1.0110702542831842,5.992031531062318,0.8165969330820964,18.612015056705424,1998.2453944729732,5.280765984355254,1.0632516038160884,0.8112800068833039,Web/CloudOps,2 -bitbrains_fast_storage/H/short,sundial_base_128m,3002624.164511734,2752943.8191741854,339.0138164151865,1.1499014166635186,4.92548112469647,0.6058981282372182,26.78099747530234,1732.80817302774,4.939066039798932,0.9662994748883101,0.7644337983354259,Web/CloudOps,2 -bitbrains_rnd/5T/short,sundial_base_128m,1401344.1824515034,1540852.046103743,126.1733071134422,1.7146683452862685,1.251046626626142,0.6860893218903396,54.78645586026391,1183.7838411008588,4.843701719118544,0.5162647464455313,0.43335403099156355,Web/CloudOps,2 -bitbrains_rnd/5T/medium,sundial_base_128m,2428639.0456661023,2380889.5423860047,192.43698043853752,4.562192427864367,2.3730152582257915,0.7679559803279247,162.52809085259724,1558.4091393681258,6.557020809910089,0.8096803679188941,0.7302818091340633,Web/CloudOps,2 -bitbrains_rnd/5T/long,sundial_base_128m,2565960.3198869433,2498272.9553300245,213.09548285627994,3.5224942814874347,3.1000865366677197,0.7565202479385937,118.99906619854494,1601.8615170753505,6.136370845058673,0.8163208206664636,0.7151753520857796,Web/CloudOps,2 -bitbrains_rnd/H/short,sundial_base_128m,1869378.5397041899,1716389.989173522,185.33467243510506,5.979679667159194,3.1037654265433465,0.628305893623993,215.97912400831697,1367.252185847289,5.983764780277694,0.8111152404515893,0.7254302314523267,Web/CloudOps,2 -bizitobs_application/10S/short,sundial_base_128m,774527.2177777778,768104.0355555556,471.55090277777776,1.4287244579270495,0.020714524586995443,0.020905992719862196,35.81248179934669,880.0722798598862,0.033940870084758545,0.018185833477313076,0.01629924027964623,Web/CloudOps,2 -bizitobs_application/10S/medium,sundial_base_128m,5238368.0,5198286.08,1243.2520833333333,2.856535846734542,0.04106525739034017,0.041892293294270834,78.73858005059354,2288.7481294367017,0.08987240285474259,0.04881885020298451,0.045652348909626786,Web/CloudOps,2 -bizitobs_application/10S/long,sundial_base_128m,7611294.72,7586994.062222222,1659.1941666666667,3.705243579739328,0.053602634006076386,0.05475409613715278,116.51264873478618,2758.8575026630137,0.10639810640911743,0.06398848701971185,0.06144815995612539,Web/CloudOps,2 -bizitobs_l2c/5T/short,sundial_base_128m,13.797112165178572,13.8958984375,2.3654639834449407,0.24810980377841865,0.14426095549885387,0.18704002017066593,2.571659952682548,3.7144464143635956,0.12756912481367788,0.08123960786712507,0.06698296806393049,Web/CloudOps,7 -bizitobs_l2c/5T/medium,sundial_base_128m,66.03551232993198,65.80960884353742,5.367345610119048,0.5301663187479796,0.38707447388300453,0.6539383868781887,6.148770555055706,8.126223743531307,0.4275588702051335,0.2824013093276875,0.23410610413208455,Web/CloudOps,7 -bizitobs_l2c/5T/long,sundial_base_128m,100.55199404761905,100.30006944444445,6.6023542906746036,0.6350142264434739,0.5411881561194837,0.7743142361111112,8.052863430563646,10.027561719960593,0.5630325993440862,0.3707123228839833,0.3100932988678433,Web/CloudOps,7 -bizitobs_l2c/H/short,sundial_base_128m,58.68516322544643,58.96220858134921,4.811042906746032,0.47643937627355415,0.43927799654807403,0.6333610292464967,6.710387912500496,7.660624205993035,0.41292726390068957,0.25932753396744096,0.22284932806442695,Web/CloudOps,7 -bizitobs_l2c/H/medium,sundial_base_128m,80.02613467261905,81.70883556547619,5.414339192708334,0.550211607426368,0.5420053790474771,0.7827382405598958,7.306465746157197,8.945732763313414,0.5416777066064108,0.3278464620274622,0.2763385372212042,Web/CloudOps,7 -bizitobs_l2c/H/long,sundial_base_128m,97.68327132936508,99.13186383928571,6.222073800223214,0.6649538160803763,0.7405663072226014,0.8124540298704117,10.485610887419583,9.883484776604105,0.6037117756486757,0.3800622257284957,0.3248376979085573,Web/CloudOps,7 -bizitobs_service/10S/short,sundial_base_128m,9121.87343915344,8756.308994708994,24.69885582010582,0.8388123634083924,0.08373284556878306,0.060550466063161376,17.905196376640344,95.50849930322138,0.0707557496138025,0.018297701994107993,0.016323560876514015,Web/CloudOps,2 -bizitobs_service/10S/medium,sundial_base_128m,97587.21015873016,98052.56126984127,62.335679563492064,1.2720799283158783,0.11873136005704366,0.08927653963603671,25.27081604263379,312.3895167234812,0.2351117117793331,0.046915301386613345,0.043992180562493655,Web/CloudOps,2 -bizitobs_service/10S/long,sundial_base_128m,131941.80402116402,132003.10857142857,80.13044312169312,1.4572878609645141,0.14303743231233465,0.09928294994212963,33.667228398243935,363.2379440823384,0.2690982814013858,0.0593631939704274,0.05684489042419871,Web/CloudOps,2 -car_parts/M/short,sundial_base_128m,1.3770329361499123,1.3562324800639574,0.551516586999043,0.9567785611036184,0.70865921627573,1.8251849331988177,22.831523029638067,1.1734704666713656,2.8138618039615704,1.3224802008097474,1.1888484578014162,Sales,1 -covid_deaths/D/short,sundial_base_128m,4750637.297243108,4477442.309774436,402.08527568922307,60.37512053436367,0.18997728960451823,0.49796751399984335,1672.7840554639606,2179.5956728813508,0.8200834790539754,0.15128654175005096,0.13106394889424736,Healthcare,1 -electricity/15T/short,sundial_base_128m,124519.82138513513,123738.63013513513,48.25886278856982,0.8949515637448004,0.15680433652720963,0.1562521980904244,15.645124261570613,352.873662073461,0.6955835443448973,0.09512773106183055,0.08361455153603606,Energy,1 -electricity/15T/medium,sundial_base_128m,272671.48163963965,271096.7338378378,55.858921030405405,0.8541111431489306,0.13563336248916688,0.13590715782921595,12.673007115851096,522.179549235356,0.8997679459171861,0.09625054583289519,0.08223440436805927,Energy,1 -electricity/15T/long,sundial_base_128m,351951.0873393393,350408.16377777775,61.53619725975976,0.9062450146418705,0.1447556375475504,0.1396683082007789,12.487065442346577,593.2546563992055,0.9359319264724014,0.09708089271255115,0.0822060533974962,Energy,1 -electricity/D/short,sundial_base_128m,1541255668.3762162,1539757168.2525406,4528.206414414414,1.4558630961996544,0.36211926647797205,0.0978611301731419,20.78940248735936,39258.82917734832,0.6451529423256734,0.07441346960502421,0.0637861337580781,Energy,1 -electricity/H/short,sundial_base_128m,1351264.2378378378,1379962.9265765767,167.6371364724099,0.9319139633483812,0.19978903100319842,0.125901819177576,18.20285203101193,1162.438917895404,0.548754183518862,0.07913670003315797,0.0693888009177783,Energy,1 -electricity/H/medium,sundial_base_128m,6083808.971531532,6075464.317837838,234.6254349662162,0.9934519730818583,0.23161005280010538,0.12755011756140908,17.08206870586526,2466.537851226194,0.962771006297108,0.09158203917812302,0.07987482442231447,Energy,1 -electricity/H/long,sundial_base_128m,9736594.810234234,9653101.306618618,268.749015015015,1.0753938178294977,0.2753369705480924,0.13579960820195194,18.99597491045824,3120.351712585335,1.230665792525617,0.10599453203302554,0.09298005290378641,Energy,1 -electricity/W/short,sundial_base_128m,156600593037.14594,165368612203.47388,38040.983783783784,1.6144820919972853,0.2493729220592092,0.10377194859960058,23.430973726850713,395727.9280479784,0.9029205411614254,0.08679697142882138,0.07201883556331198,Energy,1 -ett1/15T/short,sundial_base_128m,4.865275355747768,5.149080403645834,1.1326773507254464,0.7097059923931616,0.5011566507454214,0.24652840750558036,8.353473729986172,2.2057369189791807,0.416976091831895,0.21412316715929222,0.177346765109939,Energy,7 -ett1/15T/medium,sundial_base_128m,9.479339037698413,9.522857142857143,1.6631180245535715,1.0667045190646924,0.7683557793146981,0.41591959635416664,11.184306913045495,3.0788535265092447,0.5868325263463925,0.3169919398755988,0.25999396052771984,Energy,7 -ett1/15T/long,sundial_base_128m,8.588688616071428,8.424805307539682,1.6301932973710318,1.0882545863507582,0.8187500465963538,0.42221966455853177,11.252389295554746,2.930646450200267,0.5585841110632471,0.3107164543205254,0.2526629588207522,Energy,7 -ett1/D/short,sundial_base_128m,68630.59047619048,68294.18412698412,166.50721726190477,1.9033811160400669,1.3687090192522322,0.4968551393539187,24.00443651496063,261.97440805580703,0.6880844725384713,0.43733672923921607,0.37294448291981835,Energy,7 -ett1/H/short,sundial_base_128m,95.23248697916667,95.23712797619048,4.891598656063988,0.8294853844046632,0.46782146635509675,0.26254708426339285,10.191291541613097,9.758713387489493,0.4554790783442203,0.2283109216374914,0.19024286697535567,Energy,7 -ett1/H/medium,sundial_base_128m,130.83780691964284,130.98899739583334,6.7376825241815474,1.287646484989819,3319469932639.093,0.45304470970517113,18.002215579451374,11.438435510140486,0.5478739462199896,0.32271902128820656,0.2691033786548276,Energy,7 -ett1/H/long,sundial_base_128m,138.46879133597884,138.9707671957672,7.078614831349206,1.4054141627728294,2813768040816.4478,0.4885259396184689,19.88461175593543,11.767276292157792,0.5635093754804101,0.3389795330579862,0.2827083326570119,Energy,7 -ett1/W/short,sundial_base_128m,3683896.8571428573,3667887.4285714286,1210.601841517857,1.8430188542279329,0.7876313073294503,0.569650377546038,19.434605080896997,1919.3480291866967,0.7637543913302695,0.481727367080771,0.40394643549239956,Energy,7 -ett2/15T/short,sundial_base_128m,7.541792224702381,7.563260904947916,1.7109636579241072,0.7473446713848775,0.10887933783997097,0.12903707595098587,9.339846044863958,2.746232369028954,0.1303734201583106,0.08122553151939337,0.06865643859780353,Energy,7 -ett2/15T/medium,sundial_base_128m,11.871760912698413,11.802404513888888,2.167898685515873,0.9071844187960826,0.13902399172318064,0.16579804920014882,10.156929834549702,3.4455421797880246,0.18361917964455493,0.11553124513234761,0.09575432167607961,Energy,7 -ett2/15T/long,sundial_base_128m,11.858053075396825,11.674272073412698,2.192200210813492,0.9212435305526607,0.14248933317189483,0.16945643833705357,10.670394862380515,3.443552391847237,0.18351314024341298,0.11682631740440864,0.09767482003230302,Energy,7 -ett2/D/short,sundial_base_128m,123608.8634920635,121005.04126984127,214.6015625,1.5070957175381101,0.4611456976996528,0.15564621213882687,16.407948600654866,351.5805220601157,0.20121880891418592,0.1228221362899896,0.10264152652572658,Energy,7 -ett2/H/short,sundial_base_128m,116.91236979166666,113.46793154761905,6.692149135044643,0.7705117287530697,0.12544639634027052,0.11536166781470888,10.325377422224458,10.812602359823774,0.13713388294356296,0.08487507129051912,0.07206166053668694,Energy,7 -ett2/H/medium,sundial_base_128m,249.36175595238095,247.17736235119048,9.998749069940477,1.1150264396546699,0.18467223795585141,0.19034965151832217,12.937642203068034,15.791192353726204,0.21279647190226167,0.1347395736723634,0.11361835317008559,Energy,7 -ett2/H/long,sundial_base_128m,265.9783399470899,262.0705687830688,10.183081183862434,1.1394240030514897,0.19260719409961238,0.2007253818411045,12.836588365728401,16.308842385254998,0.22333704425394743,0.1394494593348947,0.1168439160691367,Energy,7 -ett2/W/short,sundial_base_128m,5142077.142857143,4744008.571428572,1413.6238839285713,0.9364579000863359,0.1392535056386675,0.15048468112945557,10.745230062350288,2267.614857698975,0.1902181219063579,0.11858137168660238,0.09822798552528894,Energy,7 -hierarchical_sales/D/short,sundial_base_128m,27.632022529971064,27.589404195948738,2.4153553250310047,0.7895142034181349,0.7173977498846464,0.9863113308701943,13.2086148556627,5.256617023330791,1.6126718892360312,0.7410042652728633,0.6491728654993807,Sales,1 -hierarchical_sales/W/short,sundial_base_128m,488.66101694915255,457.8660288665254,9.61986477092161,0.7509810161145285,0.684453321534199,0.4777563062764831,12.34329558035152,22.10567838699262,1.0148825074383763,0.4416526970588593,0.390210413147802,Sales,1 -hospital/M/short,sundial_base_128m,3398.8683181225556,3304.418948283355,19.89609476857888,0.8373906478027656,0.21264946849280883,0.18976043608332654,11.769844117897033,58.29981404878197,0.21164186303998,0.07222744416506434,0.060909019611888625,Healthcare,1 -jena_weather/10T/short,sundial_base_128m,714.0433531746032,760.1094246031746,6.064236886160714,0.2972308027391687,0.38476208500555925,0.5452390640500993,3.2718569348065434,26.721589645352374,0.16597456243755457,0.03766651150087034,0.03083460709230938,Nature,21 -jena_weather/10T/medium,sundial_base_128m,1463.590909090909,1480.0989898989899,10.606262400793652,0.638583318706897,0.9757086962325224,0.6651314343208874,8.155205430508937,38.25690668481848,0.23478198231038327,0.06509045103614167,0.05419848017485068,Nature,21 -jena_weather/10T/long,sundial_base_128m,1526.9175925925927,1558.02791005291,11.005598958333334,0.6789694776833581,1.0108914967660652,0.6608337079406416,8.533353195159828,39.075792923401984,0.23911147728185272,0.06734514717224567,0.05596186914190761,Nature,21 -jena_weather/D/short,sundial_base_128m,335.6586061507937,334.7327628968254,9.400784350198412,0.9311451482969424,0.7852221390519247,0.4243041023375496,12.1090083589597,18.32098813248875,0.11032229282421843,0.05660808665777112,0.04789903735717624,Nature,21 -jena_weather/H/short,sundial_base_128m,1175.9592731829573,1197.6470342522973,9.223842483813701,0.538848642518242,1.7519873990653612,0.6039663265422671,7.101045333338838,34.292262584772054,0.21020936201829357,0.05654156062425354,0.049646022452524335,Nature,21 -jena_weather/H/medium,sundial_base_128m,1254.0208333333333,1268.9168650793652,11.169922650049603,0.8685233040993711,3.727921892923987,0.6717536078559028,9.29711649459636,35.41215657557915,0.2166932665415237,0.06835073771600188,0.058378717407223495,Nature,21 -jena_weather/H/long,sundial_base_128m,1318.643253968254,1302.2162698412699,12.913236813822751,1.0881605488875692,3.741977550866237,0.6666673125413359,16.21339153163842,36.31312784611447,0.21864526188016492,0.0777519925257821,0.06572756237882615,Nature,21 -kdd_cup_2018/D/short,sundial_base_128m,2643.5459804508278,2724.824522907108,20.464195175876057,1.1735723109796423,0.6069453145779307,0.4520561319215872,15.39534679028357,51.41542551074364,1.1517600905129683,0.4584196873585527,0.39564856635318785,Nature,1 -kdd_cup_2018/H/short,sundial_base_128m,4073.6266219234467,4104.7201669204405,19.309099624056028,0.8005728901664165,0.7581389280063812,0.43962308808986017,12.435813682322742,63.82496864020731,1.3359721898774635,0.40417442670017184,0.35148524836753353,Nature,1 -kdd_cup_2018/H/medium,sundial_base_128m,4404.977002365145,4450.69979332406,20.754435195464538,0.8408122037406998,0.8275329345817771,0.46384470316113263,12.144618647993806,66.3700007711703,1.3892443178118112,0.4344279166743208,0.377101024069857,Nature,1 -kdd_cup_2018/H/long,sundial_base_128m,3238.069802731411,3268.87309073652,18.44123979822007,0.7747499424036353,0.7683727548419232,0.4688291406922856,11.506386680401688,56.90404030234946,1.3351987287102756,0.43270600476866583,0.3747144138264249,Nature,1 -m4_daily/D/short,sundial_base_128m,558540.8372030146,569714.9005373619,198.68158458379804,3.7149470831998554,0.03920724474206129,0.033353801738769055,56.307273731262825,747.3558972825562,0.11544583260675516,0.030690814161374223,0.02663964652326942,Econ/Fin,1 -m4_hourly/H/short,sundial_base_128m,1133904.4895330113,1233038.6344605475,210.26280696457326,0.8692680665321981,0.10755592383048385,0.0946229698193438,11.960330569851838,1064.849514970548,0.1453754629157268,0.028705514222148454,0.022980994079973973,Econ/Fin,1 -m4_monthly/M/short,sundial_base_128m,2240086.612148148,2205808.108740741,651.5672013888889,1.0992059769425646,0.19644105455610486,0.15443810484144424,16.191359550291462,1496.6918895177284,0.3110639567967747,0.1354180330651334,0.11625780995502473,Econ/Fin,1 -m4_quarterly/Q/short,sundial_base_128m,2086986.472,2065333.488,646.3814778645833,1.4574114966025733,0.1290917205810547,0.11715818134943644,22.36748309124086,1444.6406030566911,0.24180739765731493,0.10819287698657507,0.0930524818903939,Econ/Fin,1 -m4_weekly/W/short,sundial_base_128m,348123.5226055282,332966.698950075,291.571110992072,2.39610125632833,0.0751076570179418,0.07227586509585253,25.402798188491037,590.0199340747126,0.10749248824941905,0.05311973784637697,0.04345332274527163,Econ/Fin,1 -m4_yearly/A/short,sundial_base_128m,4997100.187690433,4880675.977133571,1120.7643876411016,4.329944223633436,0.2032089580984097,0.19126191826505415,87.54221907098058,2235.4194657134112,0.35845987326478984,0.17971976468644776,0.16001316958189987,Econ/Fin,1 -restaurant/D/short,sundial_base_128m,142.67255066778742,143.40200336228637,7.351215910152237,0.7043214521479607,0.7502271075581395,0.4040559494518773,9.366935736069946,11.944561551927615,0.5497163978907575,0.3383199887813349,0.2856917953281462,Sales,1 -saugeen/D/short,sundial_base_128m,1033.1979166666667,1043.4395833333333,12.527005208333334,2.783229391019545,0.3125386047363281,0.3236430867513021,75.86281309813374,32.14339615950167,1.0411322602659936,0.40575268345005383,0.37915594576518236,Nature,1 -saugeen/M/short,sundial_base_128m,376.7884579613095,385.25225539434524,12.846944173177084,0.7528023615422877,0.4215218226114909,0.3763252894083659,10.33956776215654,19.411039589916598,0.5827776410626605,0.38570380455032605,0.3319812578629679,Nature,1 -saugeen/W/short,sundial_base_128m,1053.62548828125,1092.30908203125,14.770535278320313,1.1982786164986492,0.3316983222961426,0.371807599067688,27.377672970743134,32.45959778372569,0.9798199500794474,0.44586088945028773,0.40576197921446977,Nature,1 -solar/10T/short,sundial_base_128m,17.38408416970803,18.219959892031632,1.918618128421533,0.8373387888683936,4.711767718096245,1.5110303447194342,7.535477779227135,4.1694225223294445,1.2128845602252192,0.558125805784385,0.4444252264450095,Energy,1 -solar/10T/medium,sundial_base_128m,18.074612226277374,18.58121198296837,2.1586906070283125,0.9408304151961179,3.513908973666567,1.4718986742424243,6.205610561348746,4.251424728991139,0.9434079950279075,0.47902200021907093,0.37262989784620915,Energy,1 -solar/10T/long,sundial_base_128m,18.170098337388485,18.24339834245742,2.1740717904121047,0.9494114377480228,3.4878601122249973,1.4685918333079886,6.2731816205260404,4.262639832004163,0.9213013122809925,0.4698907889100312,0.3648130201629169,Energy,1 -solar/D/short,sundial_base_128m,122735.82871046229,125002.05936739659,276.988899026764,1.0812809630437707,0.9917641447118309,0.47106722718027677,9.725230133388655,350.3367361703056,0.5061172575408385,0.40015461546268866,0.3237486423313621,Energy,1 -solar/H/short,sundial_base_128m,542.3035359841209,549.8866372134717,10.220167384988475,0.7869252373242515,3.7249074247143055,1.3971023706120342,13.77821861863638,23.28741153464938,0.8583817689046194,0.3767187841110908,0.3286122560749378,Energy,1 -solar/H/medium,sundial_base_128m,500.6115875912409,502.3263077858881,10.058445483576643,0.766729353250346,3.6976240779496226,1.3902708475897203,12.006373891289906,22.374351109948215,0.8059763489713974,0.3623286828450824,0.30919993527455886,Energy,1 -solar/H/long,sundial_base_128m,427.0271289537713,427.0316707218167,9.861896162814274,0.7493449101098908,4.299106481595773,1.3845172953025917,11.844666051847392,20.66463474039092,0.7164805466759955,0.3419299127598036,0.29303772239399983,Energy,1 -solar/W/short,sundial_base_128m,1235502.9489051094,1256159.5328467153,902.0643818430657,0.9808747227904373,0.19737641828773667,0.17860336025265883,6.82843081120576,1111.5318029211353,0.22689143064276918,0.1841338930567514,0.14782531849850927,Energy,1 -temperature_rain/D/short,sundial_base_128m,165.39650524032874,170.26629099021108,6.104377582519322,1.429880420844037,51.386792891286774,1.4162385989179627,27.010122751001916,12.860657263154506,1.5140393681354736,0.7186466281452656,0.6199938727217345,Nature,1 -us_births/D/short,sundial_base_128m,157040.92,157050.24,264.0079166666667,0.3886940244462994,0.02511056900024414,0.02492879867553711,6.858113080172684,396.2838881408125,0.03714839885492529,0.024748599886822296,0.02157825531986882,Healthcare,1 -us_births/M/short,sundial_base_128m,150176117.33333334,150676010.66666666,10277.453125,1.158132553474985,0.031759614745775856,0.032484558721383415,22.12468864982641,12254.636564718407,0.0380628176189588,0.03192169933545982,0.02829970829691161,Healthcare,1 -us_births/W/short,sundial_base_128m,3689235.4285714286,3637910.285714286,1498.2752511160713,1.362387484772567,0.020157682044165477,0.020313912204333713,19.911947005998787,1920.7382509263016,0.026073833419438506,0.02033893962138144,0.017341925093980613,Healthcare,1 diff --git a/results/sundial_base_128m/config.json b/results/sundial_base_128m/config.json deleted file mode 100644 index a8372631185a9c8feee07dbcbc4f4539aef14b7e..0000000000000000000000000000000000000000 --- a/results/sundial_base_128m/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "sundial_base_128m", - "model_type": "pretrained", - "model_dtype": "float32", - "model_link": "https://huggingface.co/thuml/sundial-base-128m", - "org": "THUML @ Tsinghua University", - "testdata_leakage": "No" -} \ No newline at end of file diff --git a/results/tabpfn_ts/all_results.csv b/results/tabpfn_ts/all_results.csv index 1ccae012ee7cec23c3a77da80b0b5c0c164ace66..064c978c187a8586ab3d53f89eedc4440852690f 100644 --- a/results/tabpfn_ts/all_results.csv +++ b/results/tabpfn_ts/all_results.csv @@ -1,98 +1,98 @@ dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bitbrains_fast_storage/5T/long,TabPFN-TS,5094671.185438427,5094671.185438427,483.1624726930903,1.1526641321691184,6.84183501453906,0.8053138505812252,70.21928659273716,2257.1378304034574,5.964941308843822,1.2768541439646968,0.8850161893008147,Web/CloudOps,2 -bitbrains_fast_storage/5T/medium,TabPFN-TS,4622063.369807966,4622063.369807966,441.63762099470586,1.307633919865287,6.807939859955235,0.805070973047274,98.99555171639452,2149.8984556969117,6.531985259967827,1.3418170625410066,0.9486319340358308,Web/CloudOps,2 -bitbrains_fast_storage/5T/short,TabPFN-TS,2246919.058831472,2246919.058831472,263.97642698274865,0.9980513501588254,4.24383417854469,0.7465228041656109,83.52859027541167,1498.9726678066788,4.706403428343518,0.8288206900872653,0.6618188415356218,Web/CloudOps,2 -bitbrains_fast_storage/H/short,TabPFN-TS,3140360.326601572,3140360.326601572,318.8959748516379,1.1840948063141727,4.487001069476268,0.5588612078779172,23.623995660482628,1772.106183782894,5.05107832465668,0.9089571274757822,0.6698861614157302,Web/CloudOps,2 -bitbrains_rnd/5T/long,TabPFN-TS,2963661.558287043,2963661.558287043,265.86029517843235,3.874637617567236,4.921695993468071,0.7496938695174797,253.21231299800388,1721.5288432922182,6.594789314139544,1.0184509196742515,0.8191243201309302,Web/CloudOps,2 -bitbrains_rnd/5T/medium,TabPFN-TS,2541585.941135939,2541585.941135939,232.50274768348632,4.831154957418062,4.022562604208175,0.7654880519867127,388.8164612185016,1594.2352213948666,6.707760153017819,0.9782575654136576,0.8190933026377596,Web/CloudOps,2 -bitbrains_rnd/5T/short,TabPFN-TS,1957817.1753560656,1957817.1753560656,171.3280490517959,2.0308800366527606,2.408422210268895,0.6986985656768337,300.9469489327098,1399.22020259717,5.725205164718139,0.7010249205033943,0.607620615332135,Web/CloudOps,2 -bitbrains_rnd/H/short,TabPFN-TS,2116447.4668147573,2116447.4668147573,202.59939406173635,6.680151206331231,3.3270626716840086,0.6266154306757415,212.599432283774,1454.8015214505233,6.366923246390955,0.8866740739109548,0.7424817493089628,Web/CloudOps,2 -bizitobs_application/10S/long,TabPFN-TS,5377315.84,5377315.84,1395.8348611111112,3.094252917003883,0.044311006334092885,0.04513220893012153,52.85873738124467,2318.904016987335,0.08943085901094665,0.05383177128166312,0.04856447852707176,Web/CloudOps,2 -bizitobs_application/10S/medium,TabPFN-TS,4487442.773333333,4487442.773333333,1172.7091666666668,2.490061805199081,0.03584209442138672,0.03663747787475586,36.39132929151046,2118.3585091606506,0.08318170395311869,0.0460488374856937,0.04078035676061784,Web/CloudOps,2 -bizitobs_application/10S/short,TabPFN-TS,930296.4622222222,930296.4622222222,488.9087152777778,1.263384822147888,0.018171200222439236,0.018381890190972226,8.707376745126052,964.5187723534584,0.03719763375795841,0.018855254924278652,0.014849729690675231,Web/CloudOps,2 -bizitobs_l2c/5T/long,TabPFN-TS,99.68172619047618,99.68172619047618,6.976498635912699,0.6649990805871352,0.6381057311584813,0.8647107514880953,6.210544167714041,9.984073627056052,0.5605908079423119,0.3917199685223014,0.3059204801306481,Web/CloudOps,7 -bizitobs_l2c/5T/medium,TabPFN-TS,75.91723533163265,75.91723533163265,6.2581818664965985,0.6375318293573421,0.4760612674560484,0.7679077978847789,5.821976683337321,8.713049714745845,0.4584345459406488,0.32927239672763153,0.2612696761653268,Web/CloudOps,7 -bizitobs_l2c/5T/short,TabPFN-TS,23.84475795200893,23.84475795200893,2.981525384812128,0.30632387126659066,0.1701873925698132,0.21864854494730632,3.0561285239501843,4.883109455255834,0.16770573733284233,0.1023976459299554,0.08410266668617247,Web/CloudOps,7 -bizitobs_l2c/H/long,TabPFN-TS,98.97005208333331,98.97005208333331,6.150545247395834,0.6640742576152979,0.8472777875958788,0.846485343812004,4.987145840575011,9.94836931779944,0.607675110693205,0.3756930552776827,0.29232377484739197,Web/CloudOps,7 -bizitobs_l2c/H/medium,TabPFN-TS,78.76822916666667,78.76822916666667,4.93990246000744,0.4894740456225888,0.4643175915493395,0.7718949817475819,3.567479099723273,8.875146712402374,0.5374036140096928,0.2991185972344085,0.2373706201778537,Web/CloudOps,7 -bizitobs_l2c/H/short,TabPFN-TS,59.09327721974206,59.09327721974206,4.940431625124008,0.4850944799297807,0.5703324791338804,0.6686241513206845,3.649819219429149,7.687215179747609,0.41436058548672183,0.26630191725824864,0.2095964366848913,Web/CloudOps,7 -bizitobs_service/10S/long,TabPFN-TS,130439.41409575974,130439.41409575974,75.71636338076421,1.3653844836902598,0.0781809611458441,0.07604072143228141,22.041272650206317,361.1639712038837,0.26756181598978024,0.05609310258376492,0.05160131865096251,Web/CloudOps,2 -bizitobs_service/10S/medium,TabPFN-TS,96964.0729103637,96964.0729103637,60.81208243819929,1.2255220534460762,0.0723566747968883,0.07036805625353214,18.230412141337187,311.39054723989886,0.23435986380516352,0.04576860628638456,0.04089325236933028,Web/CloudOps,2 -bizitobs_service/10S/short,TabPFN-TS,24200.27255323497,24200.27255323497,33.68946281559726,0.8831190081752103,0.059359659958974006,0.05050596377240031,10.002682447876992,155.56436787784975,0.11524706193264458,0.024958232148864053,0.01928787319334337,Web/CloudOps,2 -car_parts/M/short,TabPFN-TS,1.4236728433498649,1.4236728433498649,0.46870565286262467,0.8476360552543757,0.8072569896537893,1.8997551115703168,20.796016600345137,1.1931776243920538,2.8611175466068777,1.123908075532646,0.9696277332778144,Sales,1 -covid_deaths/D/short,TabPFN-TS,582148.6825669118,582148.6825669118,131.52239363470866,39.24224311247435,0.10336535201007536,0.1573858522759746,693.2680572368323,762.9866857074976,0.28707744008204383,0.04948593833338505,0.0410740544263699,Healthcare,1 -electricity/15T/long,TabPFN-TS,424692.1608811323,424692.1608811323,63.71647982085639,0.9447757940923152,0.1490332890817207,0.13456292517572793,8.252942553633748,651.684095924653,1.028111539984608,0.10052055681683048,0.0806289122434531,Energy,1 -electricity/15T/medium,TabPFN-TS,368464.5532976957,368464.5532976957,58.99590192588996,0.8891286550107368,0.13686671560267977,0.13146080236911156,8.060963797324014,607.0128114773985,1.045944184165752,0.10165587833115326,0.08271194794165343,Energy,1 -electricity/15T/short,TabPFN-TS,180195.4665631592,180195.4665631592,59.99907240548513,1.1509364225225842,0.1950904271022857,0.174021023029497,12.587366719604278,424.49436576138345,0.8367620891213996,0.11827000125504854,0.09677125876773451,Energy,1 -electricity/D/short,TabPFN-TS,1609175176.344216,1609175176.344216,4549.606968468468,1.4917032121138645,0.23848364907625683,0.0954992348610818,15.13704799318191,40114.52575245302,0.6592149100615675,0.07476515532159875,0.06272141383204029,Energy,1 -electricity/H/long,TabPFN-TS,18510127.710963145,18510127.710963145,340.8466862517592,1.3126972825030705,0.3020939283528691,0.14370380876468228,11.204222553381076,4302.339794921264,1.6968416119329297,0.13442983773716952,0.1082691929278382,Energy,1 -electricity/H/medium,TabPFN-TS,10527303.475523552,10527303.475523552,277.4008767648814,1.1674595528269889,0.2434614956055417,0.13190289188954438,10.06489280908614,3244.580631687792,1.2664667356915,0.10827870309132294,0.08766377053562428,Energy,1 -electricity/H/short,TabPFN-TS,1960999.6209848511,1960999.6209848511,185.3523408889556,1.0364160710610155,0.19704922343224884,0.12920845526522548,11.457480338062892,1400.3569619867826,0.6610684965538965,0.0874995423669438,0.07155646763061667,Energy,1 -electricity/W/short,TabPFN-TS,55052578878.05669,55052578878.05669,30331.729800421108,1.547049206297067,0.141604463996508,0.09870668188753938,12.15276333611517,234632.85975765777,0.5353547750270788,0.06920700024821061,0.054579121515172774,Energy,1 -ett1/15T/long,TabPFN-TS,10.810811011904764,10.810811011904764,1.7222476438492065,1.1184637386647829,0.8913237899858845,0.418278304811508,9.161866535766084,3.2879797766873144,0.6266921964023711,0.32826210377732207,0.2594954561050949,Energy,7 -ett1/15T/medium,TabPFN-TS,10.776283191499257,10.776283191499257,1.7095601302858383,1.091773313351892,0.8240003332318827,0.4108899537343827,8.169808213732072,3.2827249643397263,0.6256906424593214,0.32584386077452987,0.2525006817913073,Energy,7 -ett1/15T/short,TabPFN-TS,5.212318674723307,5.212318674723307,1.1454845610119049,0.741309439205773,0.5129581426423955,0.25125504902430945,5.135743695739446,2.283050300524127,0.43159155098213625,0.21654426895444345,0.16685967571622629,Energy,7 -ett1/D/short,TabPFN-TS,44357.80952380953,44357.80952380953,140.70858134920636,1.6395479314177053,1.6393998403397818,0.46450670030381946,14.904811845730327,210.61293769331817,0.5531818669540168,0.3695757562650177,0.29776790612808235,Energy,7 -ett1/H/long,TabPFN-TS,178.98387896825398,178.98387896825398,7.994620639054233,1.4725737522845348,4706846393031.971,0.5004761710999504,11.333256034107908,13.378485675451238,0.6406667032090834,0.3828450672581384,0.29497878110262604,Energy,7 -ett1/H/medium,TabPFN-TS,163.67566964285714,163.67566964285714,7.655423990885416,1.4045375913078184,3339463150815.331,0.47588424682617186,10.659145898074607,12.793579235024776,0.6127821182850777,0.36667666204484833,0.283164148519388,Energy,7 -ett1/H/short,TabPFN-TS,109.13167782738095,109.13167782738095,5.290147472563244,0.8871397861227843,0.49000859487624393,0.2719866979689825,6.356439133275721,10.446610829708408,0.4875860398201332,0.24691281203624557,0.19394053714174952,Energy,7 -ett1/W/short,TabPFN-TS,1451985.142857143,1451985.142857143,929.0714285714286,1.6585971126590675,0.6431011472429548,0.5415669168744769,11.965506490709258,1204.9834616529567,0.4794916796344471,0.3696996962721728,0.2840047608954231,Energy,7 -ett2/15T/long,TabPFN-TS,13.555186011904762,13.555186011904762,2.363949497767857,0.9772597904462405,0.1481694417712973,0.17701786101810515,7.784936272726835,3.681736820021872,0.1962064195659214,0.12597914779496214,0.1011020985320766,Energy,7 -ett2/15T/medium,TabPFN-TS,14.45674851190476,14.45674851190476,2.3854561554439484,0.9810684377348272,0.14774220428808144,0.1766355514526367,7.469608562271954,3.8022031129208185,0.20262627298353544,0.1271252681111848,0.10034116464990252,Energy,7 -ett2/15T/short,TabPFN-TS,9.42276349748884,9.42276349748884,1.9720010303315665,0.8412907225001276,0.1146630124965249,0.1384262584504627,5.529071809687602,3.0696520156996363,0.14572731340126654,0.09361791196688794,0.07284079213293028,Energy,7 -ett2/D/short,TabPFN-TS,124689.80317460318,124689.80317460318,218.28549107142857,1.4298486930788783,0.4788421630859375,0.14597940596323164,27.91695960351564,353.1144335404646,0.20209670692527887,0.12493054580859496,0.1258210568162837,Energy,7 -ett2/H/long,TabPFN-TS,349.1736111111111,349.1736111111111,12.713215112433865,1.4446015639472831,0.24456672487448208,0.2568363734654018,11.838650179260124,18.6861877094048,0.2558929587280855,0.1740976961517922,0.13917833955228112,Energy,7 -ett2/H/medium,TabPFN-TS,260.9357886904762,260.9357886904762,10.916395786830355,1.2487580741246136,0.21442902452607412,0.2176322028750465,10.592664800499469,16.153507009020554,0.21767889487818903,0.1471054532989758,0.12063006509927268,Energy,7 -ett2/H/short,TabPFN-TS,126.3329613095238,126.3329613095238,7.186092558361235,0.8255326354728952,0.1333656464555541,0.12338819957914808,6.245625044327679,11.239793650664758,0.14255185712990415,0.09113964824800784,0.07305442987134397,Energy,7 -ett2/W/short,TabPFN-TS,3352975.4285714286,3352975.4285714286,1215.9320591517858,0.7651817774023886,0.12514420918055943,0.12832092387335642,13.173712071216404,1831.1131665114065,0.15360232199450594,0.1019980583882235,0.09869280067398715,Energy,7 -hierarchical_sales/D/short,TabPFN-TS,31.00775791907813,31.00775791907813,2.370928715378255,0.7595246942482423,0.6425987223410554,1.0572101569362986,8.035389477271963,5.56846100094794,1.7083421681052473,0.7273746734264227,0.5917485761396661,Sales,1 -hierarchical_sales/W/short,TabPFN-TS,462.451374635858,462.451374635858,8.994814727265956,0.7311473691538796,0.5269374437620046,0.46027796147233346,6.669578861706579,21.50468262113761,0.9872904978590784,0.41295634381991136,0.34547572723646164,Sales,1 -hospital/M/short,TabPFN-TS,3655.2802314211217,3655.2802314211217,18.772015097036615,0.7644714645492778,0.19299344092231893,0.17395839997863105,5.254385536046241,60.45891358121746,0.2194798957162256,0.06814677393013854,0.05375192910322669,Healthcare,1 -jena_weather/10T/long,TabPFN-TS,1693.3583560681218,1693.3583560681218,10.627256944444444,0.6669669872039639,0.773788063451945,0.6517540018394511,6.516622821458388,41.15043567288349,0.25180656563872267,0.06503000586951763,0.05302131807666021,Nature,21 -jena_weather/10T/medium,TabPFN-TS,1750.3887817911254,1750.3887817911254,10.549468315972222,0.6257176279534998,0.7843234355751396,0.6484787581802962,6.560058680442422,41.83764789984166,0.25675694414401257,0.06474191029183934,0.053889677251242135,Nature,21 -jena_weather/10T/short,TabPFN-TS,1029.9804992191375,1029.9804992191375,6.5642832619803295,0.3103315819396469,0.3655124845059705,0.5538089704418939,2.7476479117092367,32.09330925939451,0.1993396907652391,0.040772429697493937,0.0339592102375841,Nature,21 -jena_weather/D/short,TabPFN-TS,397.21959325396824,397.21959325396824,10.065989273313493,1.228557952722909,0.6865152271871034,0.48159678625682045,7.6517322873356175,19.93036861811563,0.1200133937581282,0.0606137075219578,0.047143761671973934,Nature,21 -jena_weather/H/long,TabPFN-TS,3960.9719576719576,3960.9719576719576,18.149107142857144,1.406140600615272,4.157182337832393,0.79033203125,11.165177368080114,62.93625312704878,0.3789459725153458,0.1092777328617197,0.10253437348096008,Nature,21 -jena_weather/H/medium,TabPFN-TS,1468.8757936507936,1468.8757936507936,11.595155164930556,1.08814100404732,2.9812427832367323,0.7352610754588293,7.7572016693125905,38.32591543134741,0.2345230737441197,0.07095281089086086,0.05799743420199071,Nature,21 -jena_weather/H/short,TabPFN-TS,1157.0227521929824,1157.0227521929824,8.288363680604506,0.5490473254936928,1.0643030052146252,0.6152072928802311,4.901563186782991,34.01503714819348,0.2085099962811183,0.050807137816434395,0.04210943877353269,Nature,21 -kdd_cup_2018/D/short,TabPFN-TS,2731.1231257064965,2731.1231257064965,20.537550805405942,1.1739298514294911,0.5386272222170274,0.45167684057258417,9.678648517860324,52.26014854271366,1.170682813452448,0.4600629433466736,0.3619470425522957,Nature,1 -kdd_cup_2018/H/long,TabPFN-TS,4164.16667203731,4164.16667203731,25.603662863763393,1.0932322336358322,1.2056718454704942,0.6444264480870014,8.694193144080682,64.5303546560633,1.514142939122559,0.6007654157096626,0.477949904103166,Nature,1 -kdd_cup_2018/H/medium,TabPFN-TS,5238.046655667353,5238.046655667353,27.09359559639831,1.1284777650578837,1.044518417287213,0.6036829312783302,9.260084449300775,72.37435081344324,1.514926251625949,0.5671180295035835,0.4495885779293134,Nature,1 -kdd_cup_2018/H/short,TabPFN-TS,4678.4982992292435,4678.4982992292435,24.941407209880296,1.0505311589202366,1.048782773133547,0.5517051294271736,8.468461581709045,68.39954896948696,1.4317264336898177,0.5220688225844491,0.41801589098488257,Nature,1 -loop_seattle/5T/long,TabPFN-TS,116.89548084365325,116.89548084365325,6.416364829237616,1.0065567656849457,0.22339298842738164,0.14145765512993735,8.568140909437178,10.811821347194618,0.1911552234581402,0.11344264886878824,0.09023827778424431,Transport,1 -loop_seattle/5T/medium,TabPFN-TS,108.92337177099458,108.92337177099458,6.126644162393576,0.9637143945419536,0.21405275193284293,0.13691525135493007,8.13248716007984,10.4366360371048,0.1856891996347391,0.1090055882869805,0.08690720550103873,Transport,1 -loop_seattle/5T/short,TabPFN-TS,48.56773726991825,48.56773726991825,3.801889258543158,0.5976997857380476,0.11060900648804026,0.08033697568226156,5.304121093302596,6.969055694275821,0.1195039192617756,0.06519400718354651,0.05256348721117568,Transport,1 -loop_seattle/D/short,TabPFN-TS,19.406321453334623,19.406321453334623,3.0272181523719683,0.907428724621304,0.05706404576729695,0.05580497363772555,7.002619763110791,4.405260656684757,0.07872789401866613,0.054100433196767675,0.043121840345089806,Transport,1 -loop_seattle/H/long,TabPFN-TS,70.10133339245786,70.10133339245786,4.439102173901359,0.9260622979769342,0.14156941117640004,0.10133589307262104,6.977403521697585,8.37265390377853,0.14963051021595694,0.07933268600555231,0.06256499189469966,Transport,1 -loop_seattle/H/medium,TabPFN-TS,77.16381357230392,77.16381357230392,4.755659628160474,0.9784901403759688,0.1191087661758911,0.10848097274670045,7.622978827805903,8.784293572752674,0.15526140687770348,0.08405575227927105,0.06681332001279122,Transport,1 -loop_seattle/H/short,TabPFN-TS,75.32349253407864,75.32349253407864,4.495341405754054,0.91095777978495,0.1251505313086151,0.10250271914816066,7.682045156903769,8.678910791918456,0.15362554967485056,0.07957211578648182,0.06341097341290575,Transport,1 -m4_daily/D/short,TabPFN-TS,702442.9924972118,702442.9924972118,187.73180634589713,4.194442221931702,0.03708484540331653,0.03092866981898531,30.152829872241675,838.1187221970475,0.12946617938967445,0.028999375713517687,0.02259971410479323,Econ/Fin,1 -m4_hourly/H/short,TabPFN-TS,2216507.236413823,2216507.236413823,285.81631179170716,0.7386401357780685,0.09308361316072768,0.08753064174007102,5.496123608735736,1488.7938864778507,0.20325325838709515,0.039020241283554385,0.02960508205927616,Econ/Fin,1 -m4_monthly/M/short,TabPFN-TS,1909204.3714108795,1909204.3714108795,555.5572942889178,0.9576589316497056,0.15374823170679586,0.1294481470540718,8.502672632199085,1381.739617804628,0.28717292636452374,0.11546387749783972,0.09366665886576336,Econ/Fin,1 -m4_quarterly/Q/short,TabPFN-TS,1901198.214072917,1901198.214072917,580.5208575846355,1.2241822728860832,0.12060847956935564,0.10457869998613992,9.947300963299902,1378.83944463194,0.2307934429356135,0.09716896912073542,0.07826275149269904,Econ/Fin,1 -m4_weekly/W/short,TabPFN-TS,217483.00664238268,217483.00664238268,248.66107108420823,2.04974390202117,0.061852850136471835,0.06312812623718143,18.73303138685925,466.3507335068561,0.0849618698162227,0.04530219002970306,0.0370084480828777,Econ/Fin,1 -m4_yearly/A/short,TabPFN-TS,3555054.903659209,3555054.903659209,893.0699246634783,3.3084560346745997,0.17128991236630944,0.1483512422599227,33.0173202698287,1885.485323108936,0.3023463106417499,0.1432079017310429,0.11767364143536849,Econ/Fin,1 -m_dense/D/short,TabPFN-TS,7652.655555555555,7652.655555555555,42.433567708333335,0.6783292595875375,0.12104778134266124,0.09753263120298032,6.526921713728361,87.47945790615964,0.15143988154244065,0.07345878245000709,0.061106410525425904,Transport,1 -m_dense/H/long,TabPFN-TS,75694.39604938272,75694.39604938272,120.74972222222222,1.02275823938943,0.32039782336527445,0.2554198615933642,8.829765092160066,275.1261457029897,0.4762840292517426,0.20903561922142588,0.16524406701635594,Transport,1 -m_dense/H/medium,TabPFN-TS,71212.77555555555,71212.77555555555,116.21049045138888,0.993734534883084,0.3077546622676556,0.24954473707411023,8.673153554524918,266.85721941809174,0.4642978479599947,0.20219157174991295,0.160040826351846,Transport,1 -m_dense/H/short,TabPFN-TS,68285.90555555555,68285.90555555555,106.8298410373264,0.906291178560046,0.30428158067341887,0.23916954464382595,10.038621879613546,261.3157200697186,0.46350902522521975,0.18948953959205128,0.15486162614965193,Transport,1 -restaurant/D/short,TabPFN-TS,147.50485234907654,147.50485234907654,7.279157133165657,0.6978371816511807,0.6452601482590519,0.3987383605096257,4.8369283043616305,12.145157567898266,0.5589482913220262,0.3350036769059037,0.26285464069074244,Sales,1 -saugeen/D/short,TabPFN-TS,996.2383333333332,996.2383333333332,14.171375325520833,3.1471346641095788,0.48842748006184894,0.41242085774739584,27.22096017470464,31.56324339058541,1.0223409738504536,0.4590142233422909,0.3731311321710199,Nature,1 -saugeen/M/short,TabPFN-TS,376.1626441592262,376.1626441592262,11.992789132254464,0.7027332947267324,0.3269654228573754,0.3455390476045154,4.861786421786346,19.39491284226939,0.5822934677185015,0.360059507780706,0.2762319185074306,Nature,1 -saugeen/W/short,TabPFN-TS,967.20927734375,967.20927734375,16.254945373535158,1.3185971950086537,0.4728340625762939,0.43860502243041993,10.0293112825053,31.099988381730142,0.9387790097305582,0.4906690424989388,0.3952868544502755,Nature,1 -solar/10T/long,TabPFN-TS,24.99329126115166,24.99329126115166,1.9885748127661191,0.8706305500987933,3.3978985345314263,1.4695288541349858,5.422036201988414,4.9993290811019495,1.0805249842097455,0.4297986256368609,0.3311104604624895,Energy,1 -solar/10T/medium,TabPFN-TS,23.20675610173358,23.20675610173358,1.9164985034033883,0.8397775518236055,3.5864761297978434,1.4588056166492134,5.404413331269677,4.817339110103583,1.0689867060696263,0.42527863941397,0.3259753271245175,Energy,1 -solar/10T/short,TabPFN-TS,25.936860660450883,25.936860660450883,2.1414197193155036,0.9439361088822628,4.492088894725105,1.5343012306231942,4.045348817118251,5.092824428590768,1.4815020459581518,0.6229387523376522,0.4579774575911444,Energy,1 -solar/D/short,TabPFN-TS,113549.29051094892,113549.29051094892,253.51575425790753,0.987481598204353,0.9896838705615116,0.4369196314011177,5.250447637876118,336.97075616579684,0.4868079796401177,0.3662439243733971,0.26917233844623834,Energy,1 -solar/H/long,TabPFN-TS,985.2254866180048,985.2254866180048,14.191483868106245,1.0832552757402476,3.761211700292542,1.4626715914563058,4.509945094067163,31.388301747912468,1.088289768604162,0.4920446738090314,0.3505921321640143,Energy,1 -solar/H/medium,TabPFN-TS,806.1380322384429,806.1380322384429,11.32804896593674,0.8642389418245519,4.488747114193476,1.4055882066415757,4.4199894526363,28.39257001820094,1.0227666406383358,0.40806276354872306,0.31347488384144956,Energy,1 -solar/H/short,TabPFN-TS,849.0887552623575,849.0887552623575,11.64371880399294,0.8972739471619915,4.454845891374051,1.4148532292467024,5.817322540440987,29.13912756522332,1.0740779514829115,0.42919135491076416,0.33566385955464595,Energy,1 -solar/W/short,TabPFN-TS,975858.0729927009,975858.0729927009,733.5384352189781,0.7927547220600458,0.1474037866522796,0.14471499241181535,6.555031527149288,987.8552894997832,0.20164595749357925,0.149733530508271,0.12002529687191076,Energy,1 -sz_taxi/15T/long,TabPFN-TS,17.683922053062677,17.683922053062677,2.953506095864494,0.5596229152773028,5905908731598.259,0.42311587428774927,7.194725257547046,4.205225565063387,0.3892281573916414,0.27337124198259016,0.2424540578815739,Transport,1 -sz_taxi/15T/medium,TabPFN-TS,17.31587122729701,17.31587122729701,2.9009017553084937,0.5657806694261462,9231205100342.7,0.4234086648011819,6.352440033271851,4.1612343393874145,0.3872723050167596,0.2699773235003315,0.2304960350862857,Transport,1 -sz_taxi/15T/short,TabPFN-TS,17.14560256976639,17.14560256976639,2.7944552709185886,0.5582249003801146,1157938620500.9626,0.4043481684720851,4.390036050528658,4.14072488457835,0.3871754966467506,0.26129352650416643,0.20867625780444496,Transport,1 -sz_taxi/H/short,TabPFN-TS,7.340465578258547,7.340465578258547,1.8834059462588055,0.5726814850300137,1.4804094673221946,0.3035417825747759,4.395105016649191,2.709329359501821,0.2523907457387066,0.17545088404917308,0.1403426654750027,Transport,1 -temperature_rain/D/short,TabPFN-TS,192.348170645723,192.348170645723,5.973571384244577,1.3788383285690875,32.83221152661815,1.5353158278965455,21.838367970591225,13.868964296072113,1.6327437788569656,0.7032472870339427,0.5690453839767815,Nature,1 -us_births/D/short,TabPFN-TS,112759.78666666668,112759.78666666668,215.285703125,0.31694359235351605,0.020599613189697264,0.02048578898111979,2.2508974742308343,335.79724040954636,0.03147826645095531,0.020181287725250033,0.015855619215830008,Healthcare,1 -us_births/M/short,TabPFN-TS,59338602.666666664,59338602.666666664,6332.360677083333,0.7153333155562595,0.019422224412361785,0.019690368324518204,4.7156280861706295,7703.155370798817,0.0239259480622513,0.01966826908953116,0.015286944232309424,Healthcare,1 -us_births/W/short,TabPFN-TS,1737954.0,1737954.0,983.3291015625,0.8942316441355983,0.013318025640078954,0.013351580926350184,6.532085345899004,1318.3148334142345,0.0178959945969944,0.01334859613394065,0.010610700543554966,Healthcare,1 +bitbrains_fast_storage/5T/long,TabPFN-TS,3940977.473961663,3940977.473961663,396.2051671955408,1.0388353010633071,5.466466474860468,0.7927045072161448,67.23911919744833,1985.189530992359,5.246263156727107,1.0470519508149014,0.7602884074722032,Web/CloudOps,2 +bitbrains_fast_storage/5T/medium,TabPFN-TS,3247133.159062092,3247133.159062092,324.4935908220456,1.1867037716263282,5.815028533985838,0.7800321416939235,95.43680935654676,1801.9803436947063,5.474913948877669,0.9859011464411452,0.7353447476277656,Web/CloudOps,2 +bitbrains_fast_storage/5T/short,TabPFN-TS,2045266.0192445407,2045266.0192445407,190.9517174659572,0.8882362319520551,2.484373134741108,0.6999759874291243,76.38956860526716,1430.127973030575,4.490248114456697,0.5995411637790902,0.4558547292997618,Web/CloudOps,2 +bitbrains_fast_storage/H/short,TabPFN-TS,2405371.011957238,2405371.011957238,272.2178745663353,1.1461108043110697,3.706066647986907,0.5534585983280202,21.973896483964754,1550.925856370071,4.420642537084117,0.775909377434113,0.591312581984499,Web/CloudOps,2 +bitbrains_rnd/5T/long,TabPFN-TS,3143232.871647869,3143232.871647869,241.6739175312724,3.5093995299063607,4.050204866894866,0.7111483674880203,242.29249979536945,1772.916487499586,6.791643806713404,0.9257983536270856,0.7300597264192352,Web/CloudOps,2 +bitbrains_rnd/5T/medium,TabPFN-TS,2090599.1775186085,2090599.1775186085,199.0785499293046,4.589946127888775,2.832675134166589,0.746569956612344,371.1720930175657,1445.890444507677,6.083598065793586,0.8376249292547827,0.7095510579591657,Web/CloudOps,2 +bitbrains_rnd/5T/short,TabPFN-TS,1682744.407120047,1682744.407120047,135.30342227375482,1.8673759793901137,1.2023257691773783,0.6515096053863655,284.946804834687,1297.2063857073963,5.307794073707693,0.5536225467355949,0.4553597018381872,Web/CloudOps,2 +bitbrains_rnd/H/short,TabPFN-TS,1768178.1854349263,1768178.1854349263,167.11547426876976,5.957801473087373,1.9668859182296852,0.552835291775346,203.4573914538545,1329.7286134527324,5.819543006761521,0.7313790797335689,0.6370421458232746,Web/CloudOps,2 +bizitobs_application/10S/long,TabPFN-TS,21300886.186666667,21300886.186666667,2472.8352777777777,7.728085487915151,0.1032213168674045,0.1071828460693359,143.1277644315143,4615.288310243106,0.1779932222914751,0.0953673724588003,0.0879505832371775,Web/CloudOps,2 +bizitobs_application/10S/medium,TabPFN-TS,21666054.826666668,21666054.826666668,2030.06125,6.925782643420624,0.0924721717834472,0.0951119867960612,148.49438643019815,4654.680958633649,0.1827756217009251,0.0797145304602413,0.0700137998283303,Web/CloudOps,2 +bizitobs_application/10S/short,TabPFN-TS,4943498.24,4943498.24,967.4614583333332,3.1888377415438267,0.0462684885660807,0.0465743086073133,49.96181592052012,2223.3979041098337,0.0857475699861046,0.0373111213939501,0.0313406007204244,Web/CloudOps,2 +bizitobs_l2c/5T/long,TabPFN-TS,339.5494444444444,339.5494444444444,13.08852554563492,1.2160180268733982,0.9811188743835858,1.2126699683779762,7.2539512545927,18.42686746152054,1.0346410597480369,0.7349011420065599,0.511339388914521,Web/CloudOps,7 +bizitobs_l2c/5T/medium,TabPFN-TS,144.35393282312924,144.35393282312924,8.755759061437075,0.8700326557683994,0.6392903928649001,0.9488015518707484,6.532302877185926,12.014738150418813,0.632151911088195,0.4606816856447503,0.3448825766018137,Web/CloudOps,7 +bizitobs_l2c/5T/short,TabPFN-TS,29.177198428199404,29.177198428199404,3.351733107793899,0.3376520522696331,0.1869919781934289,0.226708325885591,4.020672322763404,5.401592212320309,0.1855125331592748,0.1151120771172689,0.0989730973449374,Web/CloudOps,7 +bizitobs_l2c/H/long,TabPFN-TS,144.74971478174604,144.74971478174604,8.725088355654762,0.9194449990311916,1.3923079579433175,0.8217161148313492,7.88847966834854,12.031197562244005,0.7349002712763818,0.5329535789191325,0.4400927339795309,Web/CloudOps,7 +bizitobs_l2c/H/medium,TabPFN-TS,114.09707961309525,114.09707961309525,7.254261997767857,0.7482961914782938,1.0562951228521056,0.8080075218563988,7.184395907292426,10.68162345400245,0.6467885245941795,0.4392565825560715,0.3800913288674146,Web/CloudOps,7 +bizitobs_l2c/H/short,TabPFN-TS,85.44098772321429,85.44098772321429,6.418287004743304,0.6344522765963027,0.7020613489677009,0.7020347231910342,6.581317614660395,9.243429435183367,0.4982445193882983,0.3459621070727685,0.2898378847142646,Web/CloudOps,7 +bizitobs_service/10S/long,TabPFN-TS,208415.56110967736,208415.56110967736,131.40309476984555,3.902102389054756,0.2514900850213503,0.1719557895671443,67.5938471682915,456.5255317172057,0.3382087086505102,0.0973476134568513,0.0905342005234128,Web/CloudOps,2 +bizitobs_service/10S/medium,TabPFN-TS,153786.42608814148,153786.42608814148,107.66579814488334,3.777267746897072,0.3485729430163631,0.1520390156208532,70.09273852151425,392.1561246342348,0.2951459406018347,0.081031816840187,0.0670881764641895,Web/CloudOps,2 +bizitobs_service/10S/short,TabPFN-TS,23223.54763853943,23223.54763853943,49.30270014662591,2.003461979848056,0.2334731027970346,0.0912925157320221,33.28855091714436,152.39274142340057,0.112897419560105,0.0365250180022352,0.0308080142329048,Web/CloudOps,2 +car_parts/M/short,TabPFN-TS,1.4217560750180898,1.4217560750180898,0.4650315940521182,0.8429977450172786,0.8077849812660705,1.899363879513,20.097226810718563,1.1923741338263296,2.859190858652686,1.1150980593916195,0.9545496300490686,Sales,1 +covid_deaths/D/short,TabPFN-TS,608643.349030727,608643.349030727,132.21949222148453,37.798524709251794,0.0992231984946835,0.1511093960201699,677.8435674633554,780.1559773729399,0.29353746932196,0.0497482250567645,0.0399575272446472,Healthcare,1 +electricity/15T/long,TabPFN-TS,479385.9336663729,479385.9336663729,69.25283177134297,1.0202781447783311,0.1595782741210479,0.1449488764255062,9.295101062040777,692.3770169975119,1.0923096108172683,0.1092548306241962,0.0888787687381395,Energy,1 +electricity/15T/medium,TabPFN-TS,412523.4689212973,412523.4689212973,65.27677076596724,0.976918207169049,0.1516940762467859,0.1445092451330036,8.89593020035391,642.2798992038419,1.106712926772942,0.1124784476584751,0.0923857961554726,Energy,1 +electricity/15T/short,TabPFN-TS,194473.3980942518,194473.3980942518,65.37218991744707,1.2266456914333352,0.2023727743818738,0.1846499673239442,11.928914973863955,440.9913809750161,0.8692809586938318,0.1288614752463225,0.1036693276639552,Energy,1 +electricity/D/short,TabPFN-TS,1471778340.7486846,1471778340.7486846,4609.131754504505,1.4894429943811094,0.2354444740752776,0.0952267406566723,13.423388704270328,38363.76338093911,0.6304440686362233,0.0757433452848885,0.0600270541999052,Energy,1 +electricity/H/long,TabPFN-TS,19914932.00037977,19914932.00037977,352.74562123382617,1.3377679338170498,0.3050388547866177,0.1455674919289879,11.358791576901645,4462.614928534589,1.7600540798077915,0.1391227743664636,0.1124445719696832,Energy,1 +electricity/H/medium,TabPFN-TS,11796934.140468594,11796934.140468594,286.1099377427443,1.180699742250593,0.2148767347608029,0.1329174415781638,10.214039705176262,3434.666525365832,1.3406635236881663,0.111678136571216,0.0908690358272918,Energy,1 +electricity/H/short,TabPFN-TS,2003726.408484706,2003726.408484706,186.82288506429205,1.044375818547214,0.1971453508145225,0.1307061697499337,11.382491212876442,1415.5304336130346,0.6682314588183454,0.0881937442408192,0.0720731827284665,Energy,1 +electricity/W/short,TabPFN-TS,43809522548.24645,43809522548.24645,28410.723933908583,1.4485545546987582,0.1054109967005592,0.0939584561545392,11.252347284933158,209307.2443759328,0.4775700762465698,0.0648238986461813,0.0514000138896149,Energy,1 +ett1/15T/long,TabPFN-TS,10.510729166666666,10.510729166666666,1.6884466455853175,1.1075089883945437,0.9056581486460964,0.4095517113095238,9.189091726171444,3.2420254728590066,0.6179332606557163,0.3218196000877369,0.2595610162075561,Energy,7 +ett1/15T/medium,TabPFN-TS,9.810387369791666,9.810387369791666,1.6322027781653026,1.0538546298293985,0.7946897001730433,0.3998653339204334,8.562064287161736,3.132153790890809,0.5969916270759061,0.3110994725382204,0.2479122114007532,Energy,7 +ett1/15T/short,TabPFN-TS,6.399552481515067,6.399552481515067,1.2574796222505116,0.786973072399437,0.5675219319419111,0.2630254836309524,5.647841580246456,2.5297336779817488,0.4782249788369692,0.2377159979221376,0.1834630230394784,Energy,7 +ett1/D/short,TabPFN-TS,43568.688888888886,43568.688888888886,141.97459077380952,1.772226457572935,1.6984561011904762,0.474870130750868,13.765077920690596,208.73114019927377,0.5482392634160486,0.3729009720126996,0.2918356857005129,Energy,7 +ett1/H/long,TabPFN-TS,172.94669312169313,172.94669312169313,7.836870659722222,1.457595714011079,2736247275849.4355,0.4894441473420965,11.165649758363148,13.150919858386072,0.6297690691031738,0.3752907624106631,0.2897646542062353,Energy,7 +ett1/H/medium,TabPFN-TS,154.01130952380953,154.01130952380953,7.3551025390625,1.3585784429613703,2668500988477.909,0.4610775538853237,10.45425370159929,12.410129311325065,0.5944157759046996,0.3522919764120212,0.2764819505676313,Energy,7 +ett1/H/short,TabPFN-TS,112.4321765718006,112.4321765718006,5.309052894228981,0.8908363269123161,0.5164939834958031,0.2685810225350516,6.503018582568272,10.603404008704024,0.4949042185542229,0.2477952053627885,0.1937159661760259,Energy,7 +ett1/W/short,TabPFN-TS,1174346.5714285714,1174346.5714285714,849.2448381696429,1.577096937888621,0.6470521518162319,0.5367565836225238,10.057570618317031,1083.6727233941858,0.431219241467038,0.3379347906702841,0.256419494681339,Energy,7 +ett2/15T/long,TabPFN-TS,12.83592013888889,12.83592013888889,2.301945064484127,0.9578073986429992,0.1473062529443106,0.178276619078621,8.571694903641015,3.582725239100661,0.1909299131946711,0.1226748193091084,0.1009458616883307,Energy,7 +ett2/15T/medium,TabPFN-TS,12.95630867125496,12.95630867125496,2.277454136439732,0.9392566587102824,0.141396593654979,0.1718650009518578,8.060023977046217,3.599487278940566,0.1918231799623912,0.1213696454009845,0.0983757090868891,Energy,7 +ett2/15T/short,TabPFN-TS,8.865403820219495,8.865403820219495,1.944286382765997,0.8450960376199323,0.1162656731139621,0.1411288760957263,6.116299433959818,2.9774827993154718,0.1413517124493446,0.0923021989443899,0.0728750638145369,Energy,7 +ett2/D/short,TabPFN-TS,162956.95238095237,162956.95238095237,241.52847222222223,1.535311945472873,0.4684700133308532,0.1532284085712735,27.805417218699205,403.67926919889305,0.2310362964807856,0.1382331171665665,0.1285422202240154,Energy,7 +ett2/H/long,TabPFN-TS,363.742328042328,363.742328042328,12.264203042328042,1.3653142407810173,0.2206563951497026,0.2301058976107804,12.436676967686727,19.072029992696844,0.2611767718316169,0.1679488214369047,0.1356672864670785,Energy,7 +ett2/H/medium,TabPFN-TS,329.2125744047619,329.2125744047619,11.717957124255951,1.277450790102998,0.2048852038421092,0.2153702872140067,10.86438040815514,18.144216004136467,0.244504978764426,0.1579070077855928,0.1280160716008789,Energy,7 +ett2/H/short,TabPFN-TS,117.8634998139881,117.8634998139881,6.839659045991444,0.7870254439803768,0.1262451481190145,0.1175918352036249,6.192682885553366,10.856495742825496,0.1376905731691205,0.0867459073933945,0.0699977966289662,Energy,7 +ett2/W/short,TabPFN-TS,14502475.42857143,14502475.42857143,1635.7318638392858,0.9589009824999204,0.1474132367542811,0.145101819719587,13.189927237151512,3808.2115787560215,0.319450568015805,0.1372128260782456,0.1202574726141917,Energy,7 +hierarchical_sales/D/short,TabPFN-TS,30.57163511846321,30.57163511846321,2.3798296243605312,0.7660793450404748,0.6310163139953155,1.0490738299820432,7.771583528614682,5.529162243818064,1.6962857446252833,0.7301053737305642,0.5930043850043732,Sales,1 +hierarchical_sales/W/short,TabPFN-TS,468.4471890724312,468.4471890724312,8.973932945122153,0.723416703409247,0.5185541231281445,0.4571566258446645,6.623818653369418,21.64364084604139,0.9936701379339342,0.41199765098765,0.3417650702357695,Sales,1 +hospital/M/short,TabPFN-TS,2679.1871536831814,2679.1871536831814,17.352619912198502,0.7531851000735269,0.1923880595738138,0.1719025956294372,5.198411914036882,51.760865078582114,0.18790395984155,0.0629940398055021,0.0504805609674915,Healthcare,1 +jena_weather/10T/long,TabPFN-TS,1580.603511491402,1580.603511491402,10.20437644675926,0.656773253670893,0.7720577018595126,0.6560148370329034,6.982261500751263,39.756804593571175,0.2432786983121498,0.0624423276576957,0.0548977041176505,Nature,21 +jena_weather/10T/medium,TabPFN-TS,1886.342711715368,1886.342711715368,10.978890326253609,0.6245848971006992,0.7680008366190657,0.6628134899497204,6.55998411025622,43.4320470587718,0.2665417450671599,0.0673772659736879,0.0567937425736504,Nature,21 +jena_weather/10T/short,TabPFN-TS,1014.2037315005348,1014.2037315005348,6.599587485903785,0.3250009779636542,0.3718861119674825,0.5649007031841884,3.7936287448317167,31.84656545846875,0.197807102381248,0.0409917131943352,0.0354390154086276,Nature,21 +jena_weather/D/short,TabPFN-TS,392.590625,392.590625,9.983251178075395,1.2287532245574706,0.6514123184023372,0.4747197711278522,7.597867713507752,19.81389979282221,0.1193120610703935,0.0601154889594781,0.0465159837615308,Nature,21 +jena_weather/H/long,TabPFN-TS,1634.8796296296296,1634.8796296296296,12.753626198743389,1.0999403769809355,2.446339739877115,0.6785660032242063,9.942908231388692,40.43364477300593,0.243455339006535,0.0767909597863064,0.0659166328078663,Nature,21 +jena_weather/H/medium,TabPFN-TS,1497.2289682539683,1497.2289682539683,11.262710813492063,0.954497320291504,2.871433126547123,0.6940052819630457,7.893160249520058,38.69404305902872,0.2367757119864755,0.0689185249443742,0.0595068926034612,Nature,21 +jena_weather/H/short,TabPFN-TS,1140.7981199744152,1140.7981199744152,8.439706545824196,0.5951480288792269,1.1638988677264472,0.6275021163443276,5.439047960494513,33.77570310111123,0.2070428939213693,0.0517348598743774,0.0433749607608675,Nature,21 +kdd_cup_2018/D/short,TabPFN-TS,2724.3329307342574,2724.3329307342574,20.274988155794933,1.1739335474857984,0.5149150721308192,0.450249306230883,9.779238874612522,52.195142788714136,1.169226615544389,0.4541812612251059,0.3591637715617976,Nature,1 +kdd_cup_2018/H/long,TabPFN-TS,4021.7201519208343,4021.7201519208343,24.687106447847157,1.0575396852617072,1.2271145815783495,0.6184559653223703,8.973319969408267,63.41703361022837,1.4880199275644768,0.5792592976530704,0.4624436577729628,Nature,1 +kdd_cup_2018/H/medium,TabPFN-TS,5288.079190672816,5288.079190672816,27.137685229471725,1.136105934900563,1.229284216293004,0.5858576113913657,10.24346547878567,72.71918034929172,1.5221441583887936,0.5680409053818023,0.4624805251220042,Nature,1 +kdd_cup_2018/H/short,TabPFN-TS,4759.803006921011,4759.803006921011,26.16721637525615,1.0963647304693052,1.20523590775101,0.5491535691464683,8.96822139561829,68.99132559185257,1.4441133901222132,0.547727228395148,0.4366137476521748,Nature,1 +loop_seattle/5T/long,TabPFN-TS,122.0583144851508,122.0583144851508,6.643171560565016,1.046792008334055,0.2157579654133274,0.1466260668449808,9.325475135533132,11.04800047452709,0.1953309189687861,0.1174526384918547,0.0935730075998391,Transport,1 +loop_seattle/5T/medium,TabPFN-TS,108.86982964073788,108.86982964073788,6.15152889476103,0.9716574976595126,0.2035810928817135,0.1374303472890942,8.337243273977046,10.43407061700935,0.1856435555399793,0.1094483388073597,0.0873598980952376,Transport,1 +loop_seattle/5T/short,TabPFN-TS,46.96549013686992,46.96549013686992,3.739100572962018,0.587682274689241,0.1026436058237333,0.0796069208682506,5.314526420206936,6.853137247777102,0.1175161738226465,0.0641173198472113,0.051947251305268,Transport,1 +loop_seattle/D/short,TabPFN-TS,18.49710145365712,18.49710145365712,2.988963810750452,0.8988332716750893,0.0563158649409149,0.0551457534018438,7.200364515864249,4.3008256711539845,0.0768615012865513,0.0534167769984987,0.0438373093456756,Transport,1 +loop_seattle/H/long,TabPFN-TS,67.59270430211558,67.59270430211558,4.412962901374914,0.9223339565898164,0.1415486417441668,0.1011307854483453,7.48961052275084,8.221478230957956,0.1469287989883885,0.0788655422862782,0.0633112868629268,Transport,1 +loop_seattle/H/medium,TabPFN-TS,71.01074400154799,71.01074400154799,4.58358863539248,0.9478641312684954,0.1195942428951046,0.1041414400618388,7.735268938960752,8.426787288258081,0.148942523265887,0.0810144167183933,0.0651342563580121,Transport,1 +loop_seattle/H/short,TabPFN-TS,74.5299885470982,74.5299885470982,4.490799522058028,0.9121818732427218,0.1258212686213086,0.102138436771049,8.036389745904458,8.63307526592339,0.1528142142383595,0.0794917198248121,0.0637984701807385,Transport,1 +m4_daily/D/short,TabPFN-TS,698541.3674169455,698541.3674169455,197.73342738221973,4.310262159241137,0.0385161610501193,0.0324561628228029,29.27039887526933,835.7878722600284,0.1291061274923901,0.030544349747604,0.0235298029984748,Econ/Fin,1 +m4_hourly/H/short,TabPFN-TS,1974169.442342405,1974169.442342405,276.0602304946973,0.7804866407185013,0.0963661524792608,0.0932995079028241,5.657858857138344,1405.051402028554,0.1918205590830847,0.0376883206391206,0.0281976469731856,Econ/Fin,1 +m4_monthly/M/short,TabPFN-TS,1756663.0326296296,1756663.0326296296,526.6686806369357,0.895008061940836,0.14536284466143,0.1226754361655977,7.925692832251994,1325.3916525426096,0.275461884811926,0.1094598318627909,0.0882623211716558,Econ/Fin,1 +m4_quarterly/Q/short,TabPFN-TS,1773423.9763385416,1773423.9763385416,555.0584905192057,1.1712220135297755,0.1143357895165681,0.1000043484320243,9.74443607939694,1331.6996569566809,0.222903072567125,0.0929070172428722,0.0750751456929036,Econ/Fin,1 +m4_weekly/W/short,TabPFN-TS,200816.425112492,200816.425112492,244.96552255196056,2.070145368591285,0.0603222059238025,0.0608874742545942,16.100915789618952,448.1254568895768,0.0816415071190786,0.0446289184108609,0.0358837513094015,Econ/Fin,1 +m4_yearly/A/short,TabPFN-TS,3351283.2554191696,3351283.2554191696,858.1312017570586,3.160159979266896,0.1653807364774557,0.1417295112996192,32.49565409116194,1830.651046873535,0.2935533802946987,0.1376053155746721,0.113378683259915,Econ/Fin,1 +m_dense/D/short,TabPFN-TS,6584.881481481481,6584.881481481481,39.23614293981481,0.6343391992101958,0.1154036016443025,0.0916553243001302,6.472099200744528,81.14728265001534,0.1404779495226384,0.067923567214623,0.0571463211874774,Transport,1 +m_dense/H/long,TabPFN-TS,72915.27901234568,72915.27901234568,120.79466049382717,1.0390647350813125,0.3313089644319483,0.2607480951003086,8.591854018250109,270.02829298491235,0.4674588925970861,0.2091134140126585,0.164429129974029,Transport,1 +m_dense/H/medium,TabPFN-TS,69178.59333333334,69178.59333333334,116.22125,1.016694269579333,0.3125688394564185,0.255682127210829,7.960831761937791,263.01823764395755,0.4576185046768153,0.2022102920051717,0.1585943408314112,Transport,1 +m_dense/H/short,TabPFN-TS,71334.99722222223,71334.99722222223,109.0473996310764,0.9158806250212368,0.3099409964832582,0.2445430331759982,10.001841796408856,267.08612323035845,0.4737442837218426,0.1934229364114031,0.1543850478362672,Transport,1 +restaurant/D/short,TabPFN-TS,178.78818526285374,178.78818526285374,8.29543477096978,0.7821100931962905,0.7700873702984403,0.4375370417899797,5.118488585776721,13.371169928725523,0.6153722208093952,0.3817751284892775,0.2965685918933398,Sales,1 +saugeen/D/short,TabPFN-TS,947.38125,947.38125,14.878590494791666,3.303358954426323,0.503057352701823,0.4162318166097005,27.681241412470868,30.779558963701867,0.996957248538798,0.4819210911798914,0.3842106395309255,Nature,1 +saugeen/M/short,TabPFN-TS,376.4469401041667,376.4469401041667,12.06310017903646,0.706898496538842,0.3438984553019206,0.3432963916233608,4.669924664301697,19.4022405949459,0.5825134688369025,0.3621704563362662,0.2778272956964877,Nature,1 +saugeen/W/short,TabPFN-TS,1006.3185546875,1006.3185546875,15.425779724121094,1.2511660036320285,0.4077761650085449,0.4090907096862793,9.909876875391326,31.72252440597217,0.9575707772767752,0.4656399878991368,0.3804498298468377,Nature,1 +solar/10T/long,TabPFN-TS,25.8354749594485,25.8354749594485,2.0935281349794708,0.9152877937539702,3.283013031614603,1.4939689741420823,5.0969315666695385,5.082860903019922,1.098579050884435,0.4524826068246617,0.3523350277353385,Energy,1 +solar/10T/medium,TabPFN-TS,26.23753896745742,26.23753896745742,2.134519475520488,0.9354003397464944,3.687648892317398,1.4898048797452168,5.059326482355148,5.122259166369603,1.1366496791575578,0.4736583601499977,0.3587467328055458,Energy,1 +solar/10T/short,TabPFN-TS,34.44245373802463,34.44245373802463,2.4745698469398665,1.0809372089579272,4.177828026682609,1.5824794908509636,4.971023404696623,5.86876935464537,1.7072243365297972,0.7198520865016745,0.5451162209570433,Energy,1 +solar/D/short,TabPFN-TS,115559.18248175182,115559.18248175182,252.5994449513382,0.9850887951305594,1.015654671105155,0.4335812756614963,5.094867900990621,339.93996893826977,0.4910974808637134,0.3649201694952824,0.2688506624469601,Energy,1 +solar/H/long,TabPFN-TS,877.5276763990267,877.5276763990267,12.818477861415248,0.9770759245501968,4.404037612326265,1.4354754742624696,4.430542735515225,29.62309363316105,1.02708677819807,0.444440047049852,0.3243015754362526,Energy,1 +solar/H/medium,TabPFN-TS,867.4373935523114,867.4373935523114,12.00386110477494,0.9209503012117918,4.588050238841548,1.415317507555885,4.401836804057126,29.452290124068643,1.0609402322505224,0.4324070941429265,0.3243953703339113,Energy,1 +solar/H/short,TabPFN-TS,919.4725602069728,919.4725602069728,12.126564822134066,0.936711092050702,4.2082555663250165,1.4261984386094817,8.184709098187527,30.32280594217779,1.1177087308701164,0.4469892200282378,0.3582317194650038,Energy,1 +solar/W/short,TabPFN-TS,1083859.6350364964,1083859.6350364964,812.4620067290145,0.8776728357098831,0.1699104378693295,0.1593432600480796,6.179114064222528,1041.0857961938086,0.2125116344852166,0.165843804265081,0.1238329806565449,Energy,1 +sz_taxi/15T/long,TabPFN-TS,17.098993945868944,17.098993945868944,2.960354205617877,0.559861515226893,6013879437233.814,0.4328146040609419,7.534451973562091,4.135092979108081,0.3827368106654918,0.2740050907737393,0.2478785901755847,Transport,1 +sz_taxi/15T/medium,TabPFN-TS,17.602143012152776,17.602143012152776,2.996658194778312,0.5854737021918714,10236354323510.264,0.4397119766626602,7.082834116822467,4.195490795145757,0.3904604399549113,0.2788890583388779,0.2448335839801526,Transport,1 +sz_taxi/15T/short,TabPFN-TS,17.541289314391122,17.541289314391122,2.853190160059667,0.5712382957982439,992721185973.3234,0.4160638187394474,4.784492880911788,4.188232242174629,0.3916176378863725,0.2667854899906525,0.214915409859324,Transport,1 +sz_taxi/H/short,TabPFN-TS,7.419267312074319,7.419267312074319,1.9177468210204036,0.5822117538174562,1.5181874943594649,0.310376113296574,4.562929717078705,2.723833201955347,0.2537418681484218,0.1786499484080345,0.1440468609014281,Transport,1 +temperature_rain/D/short,TabPFN-TS,190.3859944198081,190.3859944198081,5.962294301692679,1.3824531132761009,36.76048413464811,1.533545632917945,21.54886608943876,13.798043137336832,1.6243944833910349,0.7019196762630732,0.5654525014897981,Nature,1 +us_births/D/short,TabPFN-TS,111350.18666666666,111350.18666666666,217.17235677083332,0.3196481603311801,0.020750192006429,0.0206375964482625,2.813680058790136,333.6917539686389,0.0312808941821572,0.0203581461952356,0.0176613012395234,Healthcare,1 +us_births/M/short,TabPFN-TS,41925221.333333336,41925221.333333336,5200.2734375,0.5880410553209511,0.0159080078204472,0.0160710861285527,3.327505102123412,6474.968828753798,0.0201112090363295,0.0161520138418586,0.0125006102341928,Healthcare,1 +us_births/W/short,TabPFN-TS,1749773.857142857,1749773.857142857,1021.7772739955356,0.9289999495036037,0.0138602065188544,0.0138870137078421,5.523556437789926,1322.7901788049596,0.017956746971846,0.0138705263047056,0.0107906323842147,Healthcare,1 \ No newline at end of file diff --git a/results/tabpfn_ts/config.json b/results/tabpfn_ts/config.json index 3458e88d0259448860c715ddc2349a90eb5b4363..5be65acc213a78dd7f8478933c42710e84717d9d 100644 --- a/results/tabpfn_ts/config.json +++ b/results/tabpfn_ts/config.json @@ -2,7 +2,5 @@ "model": "TabPFN-TS", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://github.com/liam-sbhoo/tabpfn-time-series/tree/main", - "org": "PriorLabs", - "testdata_leakage": "No" + "model_link": "https://github.com/liam-sbhoo/tabpfn-time-series/tree/main" } \ No newline at end of file diff --git a/results/tempo_ensemble/all_results.csv b/results/tempo_ensemble/all_results.csv deleted file mode 100644 index d73f358b240139ea0696a116873dac4a118d2537..0000000000000000000000000000000000000000 --- a/results/tempo_ensemble/all_results.csv +++ /dev/null @@ -1,98 +0,0 @@ -dataset,model,eval_metrics/MSE[mean],eval_metrics/MSE[0.5],eval_metrics/MAE[0.5],eval_metrics/MASE[0.5],eval_metrics/MAPE[0.5],eval_metrics/sMAPE[0.5],eval_metrics/MSIS,eval_metrics/RMSE[mean],eval_metrics/NRMSE[mean],eval_metrics/ND[0.5],eval_metrics/mean_weighted_sum_quantile_loss,domain,num_variates -bizitobs_application/10S/long,TEMPO_ENSEMBLE,487627.2711111111,487627.2711111111,396.0836111111111,1.2993675903516406,0.0178912925720214,0.0178272904290093,9.800100249012653,698.3031369764217,0.0269307607957717,0.0152753616891155,0.0120698276506844,Web/CloudOps,2.0 -bizitobs_application/10S/medium,TEMPO_ENSEMBLE,307498.0266666667,307498.0266666667,311.76875,0.9177696033266344,0.0131387344996134,0.013129768371582,5.872428510396111,554.5250460228705,0.0217745664925907,0.0122422412222421,0.0094315406776331,Web/CloudOps,2.0 -bizitobs_application/10S/short,TEMPO_ENSEMBLE,525641.8133333334,525641.8133333334,390.0379861111112,0.9921897290135696,0.0150116687350802,0.0149332237243652,6.206112108526064,725.0115953095739,0.0279607992769224,0.0150422060979192,0.0116365011231077,Web/CloudOps,2.0 -bizitobs_service/10S/long,TEMPO_ENSEMBLE,128864.03386243386,128864.03386243386,71.79506613756614,1.3563611507860072,0.1136272192253637,0.0937695441674933,21.32547687545452,358.976369504225,0.2659411707148045,0.0531880802252819,0.0497378287376309,Web/CloudOps,2.0 -bizitobs_service/10S/medium,TEMPO_ENSEMBLE,98009.9961904762,98009.9961904762,66.84960813492063,1.429575874324731,0.1006226942274305,0.089753195141989,16.277174249724983,313.0654822724412,0.2356204593806827,0.0503125904006926,0.0436408573760635,Web/CloudOps,2.0 -bizitobs_service/10S/short,TEMPO_ENSEMBLE,8438.823280423281,8438.823280423281,29.30463913690476,1.2884984418523877,0.0968797762431795,0.0843838533270295,10.922623337489345,91.86306809824762,0.0680550969687549,0.0217098133056419,0.0179825354410236,Web/CloudOps,2.0 -car_parts/M/short,TEMPO_ENSEMBLE,1.448840955093402,1.448840955093402,0.4170860887539882,17.86660919142645,0.9999799200634808,1.9999561636701624,713.7590747571373,1.2036780944643806,2.886296596697481,1.0001296559577155,1.0000739200613302,Sales,1.0 -covid_deaths/D/short,TEMPO_ENSEMBLE,3359486.528320802,3359486.528320802,376.5356868734336,115.6153349693266,0.1948706839966014,0.4917750161531234,1608.2690771273951,1832.8902117477744,0.6896338295236072,0.1416733779396501,0.1157821881701,Healthcare,1.0 -hospital/M/short,TEMPO_ENSEMBLE,6031.719578444155,6031.719578444155,23.99147407037701,4.696532893791447,0.2181512732342086,0.1840471568598741,37.147902714383534,77.6641460291952,0.2819388848011488,0.0870946220356949,0.0701390741514011,Healthcare,1.0 -m4_daily/D/short,TEMPO_ENSEMBLE,1121130.894859576,1121130.894859576,516.3883590607996,7.987928627932832,0.1088423646593421,0.0810604800101247,64.81370332574242,1058.8346872196698,0.163560696084881,0.0797677300125995,0.0606080461596886,Econ/Fin,1.0 -m4_hourly/H/short,TEMPO_ENSEMBLE,1724030.044535024,1724030.044535024,289.1650402419233,2.2278853386116246,0.1401686307504749,0.1173738365971907,16.376050421624175,1313.023245999485,0.1792566653066635,0.0394774129056468,0.031167746098992,Econ/Fin,1.0 -m4_monthly/M/short,TEMPO_ENSEMBLE,2815265.2656666664,2815265.2656666664,867.5761620008681,4.678383723019503,0.235788005510966,0.1978487661213037,57.8058061415511,1677.8752235093843,0.3476662674377067,0.1797672209080605,0.1494297959225055,Econ/Fin,1.0 -m4_quarterly/Q/short,TEMPO_ENSEMBLE,2514810.428,2514810.428,810.4905891927083,2.182249153802416,0.185947568098704,0.1463373041947682,23.608043900790165,1585.8153826974942,0.265437569067886,0.1356618520005551,0.1109355578587279,Econ/Fin,1.0 -m4_weekly/W/short,TEMPO_ENSEMBLE,541471.1017784444,541471.1017784444,439.3405694236126,4.832172082392305,0.1009065322761543,0.1079463725067849,45.31173448032055,735.8472000207953,0.1340599430870445,0.0800410353278435,0.0637560156785005,Econ/Fin,1.0 -m4_yearly/A/short,TEMPO_ENSEMBLE,3878844.870897536,3878844.870897536,945.4389228040392,2.3981068438654534,0.2092134684314751,0.158700357189562,30.97324149680404,1969.478324556413,0.3158149780005754,0.1516055134414823,0.1267824542222521,Econ/Fin,1.0 -restaurant/D/short,TEMPO_ENSEMBLE,154.6462244326142,154.6462244326142,7.553120731472402,1.9378580999513804,0.6267567772908494,0.4301861248252114,17.613543161794635,12.43568351288397,0.5723189684521193,0.3476121164672595,0.2785855176058847,Sales,1.0 -temperature_rain/D/short,TEMPO_ENSEMBLE,196.24055072576525,196.24055072576525,5.971279438835857,1.1188085728690171,25.89721481389996,1.5564323267332705,23.68132606745661,14.008588463002448,1.6491812374036674,0.7029774655619903,0.6018449486123362,Nature,1.0 -loop_seattle/5T/long,TEMPO_ENSEMBLE,45.255009312113,45.255009312113,3.82509889249011,0.609772921780666,0.115840638970689,0.0808924666786259,5.58609892597957,6.727184352469688,0.1138146269817346,0.064715366906391,0.0521126066018827,Transport,1.0 -loop_seattle/5T/medium,TEMPO_ENSEMBLE,33.792408340186405,33.792408340186405,3.272990517316096,0.5214548025956122,0.0851551320653704,0.0641574094711319,4.822810274952749,5.813123802241477,0.0960772879128691,0.0540948486503963,0.0436714854292893,Transport,1.0 -loop_seattle/5T/short,TEMPO_ENSEMBLE,25.731941133386947,25.731941133386947,2.6942812042708737,0.4298290643570778,0.07453944464094,0.0555199753512292,3.882450724435325,5.072666077457391,0.0858552006954052,0.0456008832417526,0.0365482163898581,Transport,1.0 -loop_seattle/D/short,TEMPO_ENSEMBLE,201.8897832817337,201.8897832817337,11.740500435371516,2.774823575326353,0.2117016866111165,0.1850822204652831,53.69285838634296,14.208792463884246,0.2539301027317229,0.2098184268123754,0.1896011897345642,Transport,1.0 -loop_seattle/H/long,TEMPO_ENSEMBLE,107.18005890952873,107.18005890952873,5.884305998989508,1.2059726001740343,0.2284862642973856,0.1319734503159131,10.89359255795443,10.35278025022886,0.1850180328102056,0.105160420106692,0.0858770469329422,Transport,1.0 -loop_seattle/H/medium,TEMPO_ENSEMBLE,102.0016705366357,102.0016705366357,5.591995009352425,1.1218170402210388,0.2061506399306227,0.1263956800210832,10.534682855933928,10.099587641910718,0.1785090812377754,0.0988378859413388,0.0806802675836623,Transport,1.0 -loop_seattle/H/short,TEMPO_ENSEMBLE,63.93484438226468,63.93484438226468,4.569113424888654,0.9321286612874126,0.1389670075459092,0.1052935476325374,7.722207374375397,7.995926736924537,0.1415360365351048,0.0808779552283411,0.0647137448640539,Transport,1.0 -m_dense/D/short,TEMPO_ENSEMBLE,5768.324444444444,5768.324444444444,37.38796875,0.4039148995409595,0.1074763044239946,0.0843358188205295,5.089627880380552,75.94948613680309,0.1314797949034482,0.0647241043112796,0.0539366130007633,Transport,1.0 -m_dense/H/long,TEMPO_ENSEMBLE,120260.29827160494,120260.29827160494,162.72006172839505,1.3319808133687625,0.4192954648363726,0.3339448905285493,10.119297399210735,346.7856661853326,0.6003372524100504,0.2816924812509744,0.221528231064794,Transport,1.0 -m_dense/H/medium,TEMPO_ENSEMBLE,95303.87555555555,95303.87555555555,142.89605034722223,1.1974521911956586,0.3782484602009441,0.3119469536675347,9.506814665422915,308.7132578227821,0.5371220668355627,0.2486210746063868,0.1993907938331504,Transport,1.0 -m_dense/H/short,TEMPO_ENSEMBLE,39899.67444444444,39899.67444444444,87.75685763888889,0.7629589384918614,0.2737046016924269,0.2153883404201931,6.4551554438991126,199.749028644558,0.3543050243689524,0.1556588074308831,0.1239959977566907,Transport,1.0 -sz_taxi/15T/long,TEMPO_ENSEMBLE,17.11225683315527,17.11225683315527,2.807824567919783,0.5592388252412251,891763845532.28,0.4074579570368145,4.479720970305226,4.136696367048864,0.3854328199908255,0.2616164313807031,0.2072789439258874,Transport,1.0 -sz_taxi/15T/medium,TEMPO_ENSEMBLE,17.181647858796296,17.181647858796296,2.802406906606125,0.5587890281052588,980879675330.1469,0.4046340985855146,4.435455309207469,4.145075133070123,0.3862134936765363,0.2611116390794976,0.2067451524244509,Transport,1.0 -sz_taxi/15T/short,TEMPO_ENSEMBLE,16.188086902585805,16.188086902585805,2.710469068217481,0.5421277246410363,1559028962787.2595,0.3815988051585662,3.9974291375358737,4.023442170901156,0.3754715629735235,0.252943627423149,0.1990819241581065,Transport,1.0 -sz_taxi/H/short,TEMPO_ENSEMBLE,7.189825987204527,7.189825987204527,1.849522386860644,0.5637851957501169,1.014901242704473,0.2964422356369149,4.431838456166187,2.681385087450985,0.2497875734919494,0.1722944277176315,0.136247323200995,Transport,1.0 -bitbrains_fast_storage/5T/long,TEMPO_ENSEMBLE,2440224.267545323,2440224.267545323,229.6813110807334,1.117978597062503,2.251204239869468,0.8221634438380008,27.51323770956846,1562.121719823818,4.745113207690169,0.6976817548453914,0.5854367008276827,Web/CloudOps,2.0 -bitbrains_fast_storage/5T/medium,TEMPO_ENSEMBLE,2631657.734381467,2631657.734381467,232.32689319711096,1.4213753528808146,2.3469343026571554,0.8229977424824282,36.54996504533897,1622.238494914193,4.928808505230026,0.7058732552417682,0.5948226525152782,Web/CloudOps,2.0 -bitbrains_fast_storage/5T/short,TEMPO_ENSEMBLE,2230525.403204316,2230525.403204316,205.1374160586084,1.064509249551392,1.617535049043656,0.7656597272656195,19.9251692395756,1493.4943599506212,3.9417512832669366,0.5414152839665344,0.4378417919631492,Web/CloudOps,2.0 -bitbrains_fast_storage/H/short,TEMPO_ENSEMBLE,2005001.4583279784,2005001.4583279784,235.38284965169785,1.468075876452505,2.1154490817398,0.6492987395299152,30.887620124678627,1415.9807408040472,4.036005106798449,0.6709175879804512,0.5386730388224907,Web/CloudOps,2.0 -bitbrains_rnd/5T/long,TEMPO_ENSEMBLE,2049491.817123375,2049491.817123375,154.23860818175223,5.354332726402957,1.044259314751908,0.7570510199845623,190.4855143937792,1431.604630169718,5.963076180299257,0.6424515198879186,0.5788157952641853,Web/CloudOps,2.0 -bitbrains_rnd/5T/medium,TEMPO_ENSEMBLE,2012628.349056859,2012628.349056859,154.0856175924186,2.0206660007438004,1.2854108619095217,0.7562830102005815,50.702933072419086,1418.6713322883702,5.969073466936588,0.6483167388205798,0.5803418284708388,Web/CloudOps,2.0 -bitbrains_rnd/5T/short,TEMPO_ENSEMBLE,1092676.6090575084,1092676.6090575084,102.72774254536428,1.4738776261427329,0.6909973121152531,0.724109150334899,27.830770596585733,1045.3117281737102,4.614003515916433,0.4534400146017512,0.3903323828435197,Web/CloudOps,2.0 -bitbrains_rnd/H/short,TEMPO_ENSEMBLE,1373237.396530412,1373237.396530412,140.5587084683091,6.4469252489463615,1.230340532002289,0.6496405541543678,212.9778414003044,1171.8521222963298,5.128598282316404,0.6151536845814527,0.5300669828610838,Web/CloudOps,2.0 -bizitobs_l2c/5T/long,TEMPO_ENSEMBLE,28.02539958112875,28.02539958112875,3.134216545414462,0.2992086737079788,0.2278988936816867,0.5175256799768518,1.9222874573646005,5.293902112915269,0.2724096828546203,0.1612781870392175,0.1255549472781602,Web/CloudOps,7.0 -bizitobs_l2c/5T/medium,TEMPO_ENSEMBLE,23.050455729166668,23.050455729166668,2.8030650111607143,0.28310808843281,0.217644268352139,0.6169422549293154,2.255901530024711,4.801089014918039,0.2471325504016436,0.1442857241341077,0.1131950526630068,Web/CloudOps,7.0 -bizitobs_l2c/5T/short,TEMPO_ENSEMBLE,3.1966607956659225,3.1966607956659225,1.1986464727492558,0.1151517649409104,0.0398835409255254,0.038441853296189,0.667002474954807,1.7879208024031497,0.0520254942308455,0.034878600365867,0.0269425800159113,Web/CloudOps,7.0 -bizitobs_l2c/H/long,TEMPO_ENSEMBLE,16.538295200892858,16.538295200892858,2.070875573536706,0.2180866660308821,0.2278338753179033,0.6110211084759425,1.3445094546037508,4.066730283765184,0.2484076230387728,0.1264950569465923,0.097937862969621,Web/CloudOps,7.0 -bizitobs_l2c/H/medium,TEMPO_ENSEMBLE,12.969489397321428,12.969489397321428,1.750689697265625,0.1794982441293087,0.1523565293116185,0.6096963065011161,1.113684244320353,3.6013177306815667,0.2180652586802799,0.1060069203143813,0.0810585643608789,Web/CloudOps,7.0 -bizitobs_l2c/H/short,TEMPO_ENSEMBLE,4.876285129123264,4.876285129123264,1.302290780203683,0.13201672288296,0.1161148839717288,0.436991464524042,0.9056123725875996,2.2082312218432345,0.1190293182352498,0.0701968082773084,0.0558677005802573,Web/CloudOps,7.0 -electricity/15T/long,TEMPO_ENSEMBLE,133835.22996771772,133835.22996771772,53.456839245495495,0.9939385780309388,0.1802039587686282,0.1575861328858155,9.52136853864833,365.8349764138439,0.6912502396245063,0.1010074359216439,0.0823081285199345,Energy,1.0 -electricity/15T/medium,TEMPO_ENSEMBLE,147086.25630236487,147086.25630236487,58.9102391258446,1.1008310035581168,0.2031415438085511,0.1748311696095509,10.063220095437282,383.5182607156599,0.731616319583604,0.1123797658411746,0.0906301323705994,Energy,1.0 -electricity/15T/short,TEMPO_ENSEMBLE,53514.92639850788,53514.92639850788,42.72801652238176,0.8701977534359403,0.1565123488839787,0.1385838226799492,6.664328952354419,231.33293409825563,0.424854101914554,0.078472065194558,0.0622887006592235,Energy,1.0 -electricity/D/short,TEMPO_ENSEMBLE,1773811077.968721,1773811077.968721,4735.994360360361,1.115364338640572,0.5424773373398062,0.1024850777290962,10.020402740809686,42116.6365937348,0.6921162106217391,0.0778281158070003,0.0615318072875596,Energy,1.0 -electricity/H/long,TEMPO_ENSEMBLE,21656666.85463063,21656666.85463063,496.1748430930931,2.0531234768133286,0.3964748661654531,0.2308328186585022,13.780891404513913,4653.672405168914,1.835406963645144,0.1956912053343266,0.1479181783703783,Energy,1.0 -electricity/H/medium,TEMPO_ENSEMBLE,11332664.060585586,11332664.060585586,370.4004170185811,1.6193840624647802,0.2917618788278519,0.192386435431403,11.603075966381072,3366.402242838129,1.3140177023969903,0.144579485702608,0.1126590158033672,Energy,1.0 -electricity/H/short,TEMPO_ENSEMBLE,1832069.8023648649,1832069.8023648649,198.91421179969032,1.0666581008254952,0.2378714794262591,0.1420216092977438,10.306280643530442,1353.5397306192622,0.6389674211684946,0.0939017142032927,0.0757270380140367,Energy,1.0 -electricity/W/short,TEMPO_ENSEMBLE,46064089239.29369,46064089239.29369,29281.876126126128,6.7051638386670405,0.2451873894841502,0.1105028925715266,78.41860400355218,214625.4627002437,0.4897044792472421,0.0668115782690427,0.0571458143436201,Energy,1.0 -ett1/15T/long,TEMPO_ENSEMBLE,2.426123046875,2.426123046875,0.8594006541418651,0.6251000870679456,0.2925888486169558,0.2088383556547619,4.284138746297137,1.5576016971212503,0.2899107367971693,0.1599571169617456,0.1245837835659594,Energy,7.0 -ett1/15T/medium,TEMPO_ENSEMBLE,3.665840308779762,3.665840308779762,0.9739117140997025,0.6656957195358719,0.3419423155295361,0.2164743913922991,4.975116861543028,1.9146384276880484,0.3483354661955551,0.1771864525741701,0.140029793907285,Energy,7.0 -ett1/15T/short,TEMPO_ENSEMBLE,0.6365459623790922,0.6365459623790922,0.4495159694126674,0.3504219026354622,0.148824625845035,0.1175648008074079,2.5677515620110003,0.7978383059110989,0.1424446592250454,0.0802557969513414,0.0629222896778817,Energy,7.0 -ett1/D/short,TEMPO_ENSEMBLE,8428.484126984127,8428.484126984127,50.722237723214285,0.416001643493369,0.6177615211123512,0.1943805270724826,2.4777156566549183,91.806776040683,0.241133542532727,0.1332236398538365,0.1027991523160863,Energy,7.0 -ett1/H/long,TEMPO_ENSEMBLE,186.56152447089949,186.56152447089949,6.966465153769842,1.1478718553875968,706542928866.8086,0.434202228525959,7.452181852979635,13.658752668926232,0.6540880825104363,0.3336089278980483,0.2606986690022358,Energy,7.0 -ett1/H/medium,TEMPO_ENSEMBLE,210.31737351190475,210.31737351190475,7.783360072544643,1.2921248816752966,2394988688740.979,0.4857444036574591,7.670278067717789,14.502323038461968,0.6946268959068199,0.3728044970849133,0.2862649623279022,Energy,7.0 -ett1/H/short,TEMPO_ENSEMBLE,36.55992838541667,36.55992838541667,3.31173822312128,0.6173025444422964,0.308757091703869,0.1946493421282087,4.307629355652223,6.0464806611298005,0.2822139896318907,0.1545723717553161,0.1206314820077215,Energy,7.0 -ett1/W/short,TEMPO_ENSEMBLE,680088.7142857143,680088.7142857143,625.6919642857143,6.086432786812403,0.5392628056662423,0.3573795161021351,37.66618612480504,824.6749143060642,0.328157831536192,0.248977766448001,0.1820576488837628,Energy,7.0 -ett2/15T/long,TEMPO_ENSEMBLE,5.956928323412699,5.956928323412699,1.6044109623015872,0.7047809146350211,0.1055386204469079,0.1197530691964285,4.620370019558849,2.440681938191189,0.123840215227059,0.0814078212220199,0.0636566074806678,Energy,7.0 -ett2/15T/medium,TEMPO_ENSEMBLE,5.548760230654762,5.548760230654762,1.5561342075892857,0.671412971844876,0.0967213048749063,0.1097720918201264,4.294814353598369,2.3555806567924527,0.1146702656271786,0.0757530091025556,0.0591136068282064,Energy,7.0 -ett2/15T/short,TEMPO_ENSEMBLE,2.1440130324590774,2.1440130324590774,1.039876229422433,0.4684627317471617,0.0646613530082772,0.1166317985171363,2.836496571230992,1.4642448676567308,0.0650103948386862,0.0461690293415903,0.0356368146844352,Energy,7.0 -ett2/D/short,TEMPO_ENSEMBLE,50878.18095238095,50878.18095238095,143.57472718253968,0.5795335298098135,0.115604255312965,0.0856832716200086,3.9209683971288687,225.561922656243,0.1290950395867559,0.0821716044579683,0.0652697198570775,Energy,7.0 -ett2/H/long,TEMPO_ENSEMBLE,146.3332175925926,146.3332175925926,7.887528935185185,0.938929495976326,0.1597060509673862,0.160266290896784,6.476700944863757,12.09682675715382,0.1656567320337467,0.1080136380767531,0.0849789777142148,Energy,7.0 -ett2/H/medium,TEMPO_ENSEMBLE,116.4034412202381,116.4034412202381,7.034888857886905,0.8417988347064325,0.1428101438678359,0.1461324146815708,5.497659780958897,10.789042646140484,0.1453892878304396,0.0947996513277619,0.0746216000768869,Energy,7.0 -ett2/H/short,TEMPO_ENSEMBLE,83.8348400297619,83.8348400297619,5.616767810639881,0.657868124921435,0.1062725054061009,0.1005470820835658,4.51715751563887,9.15613674153908,0.1161252899482322,0.0712362439524702,0.0559794964806631,Energy,7.0 -ett2/W/short,TEMPO_ENSEMBLE,50153408.0,50153408.0,5750.481584821428,18.879777231189173,0.5354306357247489,0.7778287484025764,145.92744525176246,7081.907087783629,0.5940634324121872,0.4823772447106784,0.3722205310770798,Energy,7.0 -hierarchical_sales/D/short,TEMPO_ENSEMBLE,26.144174374741628,26.144174374741628,2.2159887987998657,0.6776781916278962,0.6214346639809414,1.0697553971327254,7.844630481376959,5.11313742967482,1.5686539388430278,0.6798408228762413,0.5596516023662144,Sales,1.0 -hierarchical_sales/W/short,TEMPO_ENSEMBLE,208.41687632415253,208.41687632415253,6.893525980286679,1.5405539950087042,0.501563509229671,0.3862917948577363,12.07970736684027,14.436650453763592,0.6627936838232559,0.3164851496293178,0.2529514701591871,Sales,1.0 -jena_weather/10T/long,TEMPO_ENSEMBLE,1494.6670006613756,1494.6670006613756,9.313304398148148,0.6309415613166219,0.5533342606620286,0.6102226200810185,7.968598337655945,38.66092343260021,0.2365608812169572,0.0569868306252071,0.0484899654114696,Nature,21.0 -jena_weather/10T/medium,TEMPO_ENSEMBLE,1428.2385615079363,1428.2385615079363,9.73109343998016,0.770657587644626,0.45034885466974,0.5904903351314484,7.660509410812896,37.79204362703791,0.2300681949417704,0.0592403820402081,0.0492121746241922,Nature,21.0 -jena_weather/10T/short,TEMPO_ENSEMBLE,356.6833209325397,356.6833209325397,6.77793705047123,0.6259220489380137,0.1543256824463067,0.4635223751976376,4.5303511138071695,18.886061551645422,0.1160439813650122,0.0416465232111659,0.0315293674377684,Nature,21.0 -jena_weather/D/short,TEMPO_ENSEMBLE,409.1419642857143,409.1419642857143,9.749877542162698,1.3035383788786572,0.5480121025175815,0.3650206308516245,13.435620298887152,20.22725795271604,0.1218011527955401,0.0587101982397621,0.049186717650696,Nature,21.0 -jena_weather/H/long,TEMPO_ENSEMBLE,3616.809523809524,3616.809523809524,16.37919560185185,1.02478500142865,2.041743679492345,0.7062990864748677,8.303475084254293,60.13991622715752,0.3621089262445626,0.0986209044544351,0.0830668877325188,Nature,21.0 -jena_weather/H/medium,TEMPO_ENSEMBLE,2571.667857142857,2571.667857142857,15.079837859623016,0.948315476755886,1.4699010265727828,0.7289495558965774,10.947285563727604,50.7116146177861,0.3103133637080175,0.0922762023189436,0.0783835088305258,Nature,21.0 -jena_weather/H/short,TEMPO_ENSEMBLE,1018.5410792606516,1018.5410792606516,8.795254330487678,0.6307897479895602,1.1392179782126224,0.6025232810623564,6.694776691348025,31.91459038215361,0.1956343923102162,0.0539143449925314,0.0444314398134425,Nature,21.0 -kdd_cup_2018/D/short,TEMPO_ENSEMBLE,1516.345084031518,1516.345084031518,11.394115079439292,0.8988963505503572,0.3130111050055663,0.2621102319434963,8.453911474684531,38.94027585972547,0.8723035410275897,0.2552402804354663,0.2098857562368817,Nature,1.0 -kdd_cup_2018/H/long,TEMPO_ENSEMBLE,3226.9120112959263,3226.9120112959263,19.277435912975083,0.8769099918367069,1.0711117767331302,0.4946325637606668,8.774512919523671,56.80591528437797,1.3328962906449948,0.4523265348849569,0.3713646975262221,Nature,1.0 -kdd_cup_2018/H/medium,TEMPO_ENSEMBLE,4411.474204999715,4411.474204999715,21.78959218251548,0.9389850331762508,0.906548250640067,0.4969745987220428,8.641128078775742,66.41892956830692,1.3902684103310523,0.4560956022965295,0.3721063901737007,Nature,1.0 -kdd_cup_2018/H/short,TEMPO_ENSEMBLE,4213.191972082293,4213.191972082293,18.50047424942846,0.7755335648478038,0.7064144683127912,0.4231625609033334,7.290873307077078,64.9091054635811,1.358665135900395,0.3872484327245064,0.317560729859845,Nature,1.0 -saugeen/D/short,TEMPO_ENSEMBLE,100.51135416666666,100.51135416666666,2.6642049153645835,0.3402823763010718,0.0503207906087239,0.0514753182729085,2.1419360602772457,10.025535106250771,0.3247294708297791,0.0862942319965577,0.0661465559255415,Nature,1.0 -saugeen/M/short,TEMPO_ENSEMBLE,4.164726983933222,4.164726983933222,1.347757066999163,0.0789051595695627,0.0377655313128516,0.0387396386691502,0.8427224228584025,2.04076627371515,0.0612699257784091,0.040463710384647,0.0340753637601601,Nature,1.0 -saugeen/W/short,TEMPO_ENSEMBLE,17.543472290039062,17.543472290039062,2.67932243347168,0.1533740309245224,0.0759897470474243,0.0801339026334025,1.5005436040659956,4.188492842304862,0.1264331392828559,0.0808775755788074,0.0707038225619495,Nature,1.0 -solar/10T/long,TEMPO_ENSEMBLE,15.809275712185723,15.809275712185723,1.8056089219637064,0.7935415450083824,2.3277865434987204,1.4880007555885038,5.265772326000681,3.976087991001422,0.9494284174682992,0.4311515301538582,0.3259064764784643,Energy,1.0 -solar/10T/medium,TEMPO_ENSEMBLE,13.317824712971412,13.317824712971412,1.4643280073848084,0.6403629744856407,1.6871777505154564,1.4886846107055962,5.860875318184641,3.64935949352368,0.9297285097230794,0.3730592994380783,0.2855840260224301,Energy,1.0 -solar/10T/short,TEMPO_ENSEMBLE,4.579851478149712,4.579851478149712,0.9389028396049556,0.4148577858316679,1.290939476594771,1.4926070684354091,3.4334659028362795,2.140058755770437,1.3168532900106638,0.5777398821412396,0.461325856836472,Energy,1.0 -solar/D/short,TEMPO_ENSEMBLE,227997.78199513385,227997.78199513385,364.8942746350365,1.1822038893743294,1.1243099463247035,0.5479662967134277,26.830195352743505,477.4911328968673,0.6898120741982009,0.5271479596321818,0.4800547635005184,Energy,1.0 -solar/H/long,TEMPO_ENSEMBLE,2874.481508515815,2874.481508515815,33.32977699842863,2.5475516817625423,6.921687556510294,1.6789198232683242,20.71382607291551,53.61419129778808,1.858901945988148,1.1556042499569716,0.9908022974231144,Energy,1.0 -solar/H/medium,TEMPO_ENSEMBLE,3328.179333941606,3328.179333941606,27.74560725079836,2.1192776144205463,0.982378255220553,1.972083945169936,13.399058473722926,57.69037470793205,2.0781418580377284,0.9994614889653576,0.7343401482401601,Energy,1.0 -solar/H/short,TEMPO_ENSEMBLE,406.0480185122935,406.0480185122935,9.035267002742229,0.6971424677787719,2.1446248619803527,1.4069967350253358,7.611412271966008,20.150633203755497,0.7427590622937399,0.3330429559543392,0.2607230327213033,Energy,1.0 -solar/W/short,TEMPO_ENSEMBLE,25620636.96350365,25620636.96350365,4588.206660583942,59.20451480380479,0.9345781089615648,1.7547384193792417,517.900837376737,5061.683214455805,1.0332160923962808,0.9365676902529754,0.6280375794904386,Energy,1.0 -us_births/D/short,TEMPO_ENSEMBLE,186916.96,186916.96,239.07765625,0.2262047828348501,0.0234775368372599,0.0226943778991699,2.5604752821394974,432.3389411098658,0.0405282675008943,0.0224115902701547,0.0185739392260598,Healthcare,1.0 -us_births/M/short,TEMPO_ENSEMBLE,145867776.0,145867776.0,9889.141927083334,1.2073705555270309,0.0316123714049657,0.0308461654110728,8.33813811617069,12077.573266182242,0.0375128602209077,0.0307156074022029,0.0255509986968776,Healthcare,1.0 -us_births/W/short,TEMPO_ENSEMBLE,21440150.85714286,21440150.85714286,4361.426897321428,3.115257411522497,0.0587949837957109,0.0607828675922443,41.82710839191516,4630.351051177746,0.0628565614933249,0.0592059424739289,0.0511111792570683,Healthcare,1.0 \ No newline at end of file diff --git a/results/tempo_ensemble/config.json b/results/tempo_ensemble/config.json deleted file mode 100644 index 895495f909c32f9d4c15139ceddb60fcd4a19654..0000000000000000000000000000000000000000 --- a/results/tempo_ensemble/config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "model": "TEMPO_ENSEMBLE", - "model_type": "fine-tuned", - "model_dtype": "float32", - "model_link": "https://huggingface.co/Melady/TEMPO", - "org": "Melady Lab @ USC", - "testdata_leakage": "Yes" -} \ No newline at end of file diff --git a/results/tft/config.json b/results/tft/config.json index 93c64bcee69aa2574edcf6ec754926eaf6072f65..d5357d9b333d4c161a005369bd344ad93e2e574f 100644 --- a/results/tft/config.json +++ b/results/tft/config.json @@ -1,7 +1,5 @@ { "model": "TFT", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Google Research", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/tide/config.json b/results/tide/config.json index 3a6cae46535106f7843ef58a6567393e736f6f49..678beca2330d8f0601496cebca4e9c3421e5acd4 100644 --- a/results/tide/config.json +++ b/results/tide/config.json @@ -1,7 +1,5 @@ { "model": "TIDE", "model_type": "deep-learning", - "model_dtype": "float32", - "org": "Google Research", - "testdata_leakage": "No" + "model_dtype": "float32" } \ No newline at end of file diff --git a/results/timer_small/config.json b/results/timer_small/config.json new file mode 100644 index 0000000000000000000000000000000000000000..05b27811e7cca7b1474dd39296a6a109d9747557 --- /dev/null +++ b/results/timer_small/config.json @@ -0,0 +1,5 @@ +{ + "model": "timer_small", + "model_type": "pretrained", + "model_dtype": "float32" +} \ No newline at end of file diff --git a/results/timesfm/config.json b/results/timesfm/config.json index 6b1b2e57e782d14c74258cb366f6d20e2c98895d..8e553d7324a1dcd00a4cfec4514f31da50434568 100644 --- a/results/timesfm/config.json +++ b/results/timesfm/config.json @@ -2,7 +2,5 @@ "model": "TimesFM", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://huggingface.co/google/timesfm-1.0-200m", - "org": "Google Research", - "testdata_leakage": "Yes" + "model_link": "https://huggingface.co/google/timesfm-1.0-200m" } \ No newline at end of file diff --git a/results/timesfm_2_0_500m/config.json b/results/timesfm_2_0_500m/config.json index 3d57139772f3c001cb2b860fe0e00ea96eea060e..8ef72136c33397d0cd202892c4270a690552c3c5 100644 --- a/results/timesfm_2_0_500m/config.json +++ b/results/timesfm_2_0_500m/config.json @@ -3,7 +3,5 @@ "model_type": "pretrained", "model_dtype": "float32", "model_link": "https://huggingface.co/google/timesfm-2.0-500m-jax", - "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm.ipynb", - "org": "Google Research", - "testdata_leakage": "Yes" + "code_link": "https://github.com/SalesforceAIResearch/gift-eval/blob/main/notebooks/timesfm.ipynb" } \ No newline at end of file diff --git a/results/visionts/config.json b/results/visionts/config.json index caa2d1b48a2e814034e39ff26247bbdd89b1ac69..ca0ac5b4842d868567d6c623a0d63a27831b1bee 100644 --- a/results/visionts/config.json +++ b/results/visionts/config.json @@ -2,7 +2,5 @@ "model": "VisionTS", "model_type": "pretrained", "model_dtype": "float32", - "model_link": "https://github.com/Keytoyze/VisionTS", - "org": "Zhejiang University", - "testdata_leakage": "No" + "model_link": "https://github.com/Keytoyze/VisionTS" } \ No newline at end of file diff --git a/src/about.py b/src/about.py index cc2699cd4ad3bf408c9132093ea3f4d02c1a9631..d12d4e1419206f58a37f82ac704ddbd4812fbb03 100644 --- a/src/about.py +++ b/src/about.py @@ -42,16 +42,7 @@ points, spanning seven domains, 10 frequencies, multivariate inputs, and predict # Which evaluations are you running? how can people reproduce what you have? LLM_BENCHMARKS_TEXT = f""" -## Update Log - -### 2025โ€‘07โ€‘24 -- Corrected the Naive and Seasonalโ€ฏNaive scores to match the latest GIFTโ€‘Eval notebooks. Most model rankings remain unchanged; only a few near the bottom shifted slightly (AutoETS and Timer each dropped two places now at 35th and 36th places respectively, while NBEATS moved up one now at 27th place). - -### 2025-08-05 -- Added new columns to the leaderboard: Organization, TestData Leakage, and MASE_Rank. TestData Leakage is a binary indicator specifying whether any test data was present in the training set. MASE_Rank reflects the model's ranking based on the MASE metric, aligned with the ranking scheme used for CRPS_Rank. These additions were made in response to multiple requests from independent groups seeking fairer comparisons. With these updates, the leaderboard now supports sorting by models that do not leak test data, and viewers can choose to rank models based on either MASE_Rank or CRPS_Rank, depending on their use case. -- Added new model type: Agentic to indicate submissions that use agentic system to generate the forecasts. - -## How It Works +How It Works To participate in the GIFT-Eval leaderboard, follow these steps to evaluate your Time Series Model: @@ -70,6 +61,7 @@ cd gift-eval Follow the instruction in the README.md file to install the required dependencies, set up your environment and obtain the evaluation results. + """ EVALUATION_QUEUE_TEXT = """ diff --git a/src/display/utils.py b/src/display/utils.py index d2d7725c6c3a51071ae4a9afda89740d6f631911..85f2cdb969c269c3b61dbbc3044d6029bfc66c7b 100644 --- a/src/display/utils.py +++ b/src/display/utils.py @@ -34,8 +34,6 @@ model_info_dict.append(["license", ColumnContent, ColumnContent("Hub License", " model_info_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False, True)]) model_info_dict.append(["likes", ColumnContent, ColumnContent("Hub โค๏ธ", "number", False, True)]) model_info_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)]) -model_info_dict.append(["org", ColumnContent, ColumnContent("Organization", "str", True, hidden=False)]) -model_info_dict.append(["testdata_leakage", ColumnContent, ColumnContent("TestData Leakage", "str", True, hidden=False)]) # model_info_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) # We use make dataclass to dynamically fill the scores from Tasks @@ -61,11 +59,9 @@ class ModelDetails: class ModelType(Enum): PT = ModelDetails(name="๐ŸŸข pretrained", symbol="๐ŸŸข") - FT = ModelDetails(name="๐ŸŸฃ fine-tuned", symbol="๐ŸŸฃ") - AG = ModelDetails(name="๐ŸŸก agentic", symbol="๐ŸŸก") + FT = ModelDetails(name="๐Ÿ”ถ fine-tuned", symbol="๐Ÿ”ถ") DL = ModelDetails(name="๐Ÿ”ท deep-learning", symbol="๐Ÿ”ท") - ST = ModelDetails(name="๐Ÿ”ถ statistical", symbol="๐Ÿ”ถ") - + ST = ModelDetails(name="๐ŸŸฃ statistical", symbol="๐ŸŸฃ") Unknown = ModelDetails(name="", symbol="?") @@ -78,8 +74,6 @@ class ModelType(Enum): return ModelType.FT if "pretrained" in type or "๐ŸŸข" in type: return ModelType.PT - if "agentic" in type or "๐ŸŸก" in type: - return ModelType.AG if "deep-learning" in type or "๐ŸŸฆ" in type: return ModelType.DL if "statistical" in type or "๐ŸŸฃ" in type: diff --git a/src/leaderboard/read_evals.py b/src/leaderboard/read_evals.py index 99710a0161d3ae29d835b8c6285055ffc08a9ca4..49659668c33dbbef683cf21d872b9e5321d2c046 100644 --- a/src/leaderboard/read_evals.py +++ b/src/leaderboard/read_evals.py @@ -21,11 +21,9 @@ class ModelConfig: model_type: ModelType = ModelType.Unknown code_link: str = "" precision: Precision = Precision.Unknown - org: str = "" license: str = "?" likes: int = 0 num_params: int | str = 0 - testdata_leakage: str = "NA" @classmethod def init_from_json_file(cls, json_filepath): @@ -41,10 +39,7 @@ class ModelConfig: model = data.get("model", "") model_link = data.get("model_link", "") code_link = data.get("code_link", "") - org = data.get("org", "") - testdata_leakage = data.get("testdata_leakage", "N/A") - return cls(model=model, model_link=model_link, model_type=model_type, code_link=code_link, org=org, - precision=precision, testdata_leakage=testdata_leakage) + return cls(model=model, model_link=model_link, model_type=model_type, code_link=code_link, precision=precision) def to_dict(self): """Converts the model info to a dict compatible with our dataframe display""" @@ -58,8 +53,6 @@ class ModelConfig: ModelInfoColumn.license.name: self.license, ModelInfoColumn.likes.name: self.likes, ModelInfoColumn.params.name: self.num_params, - ModelInfoColumn.org.name: self.org, - ModelInfoColumn.testdata_leakage.name: self.testdata_leakage, } return data_dict @@ -70,14 +63,14 @@ class EvalResult: """ eval_name: str # org_model_precision (uid) full_model: str # org/model (path on hub) - org: str + org: str model: str revision: str # commit hash, "" if main results: dict precision: Precision = Precision.Unknown model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ... weight_type: WeightType = WeightType.Original # Original or Adapter - architecture: str = "Unknown" + architecture: str = "Unknown" license: str = "?" likes: int = 0 num_params: int = 0 @@ -137,7 +130,7 @@ class EvalResult: org=org, model=model, results=results, - precision=precision, + precision=precision, revision= config.get("model_sha", ""), still_on_hub=still_on_hub, architecture=architecture @@ -244,7 +237,6 @@ def get_model_info(results_path: str, requests_path: str) -> list[ModelConfig]: for v in model_infos.values(): try: v.to_dict() # we test if the dict version is complete - # ipdb.set_trace() results.append(v) except KeyError: # not all eval values present continue diff --git a/src/utils.py b/src/utils.py index 08458bca90bbee059252027012c505504cf16aec..b23d44871a222b1804598123234fdce9f6dc75d5 100644 --- a/src/utils.py +++ b/src/utils.py @@ -48,8 +48,7 @@ def rename_metrics(df): df = df.rename(columns={ 'eval_metrics/MASE[0.5]': 'MASE', 'eval_metrics/mean_weighted_sum_quantile_loss': 'CRPS', - 'rank': 'CRPS_Rank', - 'Rank_MASE': 'MASE_Rank', + 'rank': 'Rank' }) return df @@ -106,59 +105,6 @@ def pivot_existed_df(df, tab_name): return df_pivot -def get_all_res_dfs(root_dir='results', ds_properties='results/dataset_properties.csv'): - df_list = [] - - # Walk through all folders and subfolders in the root directory - for subdir, _, files in os.walk(root_dir): - for file in files: - if file == 'all_results.csv': - file_path = os.path.join(subdir, file) - df = pd.read_csv(file_path) - df_list.append(df) - # Concatenate all dataframes into one - all_results_df = pd.concat(df_list, ignore_index=True) - all_results_df = all_results_df.sort_values(by=['model', 'dataset']).reset_index(drop=True) - all_results_df[['dataset', 'frequency', 'term_length']] = all_results_df['dataset'].str.split('/', expand=True) - - dataset_properties = pd.read_csv(ds_properties) - # Reforemat the the first element of each row after the header following these rules: - # 1. make all characters lowercase - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.lower()) - # 2. replace all spaces with underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.replace(' ', '_')) - # 3. Replace all dashes with underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.replace('-', '_')) - # 4. Replace consecutive underscores with a single underscore. There maybe more than 2 consecutive underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: re.sub('_+', '_', x)) - # 5. Remove all leading and trailing underscores - dataset_properties['dataset'] = dataset_properties['dataset'].apply(lambda x: x.strip('_')) - - df = all_results_df - - # convert it to a dictionary, with dataset as the key, and the value as another dictionary. The inner dictionary has the column names as the key, and the value as the value. - dataset_properties_dict = dataset_properties.set_index('dataset').T.to_dict('dict') - dataset_properties_dict.keys() - - # # match the dataset name in model_properties_dict with the dataset name in df and add a new column for each key value pair in the inner dictionary. - for dataset in dataset_properties_dict.keys(): - for key in dataset_properties_dict[dataset].keys(): - # set the row with the dataset name to the value of the key think step by step - # First, get the row with the dataset name - # Second, set the value of the key to the value of the key - if key == 'frequency': - # only set the frequency if the frequency column for all rows for the dataset is empty string - if all(df[df['dataset'] == dataset]['frequency'].isna()): - df.loc[df['dataset'] == dataset, key] = dataset_properties_dict[dataset][key] - else: - df.loc[df['dataset'] == dataset, key] = dataset_properties_dict[dataset][key] - - # unify the frequency - df = unify_freq(df) - # standardize by seasonal naive - df = standardize_df(df) - ipdb.set_trace() - return None def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_properties.csv'): df_list = [] @@ -217,24 +163,6 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie RANKING_METRIC = "eval_metrics/mean_weighted_sum_quantile_loss" df['rank'] = df.groupby(['dataset', 'term_length', 'frequency'])[f'{RANKING_METRIC}'].rank(method='first', ascending=True) - MASE_RANKING_METRICS = "eval_metrics/MASE[0.5]" - df['Rank_MASE'] = df.groupby(['dataset', 'term_length', 'frequency'])[f'{MASE_RANKING_METRICS}'].rank(method='first', - ascending=True) - # Add a new column to identify top 5 ranks - df['is_top5_CRPS'] = df['rank'] <= 5 - - # Count top 3 performances for each model - top5_counts = df.groupby('model')['is_top5_CRPS'].sum().reset_index() - top5_counts = top5_counts.rename(columns={'is_top5_CRPS': 'count_top5_CRPS'}) - - # Add a new column to identify top 5 ranks - df['is_top5_MASE'] = df['Rank_MASE'] <= 5 - - # Count top 3 performances for each model - top5_counts_MASE = df.groupby('model')['is_top5_MASE'].sum().reset_index() - top5_counts_MASE = top5_counts_MASE.rename(columns={'is_top5_MASE': 'count_top5_MASE'}) - - # ipdb.set_trace() # create a new column called rank metric_columns.append('rank') # create a new column called univariate. Set it to true if column num_variates is 1, otherwise set it to false @@ -245,20 +173,7 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie # ipdb.set_trace() grouped_results_overall = df.groupby(['model'])[METRIC_CHOICES].agg(stats.gmean) grouped_results_overall_rank = df.groupby(['model'])[['rank']].mean() - grouped_results_overall_rank_mase = df.groupby(['model'])[['Rank_MASE']].mean() - grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_rank,grouped_results_overall_rank_mase], axis=1) - - # grouped_results_overall_avg_rank = pd.concat([grouped_results_overall_rank, grouped_results_overall_rank_mase], axis=1) - # grouped_results_overall_avg_rank['Avg_Rank'] = grouped_results_overall_avg_rank.mean(axis=1) - # grouped_results_overall_avg_rank = grouped_results_overall_avg_rank[['Avg_Rank']] - # grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_avg_rank], axis=1) - # switch the order to ['model', 'eval_metrics/MASE[0.5]', 'Rank_MASE', 'eval_metrics/mean_weighted_sum_quantile_loss', 'rank'] - # grouped_results_overall = grouped_results_overall[['eval_metrics/MASE[0.5]', 'Rank_MASE', 'eval_metrics/mean_weighted_sum_quantile_loss', 'rank', 'Avg_Rank']] - grouped_results_overall = grouped_results_overall[['eval_metrics/MASE[0.5]', 'Rank_MASE', 'eval_metrics/mean_weighted_sum_quantile_loss', 'rank']] - # ipdb.set_trace() - # Add top3 performance statistics to the overall results - # grouped_results_overall = pd.merge(grouped_results_overall, top5_counts, on='model') - # grouped_results_overall = pd.merge(grouped_results_overall, top5_counts_MASE, on='model') + grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_rank], axis=1) # grouped_results_overall = grouped_results_overall.rename(columns={'model':'Model'}) # grouped_results.to_csv(f'artefacts/grouped_results_by_model.csv')