updating weekly stats

data/all_fpmms.parquet

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f592b1f5306ed3bacf492bdfc2f1c141abf63a65a1de5dd699d70d9be7cba684
+size 411157

data/all_trades_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e3f78f46ebb99a74a0c6965c091251c81bd283078c5a2731fa29b915e85950f
-size 1727275
+oid sha256:a12ff001752f6ca93c5ebbbf1ba39aa2c9a194d798cd4136c10bb096b8eb5490
+size 698837

data/fpmmTrades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec5e9eac58311efb3492d93bf253993863b073b420cae74944a293c92c17fcdc
-size 5442383
+oid sha256:9cf10df0a23d3a587cccec8ed841f307feaf2d8f9b1a9e25b02468e6541fe382
+size 2230205

data/fpmms.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a7edf74b65cbed42947158ae9603a5c177d57792ff455b87dfa55f106d668cc
-size 419359
+oid sha256:b393aa9eb4ee0eb927dc87405050a0a75aa553f3b5ee05f88e9595ae535a00a0
+size 428124

data/invalid_trades.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34b27ad37752126eabf389ebadfd04d9c8da718b77355f6d26918ef8a5c98adf
-size 239854
+oid sha256:9bc7524e360c8352de0f4c4d213d10b76991196373c819a4ea00822d68e48182
+size 90771

data/summary_profitability.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0b4f955e507698c53a1e04b654b098491ef0bc10c02ee55da493d8fbd5669498
-size 36397
+oid sha256:d969e863e793be8a5e871659eef73cf4029af37a3100b64243ca1cd5f74a1d1e
+size 35519

data/t_map.pkl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbc1ee9d8b1b348606f10221c56182b4341575d84d787fb136e94ed9fbbaca9c
-size 16921070
+oid sha256:acb6bae97c67972e8943ee28cded53b7102248ea95b1d52006ed36bc1c749336
+size 17302848

data/tools.parquet

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3672394d5e46d6ba2da521ee3ba098a3ac33fed4e185800ed5a2892aaf84bdf9
-size 394013019
+oid sha256:4d30486693a66156d21e5149c2796a86d341f3c433aac890ea22bfa0f0fdb80f
+size 365753056

data/tools_accuracy.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d191b7e3acb6f904b21cb0ee4ef02870500cbf9b03d59d5cd3a94469322013c0
-size 999
+oid sha256:4335a38309dcebbffe7e61d5857f3a95ca5d13f1e5c833e5aac69b6e21eaebea
+size 1012

scripts/get_mech_info.py

@@ -102,7 +102,7 @@ def fetch_block_number(timestamp_from: int, timestamp_to: int) -> dict:
     )
     # print(f"block query: {query}")
     result_json = response.json()
-    # print(f"Response of the query={result_json}")
+    print(f"Response of the query={result_json}")
     blocks = result_json.get("data", {}).get("blocks", "")
     if len(blocks) == 0:
         raise ValueError(f"The query {query} did not return any results")

scripts/mech_request_utils.py

@@ -18,11 +18,9 @@
 # ------------------------------------------------------------------------------
 
 """Script for retrieving mech requests and their delivers."""
-import os
 import json
 import time
 import pickle
-from collections import defaultdict
 from typing import Any, Dict, List, Tuple
 from pathlib import Path
 import requests
@@ -338,12 +336,6 @@ def write_mech_events_to_file(
         last_write_time = now
 
 
-def save_final_tools_json_file(data: Dict[str, Any], filename: str):
-    filename_path = DATA_DIR / filename
-    with open(filename_path, "w", encoding="utf-8") as file:
-        json.dump(data, file, indent=2)
-
-
 def save_json_file(data: Dict[str, Any], filename: str):
     """Function to save the content into a json file"""
     filename_path = JSON_DATA_DIR / filename
@@ -499,7 +491,7 @@ def get_ipfs_data():
             partial_dict = future.result()
             updated_mech_requests.update(partial_dict)
 
-    save_final_tools_json_file(updated_mech_requests, "tools_info.json")
+    save_json_file(updated_mech_requests, "tools_info.json")
 
     # delivers
     print("UPDATING IPFS CONTENTS OF DELIVERS")
@@ -525,7 +517,7 @@ def get_ipfs_data():
             partial_dict = future.result()
             final_tools_content.update(partial_dict)
 
-    save_final_tools_json_file(final_tools_content, "tools_info.json")
+    save_json_file(final_tools_content, "tools_info.json")
 
 
 def only_delivers_loop():
@@ -557,4 +549,4 @@ def only_delivers_loop():
             partial_dict = future.result()
             final_tools_content.update(partial_dict)
 
-    save_final_tools_json_file(final_tools_content, "tools_info.json")
+    save_json_file(final_tools_content, "tools_info.json")

scripts/profitability.py

@@ -441,7 +441,7 @@ def analyse_trader(
                 )
             except Exception:
                 print(f"Error while getting the number of mech calls")
-                num_mech_calls = 0  # No info
+                num_mech_calls = 2  # Average value
 
             net_earnings = (
                 earnings

scripts/pull_data.py

@@ -136,8 +136,7 @@ def weekly_analysis():
     logging.info("Finished generating the mech json files")
 
     # Run tools ETL
-    logging.info("Running new tools ETL")
-    get_mech_events_last_60_days()
+    logging.info("Generate and parse the tools content")
     generate_tools_file()
     logging.info("Tools ETL completed")
 

scripts/tools.py

@@ -56,6 +56,7 @@ from utils import (
     reduce_window,
     limit_text,
     DATA_DIR,
+    JSON_DATA_DIR,
     REQUEST_ID_FIELD,
     MechEvent,
     MechEventName,
@@ -600,7 +601,7 @@ def parse_store_json_events_parallel(
 def generate_tools_file():
     """Function to parse the json mech events and generate the parquet tools file"""
     try:
-        with open(DATA_DIR / "tools_info.json", "r") as file:
+        with open(JSON_DATA_DIR / "tools_info.json", "r") as file:
             file_contents = json.load(file)
             parse_store_json_events_parallel(file_contents)
     except Exception as e:

scripts/utils.py

@@ -19,6 +19,7 @@ REQUEST_ID_FIELD = "request_id"
 SCRIPTS_DIR = Path(__file__).parent
 ROOT_DIR = SCRIPTS_DIR.parent
 DATA_DIR = ROOT_DIR / "data"
+JSON_DATA_DIR = ROOT_DIR / "json_data"
 BLOCK_FIELD = "block"
 CID_PREFIX = "f01701220"
 REQUEST_ID = "requestId"