Spaces:
Build error
Build error
Upload folder using huggingface_hub
Browse files- .gitattributes +1 -0
- Dockerfile +7 -0
- README.md +5 -10
- tracklight_server/__pycache__/main.cpython-313.pyc +0 -0
- tracklight_server/api/__pycache__/artifacts.cpython-313.pyc +0 -0
- tracklight_server/api/__pycache__/auth.cpython-313.pyc +0 -0
- tracklight_server/api/__pycache__/dashboard.cpython-313.pyc +0 -0
- tracklight_server/api/__pycache__/ingest.cpython-313.pyc +0 -0
- tracklight_server/api/__pycache__/sync.cpython-313.pyc +0 -0
- tracklight_server/api/artifacts.py +43 -0
- tracklight_server/api/auth.py +21 -0
- tracklight_server/api/dashboard.py +47 -0
- tracklight_server/api/ingest.py +44 -0
- tracklight_server/api/sync.py +34 -0
- tracklight_server/db/__pycache__/duckdb.cpython-313.pyc +0 -0
- tracklight_server/db/duckdb.py +87 -0
- tracklight_server/hf/__pycache__/pull.cpython-313.pyc +0 -0
- tracklight_server/hf/__pycache__/push.cpython-313.pyc +0 -0
- tracklight_server/hf/pull.py +40 -0
- tracklight_server/hf/push.py +31 -0
- tracklight_server/main.py +33 -0
- tracklight_server/requirements.txt +8 -0
- tracklight_server/templates/dashboard.html +188 -0
- tracklight_server/templates/login.html +37 -0
- tracklight_server/templates/project_list.html +36 -0
- tracklight_server/tracklight.db +3 -0
.gitattributes
CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
36 |
+
tracklight_server/tracklight.db filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
FROM python:3.9-slim
|
3 |
+
WORKDIR /app
|
4 |
+
COPY ./tracklight_server /app/tracklight_server
|
5 |
+
RUN pip install "tracklight @ git+https://github.com/your_username/your_repo.git" # Replace with your repo
|
6 |
+
EXPOSE 7860
|
7 |
+
CMD ["uvicorn", "tracklight_server.main:app", "--host", "0.0.0.0", "--port", "7860"]
|
README.md
CHANGED
@@ -1,10 +1,5 @@
|
|
1 |
-
|
2 |
-
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
sdk: docker
|
7 |
-
pinned: false
|
8 |
-
---
|
9 |
-
|
10 |
-
Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
|
|
|
1 |
+
|
2 |
+
---
|
3 |
+
title: Tracklight Server
|
4 |
+
sdk: docker
|
5 |
+
---
|
|
|
|
|
|
|
|
|
|
tracklight_server/__pycache__/main.cpython-313.pyc
ADDED
Binary file (1.83 kB). View file
|
|
tracklight_server/api/__pycache__/artifacts.cpython-313.pyc
ADDED
Binary file (2.46 kB). View file
|
|
tracklight_server/api/__pycache__/auth.cpython-313.pyc
ADDED
Binary file (956 Bytes). View file
|
|
tracklight_server/api/__pycache__/dashboard.cpython-313.pyc
ADDED
Binary file (2.76 kB). View file
|
|
tracklight_server/api/__pycache__/ingest.cpython-313.pyc
ADDED
Binary file (2.62 kB). View file
|
|
tracklight_server/api/__pycache__/sync.cpython-313.pyc
ADDED
Binary file (2.06 kB). View file
|
|
tracklight_server/api/artifacts.py
ADDED
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/api/artifacts.py
|
2 |
+
|
3 |
+
from fastapi import APIRouter, Depends, UploadFile, File
|
4 |
+
from fastapi.responses import FileResponse
|
5 |
+
from .auth import verify_token
|
6 |
+
import os
|
7 |
+
import shutil
|
8 |
+
|
9 |
+
router = APIRouter()
|
10 |
+
|
11 |
+
# Get the absolute path to the artifacts directory
|
12 |
+
SERVER_DIR = os.path.dirname(os.path.dirname(__file__))
|
13 |
+
ARTIFACTS_DIR = os.path.join(SERVER_DIR, "artifacts")
|
14 |
+
|
15 |
+
# Create artifacts directory if it doesn't exist
|
16 |
+
os.makedirs(ARTIFACTS_DIR, exist_ok=True)
|
17 |
+
|
18 |
+
@router.post("/upload/{run_id}")
|
19 |
+
async def upload_artifact(run_id: str, file: UploadFile = File(...), token: str = Depends(verify_token)):
|
20 |
+
"""
|
21 |
+
Uploads an artifact file for a given run.
|
22 |
+
"""
|
23 |
+
run_artifact_dir = os.path.join(ARTIFACTS_DIR, run_id)
|
24 |
+
os.makedirs(run_artifact_dir, exist_ok=True)
|
25 |
+
|
26 |
+
file_path = os.path.join(run_artifact_dir, file.filename)
|
27 |
+
|
28 |
+
with open(file_path, "wb") as buffer:
|
29 |
+
shutil.copyfileobj(file.file, buffer)
|
30 |
+
|
31 |
+
return {"filename": file.filename, "run_id": run_id}
|
32 |
+
|
33 |
+
@router.get("/download/{run_id}/{filename}")
|
34 |
+
async def download_artifact(run_id: str, filename: str, token: str = Depends(verify_token)):
|
35 |
+
"""
|
36 |
+
Downloads an artifact file for a given run.
|
37 |
+
"""
|
38 |
+
file_path = os.path.join(ARTIFACTS_DIR, run_id, filename)
|
39 |
+
|
40 |
+
if not os.path.exists(file_path):
|
41 |
+
return {"error": "File not found"}
|
42 |
+
|
43 |
+
return FileResponse(file_path)
|
tracklight_server/api/auth.py
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/api/auth.py
|
2 |
+
|
3 |
+
from fastapi import Depends, HTTPException, status
|
4 |
+
from fastapi.security import OAuth2PasswordBearer
|
5 |
+
import os
|
6 |
+
|
7 |
+
# This is a simple example of a bearer token.
|
8 |
+
# In a real application, you would use a more secure method.
|
9 |
+
API_TOKEN = os.environ.get("TRACKLIGHT_API_TOKEN", "secret-token")
|
10 |
+
|
11 |
+
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
|
12 |
+
|
13 |
+
def verify_token(token: str = Depends(oauth2_scheme)):
|
14 |
+
"""Verifies the provided bearer token."""
|
15 |
+
if token != API_TOKEN:
|
16 |
+
raise HTTPException(
|
17 |
+
status_code=status.HTTP_401_UNAUTHORIZED,
|
18 |
+
detail="Invalid authentication credentials",
|
19 |
+
headers={"WWW-Authenticate": "Bearer"},
|
20 |
+
)
|
21 |
+
return token
|
tracklight_server/api/dashboard.py
ADDED
@@ -0,0 +1,47 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/api/dashboard.py
|
2 |
+
|
3 |
+
from fastapi import APIRouter, Depends, Request
|
4 |
+
from fastapi.responses import HTMLResponse
|
5 |
+
from fastapi.templating import Jinja2Templates
|
6 |
+
from ..db import duckdb
|
7 |
+
from .auth import verify_token
|
8 |
+
import os
|
9 |
+
|
10 |
+
router = APIRouter()
|
11 |
+
|
12 |
+
# Get the absolute path to the templates directory
|
13 |
+
SERVER_DIR = os.path.dirname(os.path.dirname(__file__))
|
14 |
+
TEMPLATES_DIR = os.path.join(SERVER_DIR, "templates")
|
15 |
+
templates = Jinja2Templates(directory=TEMPLATES_DIR)
|
16 |
+
|
17 |
+
@router.get("/login", response_class=HTMLResponse)
|
18 |
+
async def get_login_page(request: Request):
|
19 |
+
"""
|
20 |
+
Serves the login page.
|
21 |
+
"""
|
22 |
+
return templates.TemplateResponse("login.html", {"request": request})
|
23 |
+
|
24 |
+
@router.get("/dashboard", response_class=HTMLResponse)
|
25 |
+
async def get_dashboard(request: Request):
|
26 |
+
"""
|
27 |
+
Serves the main dashboard page with a list of projects.
|
28 |
+
"""
|
29 |
+
projects = duckdb.get_projects()
|
30 |
+
return templates.TemplateResponse("project_list.html", {"request": request, "projects": projects})
|
31 |
+
|
32 |
+
@router.get("/project/{project_name}", response_class=HTMLResponse)
|
33 |
+
async def get_project_page(request: Request, project_name: str, user: str):
|
34 |
+
"""
|
35 |
+
Serves the project-specific page with a list of runs.
|
36 |
+
"""
|
37 |
+
runs = duckdb.get_runs(project_name, user)
|
38 |
+
return templates.TemplateResponse("dashboard.html", {"request": request, "project": project_name, "user": user, "runs": runs})
|
39 |
+
|
40 |
+
@router.get("/api/metrics")
|
41 |
+
async def get_metrics(run_id: str, token: str = Depends(verify_token)):
|
42 |
+
"""
|
43 |
+
Returns metrics and config for a given run.
|
44 |
+
"""
|
45 |
+
metrics = duckdb.get_metrics_for_run(run_id)
|
46 |
+
config = duckdb.get_config_for_run(run_id)
|
47 |
+
return {"metrics": metrics, "config": config}
|
tracklight_server/api/ingest.py
ADDED
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/api/ingest.py
|
2 |
+
|
3 |
+
from fastapi import APIRouter, Depends, HTTPException
|
4 |
+
from typing import List
|
5 |
+
from pydantic import BaseModel
|
6 |
+
from ..db import duckdb
|
7 |
+
from .auth import verify_token
|
8 |
+
|
9 |
+
router = APIRouter()
|
10 |
+
|
11 |
+
class Metric(BaseModel):
|
12 |
+
run_id: str
|
13 |
+
project: str
|
14 |
+
user: str
|
15 |
+
metric_name: str
|
16 |
+
value: float
|
17 |
+
timestamp: str
|
18 |
+
|
19 |
+
class Config(BaseModel):
|
20 |
+
run_id: str
|
21 |
+
config_name: str
|
22 |
+
value: str
|
23 |
+
|
24 |
+
@router.post("/log", status_code=200)
|
25 |
+
def log_metrics(metrics: List[Metric], token: str = Depends(verify_token)):
|
26 |
+
"""
|
27 |
+
Receives a list of metrics and logs them to the database.
|
28 |
+
"""
|
29 |
+
try:
|
30 |
+
duckdb.insert_metrics([metric.dict() for metric in metrics])
|
31 |
+
return {"message": "Metrics logged successfully."}
|
32 |
+
except Exception as e:
|
33 |
+
raise HTTPException(status_code=500, detail=str(e))
|
34 |
+
|
35 |
+
@router.post("/log_config", status_code=200)
|
36 |
+
def log_config(configs: List[Config], token: str = Depends(verify_token)):
|
37 |
+
"""
|
38 |
+
Receives a list of config values and logs them to the database.
|
39 |
+
"""
|
40 |
+
try:
|
41 |
+
duckdb.insert_config([config.dict() for config in configs])
|
42 |
+
return {"message": "Configs logged successfully."}
|
43 |
+
except Exception as e:
|
44 |
+
raise HTTPException(status_code=500, detail=str(e))
|
tracklight_server/api/sync.py
ADDED
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/api/sync.py
|
2 |
+
|
3 |
+
from fastapi import APIRouter, Depends, HTTPException
|
4 |
+
from pydantic import BaseModel
|
5 |
+
from ..hf import push, pull
|
6 |
+
from .auth import verify_token
|
7 |
+
|
8 |
+
router = APIRouter()
|
9 |
+
|
10 |
+
class SyncRequest(BaseModel):
|
11 |
+
repo_id: str
|
12 |
+
hf_token: str
|
13 |
+
|
14 |
+
@router.post("/sync/push")
|
15 |
+
async def sync_push(request: SyncRequest, token: str = Depends(verify_token)):
|
16 |
+
"""
|
17 |
+
Pushes local data to a Hugging Face Dataset repo.
|
18 |
+
"""
|
19 |
+
try:
|
20 |
+
push.push_to_hub(request.repo_id, request.hf_token)
|
21 |
+
return {"message": "Data push initiated."}
|
22 |
+
except Exception as e:
|
23 |
+
raise HTTPException(status_code=500, detail=str(e))
|
24 |
+
|
25 |
+
@router.post("/sync/pull")
|
26 |
+
async def sync_pull(request: SyncRequest, token: str = Depends(verify_token)):
|
27 |
+
"""
|
28 |
+
Pulls data from a Hugging Face Dataset repo.
|
29 |
+
"""
|
30 |
+
try:
|
31 |
+
pull.pull_from_hub(request.repo_id, request.hf_token)
|
32 |
+
return {"message": "Data pull initiated."}
|
33 |
+
except Exception as e:
|
34 |
+
raise HTTPException(status_code=500, detail=str(e))
|
tracklight_server/db/__pycache__/duckdb.cpython-313.pyc
ADDED
Binary file (5.18 kB). View file
|
|
tracklight_server/db/duckdb.py
ADDED
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/db/duckdb.py
|
2 |
+
|
3 |
+
import duckdb
|
4 |
+
import os
|
5 |
+
|
6 |
+
DB_FILE = "tracklight.db"
|
7 |
+
TABLE_NAME = "metrics"
|
8 |
+
|
9 |
+
# Define the database path in the user's home directory
|
10 |
+
HOME_DIR = os.path.expanduser("~")
|
11 |
+
TRACKLIGHT_DIR = os.path.join(HOME_DIR, ".tracklight")
|
12 |
+
DB_PATH = os.path.join(TRACKLIGHT_DIR, DB_FILE)
|
13 |
+
|
14 |
+
# Ensure the .tracklight directory exists
|
15 |
+
os.makedirs(TRACKLIGHT_DIR, exist_ok=True)
|
16 |
+
|
17 |
+
def get_connection():
|
18 |
+
"""Returns a connection to the DuckDB database."""
|
19 |
+
return duckdb.connect(DB_PATH)
|
20 |
+
|
21 |
+
def create_tables():
|
22 |
+
"""Creates the metrics and config tables if they don't exist."""
|
23 |
+
with get_connection() as con:
|
24 |
+
con.execute(f"""
|
25 |
+
CREATE TABLE IF NOT EXISTS {TABLE_NAME} (
|
26 |
+
run_id VARCHAR,
|
27 |
+
project VARCHAR,
|
28 |
+
user_name VARCHAR,
|
29 |
+
metric_name VARCHAR,
|
30 |
+
value DOUBLE,
|
31 |
+
timestamp TIMESTAMP
|
32 |
+
)
|
33 |
+
""")
|
34 |
+
con.execute("""
|
35 |
+
CREATE TABLE IF NOT EXISTS config (
|
36 |
+
run_id VARCHAR,
|
37 |
+
config_name VARCHAR,
|
38 |
+
value VARCHAR
|
39 |
+
)
|
40 |
+
""")
|
41 |
+
|
42 |
+
def insert_metrics(metrics: list):
|
43 |
+
"""Inserts a list of metrics into the database."""
|
44 |
+
with get_connection() as con:
|
45 |
+
con.executemany(f"INSERT INTO {TABLE_NAME} VALUES (?, ?, ?, ?, ?, ?)",
|
46 |
+
[(m['run_id'], m['project'], m['user'], m['metric_name'], m['value'], m['timestamp']) for m in metrics])
|
47 |
+
|
48 |
+
def insert_config(configs: list):
|
49 |
+
"""Inserts a list of config values into the database."""
|
50 |
+
with get_connection() as con:
|
51 |
+
con.executemany("INSERT INTO config VALUES (?, ?, ?)",
|
52 |
+
[(c['run_id'], c['config_name'], c['value']) for c in configs])
|
53 |
+
|
54 |
+
def get_projects():
|
55 |
+
"""Retrieves all projects with their creation timestamp."""
|
56 |
+
with get_connection() as con:
|
57 |
+
return con.execute(f"""
|
58 |
+
SELECT project, MIN(timestamp) as creation_time
|
59 |
+
FROM {TABLE_NAME}
|
60 |
+
GROUP BY project
|
61 |
+
ORDER BY creation_time DESC
|
62 |
+
""").fetchall()
|
63 |
+
|
64 |
+
def get_runs(project: str, user: str):
|
65 |
+
"""Retrieves all runs for a given project and user, sorted by the most recent timestamp."""
|
66 |
+
with get_connection() as con:
|
67 |
+
return con.execute(f"""
|
68 |
+
SELECT run_id
|
69 |
+
FROM {TABLE_NAME}
|
70 |
+
WHERE project = ? AND user_name = ?
|
71 |
+
GROUP BY run_id
|
72 |
+
ORDER BY MAX(timestamp) DESC
|
73 |
+
""", [project, user]).fetchall()
|
74 |
+
|
75 |
+
def get_metrics_for_run(run_id: str):
|
76 |
+
"""Retrieves all metrics for a given run, excluding config parameters."""
|
77 |
+
with get_connection() as con:
|
78 |
+
return con.execute(f"SELECT metric_name, value, timestamp FROM {TABLE_NAME} WHERE run_id = ? AND metric_name NOT LIKE 'config/%' ORDER BY timestamp", [run_id]).fetchall()
|
79 |
+
|
80 |
+
def get_config_for_run(run_id: str):
|
81 |
+
"""Retrieves all configuration parameters for a given run."""
|
82 |
+
with get_connection() as con:
|
83 |
+
return con.execute(f"SELECT config_name, value FROM config WHERE run_id = ?", [run_id]).fetchall()
|
84 |
+
|
85 |
+
|
86 |
+
# Initialize the database
|
87 |
+
create_tables()
|
tracklight_server/hf/__pycache__/pull.cpython-313.pyc
ADDED
Binary file (1.95 kB). View file
|
|
tracklight_server/hf/__pycache__/push.cpython-313.pyc
ADDED
Binary file (1.52 kB). View file
|
|
tracklight_server/hf/pull.py
ADDED
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/hf/pull.py
|
2 |
+
|
3 |
+
from datasets import load_dataset
|
4 |
+
from huggingface_hub import HfFolder
|
5 |
+
from ..db import duckdb
|
6 |
+
|
7 |
+
def pull_from_hub(repo_id: str, hf_token: str):
|
8 |
+
"""
|
9 |
+
Pulls data from a Hugging Face Dataset repo and merges it into the local DuckDB.
|
10 |
+
"""
|
11 |
+
# Authenticate with Hugging Face
|
12 |
+
HfFolder.save_token(hf_token)
|
13 |
+
|
14 |
+
try:
|
15 |
+
# Load the dataset from the Hub
|
16 |
+
dataset = load_dataset(repo_id)
|
17 |
+
except Exception as e:
|
18 |
+
print(f"Failed to load dataset from {repo_id}: {e}")
|
19 |
+
return
|
20 |
+
|
21 |
+
if 'train' not in dataset:
|
22 |
+
print(f"No 'train' split found in the dataset {repo_id}.")
|
23 |
+
return
|
24 |
+
|
25 |
+
df = dataset['train'].to_pandas()
|
26 |
+
|
27 |
+
if df.empty:
|
28 |
+
print("No data to pull.")
|
29 |
+
return
|
30 |
+
|
31 |
+
# Merge the data into the local DuckDB
|
32 |
+
with duckdb.get_connection() as con:
|
33 |
+
# A simple approach is to just insert all data.
|
34 |
+
# A more sophisticated approach would be to handle duplicates.
|
35 |
+
con.execute(f"CREATE TABLE IF NOT EXISTS temp_table AS SELECT * FROM {duckdb.TABLE_NAME} WHERE 1=0")
|
36 |
+
con.execute("INSERT INTO temp_table SELECT * FROM df")
|
37 |
+
con.execute(f"INSERT INTO {duckdb.TABLE_NAME} SELECT * FROM temp_table ON CONFLICT DO NOTHING")
|
38 |
+
con.execute("DROP TABLE temp_table")
|
39 |
+
|
40 |
+
print(f"Successfully pulled and merged data from {repo_id}")
|
tracklight_server/hf/push.py
ADDED
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/hf/push.py
|
2 |
+
|
3 |
+
from datasets import Dataset
|
4 |
+
from huggingface_hub import HfApi, HfFolder
|
5 |
+
from ..db import duckdb
|
6 |
+
import os
|
7 |
+
|
8 |
+
def push_to_hub(repo_id: str, hf_token: str):
|
9 |
+
"""
|
10 |
+
Pushes the local DuckDB data to a Hugging Face Dataset repo.
|
11 |
+
"""
|
12 |
+
# Authenticate with Hugging Face
|
13 |
+
HfFolder.save_token(hf_token)
|
14 |
+
|
15 |
+
# Get all data from DuckDB
|
16 |
+
with duckdb.get_connection() as con:
|
17 |
+
df = con.execute(f"SELECT * FROM {duckdb.TABLE_NAME}").fetchdf()
|
18 |
+
|
19 |
+
if df.empty:
|
20 |
+
print("No data to push.")
|
21 |
+
return
|
22 |
+
|
23 |
+
# Create a Hugging Face Dataset
|
24 |
+
dataset = Dataset.from_pandas(df)
|
25 |
+
|
26 |
+
# Push the dataset to the Hub
|
27 |
+
try:
|
28 |
+
dataset.push_to_hub(repo_id=repo_id, private=True)
|
29 |
+
print(f"Successfully pushed data to {repo_id}")
|
30 |
+
except Exception as e:
|
31 |
+
print(f"Failed to push data to {repo_id}: {e}")
|
tracklight_server/main.py
ADDED
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# tracklight_server/main.py
|
2 |
+
|
3 |
+
from fastapi import FastAPI, Depends
|
4 |
+
from fastapi.responses import RedirectResponse
|
5 |
+
from fastapi.security import OAuth2PasswordRequestForm
|
6 |
+
|
7 |
+
from .api import ingest, auth, dashboard, artifacts, sync
|
8 |
+
from contextlib import asynccontextmanager
|
9 |
+
from .db import duckdb
|
10 |
+
|
11 |
+
@asynccontextmanager
|
12 |
+
async def lifespan(app: FastAPI):
|
13 |
+
# Create the database and table on startup
|
14 |
+
duckdb.create_tables()
|
15 |
+
yield
|
16 |
+
|
17 |
+
app = FastAPI(title="Tracklight Server", lifespan=lifespan)
|
18 |
+
|
19 |
+
# Include the API routers
|
20 |
+
app.include_router(ingest.router, prefix="/api", tags=["ingest"])
|
21 |
+
app.include_router(artifacts.router, prefix="/api", tags=["artifacts"])
|
22 |
+
app.include_router(sync.router, prefix="/api", tags=["sync"])
|
23 |
+
app.include_router(dashboard.router, tags=["dashboard"])
|
24 |
+
|
25 |
+
@app.get("/")
|
26 |
+
def read_root():
|
27 |
+
return RedirectResponse(url="/dashboard")
|
28 |
+
|
29 |
+
# This is needed for the OAuth2PasswordBearer to work
|
30 |
+
@app.post("/token")
|
31 |
+
async def token(form_data: OAuth2PasswordRequestForm = Depends()):
|
32 |
+
# In a real app, you'd verify the username and password here
|
33 |
+
return {"access_token": auth.API_TOKEN, "token_type": "bearer"}
|
tracklight_server/requirements.txt
ADDED
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
fastapi
|
2 |
+
uvicorn
|
3 |
+
duckdb
|
4 |
+
python-multipart
|
5 |
+
jinja2
|
6 |
+
datasets
|
7 |
+
huggingface_hub
|
8 |
+
pandas
|
tracklight_server/templates/dashboard.html
ADDED
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<!DOCTYPE html>
|
2 |
+
<html lang="en">
|
3 |
+
<head>
|
4 |
+
<meta charset="UTF-8">
|
5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
6 |
+
<title>Tracklight Dashboard</title>
|
7 |
+
<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
|
8 |
+
<style>
|
9 |
+
body { font-family: sans-serif; }
|
10 |
+
.container { max-width: 1200px; margin: 0 auto; padding: 20px; }
|
11 |
+
.header { display: flex; justify-content: space-between; align-items: center; }
|
12 |
+
.run { border: 1px solid #ccc; padding: 10px; margin-bottom: 10px; }
|
13 |
+
.metric-summary { display: flex; align-items: center; }
|
14 |
+
.play-button { margin-left: 10px; }
|
15 |
+
</style>
|
16 |
+
</head>
|
17 |
+
<body>
|
18 |
+
<div class="container">
|
19 |
+
<div class="header">
|
20 |
+
<div>
|
21 |
+
<h1><a href="/dashboard">Tracklight Dashboard</a></h1>
|
22 |
+
<h2>Project: {{ project }}</h2>
|
23 |
+
<h3>User: {{ user }}</h3>
|
24 |
+
</div>
|
25 |
+
<button onclick="logout()">Logout</button>
|
26 |
+
</div>
|
27 |
+
|
28 |
+
<div id="runs">
|
29 |
+
{% for run in runs %}
|
30 |
+
<div class="run" role="region" aria-labelledby="run-heading-{{ loop.index }}">
|
31 |
+
<h4 id="run-heading-{{ loop.index }}">Run ID: {{ run[0] }}</h4>
|
32 |
+
<button onclick="loadMetrics('{{ run[0] }}')">Load Metrics</button>
|
33 |
+
<div id="config-{{ run[0] }}"></div>
|
34 |
+
<div id="plot-{{ run[0] }}" role="img" aria-label="A plot of metrics for this run."></div>
|
35 |
+
<div id="summary-{{ run[0] }}" aria-live="polite"></div>
|
36 |
+
</div>
|
37 |
+
{% endfor %}
|
38 |
+
</div>
|
39 |
+
</div>
|
40 |
+
|
41 |
+
<script>
|
42 |
+
const audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
43 |
+
const token = localStorage.getItem('tracklight-token');
|
44 |
+
|
45 |
+
// If no token, redirect to login
|
46 |
+
if (!token) {
|
47 |
+
const urlParams = new URLSearchParams(window.location.search);
|
48 |
+
window.location.href = '/login?' + urlParams.toString();
|
49 |
+
}
|
50 |
+
|
51 |
+
async function loadMetrics(runId) {
|
52 |
+
const response = await fetch(`/api/metrics?run_id=${runId}`, {
|
53 |
+
headers: {
|
54 |
+
'Authorization': `Bearer ${token}`
|
55 |
+
}
|
56 |
+
});
|
57 |
+
|
58 |
+
if (response.status === 401) {
|
59 |
+
alert('Invalid or expired token. Please log in again.');
|
60 |
+
logout();
|
61 |
+
return;
|
62 |
+
}
|
63 |
+
|
64 |
+
const data = await response.json();
|
65 |
+
|
66 |
+
// Render config table
|
67 |
+
const configDiv = document.getElementById(`config-${runId}`);
|
68 |
+
configDiv.innerHTML = generateConfigTable(data.config);
|
69 |
+
|
70 |
+
// Process and render metrics
|
71 |
+
const metrics = data.metrics;
|
72 |
+
const plotData = {};
|
73 |
+
|
74 |
+
metrics.forEach(metric => {
|
75 |
+
const [name, value, timestamp] = metric;
|
76 |
+
if (!plotData[name]) {
|
77 |
+
plotData[name] = {
|
78 |
+
x: [],
|
79 |
+
y: [],
|
80 |
+
mode: 'lines+markers',
|
81 |
+
name: name
|
82 |
+
};
|
83 |
+
}
|
84 |
+
plotData[name].x.push(new Date(timestamp));
|
85 |
+
plotData[name].y.push(value);
|
86 |
+
});
|
87 |
+
|
88 |
+
const plotDiv = document.getElementById(`plot-${runId}`);
|
89 |
+
Plotly.newPlot(plotDiv, Object.values(plotData), {title: `Metrics for run ${runId}`});
|
90 |
+
|
91 |
+
// Generate and display summary
|
92 |
+
const summaryDiv = document.getElementById(`summary-${runId}`);
|
93 |
+
summaryDiv.innerHTML = generateSummary(plotData);
|
94 |
+
}
|
95 |
+
|
96 |
+
function generateConfigTable(config) {
|
97 |
+
if (!config || config.length === 0) {
|
98 |
+
return '';
|
99 |
+
}
|
100 |
+
let table = '<h4>Configuration:</h4><table><thead><tr><th>Parameter</th><th>Value</th></tr></thead><tbody>';
|
101 |
+
config.forEach(param => {
|
102 |
+
const name = param[0].replace('config/', '');
|
103 |
+
const value = param[1];
|
104 |
+
table += `<tr><td>${name}</td><td>${value}</td></tr>`;
|
105 |
+
});
|
106 |
+
table += '</tbody></table>';
|
107 |
+
return table;
|
108 |
+
}
|
109 |
+
|
110 |
+
function generateSummary(plotData) {
|
111 |
+
let summary = '<h4>Metrics Summary:</h4><ul>';
|
112 |
+
for (const metricName in plotData) {
|
113 |
+
const metric = plotData[metricName];
|
114 |
+
const values = metric.y;
|
115 |
+
const firstVal = values[0];
|
116 |
+
const lastVal = values[values.length - 1];
|
117 |
+
|
118 |
+
// Check for constant metric
|
119 |
+
const isConstant = values.every(v => v === firstVal);
|
120 |
+
if (isConstant) {
|
121 |
+
summary += `<li><b>${metricName}</b>: is constant at ${firstVal.toFixed(4)} throughout the run.</li>`;
|
122 |
+
continue;
|
123 |
+
}
|
124 |
+
|
125 |
+
// Descriptive stats for volatile metrics
|
126 |
+
const minVal = Math.min(...values);
|
127 |
+
const maxVal = Math.max(...values);
|
128 |
+
const trend = lastVal > firstVal ? 'increased' : 'decreased';
|
129 |
+
|
130 |
+
// Momentum analysis (simple version)
|
131 |
+
const deltas = values.slice(1).map((v, i) => Math.abs(v - values[i]));
|
132 |
+
const avgDelta = deltas.reduce((a, b) => a + b, 0) / deltas.length;
|
133 |
+
const stdDev = Math.sqrt(deltas.map(d => Math.pow(d - avgDelta, 2)).reduce((a, b) => a + b, 0) / deltas.length);
|
134 |
+
|
135 |
+
let momentumDesc = '';
|
136 |
+
if (stdDev / avgDelta > 0.5) {
|
137 |
+
momentumDesc = 'volatilely';
|
138 |
+
} else if (avgDelta > (maxVal - minVal) / values.length * 2) {
|
139 |
+
momentumDesc = 'sharply';
|
140 |
+
} else {
|
141 |
+
momentumDesc = 'steadily';
|
142 |
+
}
|
143 |
+
|
144 |
+
summary += `<li class="metric-summary">
|
145 |
+
<b>${metricName}</b>: ${momentumDesc} ${trend} from ${firstVal.toFixed(4)} to ${lastVal.toFixed(4)} over ${values.length} steps.
|
146 |
+
(Min: ${minVal.toFixed(4)}, Max: ${maxVal.toFixed(4)})
|
147 |
+
<button class="play-button" onclick="playSlopeSound('${metricName}', [${values}])" aria-label="Play audio representation of the ${metricName} slope">Play Slope</button>
|
148 |
+
</li>`;
|
149 |
+
}
|
150 |
+
summary += '</ul>';
|
151 |
+
return summary;
|
152 |
+
}
|
153 |
+
|
154 |
+
function playSlopeSound(metricName, values) {
|
155 |
+
const duration = 2; // seconds
|
156 |
+
const sampleRate = audioContext.sampleRate;
|
157 |
+
const buffer = audioContext.createBuffer(1, sampleRate * duration, sampleRate);
|
158 |
+
const data = buffer.getChannelData(0);
|
159 |
+
|
160 |
+
const minVal = Math.min(...values);
|
161 |
+
const maxVal = Math.max(...values);
|
162 |
+
|
163 |
+
for (let i = 0; i < data.length; i++) {
|
164 |
+
const t = i / sampleRate;
|
165 |
+
const progress = t / duration;
|
166 |
+
const index = Math.floor(progress * (values.length - 1));
|
167 |
+
|
168 |
+
const value1 = values[index];
|
169 |
+
const value2 = values[index + 1];
|
170 |
+
const normalizedValue = (value1 + (value2 - value1) * (progress * (values.length - 1) - index) - minVal) / (maxVal - minVal);
|
171 |
+
|
172 |
+
const freq = 220 + (normalizedValue * 660); // Map value to frequency range (A3 to A5)
|
173 |
+
data[i] = Math.sin(2 * Math.PI * freq * t);
|
174 |
+
}
|
175 |
+
|
176 |
+
const source = audioContext.createBufferSource();
|
177 |
+
source.buffer = buffer;
|
178 |
+
source.connect(audioContext.destination);
|
179 |
+
source.start();
|
180 |
+
}
|
181 |
+
|
182 |
+
function logout() {
|
183 |
+
localStorage.removeItem('tracklight-token');
|
184 |
+
window.location.href = '/login';
|
185 |
+
}
|
186 |
+
</script>
|
187 |
+
</body>
|
188 |
+
</html>
|
tracklight_server/templates/login.html
ADDED
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<!DOCTYPE html>
|
2 |
+
<html lang="en">
|
3 |
+
<head>
|
4 |
+
<meta charset="UTF-8">
|
5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
6 |
+
<title>Login - Tracklight</title>
|
7 |
+
<style>
|
8 |
+
body { font-family: sans-serif; display: flex; justify-content: center; align-items: center; height: 100vh; margin: 0; }
|
9 |
+
.login-container { border: 1px solid #ccc; padding: 40px; border-radius: 8px; text-align: center; }
|
10 |
+
input { padding: 10px; margin-bottom: 20px; width: 250px; }
|
11 |
+
button { padding: 10px 20px; cursor: pointer; }
|
12 |
+
</style>
|
13 |
+
</head>
|
14 |
+
<body>
|
15 |
+
<div class="login-container">
|
16 |
+
<h1>Tracklight Login</h1>
|
17 |
+
<p>Please enter the API token to proceed.</p>
|
18 |
+
<input type="password" id="tokenInput" placeholder="API Token">
|
19 |
+
<br>
|
20 |
+
<button onclick="login()">Login</button>
|
21 |
+
</div>
|
22 |
+
|
23 |
+
<script>
|
24 |
+
function login() {
|
25 |
+
const token = document.getElementById('tokenInput').value;
|
26 |
+
if (token) {
|
27 |
+
localStorage.setItem('tracklight-token', token);
|
28 |
+
// Redirect to the dashboard, preserving query params if they exist
|
29 |
+
const urlParams = new URLSearchParams(window.location.search);
|
30 |
+
window.location.href = '/dashboard?' + urlParams.toString();
|
31 |
+
} else {
|
32 |
+
alert('Please enter a token.');
|
33 |
+
}
|
34 |
+
}
|
35 |
+
</script>
|
36 |
+
</body>
|
37 |
+
</html>
|
tracklight_server/templates/project_list.html
ADDED
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<!DOCTYPE html>
|
2 |
+
<html lang="en">
|
3 |
+
<head>
|
4 |
+
<meta charset="UTF-8">
|
5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
6 |
+
<title>Tracklight Projects</title>
|
7 |
+
<style>
|
8 |
+
body { font-family: sans-serif; }
|
9 |
+
.container { max-width: 1200px; margin: 0 auto; padding: 20px; }
|
10 |
+
table { width: 100%; border-collapse: collapse; }
|
11 |
+
th, td { border: 1px solid #ccc; padding: 8px; text-align: left; }
|
12 |
+
th { background-color: #f2f2f2; }
|
13 |
+
</style>
|
14 |
+
</head>
|
15 |
+
<body>
|
16 |
+
<div class="container">
|
17 |
+
<h1>Projects</h1>
|
18 |
+
<table>
|
19 |
+
<thead>
|
20 |
+
<tr>
|
21 |
+
<th>Project Name</th>
|
22 |
+
<th>Created At</th>
|
23 |
+
</tr>
|
24 |
+
</thead>
|
25 |
+
<tbody>
|
26 |
+
{% for project in projects %}
|
27 |
+
<tr>
|
28 |
+
<td><a href="/project/{{ project[0] }}?user=hf-user">{{ project[0] }}</a></td>
|
29 |
+
<td>{{ project[1] }}</td>
|
30 |
+
</tr>
|
31 |
+
{% endfor %}
|
32 |
+
</tbody>
|
33 |
+
</table>
|
34 |
+
</div>
|
35 |
+
</body>
|
36 |
+
</html>
|
tracklight_server/tracklight.db
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:bab1c02e7c2483e53a6560ebb41d71ade11d167a0ced73acfe0aac8605c7ed0c
|
3 |
+
size 536576
|