Spaces:
Runtime error
Runtime error
File size: 2,582 Bytes
6bec1f5 92efc51 6bec1f5 c26894e 6bec1f5 92efc51 6bec1f5 92efc51 6bec1f5 92efc51 6bec1f5 c26894e 6bec1f5 f8a5547 6bec1f5 d599f4f f8a5547 d269dc6 f8a5547 d599f4f d269dc6 d599f4f cef47be f8a5547 6bec1f5 8aabc99 6bec1f5 f8a5547 6bec1f5 92356fe 92efc51 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 |
import os
import sys
import subprocess
from huggingface_hub import hf_hub_download
def run_command(command: str, cwd: str = None) -> tuple:
"""Run a shell command in the specified directory and return the output."""
process = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd
)
stdout, stderr = process.communicate()
if process.returncode != 0:
print(f"Error: {stderr.decode()}")
else:
print(f"Output: {stdout.decode()}")
return stdout, stderr
def download_dataset():
"""Download the dataset."""
print("Downloading the autocast dataset...")
repo_id = "valory/autocast"
base_dir = os.getcwd()
output_dir = os.path.join(
base_dir, "olas-predict-benchmark", "benchmark", "data", "autocast"
)
if not os.path.exists(output_dir):
os.makedirs(output_dir, exist_ok=True)
filenames = [
"autocast_questions_filtered.json",
"autocast_questions_filtered.pkl",
]
for filename in filenames:
hf_hub_download(
repo_id=repo_id,
filename=filename,
local_dir=output_dir,
repo_type="dataset",
)
print("Dataset downloaded successfully.")
def start():
"""Start commands."""
print("Starting commands...")
base_dir = os.getcwd()
olas_dir = os.path.join(base_dir, "olas-predict-benchmark")
mech_dir = os.path.join(olas_dir, "benchmark", "mech")
commands = [
("git submodule init", base_dir),
# no updates
# ("git submodule update --init --recursive", base_dir),
# ("git submodule update --remote --recursive", base_dir),
(
'git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"',
olas_dir,
),
# no updates
("git remote update", olas_dir),
("git fetch --all", olas_dir),
("git checkout main", olas_dir),
("git pull origin main", olas_dir),
("git checkout 56ecf18a982c4548feac5efe787690a3ec37c835", mech_dir),
# ("git pull origin main", mech_dir),
("pip install -e .", os.path.join(olas_dir, "benchmark")),
("pip install -e .", mech_dir),
("pip install lxml[html_clean]", base_dir),
("pip install --upgrade huggingface_hub", base_dir),
]
for command, cwd in commands:
run_command(command, cwd=cwd)
# add benchmark to the path
sys.path.append(os.path.join(olas_dir, "benchmark"))
# Download the dataset
download_dataset()
start()
|