Spaces:
Running
Running
File size: 2,134 Bytes
45c55a7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
import os
import zipfile
import json
from json_encryption import JsonEncryption
from tqdm import tqdm
from huggingface_hub import HfApi
# Create an instance with your password
encryptor = JsonEncryption("hal1234")
# Create encrypted_files directory if it doesn't exist
encrypted_dir = "encrypted_files"
if not os.path.exists(encrypted_dir):
os.makedirs(encrypted_dir)
# Get all JSON files from evals_live directory
json_files = [f for f in os.listdir("evals_live") if f.endswith(".json")]
if not json_files:
print("No files found to encrypt and upload")
exit()
# Initialize Hugging Face API
api = HfApi(token=os.getenv("HF_TOKEN"))
# Process each file
for json_file in tqdm(json_files, desc="Encrypting files", unit="file"):
input_path = os.path.join("evals_live", json_file)
# Read the JSON to get the run_id
with open(input_path) as f:
data = json.load(f)
run_id = data["config"]["run_id"]
encrypted_path = os.path.join(encrypted_dir, json_file)
zip_path = os.path.join(encrypted_dir, f"{run_id}.zip")
try:
# Only encrypt and zip if the zip doesn't exist yet
if not os.path.exists(zip_path):
# Encrypt the JSON file
encryptor.encrypt_json_file(input_path, encrypted_path)
# Create zip archive containing the encrypted file
with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
zipf.write(encrypted_path, os.path.basename(encrypted_path))
# Remove the unzipped encrypted file
os.remove(encrypted_path)
# Upload to Hugging Face Hub (attempt for all files)
api.upload_file(
path_or_fileobj=zip_path,
path_in_repo=os.path.basename(zip_path),
repo_id="agent-evals/agent_traces",
repo_type="dataset",
commit_message=f"Add encrypted trace for {json_file}"
)
print(f"Successfully uploaded {zip_path}")
except Exception as e:
print(f"Error processing {json_file}: {str(e)}")
print(f"Processed {len(json_files)} files") |