finalzing training

.github/workflows/training.yml

@@ -0,0 +1,47 @@
+name: training script
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run every 30 minutes
+    #- cron: '*/30 * * * *'
+    - cron: '0 1 * * 1'
+
+jobs:
+  run-scraper:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r all-requirements.txt
+
+    - name: Training
+      env:
+        RAPID_API_KEY: ${{ secrets.RAPID_API_KEY }}
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      run: |
+        python llm-tagging.py
+        python filter-faults.py
+        python train.py
+    - name: List tags folder
+      run: ls -R tags || echo "tags folder not found"
+    - name: Commit and Push Changes
+      run: |
+        git config --global user.name "github-actions[bot]"
+        git config --global user.email "github-actions[bot]@users.noreply.github.com"
+        git add tags
+        git commit -m "Add tags generated by script"
+        git push
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+

llm-tagging.py

@@ -135,7 +135,7 @@ def tag_posting(job_path, output_path):
 def tag_all_today():
 
     date = datetime.today().strftime('%d-%m-%Y')
-    date = "04-01-2025"
+    # date = "04-01-2025"
 
     jobs = os.listdir(f'./job-postings/{date}')
     output_path = f'./data/tags-{date}.jsonl'

train.py

@@ -160,7 +160,7 @@ def train(json_path: str):
 def train_today():
 
     date = datetime.today().strftime('%d-%m-%Y')
-    date = "04-01-2025"
+    # date = "04-01-2025"
     json_path = os.path.join(os.getcwd(),f'data/tags-{date}.jsonl')
     print(f"Training on {json_path}")
     train(json_path=json_path)