daily_run_v2.yml

# This workflows starts scraping Routescanner and MSC each day at 05:00 UTC,
# using the v2 scrapers, and opens a PR to merge the gathered data.

name: Scrape Routescanner and MSC daily (v2)
on:
  #schedule:
  #   - cron: '0 5 * * *'
  push:
    paths:
     - ".github/workflows/daily_run_v2.yml"
     - "webscrapers/routescanner_automated_v2.py"
     - "webscrapers/msc_automated_v2.py"
  workflow_dispatch:

jobs:
  routescanner-v2:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    - run: pip install -U pandas requests fake_useragent
    - run: python routescanner_automated_v2.py
      working-directory: ./webscrapers
    - uses: stefanzweifel/git-auto-commit-action@v5
      with:
        branch: "data_staging"  
        commit_options: '--signoff'
        commit_message: "Add Routescanner v2 data"

  msc-v2:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-python@v4
      with:
        python-version: "3.11"
    - run: pip install -U pandas requests
    - run: python msc_automated_v2.py
      working-directory: ./webscrapers
    - uses: stefanzweifel/git-auto-commit-action@v5
      with:
        branch: "data_staging"  
        commit_options: '--signoff'
        commit_message: "Add MSC v2 data"