EleutherAI/lm-evaluation-harness

name: Tasks Modified

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:

env:
  TQDM_DISABLE: "1"
  HF_HUB_DISABLE_PROGRESS_BARS: "1"

# comment/edit out the above to stop/change the triggers
jobs:
  changed_files:
    runs-on: ubuntu-latest  # windows-latest || macos-latest
    timeout-minutes: 120
    name: Scan for changed tasks
    steps:
      - name: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 2  # OR "2" -> To retrieve the preceding commit.

      # Uses the tj-actions/changed-files action to check for changes.
      # The `files_yaml` input optionally takes a yaml string to specify filters,
      # and prepends the filter name to the standard output names.
      - name: Check task folders
        id: changed-tasks
        uses: tj-actions/changed-files@v47
        with:
          # tasks checks the tasks folder and api checks the api folder for changes
          files_yaml: |
            tasks:
              - lm_eval/tasks/**
            api:
              - lm_eval/api/**
          write_output_files: true

    # The next step is optional; the files are written to the workspace by default (above).
    # so it's just for debugging
      - name: Run Tests
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        run: |
          echo .github/outputs/tasks_all_changed_and_modified_files.txt >> 'GITHUB_ENV'
          echo "One or more test file(s) has changed."
          echo "List of all the files that have changed: ${{ steps.changed-tasks.outputs.tasks_all_modified_files }}"

      - name: Install uv
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          python-version: "3.10"
          activate-environment: true
      - name: Install dependencies
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true'
        run: |
          uv pip install -e '.[dev,ifeval,unitxt,math,longbench,hf]' --extra-index-url https://download.pytorch.org/whl/cpu
      - name: Test with pytest
        # if new tasks are added, run tests on them
        if: steps.changed-tasks.outputs.tasks_any_modified == 'true'
        run: pytest -x -s -vv tests/test_tasks.py
        # if api is modified, run tests on it
      - name: Test more tasks with pytest
        env:
          API: true
        if: steps.changed-tasks.outputs.api_any_modified == 'true'
        run: pytest -x -s -vv -n=auto tests/test_tasks.py

name: Publish Python distribution to PyPI

on:
  push:
    tags:
      - '*'

jobs:
  build:
    name: Build distribution
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v6
    - name: Set up Python
      uses: actions/setup-python@v6
      with:
        python-version: "3.x"

    - name: Print version
      run: |
        # Extract version from pyproject.toml
        PYPROJECT_VERSION=$(grep 'version = ' pyproject.toml | head -1 | cut -d'"' -f2)
        echo "Version in pyproject.toml: $PYPROJECT_VERSION"

    - name: Install pypa/build
      run: >-
        python3 -m
        pip install
        build
        --user
    - name: Build a binary wheel and a source tarball
      run: python3 -m build
    - name: Store the distribution packages
      uses: actions/upload-artifact@v7
      with:
        name: python-package-distributions
        path: dist/

  publish-to-pypi:
    name: >-
      Publish Python distribution to PyPI
    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
    needs:
    - build
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/lm_eval
    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing

    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v8
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish distribution to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1

  publish-to-testpypi:
    name: Publish Python distribution to TestPyPI
    needs:
    - build
    runs-on: ubuntu-latest

    environment:
      name: testpypi
      url: https://test.pypi.org/p/lm_eval

    permissions:
      id-token: write  # IMPORTANT: mandatory for trusted publishing

    steps:
    - name: Download all the dists
      uses: actions/download-artifact@v8
      with:
        name: python-package-distributions
        path: dist/
    - name: Publish distribution to TestPyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        repository-url: https://test.pypi.org/legacy/

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
# just comment out unwanted steps to turn off the test.
name: Unit Tests

on:
  push:
    branches:
      - 'main'
  pull_request:
    branches:
      - 'main'
  workflow_dispatch:

env:
  TQDM_DISABLE: "1"
  HF_HUB_DISABLE_PROGRESS_BARS: "1"

# Jobs run concurrently and steps run sequentially within a job.
# jobs: linter and cpu_tests. Add more jobs/steps as required.
jobs:
  linter:
    name: Linters
    runs-on: ubuntu-latest
    timeout-minutes: 5

    steps:
      - name: Checkout Code
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          python-version: "3.10"
          activate-environment: true
      - name: Install pip
        run: uv pip install pip
      - name: Pre-Commit
        env:
          SKIP: "no-commit-to-branch,mypy"
        uses: pre-commit/action@v3.0.1
        with:
          extra_args: --from-ref ${{ github.event.pull_request.base.sha || 'HEAD~1' }} --to-ref HEAD
  # Job 2
  testcpu:
    name: CPU Tests
    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      matrix:
        python-version: ["3.10", "3.11", "3.12"]
    timeout-minutes: 30
    steps:
      - name: Checkout Code
        uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          python-version: ${{ matrix.python-version }}
          activate-environment: true

      # Cache HuggingFace cache directory for CPU tests
      - name: Cache HuggingFace cache (CPU tests)
        uses: actions/cache@v5
        id: cache-hf-cpu
        with:
          path: ~/.cache/huggingface
          key: ${{ runner.os }}-hf-cache-cpu
          restore-keys: |
            ${{ runner.os }}-hf-cache-cpu

      - name: Install dependencies
        run: |
          uv pip install -e '.[dev,unitxt,hf]' --extra-index-url https://download.pytorch.org/whl/cpu
          uv pip install hf_xet

      - name: Test with pytest
        run: pytest -x --showlocals -s -vv -n=auto --ignore=tests/models/test_openvino.py --ignore=tests/models/test_hf_steered.py --ignore=tests/scripts/test_zeno_visualize.py

      # Save test artifacts
      - name: Archive test artifacts
        if: always()  # Upload artifacts even if tests fail
        uses: actions/upload-artifact@v7
        with:
          name: output_testcpu${{ matrix.python-version }}
          path: |
            test_logs/*

#  testmodels:
#    name: External LM Tests
#    runs-on: ubuntu-latest
#    timeout-minutes: 30
#    steps:
#      - name: Checkout Code
#        uses: actions/checkout@v4
#      - name: Set up Python 3.9
#        uses: actions/setup-python@v5
#        with:
#          python-version: 3.9
#          cache: pip
#          cache-dependency-path: pyproject.toml
#
#      # Cache HuggingFace cache directory for External LM tests
#      - name: Cache HuggingFace cache (External LM tests)
#        uses: actions/cache@v3
#        id: cache-hf-lm
#        with:
#          path: ~/.cache/huggingface
#          key: ${{ runner.os }}-hf-cache-external-lm
#          restore-keys: |
#            ${{ runner.os }}-hf-cache-external-lm
#
#      - name: Install dependencies
#        run: |
#          python -m pip install --upgrade pip
#          pip install -e '.[dev,optimum,api]' --extra-index-url https://download.pytorch.org/whl/cpu
#          pip install -U transformers peft accelerate
#
#      - name: Test with pytest
#        run: python -m pytest tests/models --showlocals -s -vv
#        continue-on-error: true  # Continue workflow even if tests fail