vibrantlabsai/ragas

name: CI

on:
  pull_request:

permissions:
  contents: read

env:
  LINES: 120
  COLUMNS: 120

# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#defaultsrun
defaults:
  run:
    shell: bash --noprofile --norc -exo pipefail {0}

jobs:
  diff:
    runs-on: ubuntu-latest
    outputs:
      related: ${{ steps.filter.outputs.related }}
      ragas: ${{ steps.filter.outputs.ragas }}
      docs: ${{ steps.filter.outputs.docs }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          base: "main"
          token: ${{ github.token }}
          filters: |
            related: &related
              - .github/workflows/ci.yaml
              - codecov.yml
              - pyproject.toml
              - Makefile
            ragas:
              - *related
              - "src/ragas/**"
              - "tests/**"
              - "examples/**"
            docs:
              - *related
              - "docs/**"

  unit_tests:
    needs:
      - diff

    strategy:
      fail-fast: false
      matrix:
        include:
          # Critical path: Latest + oldest Python on Ubuntu (full test suite)
          - os: ubuntu-latest
            python-version: "3.9"
            test-type: "full"
          - os: ubuntu-latest
            python-version: "3.12"
            test-type: "full"
          - os: ubuntu-latest
            python-version: "3.13"
            test-type: "full"
          # Cross-platform validation (essential tests only)
          - os: macos-latest
            python-version: "3.11"
            test-type: "essential"
          - os: windows-latest
            python-version: "3.10"
            test-type: "essential"

    if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.ragas == 'true') || github.event_name == 'push' }}
    name: python${{ matrix.python-version }}_unit_tests (${{ matrix.os }}, ${{ matrix.test-type }})
    runs-on: ${{ matrix.os }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # fetch all tags and branches

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          architecture: ${{ matrix.os == 'macos-latest' && 'arm64' || 'x64' }}

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Get pip cache dir
        id: cache-dir
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT

      - name: Cache dependencies (UV cache)
        uses: actions/cache@v4
        id: cache-deps
        with:
          path: |
            ${{ steps.cache-dir.outputs.dir }}
            ~/.cache/uv
          key: deps-${{ runner.os }}-py${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            deps-${{ runner.os }}-py${{ matrix.python-version }}-
            deps-${{ runner.os }}-py3.11-
            deps-${{ runner.os }}-

      - name: Install dependencies
        run: |
          # Use minimal install for fast CI runs (79 packages vs 383)
          # This uses make install-minimal for consistency with local development
          make install-minimal

      - name: Run unit tests
        run: |
          # Configure test options based on OS and test type
          if [ "${{ matrix.os }}" != 'windows-latest' ]; then
            # Use pytest-xdist to improve test run-time on Linux/macOS
            OPTS=(--dist loadfile -n auto)
          fi

          # Run different test suites based on test type
          if [ "${{ matrix.test-type }}" = "full" ]; then
            # Full test suite with notebook tests
            uv run pytest --nbmake tests/unit "${OPTS[@]}"
          else
            # Essential tests only (faster for cross-platform validation)
            uv run pytest tests/unit -k "not slow" "${OPTS[@]}"
          fi
        env:
          __RAGAS_DEBUG_TRACKING: true
          RAGAS_DO_NOT_TRACK: true

  code_quality_check:
    runs-on: ubuntu-latest
    needs:
      - diff

    if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.ragas == 'true') || github.event_name == 'push' }}

    steps:
      - uses: actions/checkout@v4

      - name: Setup python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          architecture: x64

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Get pip cache dir
        id: cache-dir
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT

      - name: Cache dependencies (UV cache)
        uses: actions/cache@v4
        id: cache-deps
        with:
          path: |
            ${{ steps.cache-dir.outputs.dir }}
            ~/.cache/uv
          key: deps-ubuntu-py3.11-codestyle-${{ hashFiles('pyproject.toml') }}
          restore-keys: |
            deps-ubuntu-py3.11-codestyle-
            deps-ubuntu-py3.11-
            deps-ubuntu-

      - name: Install dependencies
        run: |
          # Use minimal install for fast CI runs (79 packages vs 383)
          # This uses make install-minimal for consistency with local development
          make install-minimal

      - name: Format check (dry run)
        run: |
          # Check if code is properly formatted (without making changes)
          # Note: We use direct commands here instead of the standalone Makefiles
          # to have precise control over CI-specific options like --check for dry-run
          echo "Checking ragas formatting..."
          uv run ruff format --check src tests docs --exclude src/ragas/_version.py --config pyproject.toml
          uv run ruff check src docs tests --exclude src/ragas/_version.py --config pyproject.toml

      - name: Type check
        run: make type

name: Claude Code

on:
  issue_comment:
    types: [created]
  pull_request_review_comment:
    types: [created]
  issues:
    types: [opened, assigned]
  pull_request_review:
    types: [submitted]

jobs:
  claude:
    if: |
      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
      issues: write
      id-token: write
      actions: read # Required for Claude to read CI results on PRs
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Run Claude Code
        id: claude
        uses: anthropics/claude-code-action@beta
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

          # This is an optional setting that allows Claude to read CI results on PRs
          additional_permissions: |
            actions: read
          
          # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1)
          # model: "claude-opus-4-1-20250805"
          
          # Optional: Customize the trigger phrase (default: @claude)
          # trigger_phrase: "/claude"
          
          # Optional: Trigger when specific user is assigned to an issue
          # assignee_trigger: "claude-bot"
          
          # Optional: Allow Claude to run specific commands
          # allowed_tools: "Bash(npm install),Bash(npm run build),Bash(npm run test:*),Bash(npm run lint:*)"
          
          # Optional: Add custom instructions for Claude to customize its behavior for your project
          # custom_instructions: |
          #   Follow our coding standards
          #   Ensure all new code has tests
          #   Use TypeScript for new files
          
          # Optional: Custom environment variables for Claude
          # claude_env: |
          #   NODE_ENV: test

name: Claude Code Review

on:
  issue_comment:
    types: [created]

jobs:
  claude-review:
    if: |
      github.event.issue.pull_request &&
      contains(github.event.comment.body, '/claude-review')
    
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
      issues: write
      id-token: write
    
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Run Claude Code Review
        id: claude-review
        uses: anthropics/claude-code-action@beta
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}

          # Optional: Specify model (defaults to Claude Sonnet 4, uncomment for Claude Opus 4.1)
          # model: "claude-opus-4-1-20250805"

          # Customize the trigger phrase to use /claude-review
          trigger_phrase: "/claude-review"

          # Custom instructions for the review
          custom_instructions: |
            When triggered with /claude-review, please analyze this pull request and provide:

            ## Change Type Classification
            First, identify the primary type of change based on the files modified and changes made:
            - **🐛 Bug Fix**: Fixes existing functionality
            - **✨ New Feature**: Adds new functionality
            - **📚 Documentation**: Updates or adds documentation (README, docs/, comments)
            - **🔧 Refactor**: Code restructuring without changing functionality
            - **🧪 Tests**: Adds or modifies tests
            - **🏗️ Build/CI**: Changes to build process, CI/CD, dependencies
            - **🎨 Style**: Code formatting, linting fixes
            - **⚡ Performance**: Improves performance
            - **🔒 Security**: Security-related improvements
            - **🗑️ Cleanup**: Removes deprecated code, unused files
            - **🔀 Merge**: Merge commits or branch management
            - **📦 Dependencies**: Updates dependencies or package versions

            ## Code Review
            Then provide feedback on:
            - Code quality and best practices
            - Potential bugs or issues
            - Performance considerations
            - Security concerns
            - Test coverage
            
            Be constructive and helpful in your feedback.

          # Optional: Use sticky comments to make Claude reuse the same comment on subsequent pushes to the same PR
          # use_sticky_comment: true
          
          # Optional: Customize review based on file types
          # direct_prompt: |
          #   Review this PR focusing on:
          #   - For TypeScript files: Type safety and proper interface usage
          #   - For API endpoints: Security, input validation, and error handling
          #   - For React components: Performance, accessibility, and best practices
          #   - For tests: Coverage, edge cases, and test quality
          
          # Optional: Different prompts for different authors
          # direct_prompt: |
          #   ${{ github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' && 
          #   'Welcome! Please review this PR from a first-time contributor. Be encouraging and provide detailed explanations for any suggestions.' ||
          #   'Please provide a thorough code review focusing on our coding standards and best practices.' }}
          
          # Optional: Add specific tools for running tests or linting
          # allowed_tools: "Bash(npm run test),Bash(npm run lint),Bash(npm run typecheck)"
          
          # Optional: Skip review for certain conditions
          # if: |
          #   !contains(github.event.pull_request.title, '[skip-review]') &&
          #   !contains(github.event.pull_request.title, '[WIP]')

name: Claude Docs Apply

on:
  pull_request_target:
    types: [labeled]

jobs:
  apply-docs:
    if: github.event.label.name == 'update-docs'
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
      issues: write
      id-token: write

    steps:
      - name: Checkout PR branch
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.ref }}
          # Use PAT for fork PRs (requires CLAUDE_CODE_PAT secret), GITHUB_TOKEN for same-repo PRs
          token: ${{ secrets.CLAUDE_CODE_PAT || secrets.GITHUB_TOKEN }}
          fetch-depth: 0

      - name: Configure git
        run: |
          git config --global user.name "Claude Code Bot"
          git config --global user.email "noreply@anthropic.com"

      - name: Apply documentation updates
        uses: anthropics/claude-code-action@v1
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
          github_token: ${{ secrets.CLAUDE_CODE_PAT || secrets.GITHUB_TOKEN }}

          prompt: |
            REPO: ${{ github.repository }}
            PR NUMBER: ${{ github.event.pull_request.number }}
            PR TITLE: ${{ github.event.pull_request.title }}

            You are a documentation assistant for the Ragas project. Update the documentation based on the code changes in this PR.

            ## Quick Action Plan

            1. Run `gh pr diff` to review changes
            2. Identify what docs need updating (see structure below)
            3. Make focused updates efficiently
            4. Commit with clear message

            ## Documentation Structure (Diátaxis Framework)

            **Where to update:**
            - `docs/howtos/` - How-to guides (step-by-step instructions)
            - `docs/concepts/` - Concept docs (explanations and rationale)
            - `docs/getstarted/` - Tutorials (learning experiences)
            - Source code docstrings - API documentation (feeds auto-generated reference)

            **DO NOT edit:**
            - `docs/references/**` - AUTO-GENERATED by mkdocstrings

            ## Writing Guidelines

            - Use second-person ("you") and active voice
            - Code blocks must be copy-pasteable with imports
            - Use `??? "Click to expand"` for verbose outputs
            - Add blank line after text ending with colon before lists
            - Update `mkdocs.yml` nav if adding new pages
            - Keep modes separate: no theory in how-tos, no instructions in concepts

            ## Documentation Modes Reference

            1. **Tutorials** (`docs/getstarted/`) - "Can you teach me to...?"
               - Narrative learning experience with complete working examples

            2. **How-to Guides** (`docs/howtos/`) - "How do I...?"
               - Concise step-by-step from user's perspective

            3. **Reference** (`docs/references/`) - "What is...?"
               - AUTO-GENERATED - edit source docstrings instead

            4. **Explanation** (`docs/concepts/`) - "Why...?"
               - Discursive articles on design decisions and theory

            ## Completion

            After making changes, commit to this PR branch with a concise, descriptive message.

          claude_args: |
            --max-turns 30
            --allowedTools "Read,Write,Edit,Glob,Grep,Bash(git:*),Bash(gh pr diff:*),Bash(gh pr view:*)"

      - name: Remove labels after completion
        if: always()
        run: |
          # Remove both labels
          gh pr edit ${{ github.event.pull_request.number }} --remove-label "update-docs" || true
          gh pr edit ${{ github.event.pull_request.number }} --remove-label "needs-doc-update" || true

          # Comment that docs were updated
          gh pr comment ${{ github.event.pull_request.number }} --body "✅ Documentation update completed."
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

name: Claude Docs Check

on:
  pull_request_target:
    types: [opened, synchronize, reopened]
    paths:
      - "src/**/*.py"

jobs:
  check-docs:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
      issues: write
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.sha }}
          token: ${{ secrets.GITHUB_TOKEN }}
          fetch-depth: 0

      - name: Analyze PR for documentation needs
        id: analyze
        uses: anthropics/claude-code-action@v1
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          allowed_non_write_users: "*"

          prompt: |
            REPO: ${{ github.repository }}
            PR NUMBER: ${{ github.event.pull_request.number }}
            PR TITLE: ${{ github.event.pull_request.title }}

            You are a documentation analyst for the Ragas project. Analyze this PR to determine if documentation updates are needed.

            ## Quick Decision Rules

            **needs_update: false** (most common):
            - Docstrings already updated in code → no action needed (API docs are auto-generated)
            - Internal refactoring with no API changes → no action needed
            - Bug fixes with no user-facing changes → no action needed
            - Infrastructure/build changes → no action needed

            **needs_update: true** (only when necessary):
            - New user-facing features WITHOUT docstrings → need docs
            - Changed usage patterns in how-to guides → need updates
            - New core concepts without explanation → need concept docs
            - Modified getting started flow → need tutorial updates

            ## Your Task

            1. Run `gh pr diff` to review code changes
            2. Check if docstrings are present for API changes
            3. Return JSON immediately with your decision

            Return format:
            - `needs_update`: boolean
            - `reason`: brief explanation (1-2 sentences max)

            ## Documentation Structure Reference
            - `docs/howtos/` - Step-by-step guides
            - `docs/concepts/` - Conceptual explanations
            - `docs/getstarted/` - Tutorials
            - `docs/references/` - AUTO-GENERATED (never edit directly)

            IMPORTANT: Be decisive. Default to needs_update: false if docstrings are present. Return JSON within 3 turns.

          claude_args: |
            --max-turns 20
            --json-schema '{"type":"object","properties":{"needs_update":{"type":"boolean"},"reason":{"type":"string"}},"required":["needs_update","reason"]}'
            --allowedTools "Bash(gh pr diff:*),Bash(gh pr view:*),Read,Glob,Grep"

      - name: Parse analysis result
        id: parse
        run: |
          # Use heredoc to safely handle JSON with special characters
          cat <<'EOF' > /tmp/output.json
          ${{ steps.analyze.outputs.structured_output }}
          EOF

          echo "structured_output=$(cat /tmp/output.json)"
          NEEDS_UPDATE=$(jq -r '.needs_update' /tmp/output.json)
          REASON=$(jq -r '.reason' /tmp/output.json)
          echo "needs_update=$NEEDS_UPDATE" >> $GITHUB_OUTPUT

          # Use multiline string format for reason to handle special characters
          {
            echo 'reason<<EOF'
            jq -r '.reason' /tmp/output.json
            echo 'EOF'
          } >> $GITHUB_OUTPUT

      - name: Add label and comment if docs needed
        if: steps.parse.outputs.needs_update == 'true'
        run: |
          # Add the needs-doc-update label
          gh pr edit ${{ github.event.pull_request.number }} --add-label "needs-doc-update"

          # Comment with instructions
          gh pr comment ${{ github.event.pull_request.number }} --body "📝 **Documentation update may be needed**

          ${{ steps.parse.outputs.reason }}

          **To apply documentation updates:** Add the \`update-docs\` label to this PR."
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Comment if no docs needed
        if: steps.parse.outputs.needs_update == 'false'
        run: |
          gh pr comment ${{ github.event.pull_request.number }} --body "✅ No documentation update needed — ${{ steps.parse.outputs.reason }}"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

name: Issue Manager

on:
  schedule:
    - cron: "0 0 * * *"
  issue_comment:
    types:
      - created
      - edited
  issues:
    types:
      - labeled
  pull_request_target:
    types:
      - labeled
  workflow_dispatch:

jobs:
  issue-manager:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
      - uses: tiangolo/issue-manager@0.4.0
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          config: >
            {
                "$schema": "https://raw.githubusercontent.com/tiangolo/issue-manager/master/schema.json",
                "answered": {
                    "delay": "P3DT12H30M5S",
                    "message": "It seems the issue was answered, closing this now.",
                    "remove_label_on_comment": false,
                    "remove_label_on_close": false
                },
                "validated": {
                    "delay": 300,
                    "message": "The issue could not be validated after 5 minutes. Closing now.",
                    "remove_label_on_comment": true,
                    "remove_label_on_close": false
                },
                "waiting": {
                    "delay": 691200,
                    "message": "Closing after 8 days of waiting for the additional info requested.",
                    "remove_label_on_comment": true,
                    "remove_label_on_close": true
                }
            }

name: Upload ragas-examples Package

on:
  release:
    types: [published]

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest
    environment: pypi-release
    strategy:
      matrix:
        package:
          - name: ragas-examples
            directory: examples
            token: PYPI_API_TOKEN
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v3
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --upgrade setuptools setuptools_scm[toml] build 
    - name: get setuptools-scm version 
      run: python -m setuptools_scm
      working-directory: ${{ matrix.package.directory }}
    - name: Build package
      run: python -m build
      working-directory: ${{ matrix.package.directory }}
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets[matrix.package.token] }}
        packages-dir: ${{ matrix.package.directory }}/dist/
        attestations: false

# This workflow will upload Python Packages using Twine when a release is created
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries

# This workflow uses actions that are not certified by GitHub.
# They are provided by a third-party and are governed by
# separate terms of service, privacy policy, and support
# documentation.

name: Upload Python Packages

on:
  release:
    types: [published]

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest
    environment: pypi-release
    strategy:
      matrix:
        package:
          - name: ragas
            directory: .
            token: PYPI_API_TOKEN
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v3
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install --upgrade setuptools setuptools_scm[toml] build 
    - name: get setuptools-scm version 
      run: python -m setuptools_scm
      working-directory: ${{ matrix.package.directory }}
    - name: Build package
      run: python -m build
      working-directory: ${{ matrix.package.directory }}
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets[matrix.package.token] }}
        packages-dir: ${{ matrix.package.directory }}/dist/
        attestations: false