agno-agi/agno

name: Claude Code

on:
  issue_comment:
    types: [created]
  pull_request_review_comment:
    types: [created]
  issues:
    types: [opened]
  pull_request_review:
    types: [submitted]

jobs:
  claude:
    concurrency:
      group: claude-${{ github.event.issue.number || github.event.pull_request.number || github.run_id }}
      cancel-in-progress: true
    timeout-minutes: 15
    if: |
      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review' && contains(github.event.review.body || '', '@claude')) ||
      (github.event_name == 'issues' && (contains(github.event.issue.body || '', '@claude') || contains(github.event.issue.title || '', '@claude')))
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pull-requests: write
      issues: write
      id-token: write
      actions: read
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Detect mode
        id: detect
        env:
          EVENT_NAME: ${{ github.event_name }}
          PR_URL: ${{ github.event.issue.pull_request.url || '' }}
          COMMENT_BODY: ${{ github.event.comment.body || github.event.review.body || github.event.issue.body || '' }}
        run: |
          IS_PR="false"
          case "$EVENT_NAME" in
            pull_request_review_comment|pull_request_review) IS_PR="true" ;;
            issue_comment) [ -n "$PR_URL" ] && IS_PR="true" ;;
          esac

          if [ "$IS_PR" = "true" ] && echo "$COMMENT_BODY" | grep -iqE '\breview\b'; then
            echo "mode=review" >> "$GITHUB_OUTPUT"
          else
            echo "mode=assistant" >> "$GITHUB_OUTPUT"
          fi

      - name: Claude Review
        if: steps.detect.outputs.mode == 'review'
        uses: anthropics/claude-code-action@v1
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_CI_API_KEY }}
          plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
          plugins: |
            code-review@claude-code-plugins
            pr-review-toolkit@claude-code-plugins
          use_sticky_comment: true
          show_full_output: true
          claude_args: |
            --model "claude-opus-4-6"

      - name: Claude Assistant
        if: steps.detect.outputs.mode == 'assistant'
        uses: anthropics/claude-code-action@v1
        with:
          anthropic_api_key: ${{ secrets.ANTHROPIC_CI_API_KEY }}

name: Performance Comparison

on:
  workflow_dispatch:

jobs:
  langgraph-performance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y python3-venv
          python -m pip install --upgrade pip
          pip install openai agno langgraph langchain_openai

      - name: Run LangGraph Performance Test
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          echo "Running LangGraph Performance Test..."
          python evals/performance/other/langgraph_instantiation.py | tee langgraph_results.txt

      - name: Format Results
        run: |
          echo "## LangGraph Results" > langgraph_results.md
          echo "========================" >> langgraph_results.md
          cat langgraph_results.txt >> langgraph_results.md

      - name: Upload LangGraph Results
        uses: actions/upload-artifact@v4
        with:
          name: langgraph-results
          path: langgraph_results.md

  agno-performance:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y python3-venv
          python -m pip install --upgrade pip
          pip install openai agno

      - name: Run Agno Performance Test
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          echo "Running Agno Performance Test..."
          python evals/performance/instantiation_with_tool.py | tee agno_results.txt

      - name: Format Results
        run: |
          echo "## Agno Results" > agno_results.md
          echo "========================" >> agno_results.md
          cat agno_results.txt >> agno_results.md

      - name: Upload Agno Results
        uses: actions/upload-artifact@v4
        with:
          name: agno-results
          path: agno_results.md

  combine-results:
    needs: [langgraph-performance, agno-performance]
    runs-on: ubuntu-latest
    steps:
      - name: Download LangGraph Results
        uses: actions/download-artifact@v4
        with:
          name: langgraph-results

      - name: Download Agno Results
        uses: actions/download-artifact@v4
        with:
          name: agno-results

      - name: Combine Results
        run: |
          echo "Performance Test Results" > final_results.md
          echo "========================" >> final_results.md
          echo "" >> final_results.md
          cat langgraph_results.md >> final_results.md
          echo "" >> final_results.md
          cat agno_results.md >> final_results.md

      - name: Upload Combined Results
        uses: actions/upload-artifact@v4
        with:
          name: final-results
          path: final_results.md

name: PR Lint

on:
  pull_request:
    types:
      - opened
      - edited
      - synchronize

jobs:
  lint-pr:
    name: Lint PR Title and Body
    runs-on: ubuntu-latest
    steps:
      - name: Check PR Title Format
        env:
          TITLE: "${{ github.event.pull_request.title }}"
        run: |
          TITLE_LOWER=$(echo "$TITLE" | tr '[:upper:]' '[:lower:]')
          REGEX='^(\[(feat|fix|cookbook|test|refactor|chore|style|revert|release)\][[:space:]]+.+|(feat|fix|cookbook|test|refactor|chore|style|revert|release):[[:space:]]+.+|(feat|fix|cookbook|test|refactor|chore|style|revert|release)-[a-z0-9-]+)$'

          if [[ "$TITLE_LOWER" =~ $REGEX ]]; then
            echo "✅ PR Title format is valid."
          else
            echo "❌ PR Title '$TITLE' does not match the required format."
            echo "   Use one of: [feat] title, feat: title, or feat-some-title"
            exit 1
          fi

name: PR Triage

on:
  pull_request_target:
    types:
      - opened

permissions:
  pull-requests: write
  issues: read

jobs:
  triage:
    name: Triage PR
    runs-on: ubuntu-latest
    steps:
      - name: Triage PR
        uses: actions/github-script@v7
        with:
          script: |
            const pr = context.payload.pull_request;
            const author = pr.user.login;
            const body = (pr.body || '');
            const bodyLower = body.toLowerCase();
            const owner = context.repo.owner;
            const repo = context.repo.repo;
            const prNumber = pr.number;

            const labels = [];
            const findings = [];

            // --- Helper: check if author is an org member / collaborator ---
            function isOrgMember() {
              const association = pr.author_association;
              return ['MEMBER', 'OWNER', 'COLLABORATOR'].includes(association);
            }

            async function hasWriteAccess() {
              try {
                const { data: permLevel } = await github.rest.repos.getCollaboratorPermissionLevel({
                  owner, repo, username: author,
                });
                return ['admin', 'maintain', 'write'].includes(permLevel.permission);
              } catch (err) {
                console.log(`Could not check permissions for ${author}: ${err.message}`);
                return false;
              }
            }

            const skipChecks = isOrgMember() || await hasWriteAccess();

            // --- 1. First-time contributor label (applied immediately) ---
            try {
              const { data: searchResult } = await github.rest.search.issuesAndPullRequests({
                q: `repo:${owner}/${repo} type:pr author:${author}`,
                per_page: 1,
              });
              if (searchResult.total_count <= 1) {
                await github.rest.issues.addLabels({
                  owner, repo, issue_number: prNumber, labels: ['first-time-contributor'],
                });
              }
            } catch (err) {
              console.log(`Could not check/label first-time contributor: ${err.message}`);
            }

            // --- All remaining checks are skipped for org members / collaborators ---
            if (!skipChecks) {

              // --- 2. Missing issue link ---
              const hasIssueRef = /(?:fixes|closes|resolves|fix|close|resolve)\s+#\d+/.test(bodyLower)
                || /#\d+/.test(bodyLower)
                || /github\.com\/agno-agi\/agno\/issues\/\d+/.test(bodyLower);

              if (!hasIssueRef) {
                labels.push('missing-issue-link');
                findings.push(
                  '**Missing issue link:** Please link the issue this PR addresses using `fixes #<issue_number>`, '
                  + '`closes #<issue_number>`, or `resolves #<issue_number>` in the PR description. '
                  + 'If there is no existing issue, please create one first.'
                );
              }

              // --- 3. Missing tests ---
              try {
                const files = await github.paginate(github.rest.pulls.listFiles, {
                  owner, repo, pull_number: prNumber, per_page: 100,
                });

                const changedFiles = files.map(f => f.filename);
                const touchesSource = changedFiles.some(f =>
                  f.startsWith('libs/agno/agno/') && f.endsWith('.py')
                );
                const touchesTests = changedFiles.some(f =>
                  f.includes('/tests/') && f.endsWith('.py')
                );

                if (touchesSource && !touchesTests) {
                  labels.push('missing-tests');
                  findings.push(
                    '**Missing tests:** This PR modifies source code but does not include any test changes. '
                    + 'Please add or update tests to cover your changes.'
                  );
                }
              } catch (err) {
                console.log(`Could not check for missing tests: ${err.message}`);
              }

              // --- 4. Duplicate PRs and issue assignment checks ---
              const issueRefs = new Set();
              const closingPatterns = [
                /(?:fixes|closes|resolves|fix|close|resolve)\s+#(\d+)/gi,
                /github\.com\/agno-agi\/agno\/issues\/(\d+)/gi,
              ];

              for (const pattern of closingPatterns) {
                let match;
                while ((match = pattern.exec(body)) !== null) {
                  issueRefs.add(parseInt(match[1], 10));
                }
              }

              if (issueRefs.size > 0) {
                // --- 4a. Duplicate detection ---
                try {
                  const duplicates = [];
                  for (const issueNumber of issueRefs) {
                    const { data: searchResult } = await github.rest.search.issuesAndPullRequests({
                      q: `repo:${owner}/${repo} is:pr is:open ${issueNumber} in:body`,
                      per_page: 10,
                    });
                    for (const item of searchResult.items) {
                      if (item.number !== prNumber && item.pull_request) {
                        duplicates.push({ issue: issueNumber, pr: item.number });
                      }
                    }
                  }

                  if (duplicates.length > 0) {
                    labels.push('possible-duplicate');
                    const prLinks = duplicates.map(d => `- #${d.pr} (also references issue #${d.issue})`).join('\n');
                    findings.push(
                      '**Possible duplicate:** The following open PRs also reference the same issue(s):\n\n'
                      + prLinks + '\n\n'
                      + 'If this is intentional, please explain in your PR description why this approach is preferred. '
                      + 'Otherwise, consider collaborating on the existing PR instead.'
                    );
                  }
                } catch (err) {
                  console.log(`Could not check for duplicates: ${err.message}`);
                }

                // --- 4b. Linked issue assigned to someone else ---
                try {
                  const conflicts = [];
                  for (const issueNumber of issueRefs) {
                    try {
                      const { data: issue } = await github.rest.issues.get({
                        owner, repo, issue_number: issueNumber,
                      });
                      if (issue.assignees && issue.assignees.length > 0) {
                        const assignedToAuthor = issue.assignees.some(a => a.login === author);
                        if (!assignedToAuthor) {
                          const assignees = issue.assignees.map(a => `@${a.login}`).join(', ');
                          conflicts.push({ issue: issueNumber, assignees });
                        }
                      }
                    } catch (err) {
                      console.log(`Could not fetch issue #${issueNumber}: ${err.message}`);
                    }
                  }

                  if (conflicts.length > 0) {
                    labels.push('issue-assigned-to-other');
                    const details = conflicts.map(c => `- Issue #${c.issue} is assigned to ${c.assignees}`).join('\n');
                    findings.push(
                      '**Issue assigned to someone else:** This PR references an issue that is already assigned:\n\n'
                      + details + '\n\n'
                      + 'Please confirm with the assignee or a maintainer in the issue comments before proceeding.'
                    );
                  }
                } catch (err) {
                  console.log(`Could not check issue assignments: ${err.message}`);
                }
              }

              // --- 5. Low-context PR ---
              const strippedBody = body
                .replace(/## Summary/g, '')
                .replace(/## Type of change/g, '')
                .replace(/## Checklist/g, '')
                .replace(/## Additional Notes/g, '')
                .replace(/### Duplicate and AI-Generated PR Check/g, '')
                .replace(/- \[ \] .*/g, '')
                .replace(/- \[x\] .*/g, '')
                .replace(/---/g, '')
                .replace(/\(If applicable.*\)/g, '')
                .replace(/Add any important context.*/g, '')
                .replace(/Describe key changes.*/g, '')
                .trim();

              if (strippedBody.length < 30) {
                labels.push('needs-description');
                findings.push(
                  '**Needs description:** This PR has little or no description beyond the template defaults. '
                  + 'Please fill in the Summary section explaining what this PR does and why.'
                );
              }
            }

            // --- Apply all labels at once ---
            if (labels.length > 0) {
              await github.rest.issues.addLabels({
                owner, repo, issue_number: prNumber, labels,
              });
            }

            // --- Determine if PR should be auto-closed ---
            const shouldClose = labels.includes('issue-assigned-to-other') || labels.includes('possible-duplicate');

            // --- Post a single consolidated comment ---
            if (findings.length > 0) {
              const header = findings.length === 1
                ? '### PR Triage\n\n'
                : '### PR Triage\n\nA few things to address before this PR can be reviewed:\n\n';

              let comment = header + findings.join('\n\n---\n\n');

              if (shouldClose) {
                comment += '\n\n---\n\n';
                comment += '**This PR has been automatically closed.** ';
                if (labels.includes('issue-assigned-to-other') && labels.includes('possible-duplicate')) {
                  comment += 'The linked issue is already assigned to someone else and there is an existing PR for it. ';
                } else if (labels.includes('issue-assigned-to-other')) {
                  comment += 'The linked issue is already assigned to someone else. ';
                } else {
                  comment += 'There is already an open PR addressing this issue. ';
                }
                comment += 'If you believe your contribution is valuable, please comment on the original issue explaining your approach and why it might be preferred. ';
                comment += 'A maintainer can reopen this PR if appropriate.';
              }

              await github.rest.issues.createComment({
                owner, repo, issue_number: prNumber, body: comment,
              });
            }

            // --- Auto-close PR if it conflicts with existing work ---
            if (shouldClose) {
              await github.rest.pulls.update({
                owner, repo, pull_number: prNumber, state: 'closed',
              });
              console.log(`PR #${prNumber} auto-closed due to: ${labels.filter(l => l === 'issue-assigned-to-other' || l === 'possible-duplicate').join(', ')}`);
            }

name: Release Agno

on:
  release:
    types: [published]

permissions:
  contents: read

jobs:
  build-release-agno:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y python3-venv
          python -m pip install --upgrade pip build

      - name: Copy README
        run: |
          cp README.md libs/agno/

      - name: Build package
        run: |
          cd libs/agno
          python -m build

      - name: Publish to TestPyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository-url: https://test.pypi.org/legacy/
          packages-dir: libs/agno/dist

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}
          packages-dir: libs/agno/dist

  build-release-agno-infra:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y python3-venv
          python -m pip install --upgrade pip build

      - name: Build package
        run: |
          cd libs/agno_infra
          python -m build

      - name: Publish to TestPyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        continue-on-error: true
        with:
          user: __token__
          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
          repository-url: https://test.pypi.org/legacy/
          packages-dir: libs/agno_infra/dist

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        continue-on-error: true
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}
          packages-dir: libs/agno_infra/dist

name: "Mark Stale Issues and PRs"

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:

jobs:
  stale:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write

    steps:
      - uses: actions/stale@v9
        with:
          stale-issue-message: >
            This issue has been automatically marked as stale due to 30 days of inactivity.

          stale-pr-message: >
            This pull request has been automatically marked as stale due to 14 days of inactivity.
            If this is still relevant, please update it or leave a comment.

          # Mark issues as stale after 30 days
          days-before-issue-stale: 30

          # Effectively never close issues or PRs
          days-before-issue-close: 99999
          days-before-pr-close: -1

          # Mark PRs as stale after 14 days but never auto-close
          days-before-pr-stale: 14

          stale-issue-label: "stale"
          stale-pr-label: "stale"
          remove-stale-when-updated: true

name: Validation

on:
  push:
    branches:
      - "main"
      - "v2.4.0"
      - "v2.5-phase1"
      - "v2.5-phase1-prerelease-fixes"
  pull_request:
    types:
      - opened
      - synchronize
      - reopened
    branches:
      - "main"
      - "v2.4.0"
      - "v2.5-phase1"
      - "v2.5-phase1-prerelease-fixes"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  style-check-agno:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]

    defaults:
      run:
        working-directory: libs/agno

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
      - name: Install dependencies
        run: |
          python -m pip install -e .[dev]
      - name: Ruff check
        run: |
          ruff check .
      - name: Mypy
        run: |
          mypy . --config-file pyproject.toml

  style-check-cookbook:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]

    defaults:
      run:
        working-directory: cookbook

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
      - name: Install dependencies
        run: |
          python -m pip install -e "../libs/agno/"[dev]
      - name: Ruff check
        run: |
          ruff check .
      # - name: Mypy
      #   run: |
      #     mypy .


  style-check-agno-infra:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.10"]

    defaults:
      run:
        working-directory: libs/agno_infra

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
      - name: Install dependencies
        run: |
          python -m pip install -e .[dev]
      - name: Ruff check
        run: |
          ruff check .
      - name: Mypy
        run: |
          mypy . --config-file pyproject.toml

  tests:
    runs-on: ubuntu-latest
    # Skip push events if there's an open PR from the same branch
    if: github.event_name != 'push' || !github.event.pull_request
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          set -e
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run test setup
        working-directory: .
        run: |
          set -e
          ./scripts/test_setup.sh
      - name: Run tests for Agno
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          uv pip install pytest-cov
          python -m pytest --cov=agno --cov-report=json:coverage-agno.json ./libs/agno/tests/unit --ignore=libs/agno/tests/unit/models/litellm --ignore=libs/agno/tests/unit/tools/test_crawl4ai.py
          echo "AGNO_COVERAGE=$(python -c 'import json; print(json.load(open("coverage-agno.json"))["totals"]["percent_covered_display"])')" >> $GITHUB_ENV

name: Main Validation

on:
  push:
    branches:
      - "*release*" # Run on release-* format
      - "release/*" # Run on release/1.2.3 branch format
      - "v2.5-phase1"
      - "fix/ci-test-failures"
      - "fix/team-hitl-streaming"
      - "v2.5-phase2"
  workflow_dispatch: # Allows manual triggering
    inputs:
      branch:
        description: "Branch to run checks on"
        required: true
        type: string

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # Run tests for OpenAI
  test-openai:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and OpenAI tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh openai

  # Run tests for Google
  test-google:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
      GOOGLE_CLOUD_PROJECT: ${{ secrets.GOOGLE_CLOUD_PROJECT }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Google tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh google

  # Run tests for Anthropic
  test-anthropic:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Anthropic tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh anthropic

  # Run tests for AWS
  #
  # -- Some tests hang. Commented out until we revisit them.
  #
  # test-aws:
  #   runs-on: ubuntu-latest
  #   strategy:
  #     matrix:
  #       python-version: ["3.12"]
  #     fail-fast: true
  #   env:
  #     AWS_ACCESS_KEY_ID: ${{ secrets.BEDROCK_AWS_ACCESS_KEY_ID }}
  #     AWS_SECRET_ACCESS_KEY: ${{ secrets.BEDROCK_AWS_SECRET_ACCESS_KEY }}
  #     AWS_REGION: "us-east-1"
  #     EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
  #   steps:
  #     - uses: actions/checkout@v3
  #     - name: Set up Python ${{ matrix.python-version }}
  #       uses: actions/setup-python@v4
  #       with:
  #         python-version: ${{ matrix.python-version }}
  #     - name: Install uv
  #       run: |
  #         curl -LsSf https://astral.sh/uv/install.sh | sh
  #         echo "$HOME/.cargo/bin" >> $GITHUB_PATH
  #     - name: Run dev setup and AWS tests
  #       working-directory: .
  #       run: |
  #         ./scripts/run_model_tests.sh aws

  # Run tests for Cohere
  test-cohere:
    if: false # Disable cohere tests until we get a production account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      CO_API_KEY: ${{ secrets.CO_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Cohere tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh cohere

  # Run tests for Deepseek
  test-deepseek:
    if: false # Disable deepseek tests until we get a production account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Deepseek tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh deepseek

  # Run tests for Fireworks
  test-fireworks:
    if: false # Disable fireworks tests until we get a production account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/dev_setup.sh
      - name: Run Fireworks tests
        working-directory: .
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/models/fireworks

  # Run tests for Groq
  test-groq:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Groq tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh groq

  # Run tests for Mistral
  test-mistral:
    if: false # Disable mistral tests until we get a production account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Mistral tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh mistral

  # Run tests for Nvidia
  test-nvidia:
    if: false # Disable nvidia tests until we get a test account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Nvidia tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh nvidia

  # Run tests for OpenRouter
  test-openrouter:
    if: false # Disable openrouter tests until we get a test account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and OpenRouter tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh openrouter

  # Run tests for Perplexity
  test-perplexity:
    if: false # Disable perplexity tests until we get a production account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Perplexity tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh perplexity

  # Run tests for SambaNova
  test-sambanova:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      SAMBANOVA_API_KEY: ${{ secrets.SAMBANOVA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and SambaNova tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh sambanova

  # Run tests for Together
  test-together:
    if: false # The tests take too long to run
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Together tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh together

  # Run tests for xAI
  test-xai:
    runs-on: ubuntu-latest
    if: false # Disable xAI tests until we get a test account
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and XAI tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh xai

  # Run tests for v0
  test-v0:
    if: false # We don't have credits
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      V0_API_KEY: ${{ secrets.V0_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Vercel V0 tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh vercel

  test-ibm-watsonx:
    if: false # Our account is not working
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      IBM_WATSONX_PROJECT_ID: ${{ secrets.IBM_WATSONX_PROJECT_ID }}
      IBM_WATSONX_API_KEY: ${{ secrets.IBM_WATSONX_API_KEY }}
      IBM_WATSONX_URL: ${{ secrets.IBM_WATSONX_URL }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and IBM-WatsonX tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh ibm

  # Run tests for Cerebras
  test-cerebras:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Cerebras tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh cerebras

  test-deepinfra:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup and Deepinfra tests
        working-directory: .
        run: |
          ./scripts/run_model_tests.sh deepinfra

  # Run tests for AI/ML API
  test-aimlapi:
    if: false # We need more credits
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      AIMLAPI_API_KEY: ${{ secrets.AIMLAPI_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/dev_setup.sh
      - name: Run AImlAPI tests
        working-directory: .
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/models/aimlapi

  # Run tests for DashScope
  test-dashscope:
    if: false # We need more credits
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/dev_setup.sh
      - name: Run AImlAPI tests
        working-directory: .
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/models/dashscope

  # Run tests for LangDB
  test-langdb:
    if: false # We do not have a company account
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      LANGDB_API_KEY: ${{ secrets.LANGDB_API_KEY }}
      LANGDB_PROJECT_ID: ${{ secrets.LANGDB_PROJECT_ID }}
      EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/dev_setup.sh
      - name: Run LangDB tests
        working-directory: .
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/models/langdb

  test-agents:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run agent integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/agent/test_guardrails.py \
          ./libs/agno/tests/integration/agent/test_output_model.py \
          ./libs/agno/tests/integration/agent/test_input.py \
          ./libs/agno/tests/integration/agent/test_memory_impact.py \
          ./libs/agno/tests/integration/agent/test_async_tool_calling.py \
          ./libs/agno/tests/integration/agent/test_disable_storing_tool_and_history_messages.py \
          ./libs/agno/tests/integration/agent/test_agent_convenience_functions.py \
          ./libs/agno/tests/integration/agent/test_session.py \
          ./libs/agno/tests/integration/agent/test_reasoning_content_default_COT.py \
          ./libs/agno/tests/integration/agent/test_introduction.py \
          ./libs/agno/tests/integration/agent/test_followups.py \
          ./libs/agno/tests/integration/agent/test_tool_hooks.py \
          ./libs/agno/tests/integration/agent/human_in_the_loop/test_continue_run_history_context.py

  test-agents-2:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run agent integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/agent \
          --ignore=./libs/agno/tests/integration/agent/test_guardrails.py \
          --ignore=./libs/agno/tests/integration/agent/test_output_model.py \
          --ignore=./libs/agno/tests/integration/agent/test_input.py \
          --ignore=./libs/agno/tests/integration/agent/test_memory_impact.py \
          --ignore=./libs/agno/tests/integration/agent/test_async_tool_calling.py \
          --ignore=./libs/agno/tests/integration/agent/test_disable_storing_tool_and_history_messages.py \
          --ignore=./libs/agno/tests/integration/agent/test_agent_convenience_functions.py \
          --ignore=./libs/agno/tests/integration/agent/test_session.py \
          --ignore=./libs/agno/tests/integration/agent/test_reasoning_content_default_COT.py \
          --ignore=./libs/agno/tests/integration/agent/test_introduction.py

  test-teams-1:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run teams integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest \
          ./libs/agno/tests/integration/teams/test_hooks.py \
          ./libs/agno/tests/integration/teams/test_dependencies.py \
          ./libs/agno/tests/integration/teams/test_send_media_to_model.py \
          ./libs/agno/tests/integration/teams/human_in_the_loop/test_team_confirmation_flows.py \
          ./libs/agno/tests/integration/teams/human_in_the_loop/test_team_user_input_flows.py \
          ./libs/agno/tests/integration/teams/test_retries.py \
          ./libs/agno/tests/integration/teams/human_in_the_loop/test_team_external_execution_flows.py \
          ./libs/agno/tests/integration/teams/human_in_the_loop/test_continue_run_history_context.py

  test-teams-2:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run teams integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest \
          ./libs/agno/tests/integration/teams/test_output_schema_override.py \
          ./libs/agno/tests/integration/teams/test_event_streaming.py \
          ./libs/agno/tests/integration/teams/test_session_state.py \
          ./libs/agno/tests/integration/teams/test_storage_and_memory.py \
          ./libs/agno/tests/integration/teams/human_in_the_loop/test_team_tool_hitl.py \
          ./libs/agno/tests/integration/teams/test_reasoning_content_default_COT.py \
          ./libs/agno/tests/integration/teams/test_filter_tool_calls.py \
          ./libs/agno/tests/integration/teams/test_multimodal.py

  test-teams-3:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run teams integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest \
          ./libs/agno/tests/integration/teams/test_team_reasoning_new_models.py \
          ./libs/agno/tests/integration/teams/test_session.py \
          ./libs/agno/tests/integration/teams/test_input.py \
          ./libs/agno/tests/integration/teams/test_team_delegation.py \
          ./libs/agno/tests/integration/teams/test_history.py \
          ./libs/agno/tests/integration/teams/test_memory_impact.py \
          ./libs/agno/tests/integration/teams/test_team_with_output_model.py \
          ./libs/agno/tests/integration/teams/test_team_with_member_with_parser_model.py

  test-teams-4:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run teams integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest \
          ./libs/agno/tests/integration/teams/test_team_convenience_functions.py \
          ./libs/agno/tests/integration/teams/test_introduction.py \
          ./libs/agno/tests/integration/teams/test_callable_members.py \
          ./libs/agno/tests/integration/teams/test_structured_output.py \
          ./libs/agno/tests/integration/teams/test_tool_compression.py \
          ./libs/agno/tests/integration/teams/test_metrics.py \
          ./libs/agno/tests/integration/teams/test_parser_model.py \
          ./libs/agno/tests/integration/teams/test_tasks_mode_streaming.py \
          ./libs/agno/tests/integration/teams/test_followups.py \
          ./libs/agno/tests/integration/teams/test_tool_hooks.py \

  verify-split-test-coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Verify all split integration tests are assigned to CI jobs
        run: |
          status=0
          for dir in teams knowledge; do
            grep -oE "\./libs/agno/tests/integration/${dir}/[^ \\\\]+(/[^ \\\\]+)*\.py" \
              .github/workflows/test_on_release.yml \
              | grep -v '^\s*#' | sort -u > /tmp/wf.txt
            find "libs/agno/tests/integration/${dir}" -name 'test_*.py' \
              | sed 's|^|./|' | sort > /tmp/repo.txt
            if ! diff /tmp/repo.txt /tmp/wf.txt; then
              echo "::error::${dir} test coverage mismatch — see diff above"
              status=1
            fi
          done
          exit $status

  test-workflows:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run workflows integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/workflows

  test-embedder:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
      JINA_API_KEY: ${{ secrets.JINA_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run embedder tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/embedder

  test-knowledge-1:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run knowledge integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/knowledge/test_csv_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_docling_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_docx_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_json_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_md_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_pdf_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_pptx_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_text_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_async_knowledge_retriever.py

  test-knowledge-2:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run knowledge integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/knowledge/test_arxiv_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_firecrawl_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_website_knowledge.py \
          ./libs/agno/tests/integration/knowledge/test_youtube_knowledge.py \
          ./libs/agno/tests/integration/knowledge/filters/test_agentic_filtering.py

  # Run A2A tests (isolated due to dependency conflicts)
  test-a2a:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Setup environment for A2A tests (without conflicting deps)
        working-directory: .
        run: |
          VIRTUAL_ENV=.venv uv venv --python 3.12
          VIRTUAL_ENV=.venv uv pip install -r ./libs/agno/requirements.txt
          VIRTUAL_ENV=.venv uv pip install -e ./libs/agno[dev,os,integration-tests,openai]
          VIRTUAL_ENV=.venv uv pip install a2a-sdk
      - name: Run A2A integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          pip install a2a-sdk
          python -m pytest ./libs/agno/tests/integration/os/interfaces/test_a2a.py

  test-clean:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Setup environment
        working-directory: .
        run: |
          VIRTUAL_ENV=.venv uv venv --python 3.12
          VIRTUAL_ENV=.venv uv pip install agno
          VIRTUAL_ENV=.venv uv pip install -r ./libs/agno/requirements.txt
          VIRTUAL_ENV=.venv uv pip install openai chromadb sqlalchemy pytest pytest-asyncio pytest-cov pytest-mock
      - name: Run Basic Clean Test
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/test_basic.py

  test-agent-os:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run AgentOS integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest --ignore=./libs/agno/tests/integration/os/interfaces/test_a2a.py ./libs/agno/tests/integration/os


  test-clean-os:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Setup environment
        working-directory: .
        run: |
          VIRTUAL_ENV=.venv uv venv --python 3.12
          VIRTUAL_ENV=.venv uv pip install agno
          VIRTUAL_ENV=.venv uv pip install -r ./libs/agno/requirements.txt
          VIRTUAL_ENV=.venv uv pip install openai chromadb sqlalchemy fastapi pytest pytest-asyncio pytest-cov pytest-mock
      - name: Run Basic Clean Test
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./libs/agno/tests/integration/test_os_basic.py

  # Run remaining integration tests
  test-remaining:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true # Stop all matrix jobs if one fails
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Run dev setup
        working-directory: .
        run: |
          ./scripts/test_setup.sh
      - name: Run remaining integration tests
        working-directory: .
        env:
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest --ignore=./libs/agno/tests/integration/db --ignore=./libs/agno/tests/integration/models --ignore=./libs/agno/tests/integration/agent --ignore=./libs/agno/tests/integration/knowledge --ignore=./libs/agno/tests/integration/teams --ignore=./libs/agno/tests/integration/embedder --ignore=./libs/agno/tests/integration/workflows --ignore=./libs/agno/tests/integration/workflows_2 --ignore=./libs/agno/tests/integration/os/interfaces/test_a2a.py --ignore=./libs/agno/tests/integration/os ./libs/agno/tests/integration

  test-dev-setup:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Make dev_setup.sh executable
        run: chmod +x ./scripts/dev_setup.sh
      - name: Run dev setup script
        run: |
          ./scripts/dev_setup.sh
      - name: Verify virtual environment was created
        run: |
          if [ ! -d ".venv" ]; then
            echo "Error: .venv directory not found"
            exit 1
          fi
          echo "✓ Virtual environment created successfully"
      - name: Verify agno installation
        run: |
          source .venv/bin/activate
          python -c "import agno; print(f'✓ agno version: {agno.__version__}')" || exit 1
      - name: Verify agno_infra installation
        run: |
          source .venv/bin/activate
          uv pip show agno-infra || exit 1
      - name: List installed packages
        run: |
          source .venv/bin/activate
          uv pip list

  # Run AgentOS System Tests (multi-container Docker setup)
  test-agentos-system:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]
      fail-fast: true
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install uv
        run: |
          curl -LsSf https://astral.sh/uv/install.sh | sh
          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
      - name: Create virtual environment
        working-directory: ./libs/agno/tests/system
        run: |
          VIRTUAL_ENV=.venv uv venv --python 3.12
      - name: Install test dependencies
        working-directory: ./libs/agno/tests/system
        run: |
          source .venv/bin/activate
          VIRTUAL_ENV=.venv uv pip install -r requirements.txt
      - name: Start Docker Compose services
        working-directory: ./libs/agno/tests/system
        run: |
          docker compose up --build -d
      - name: Wait for services to be healthy
        working-directory: ./libs/agno/tests/system
        run: |
          echo "Waiting for services to become healthy..."
          MAX_WAIT=180
          WAIT_INTERVAL=10
          ELAPSED=0

          while [ $ELAPSED -lt $MAX_WAIT ]; do
            GATEWAY_HEALTH=$(docker inspect --format='{{.State.Health.Status}}' system-test-gateway 2>/dev/null || echo "not_found")
            REMOTE_HEALTH=$(docker inspect --format='{{.State.Health.Status}}' system-test-remote 2>/dev/null || echo "not_found")
            POSTGRES_HEALTH=$(docker inspect --format='{{.State.Health.Status}}' system-test-postgres 2>/dev/null || echo "not_found")

            echo "  Postgres: $POSTGRES_HEALTH, Remote: $REMOTE_HEALTH, Gateway: $GATEWAY_HEALTH"

            if [ "$GATEWAY_HEALTH" = "healthy" ] && [ "$REMOTE_HEALTH" = "healthy" ] && [ "$POSTGRES_HEALTH" = "healthy" ]; then
              echo "All services are healthy!"
              break
            fi

            sleep $WAIT_INTERVAL
            ELAPSED=$((ELAPSED + WAIT_INTERVAL))
          done

          if [ $ELAPSED -ge $MAX_WAIT ]; then
            echo "Timeout waiting for services to be healthy"
            docker compose logs
            exit 1
          fi
      - name: Run system tests
        working-directory: ./libs/agno/tests/system
        env:
          GATEWAY_URL: http://localhost:7001
          AGNO_TELEMETRY: false
        run: |
          source .venv/bin/activate
          python -m pytest ./tests -v --tb=short
      - name: Show container logs on failure
        if: failure()
        working-directory: ./libs/agno/tests/system
        run: |
          echo "=== Gateway Server Logs ==="
          docker compose logs gateway-server --tail=100
          echo ""
          echo "=== Remote Server Logs ==="
          docker compose logs remote-server --tail=100
          echo ""
          echo "=== A2A Server Logs ==="
          docker compose logs agno-a2a-server --tail=100
          echo ""
          echo "=== ADK Server Logs ==="
          docker compose logs adk-server --tail=100
          echo ""
      - name: Cleanup Docker resources
        if: always()
        working-directory: ./libs/agno/tests/system
        run: |
          docker compose down -v