deepset-ai/haystack

name: Approve and merge API reference sync PRs

# Automatically approve and merge API reference sync PRs from Haystack, Haystack Core Integrations,
# and Haystack Experimental

on:
  pull_request:
    branches:
      - main
    paths:
      - "docs-website/reference/**"
      - "docs-website/reference_versioned_docs/**"

permissions:
  pull-requests: write
  contents: write

env:
  GH_TOKEN: ${{ github.token }}

jobs:
  auto-approve-and-merge:
    if: |
      github.event.pull_request.user.login == 'HaystackBot' &&
      startsWith(github.event.pull_request.head.ref, 'sync-docusaurus-api-reference') &&
      github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-slim
    steps:
      - name: Approve PR
        run: gh pr review --approve ${{ github.event.pull_request.number }} --repo ${{ github.repository }}

      - name: Enable auto-merge
        run: gh pr merge --squash --auto ${{ github.event.pull_request.number }} --repo ${{ github.repository }}

name: Branch off

on:
  workflow_dispatch:
  workflow_call:
    outputs:
      bump_version_pr_url:
        description: 'URL of the bump version PR'
        value: ${{ jobs.branch-off.outputs.bump_version_pr_url }}
env:
  PYTHON_VERSION: "3.10"

jobs:
  branch-off:
    runs-on: ubuntu-slim
    outputs:
      bump_version_pr_url: ${{ steps.create-pr.outputs.pull-request-url }}

    steps:
      - name: Checkout this repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: main

      - name: Define all versions
        id: versions
        shell: bash
        run: |
          # example: 2.20.0-rc0 in VERSION.txt -> 2.20
          echo "current_version_major_minor=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT"
          # example: 2.20.0-rc0 in VERSION.txt -> 2.21.0-rc0
          echo "next_version_rc0=$(awk -F. '/[0-9]+\./{$2++;print}' OFS=. < VERSION.txt)" >> "$GITHUB_OUTPUT"

      - name: Create release branch and tag
        shell: bash
        env:
          # We use the HAYSTACK_BOT_TOKEN here so the PR created by the step will
          # trigger required workflows and can be merged by anyone
          GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"

          # Create the release branch from main
          git checkout -b v${{ steps.versions.outputs.current_version_major_minor }}.x
          git push -u origin v${{ steps.versions.outputs.current_version_major_minor }}.x

          # Tag the branch-off point with the next version rc0 to mark start of next dev cycle.
          # The tag points to this commit (before VERSION.txt is bumped).
          # This is intentional for reno to work properly.
          git tag "v${{ steps.versions.outputs.next_version_rc0 }}" -m "v${{ steps.versions.outputs.next_version_rc0 }}"
          git push --tags

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: "22"

      - name: Prepare changes for main
        shell: bash
        run: |
          git checkout main

          # Bump VERSION.txt to next version rc0
          echo "${{ steps.versions.outputs.next_version_rc0 }}" > VERSION.txt

          # Generate unstable docs for Docusaurus
          python ./.github/utils/create_unstable_docs_docusaurus.py --new-version ${{ steps.versions.outputs.current_version_major_minor }}

      - name: Create PR to bump unstable version and create unstable docs
        id: create-pr
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          commit-message: "Bump unstable version and create unstable docs"
          branch: bump-version
          base: main
          title: "Bump unstable version and create unstable docs"
          add-paths: |
            VERSION.txt
            docs-website
          body: |
            This PR:
            - Bumps the unstable version to `${{ steps.versions.outputs.next_version_rc0 }}`
            - Creates the unstable docs for Haystack ${{ steps.versions.outputs.current_version_major_minor }}

            You can inspect the docs preview (two unstable versions will be available) and merge it.
          labels: "ignore-for-release-notes"
          reviewers: "${{ github.actor }}"

name: Check API reference changes

on:
  pull_request:
    paths:
      - "haystack/**/*.py"
      - "pydoc/*.yml"

jobs:
  test-api-reference-build:
    runs-on: ubuntu-slim
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.13"

      - name: Detect API reference changes
        id: changed
        shell: python
        run: |
          import os
          import subprocess
          from pathlib import Path

          import sys
          sys.path.insert(0, ".github/utils")
          from docstrings_checksum import docstrings_checksum

          def git(*args):
              result = subprocess.run(["git", *args], capture_output=True, text=True)
              return result.stdout.strip(), result.returncode

          base_sha, _ = git("rev-parse", "HEAD^1")
          diff_output, _ = git("diff", "--name-only", f"{base_sha}...HEAD")
          changed_files = set(diff_output.splitlines())

          needs_check = False

          # If any pydoc config changed, always rebuild
          if any(f.startswith("pydoc/") and f.endswith(".yml") for f in changed_files):
              needs_check = True

          # If Python files changed, compare docstring checksums
          if not needs_check and any(f.startswith("haystack/") and f.endswith(".py") for f in changed_files):
              runner_temp = os.environ["RUNNER_TEMP"]
              base_worktree = os.path.join(runner_temp, "base")
              _, rc = git("worktree", "add", base_worktree, base_sha)

              pr_checksum = docstrings_checksum(Path(".").glob("haystack/**/*.py"))
              base_checksum = ""
              if rc == 0:
                  base_checksum = docstrings_checksum(Path(base_worktree).glob("haystack/**/*.py"))

              if pr_checksum != base_checksum:
                  needs_check = True

          print(f"API reference check needed: {needs_check}")
          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
              f.write(f"needs_check={str(needs_check).lower()}\n")

      - name: Install Hatch
        if: steps.changed.outputs.needs_check == 'true'
        run: pip install hatch

      - name: Generate API references
        if: steps.changed.outputs.needs_check == 'true'
        run: hatch run docs

      - name: Set up Node.js
        if: steps.changed.outputs.needs_check == 'true'
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: "22"

      - name: Run Docusaurus md/mdx checker
        if: steps.changed.outputs.needs_check == 'true'
        working-directory: tmp_api_reference
        run: |
          # docusaurus-mdx-checker is a package that is not frequently updated. Its dependency katex sometimes ships a
          # broken ESM build, where a __VERSION__ placeholder is left unresolved, causing a ReferenceError at import time.
          # Node 22+ prefers ESM when available. We force CJS (CommonJS) resolution to use the working katex build.
          # This should be safe because docusaurus-mdx-checker and its dependencies provide CJS builds.
          export NODE_OPTIONS="--conditions=require"
          npx docusaurus-mdx-checker -v || {
              echo ""
              echo "For common MDX problems, see https://docusaurus.io/blog/preparing-your-site-for-docusaurus-v3#common-mdx-problems"
              exit 1
            }

name: CI Metrics

on:
  workflow_run:
    workflows:
      - "end-to-end"
      - "Tests"
      - "Slow Integration Tests"
    types:
      - completed
  pull_request:
    types:
      - opened
      - closed
jobs:
  send:
    runs-on: ubuntu-slim
    steps:
      - uses: int128/datadog-actions-metrics@5a8f6d6b794600afbbaa15140a7281a60d7ab0ca # v1.156.0
        with:
          datadog-api-key: ${{ secrets.DATADOG_API_KEY }}
          datadog-site: "datadoghq.eu"
          collect-job-metrics: true

name: Docker image release

on:
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - '.github/workflows/docker_release.yml'
      - 'docker/**'
      - 'haystack/**'
      - 'pyproject.toml'
      - 'VERSION.txt'
    tags:
      - "v2.[0-9]+.[0-9]+*"

env:
  DOCKER_REPO_NAME: deepset/haystack

jobs:
  build-and-push:
    name: Build base image
    runs-on: ubuntu-latest
    if: github.repository_owner == 'deepset-ai'

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up QEMU
        uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0

      - name: Login to DockerHub
        uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0
        with:
          images: $DOCKER_REPO_NAME

      - name: Detect stable version
        run: |
          if [[ "${{ steps.meta.outputs.version }}" =~ ^v2\.[0-9]+\.[0-9]+$ ]]; then
            echo "IS_STABLE=true" >> "$GITHUB_ENV"
            echo "Stable version detected"
          else
            echo "Not a stable version"
          fi
      - name: Build base images
        uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7.0.0
        env:
          IMAGE_TAG_SUFFIX: ${{ steps.meta.outputs.version }}
          HAYSTACK_VERSION: ${{ steps.meta.outputs.version }}
        with:
          source: ./docker
          targets: base
          push: true

      - name: Test base image
        run: |
          EXPECTED_VERSION=$(cat VERSION.txt)
          if [[ $EXPECTED_VERSION == *"-"* ]]; then
            EXPECTED_VERSION=$(cut -d '-' -f 1 < VERSION.txt)$(cut -d '-' -f 2 < VERSION.txt)
          fi
          TAG="base-${{ steps.meta.outputs.version }}"

          PLATFORM="linux/amd64"
          VERSION=$(docker run --platform "$PLATFORM" --rm "deepset/haystack:$TAG" python -c"from haystack.version import __version__; print(__version__)")
          [[ "$VERSION" = "$EXPECTED_VERSION" ]] || echo "::error 'Haystack version in deepset/haystack:$TAG image for $PLATFORM is different from expected'"

          PLATFORM="linux/arm64"
          VERSION=$(docker run --platform "$PLATFORM" --rm "deepset/haystack:$TAG" python -c"from haystack.version import __version__; print(__version__)")
          [[ "$VERSION" = "$EXPECTED_VERSION" ]] || echo "::error 'Haystack version in deepset/haystack:$TAG image for $PLATFORM is different from expected'"

          # Remove image after test to avoid filling the GitHub runner and prevent its failure
          docker rmi "deepset/haystack:$TAG"

name: Test Python snippets in docs

on:
  schedule:
    - cron: '17 3 * * *'  # daily at 03:17 UTC
  workflow_dispatch:
    inputs:
      haystack_version:
        description: 'Haystack version to test against (e.g., 2.16.1, main)'
        required: false
        default: 'main'
        type: string

  # TEMPORARILY DISABLED
  # push:
  #   paths:
  #     - 'docs-website/docs/**'
  #     - 'docs-website/versioned_docs/**'
  #     - 'docs-website/scripts/test_python_snippets.py'
  #     - 'docs-website/scripts/generate_requirements.py'
  #     - '.github/workflows/docs-website-test-docs-snippets.yml'

jobs:
  test-docs-snippets:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      # TODO: We'll properly set these after migration to core project
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: '3.11'

      - name: Install base dependencies
        run: |
          python -m pip install --upgrade pip
          pip install requests toml

      - name: Generate requirements.txt
        run: |
          # Use input version or default to main
          if [ "${{ github.event.inputs.haystack_version }}" != "" ]; then
            VERSION="${{ github.event.inputs.haystack_version }}"
          else
            VERSION="main"
          fi
          echo "Generating requirements.txt for Haystack version: $VERSION"
          python docs-website/scripts/generate_requirements.py --version "$VERSION"

      - name: Install dependencies
        run: |
          if [ -f requirements.txt ]; then
            echo "Installing dependencies from requirements.txt"
            pip install -r requirements.txt
          else
            echo "Error: requirements.txt was not generated"
            exit 1
          fi

      - name: Run snippet tests (verbose)
        run: |
          # TEMPORARY: Testing with single file to make CI green
          # TODO: Expand to run all docs: --paths docs versioned_docs
          python docs-website/scripts/test_python_snippets.py docs-website/reference/haystack-api/agents_api.md

name: Docs Search Sync

on:
  workflow_dispatch: # Activate this workflow manually
  schedule:
    - cron: "0 1 * * *"

# Running this workflow multiple times in parallel can cause issues with files uploads and deletions.
concurrency:
  group: docs-search-sync
  cancel-in-progress: false

jobs:
  docs-search-sync:
    runs-on: ubuntu-latest

    steps:

      - name: Checkout Haystack repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Node.js
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: "22"

      - name: Install Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.12"

      - name: Install Docusaurus and build docs-website
        working-directory: docs-website
        run: |
          npm install
          npm run build

      - name: Install script dependencies
        # sniffio is needed because of https://github.com/deepset-ai/deepset-cloud-sdk/issues/286
        # we pin pyrate-limiter due to https://github.com/deepset-ai/deepset-cloud-sdk/issues/295
        run: pip install deepset-cloud-sdk sniffio requests "pyrate-limiter<4"

      - name: Update new docs to Search pipeline and remove outdated docs
        env:
          DEEPSET_WORKSPACE_DOCS_SEARCH: ${{ secrets.DEEPSET_WORKSPACE_DOCS_SEARCH }}
          DEEPSET_API_KEY_DOCS_SEARCH: ${{ secrets.DEEPSET_API_KEY_DOCS_SEARCH }}
        run: python ./.github/utils/docs_search_sync.py

      - name: Notify Slack on nightly failure
        if: failure() && github.event_name == 'schedule'
        uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

name: Add label on docstrings edit

on:
  pull_request_target:
    paths:
      - "haystack/**/*.py"

env:
  PYTHON_VERSION: "3.11"

jobs:
  label:
    runs-on: ubuntu-slim

    steps:
      - name: Checkout base commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.base_ref }}

      - name: Copy file
        # We copy our script after base ref checkout so we keep executing
        # the same version even after checking out the HEAD ref.
        # This is done to prevent executing malicious code in forks' PRs.
        run: cp .github/utils/docstrings_checksum.py "${{ runner.temp }}/docstrings_checksum.py"

      - name: Setup Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Get docstrings
        id: base-docstrings
        run: |
          CHECKSUM=$(python "${{ runner.temp }}/docstrings_checksum.py" --root "${{ github.workspace }}")
          echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"

      - name: Checkout HEAD commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ github.event.pull_request.head.ref }}
          # This must be set to correctly checkout a fork
          repository: ${{ github.event.pull_request.head.repo.full_name }}

      - name: Get docstrings
        id: head-docstrings
        run: |
          CHECKSUM=$(python "${{ runner.temp }}/docstrings_checksum.py" --root "${{ github.workspace }}")
          echo "checksum=$CHECKSUM" >> "$GITHUB_OUTPUT"

      - name: Check if we should label
        id: run-check
        run: echo "should_run=${{ steps.base-docstrings.outputs.checksum != steps.head-docstrings.outputs.checksum }}" >> "$GITHUB_OUTPUT"

      - name: Add label
        if: ${{ steps.run-check.outputs.should_run == 'true' }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh pr edit ${{ github.event.pull_request.html_url }} --add-label "type:documentation"

name: Sync docs with Docusaurus

on:
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - "pydoc/**"
      - "haystack/**"
      - ".github/workflows/docusaurus_sync.yml"

env:
  HATCH_VERSION: "1.16.5"
  PYTHON_VERSION: "3.11"

jobs:
  sync:
    runs-on: ubuntu-slim
    permissions:
      contents: write

    steps:
      - name: Checkout Haystack repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Generate API reference for Docusaurus
        run: hatch run docs

      - name: Sync generated API reference to docs folder
        run: |
          SOURCE_PATH="tmp_api_reference"
          DEST_PATH="docs-website/reference/haystack-api"

          echo "Syncing from $SOURCE_PATH to $DEST_PATH"
          mkdir -p $DEST_PATH
          # Using rsync to copy files. This will also remove files in dest that are no longer in source.
          rsync -av --delete --exclude='.git/' "$SOURCE_PATH/" "$DEST_PATH/"

      - name: Create Pull Request
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          commit-message: "Sync Haystack API reference on Docusaurus"
          branch: sync-docusaurus-api-reference
          base: main
          title: "docs: sync Haystack API reference on Docusaurus"
          add-paths: |
            docs-website/reference/haystack-api
          body: |
            This PR syncs the Haystack API reference on Docusaurus. Just approve and merge it.

# If you change this name also do it in ci_metrics.yml
name: end-to-end

on:
  workflow_dispatch: # Activate this workflow manually
  schedule:
    - cron: "0 0 * * *"
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
    paths:
      - "e2e/**/*.py"
      - ".github/workflows/e2e.yml"

env:
  PYTHON_VERSION: "3.10"
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  HATCH_VERSION: "1.16.5"
  # we use HF_TOKEN instead of HF_API_TOKEN to work around a Hugging Face bug
  # see https://github.com/deepset-ai/haystack/issues/9552
  HF_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}

jobs:
  run:
    timeout-minutes: 60
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: "${{ env.PYTHON_VERSION }}"

    - name: Install Hatch
      run: pip install hatch==${{ env.HATCH_VERSION }}

    - name: Run tests
      run: hatch run e2e:test

    - name: Notify Slack on nightly failure
      if: failure() && github.event_name == 'schedule'
      uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
      with:
        slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

name: Project release on Github

on:
  workflow_dispatch:  # this is useful to re-generate the release page without a new tag being pushed
  push:
    tags:
      - "v2.[0-9]+.[0-9]+*"
      # Ignore release versions tagged with -rc0 suffix
      - "!v2.[0-9]+.[0-9]-rc0"
jobs:
  generate-notes:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-tags: true
          fetch-depth: 0  # slow but needed by reno

      - name: Parse version
        id: version
        run: |
          echo "current_release=$(awk -F \\- '{print $1}' < VERSION.txt)" >> "$GITHUB_OUTPUT"
          echo "current_pre_release=$(awk -F \\- '{print $2}' < VERSION.txt)" >> "$GITHUB_OUTPUT"

      - name: Install reno
        run: |
          python -m pip install --upgrade pip
          pip install "reno<5"

      # Remove next version rc0 tag in the CI environment to prevent reno from assigning notes to future releases.
      # This ensures release notes are correctly aggregated for the current version.
      # This is a workaround. Can be removed if the release process is fully aligned with reno.
      - name: Delete next version rc0 tag in the CI environment
        run: |
          # Parse version X.Y.Z and increment Y for next minor version
          IFS='.' read -r MAJOR MINOR _ <<< "${{ steps.version.outputs.current_release }}"
          NEXT_MINOR=$((MINOR + 1))
          NEXT_TAG="v${MAJOR}.${NEXT_MINOR}.0-rc0"

          if git rev-parse --verify "$NEXT_TAG" >/dev/null 2>&1; then
            git tag -d "$NEXT_TAG"
            echo "Deleted local tag $NEXT_TAG"
          else
            echo "Tag $NEXT_TAG does not exist locally"
          fi

      - name: Generate release notes
        env:

          EARLIEST_VERSION: v${{ steps.version.outputs.current_release }}-rc1
        run: |
          reno report --no-show-source --ignore-cache --earliest-version "$EARLIEST_VERSION" -o relnotes.rst

      - name: Convert to Markdown
        uses: docker://pandoc/core:3.8
        with:
          args: "--from rst --to gfm --syntax-highlighting=none --wrap=none relnotes.rst -o relnotes.md"

      # We copy the relnotes file since the original one cannot be modified due to permissions
      - name: Copy relnotes file
        run: |
          cat relnotes.md > enhanced_relnotes.md

      - name: Add contributor list
          # only for minor releases and minor release candidates (not bugfix releases)
        if: endsWith(steps.version.outputs.current_release, '.0')
        env:
          GH_TOKEN: ${{ github.token }}
          START: v${{ steps.version.outputs.current_release }}-rc0
          END: ${{ github.ref_name }}
        run: |
          JQ_EXPR='[.commits[].author.login]
            | map(select(. != null and . != "HaystackBot" and . != "dependabot[bot]" and . != "github-actions[bot]"))
            | unique
            | sort_by(ascii_downcase)
            | map("@\(.)")
            | join(", ")'
          CONTRIBUTORS=$(gh api "repos/deepset-ai/haystack/compare/$START...$END" \
            --jq "$JQ_EXPR") || { echo "Unable to fetch contributors"; exit 1; }

          {
            echo ""
            echo "## 💙 Big thank you to everyone who contributed to this release!"
            echo ""
            echo "$CONTRIBUTORS"
          } >> enhanced_relnotes.md

      - name: Debug
        run: |
          cat enhanced_relnotes.md

      - uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0
        with:
          bodyFile: "enhanced_relnotes.md"
          prerelease: ${{ steps.version.outputs.current_pre_release != '' }}
          allowUpdates: true

name: "Labeler"
on:
- pull_request_target

permissions:
  contents: read
  pull-requests: write

jobs:
  triage:
    runs-on: ubuntu-slim
    steps:
    - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1
      with:
        repo-token: "${{ secrets.GITHUB_TOKEN }}"

name: License Compliance

on:
  pull_request:
    paths:
      - "**/pyproject.toml"
      - ".github/workflows/license_compliance.yml"
  # Since we test PRs, there is no need to run the workflow at each
  # merge on `main`. Let's use a cron job instead.
  schedule:
    - cron: "0 0 * * *" # every day at midnight
env:
  PYTHON_VERSION: "3.10"

jobs:
  license_check_direct:
    name: Direct dependencies only
    env:
      REQUIREMENTS_FILE: requirements_direct.txt
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Get direct dependencies
        run: |
          pip install toml
          python .github/utils/pyproject_to_requirements.py pyproject.toml > ${{ env.REQUIREMENTS_FILE }}

      - name: Check Licenses
        id: license_check_report
        uses: pilosus/action-pip-license-checker@e909b0226ff49d3235c99c4585bc617f49fff16a # v3.1.0
        with:
          github-token: ${{ secrets.GH_ACCESS_TOKEN }}
          requirements: ${{ env.REQUIREMENTS_FILE }}
          fail: "Copyleft,Other,Error"
          # Exclusions in the vanilla distribution must be explicitly motivated
          # - tqdm is MLP but there are no better alternatives
          # - typing_extensions>=4.13.0 has a Python Software Foundation License 2.0 but pip-license-checker does not recognize it
          #   (https://github.com/pilosus/pip-license-checker/issues/143)
          exclude: "(?i)^(tqdm|typing_extensions).*"

      # We keep the license inventory on FOSSA
      - name: Send license report to Fossa
        uses: fossas/fossa-action@c414b9ad82eaad041e47a7cf62a4f02411f427a0 # v1.8.0
        continue-on-error: true # not critical
        with:
          api-key: ${{ secrets.FOSSA_LICENSE_SCAN_TOKEN }}

      - name: Print report
        if: ${{ always() }}
        run: echo "${{ steps.license_check_report.outputs.report }}"

      - name: Notify Slack on failure
        if: failure()
        uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

name: Nightly pre-release on PyPI

on:
  schedule:
    # Run at midnight UTC every day
    - cron: "0 0 * * *"
  workflow_dispatch:

env:
  HATCH_VERSION: "1.16.5"

jobs:
  nightly-release:
    runs-on: ubuntu-latest
    # Always build from main for consistency (scheduled and manual runs)
    steps:
      - name: Checkout main
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: main
          fetch-depth: 1

      # Reads VERSION.txt, strips any -rcN suffix, and appends .devYYYYMMDDHHMMSS
      # (e.g. 2.25.0.dev20250217000000) so each run gets a unique, PEP 440–valid pre-release version.
      - name: Set nightly version
        id: set-version
        run: |
          BASE_VERSION=$(sed 's/-rc[0-9]*$//' VERSION.txt)
          TIMESTAMP=$(date +%Y%m%d%H%M%S)
          NIGHTLY_VERSION="${BASE_VERSION}.dev${TIMESTAMP}"
          echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT"
          echo "${NIGHTLY_VERSION}" > VERSION.txt
          echo "Building haystack-ai version: ${NIGHTLY_VERSION}"

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Build Haystack
        run: hatch build

      - name: Publish to PyPI
        env:
          HATCH_INDEX_USER: __token__
          HATCH_INDEX_AUTH: ${{ secrets.HAYSTACK_AI_PYPI_TOKEN }}
        run: hatch publish -y

name: Track issues with Github project

on:
  issues:
    types:
      - opened

jobs:
  add-to-project:
    name: Add new issues to project for triage
    runs-on: ubuntu-slim
    steps:
      - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2
        with:
          project-url: https://github.com/orgs/deepset-ai/projects/5
          github-token: ${{ secrets.GH_PROJECT_PAT }}

name: Release new minor version docs

on:
  push:
    tags:
      # Trigger this only for new minor version tags (e.g. v2.99.0)
      - "v[0-9]+.[0-9]+.0"
      # Exclude 1.x tags
      - "!v1.[0-9]+.[0-9]+"

env:
  PYTHON_VERSION: "3.10"

jobs:
  promote:
    runs-on: ubuntu-slim
    steps:
      - name: Checkout this repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        # use VERSION.txt file from main branch
        with:
          ref: main

      - name: Get version to release
        id: version
        shell: bash
        # We only need `major.minor`. At this point, VERSION.txt contains the next version.
        # For example, if we are releasing 2.20.0, VERSION.txt contains 2.21.0-rc0.
        run: |
          MAJOR=$(cut -d "." -f 1 < VERSION.txt)
          MINOR=$(cut -d "." -f 2 < VERSION.txt)
          MINOR=$((MINOR - 1))
          echo "version=${MAJOR}.${MINOR}" >> "$GITHUB_OUTPUT"

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Promote unstable docs for Docusaurus
        run: |
          python ./.github/utils/promote_unstable_docs_docusaurus.py --version ${{ steps.version.outputs.version }}

      - name: Create Pull Request with Docusaurus docs updates
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          commit-message: "Promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
          branch: promote-unstable-docs-${{ steps.version.outputs.version }}
          base: main
          title: "docs: promote unstable docs for Haystack ${{ steps.version.outputs.version }}"
          add-paths: |
            docs-website
          body: |
            This PR promotes the unstable docs for Haystack ${{ steps.version.outputs.version }} to stable.
            It is expected to run at the time of the release.
            You can inspect the docs preview and merge it. There should now be only one unstable version representing the next (main) branch.
          # This workflow is triggered by a tag pushed by the HaystackBot in release.yml > create-release-tag.
          # GitHub requires reviewers to be different from the PR author, so setting `github.actor`
          # would fail (it would request a review from the HaystackBot itself).
          # So we don't set any reviewers and instead notify the Release Manager
          # (see .github/utils/prepare_release_notification.sh).

name: Push release notes to website

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Haystack version (vX.Y.Z)'
        required: true
        type: string

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  VERSION: ${{ inputs.version }}

jobs:
  push-release-notes-to-website:

    runs-on: ubuntu-slim
    steps:
      - name: Checkout Haystack home repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: deepset-ai/haystack-home

      - name: Get release notes and add frontmatter
        id: release_notes

        run: |
          VERSION_NUMBER="${VERSION:1}"
          RELEASE_DATE=$(gh release view "$VERSION" --repo deepset-ai/haystack --json publishedAt --jq '.publishedAt | split("T")[0]')
          RELEASE_NOTES_PATH="content/release-notes/$VERSION_NUMBER.md"

          {
            echo "---"
            echo "title: Haystack $VERSION_NUMBER"
            echo "description: Release notes for Haystack $VERSION_NUMBER"
            echo "toc: True"
            echo "date: $RELEASE_DATE"
            echo "last_updated: $RELEASE_DATE"
            echo 'tags: ["Release Notes"]'
            echo "link: https://github.com/deepset-ai/haystack/releases/tag/$VERSION"
            echo "---"
            echo ""
          } > "$RELEASE_NOTES_PATH"

          gh release view "$VERSION" --repo deepset-ai/haystack --json body --jq '.body' >> "$RELEASE_NOTES_PATH"

          cat "$RELEASE_NOTES_PATH"

      - name: Create Pull Request to Haystack Home
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          commit-message: "Add release notes for Haystack ${{ env.VERSION }}"
          branch: add-release-notes-for-haystack-${{ env.VERSION }}
          base: main
          title: "docs: add release notes for Haystack ${{ env.VERSION }}"
          add-paths: |
            content/release-notes
          body: |
            This PR adds the release notes for Haystack ${{ env.VERSION }} to the website.
          reviewers: "${{ github.actor }}"

name: Project release on PyPi

on:
  push:
    tags:
      - "v[0-9]+.[0-9]+.[0-9]+*"
      # We must not release versions tagged with -rc0 suffix
      - "!v[0-9]+.[0-9]+.[0-9]-rc0"

env:
  HATCH_VERSION: "1.16.5"

jobs:
  release-on-pypi:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Build Haystack
        run: hatch build

      - name: Publish on PyPi
        env:
          HATCH_INDEX_USER: __token__
          HATCH_INDEX_AUTH: ${{ secrets.HAYSTACK_AI_PYPI_TOKEN }}
        run: hatch publish -y

name: Release

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Version to release (e.g., v2.99.0-rc1 or v2.99.0)'
        required: true
        type: string

# Only one release workflow runs at a time; additional runs are queued.
concurrency:
  group: release
  cancel-in-progress: false

jobs:
  parse-validate-version:
    runs-on: ubuntu-slim
    outputs:
      version: ${{ steps.parse-validate.outputs.version }}
      major_minor: ${{ steps.parse-validate.outputs.major_minor }}
      release_branch: ${{ steps.parse-validate.outputs.release_branch }}
      is_rc: ${{ steps.parse-validate.outputs.is_rc }}
      is_first_rc: ${{ steps.parse-validate.outputs.is_first_rc }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0  # needed to fetch tags and branches
      - name: Parse and validate version
        id: parse-validate
        env:
          GH_TOKEN: ${{ github.token }}
        run: .github/utils/parse_validate_version.sh "${{ inputs.version }}"

  branch-off:
    needs: ["parse-validate-version"]
    if: needs.parse-validate-version.outputs.is_first_rc == 'true'
    uses: ./.github/workflows/branch_off.yml
    # https://docs.github.com/en/actions/how-tos/reuse-automations/reuse-workflows#passing-secrets-to-nested-workflows
    secrets: inherit

  create-release-tag:
    needs: ["parse-validate-version", "branch-off"]
    if: always() && needs.parse-validate-version.result == 'success' && needs.branch-off.result != 'failure'
    runs-on: ubuntu-slim
    permissions:
      contents: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0  # needed to fetch tags and branches
          # use this token so the created tag triggers workflows (does not happen with the default github.token)
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
      - name: Update VERSION.txt and create tag
        env:
          GITHUB_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
        run: |
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"

          git checkout ${{ needs.parse-validate-version.outputs.release_branch }}
          git pull origin ${{ needs.parse-validate-version.outputs.release_branch }}

          echo "${{ needs.parse-validate-version.outputs.version }}" > VERSION.txt
          git add VERSION.txt
          git commit -m "bump version to ${{ needs.parse-validate-version.outputs.version }}"
          git push origin ${{ needs.parse-validate-version.outputs.release_branch }}

          TAG="v${{ needs.parse-validate-version.outputs.version }}"
          git tag -m "$TAG" "$TAG"
          git push origin "$TAG"

  check-artifacts:
    needs: ["parse-validate-version", "create-release-tag"]
    if: always() && needs.parse-validate-version.result == 'success' && needs.create-release-tag.result == 'success'
    runs-on: ubuntu-latest
    outputs:
      github_url: ${{ steps.set-outputs.outputs.github_url }}
      pypi_url: ${{ steps.set-outputs.outputs.pypi_url }}
      docker_url: ${{ steps.set-outputs.outputs.docker_url }}
    env:
      GH_TOKEN: ${{ github.token }}
      VERSION: ${{ needs.parse-validate-version.outputs.version }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          fetch-depth: 0  # needed to fetch tags and branches

      - name: Wait for release workflows
        run: |
          .github/utils/wait_for_workflows.sh "v${{ env.VERSION }}" \
            "Project release on PyPi" \
            "Project release on Github" \
            "Docker image release"

      - name: Check artifacts
        run: |
          check() {
            for _ in {1..5}; do curl -sf "$2" > /dev/null && echo "✅ $1" && return 0; sleep 30; done
            echo "❌ $1 not found" && return 1
          }
          check "GitHub Release" "https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ env.VERSION }}"
          check "PyPI package" "https://pypi.org/pypi/haystack-ai/${{ env.VERSION }}/json"
          check "Docker image" "https://hub.docker.com/v2/repositories/deepset/haystack/tags/base-v${{ env.VERSION }}"

      - name: Set artifact URLs
        id: set-outputs
        run: |
          {
            echo "github_url=https://github.com/${{ github.repository }}/releases/tag/v${{ env.VERSION }}"
            echo "pypi_url=https://pypi.org/project/haystack-ai/${{ env.VERSION }}/"
            echo "docker_url=https://hub.docker.com/r/deepset/haystack/tags?name=base-v${{ env.VERSION }}"
          } >> "$GITHUB_OUTPUT"

  bump-dc-pipeline-templates:
    needs: ["parse-validate-version", "check-artifacts"]
    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
    runs-on: ubuntu-slim
    outputs:
      pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
    env:
      VERSION: ${{ needs.parse-validate-version.outputs.version }}
    steps:
      - name: Checkout dc-pipeline-templates
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: deepset-ai/dc-pipeline-templates
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}

      - name: Update haystack pin
        run: sed -i "s/haystack-ai>=.*/haystack-ai>=${VERSION}/" requirements-test.txt

      - name: Create Pull Request
        id: create-pr
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          commit-message: "Bump haystack to ${{ env.VERSION }}"
          branch: bump-haystack-${{ env.VERSION }}
          base: main
          title: "chore: bump haystack to ${{ env.VERSION }}"
          add-paths: requirements-test.txt
          body: |
            Bump haystack pin to `${{ env.VERSION }}` for platform testing.

  bump-deepset-cloud-custom-nodes:
    needs: ["parse-validate-version", "check-artifacts"]
    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
    runs-on: ubuntu-slim
    outputs:
      pr_url: ${{ steps.create-pr.outputs.pull-request-url }}
    env:
      VERSION: ${{ needs.parse-validate-version.outputs.version }}
    steps:
      - name: Checkout haystack (for utils)
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: haystack
          sparse-checkout: .github/utils/update_haystack_dc_custom_nodes.py
          sparse-checkout-cone-mode: false

      - name: Checkout deepset-cloud-custom-nodes
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: deepset-ai/deepset-cloud-custom-nodes
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          path: deepset-cloud-custom-nodes

      - name: Set up Python
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "3.13"

      - name: Install tomlkit
        run: pip install tomlkit

      - name: Update haystack-ai in uv.lock
        run: python haystack/.github/utils/update_haystack_dc_custom_nodes.py "${{ env.VERSION }}" deepset-cloud-custom-nodes/uv.lock

      - name: Create Pull Request
        id: create-pr
        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
        with:
          token: ${{ secrets.HAYSTACK_BOT_TOKEN }}
          path: deepset-cloud-custom-nodes
          commit-message: "Bump haystack to ${{ env.VERSION }}"
          branch: bump-haystack-${{ env.VERSION }}
          base: main
          title: "chore: bump haystack to ${{ env.VERSION }}"
          add-paths: uv.lock
          body: |
            Bump haystack pin to `${{ env.VERSION }}` for platform testing.

  bump-dc-pipeline-images:
    needs: ["parse-validate-version", "check-artifacts"]
    if: always() && needs.check-artifacts.result == 'success' && needs.parse-validate-version.outputs.is_rc == 'true'
    runs-on: ubuntu-slim
    outputs:
      pr_url: ${{ steps.wait-pr.outputs.pr_url }}
    env:
      VERSION: ${{ needs.parse-validate-version.outputs.version }}
    steps:
      - name: Trigger "Update package version" workflow
        env:
          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
        run: |
          gh workflow run update-package-version.yaml \
            -R deepset-ai/dc-pipeline-images \
            -f haystack_version="${{ env.VERSION }}"

      - name: Wait for PR
        id: wait-pr
        env:
          GH_TOKEN: ${{ secrets.HAYSTACK_BOT_TOKEN }}
        run: |
          TRIGGER_TIME=$(date -u +%Y-%m-%dT%H:%M:%SZ)
          for i in $(seq 1 30); do
            PR_URL=$(gh pr list -R deepset-ai/dc-pipeline-images \
              --head "bump/hs${{ env.VERSION }}" \
              --json url,createdAt \
              --jq ".[] | select(.createdAt >= \"$TRIGGER_TIME\") | .url")
            if [[ -n "$PR_URL" ]]; then
              echo "pr_url=$PR_URL" >> "$GITHUB_OUTPUT"
              echo "Found PR: $PR_URL"
              exit 0
            fi
            echo "Attempt $i: PR not found yet, waiting 10s..."
            sleep 10
          done
          echo "PR not found after 5 minutes"

  notify:
    needs:
      - "parse-validate-version"
      - "branch-off"
      - "create-release-tag"
      - "check-artifacts"
      - "bump-dc-pipeline-templates"
      - "bump-deepset-cloud-custom-nodes"
      - "bump-dc-pipeline-images"
    if: always()
    runs-on: ubuntu-slim
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Prepare release notification
        env:
          VERSION: ${{ inputs.version }}
          GH_TOKEN: ${{ github.token }}
          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
          HAS_FAILURE: ${{ contains(needs.*.result, 'failure') }}
          IS_RC: ${{ needs.parse-validate-version.outputs.is_rc }}
          IS_FIRST_RC: ${{ needs.parse-validate-version.outputs.is_first_rc }}
          MAJOR_MINOR: ${{ needs.parse-validate-version.outputs.major_minor }}
          GITHUB_URL: ${{ needs.check-artifacts.outputs.github_url }}
          PYPI_URL: ${{ needs.check-artifacts.outputs.pypi_url }}
          DOCKER_URL: ${{ needs.check-artifacts.outputs.docker_url }}
          BUMP_VERSION_PR_URL: ${{ needs.branch-off.outputs.bump_version_pr_url }}
          DC_PIPELINE_TEMPLATES_PR_URL: ${{ needs.bump-dc-pipeline-templates.outputs.pr_url }}
          DC_CUSTOM_NODES_PR_URL: ${{ needs.bump-deepset-cloud-custom-nodes.outputs.pr_url }}
          DC_PIPELINE_IMAGES_PR_URL: ${{ needs.bump-dc-pipeline-images.outputs.pr_url }}
        run: .github/utils/prepare_release_notification.sh

      - name: Send release notification to Slack
        uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1
        with:
          webhook: ${{ secrets.SLACK_WEBHOOK_URL_RELEASE }}
          webhook-type: incoming-webhook
          payload-file-path: slack_payload.json

name: Check Release Notes

on:
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
      - labeled
      - unlabeled
    paths:
      - "**.py"
      - "pyproject.toml"
      - "!.github/**/*.py"
      - "releasenotes/notes/*.yaml"

jobs:
  reno:
    runs-on: ubuntu-slim
    env:
      PYTHON_VERSION: "3.10"
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # With the default value of 1, there are corner cases where tj-actions/changed-files
          # fails with a `no merge base` error
          fetch-depth: 0

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"
      - name: Get release note files
        id: changed-files
        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
        with:
          files: releasenotes/notes/*.yaml

      - name: Check release notes
        if: steps.changed-files.outputs.any_changed == 'false' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
        run: |
          # Check if any of the commit messages contain tags ci/docs/test
          if git log --pretty=%s origin/main..HEAD | grep -E '^(ci:|docs:|test:)' > /dev/null; then
            echo "Skipping release note check for commits with 'ci:', 'docs:', or 'test:' tags."
          else
            echo "::error::The release notes file is missing, please add one or attach the label 'ignore-for-release-notes' to this PR."
            exit 1
          fi

      - name: Verify release notes formatting
        if: steps.changed-files.outputs.any_changed == 'true' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
        run: |
          pip install "reno<5"
          reno lint .  # it is not possible to pass a list of files to reno lint

      - name: Check reStructuredText code formatting
        if: steps.changed-files.outputs.any_changed == 'true' && !contains( github.event.pull_request.labels.*.name, 'ignore-for-release-notes')
        shell: python
        run: |
          files = "${{ steps.changed-files.outputs.all_changed_files }}".split()
          errors = []

          for filepath in files:
            with open(filepath) as f:
              for line_no, line in enumerate(f, start=1):
                # Check for triple backticks (Markdown code blocks)
                if "```" in line:
                  err = (f"Format error in {filepath}:{line_no}: "
                         "Found triple backticks. Use reStructuredText code block directive instead: .. code:: python")
                  errors.append(err)

                # Check for single backticks (Markdown inline code)
                if "`" in line.replace("```", "").replace("``", ""):
                  err = (f"Format error in {filepath}:{line_no}: "
                         "Found single backticks. Use double backticks (``code``) for inline code.")
                  errors.append(err)

          if errors:
              raise Exception("\n".join(errors))

name: Check Release Notes

on:
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
      - labeled
      - unlabeled
    paths-ignore:
      - "**.py"
      - "pyproject.toml"
      - "!.github/**/*.py"
      - "releasenotes/notes/*.yaml"

jobs:
  reno:
    runs-on: ubuntu-slim
    steps:
      - name: Skip mandatory job
        run: echo "Skipped!"

# If you change this name also do it in ci_metrics.yml
name: Slow Integration Tests

# The workflow will always run, but the actual tests will only execute when:
# - The workflow is triggered manually
# - The workflow is scheduled
# - The PR has the "run-slow-tests" label
# - The push is to a release branch
# - There are changes to relevant files.
# Note: If no conditions are met, the workflow will complete successfully without running tests
# to satisfy Branch Protection rules.

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
  PYTHON_VERSION: "3.10"
  HATCH_VERSION: "1.16.5"
  HAYSTACK_MPS_ENABLED: false
  HAYSTACK_XPU_ENABLED: false

on:
  workflow_dispatch: # Activate this workflow manually
  schedule:
    - cron: "0 0 * * *"
  push:
    branches:
      # release branches have the form v1.9.x
      - "v[0-9].*[0-9].x"
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - labeled
      - unlabeled

jobs:
  check-if-changed:
  # This job checks if the relevant files have been changed.
  # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
  # This ensures the "Slow Integration Tests completed" job always runs, which is required by Branch Protection rules.
    name: Check if changed
    runs-on: ubuntu-slim
    permissions:
      pull-requests: read
    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
    outputs:
      changes: ${{ steps.changes.outputs.changes }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Check for changed code
        id: changes
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          # List of Python files that trigger slow integration tests when modified
          filters: |
            changes:
              - "haystack/components/audio/whisper_local.py"
              - "haystack/components/classifiers/zero_shot_document_classifier.py"
              - "haystack/components/converters/tika.py"
              - "haystack/components/embedders/hugging_face_api_document_embedder.py"
              - "haystack/components/embedders/hugging_face_api_text_embedder.py"
              - "haystack/components/embedders/backends/sentence_transformers_backend.py"
              - "haystack/components/embedders/backends/sentence_transformers_sparse_backend.py"
              - "haystack/components/embedders/image/sentence_transformers_doc_image_embedder.py"
              - "haystack/components/embedders/sentence_transformers_text_embedder.py"
              - "haystack/components/embedders/sentence_transformers_sparse_document_embedder.py"
              - "haystack/components/embedders/sentence_transformers_sparse_text_embedder.py"
              - "haystack/components/evaluators/sas_evaluator.py"
              - "haystack/components/generators/chat/hugging_face_api.py"
              - "haystack/components/generators/chat/hugging_face_local.py"
              - "haystack/components/generators/hugging_face_api.py"
              - "haystack/components/generators/hugging_face_local_generator.py"
              - "haystack/components/preprocessors/embedding_based_document_splitter.py"
              - "haystack/components/rankers/sentence_transformers_diversity.py"
              - "haystack/components/rankers/sentence_transformers_similarity.py"
              - "haystack/components/rankers/transformers_similarity.py"
              - "haystack/components/readers/extractive.py"
              - "haystack/components/retrievers/multi_query_embedding_retriever.py"
              - "haystack/components/routers/transformers_text_router.py"
              - "haystack/components/routers/zero_shot_text_router.py"

              - "test/components/audio/test_whisper_local.py"
              - "test/components/classifiers/test_zero_shot_document_classifier.py"
              - "test/components/converters/test_tika_doc_converter.py"
              - "test/components/embedders/test_hugging_face_api_document_embedder.py"
              - "test/components/embedders/test_hugging_face_api_text_embedder.py"
              - "test/components/embedders/image/test_sentence_transformers_doc_image_embedder.py"
              - "test/components/embedders/test_sentence_transformers_text_embedder.py"
              - "test/components/embedders/test_sentence_transformers_sparse_document_embedder.py"
              - "test/components/embedders/test_sentence_transformers_sparse_text_embedder.py"
              - "test/components/evaluators/test_sas_evaluator.py"
              - "test/components/generators/chat/test_hugging_face_api.py"
              - "test/components/generators/chat/test_hugging_face_local.py"
              - "test/components/generators/test_hugging_face_api.py"
              - "test/components/generators/test_hugging_face_local_generator.py"
              - "test/components/preprocessors/test_embedding_based_document_splitter.py"
              - "test/components/rankers/test_sentence_transformers_diversity.py"
              - "test/components/rankers/test_sentence_transformers_similarity.py"
              - "test/components/rankers/test_transformers_similarity.py"
              - "test/components/readers/test_extractive.py"
              - "test/components/retrievers/test_multi_query_embedding_retriever.py"
              - "test/components/routers/test_transformers_text_router.py"
              - "test/components/routers/test_zero_shot_text_router.py"

  slow-integration-tests:
    name: Slow Tests / ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    needs: check-if-changed
    timeout-minutes: 30
    # Run tests if: manual trigger, scheduled, PR has label, release branch, or relevant files changed
    if: |
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-slow-tests')) ||
      (github.event_name == 'push' && github.ref == 'refs/heads/v[0-9].*[0-9].x') ||
      (needs.check-if-changed.outputs.changes == 'true')

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        include:
          - os: ubuntu-latest
            install_cmd: "sudo apt update && sudo apt install ffmpeg"
          - os: macos-latest
            install_cmd: "brew install ffmpeg"
          - os: windows-latest
            install_cmd: "echo 'No additional dependencies needed'"

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}

      - name: Run Tika
        if: matrix.os == 'ubuntu-latest'
        run: |
          docker run -d -p 9998:9998 apache/tika:2.9.0.0

      - name: Install Whisper dependencies
        shell: bash
        run: ${{ matrix.install_cmd }}

      - name: Run tests
        run: hatch run test:integration-only-slow

      - name: Notify Slack on nightly failure
        if: failure() && github.event_name == 'schedule'
        uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

  slow-integration-tests-completed:
    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
    name: Slow Integration Tests completed
    runs-on: ubuntu-slim
    if: ${{ always() && !cancelled() }}
    needs: slow-integration-tests

    steps:
    - name: Mark tests as completed
      run: |
        if [ "${{ needs.slow-integration-tests.result }}" = "failure" ]; then
          echo "Slow Integration Tests failed!"
          exit 1
        else
          echo "Slow Integration Tests completed!"
        fi

name: 'Stalebot'
on:
  schedule:
    - cron: '30 1 * * *'

jobs:
  makestale:
    runs-on: ubuntu-slim
    steps:
      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
        with:
          any-of-labels: 'proposal,information-needed'
          stale-pr-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
          days-before-stale: 30
          days-before-close: 10

# If you change this name also do it in ci_metrics.yml
name: Tests

# The workflow will always run, but the actual tests will only execute when:
# - The workflow is triggered manually.
# - The push is to main or a release branch.
# - There are changes to relevant files on a pull request.
# Note: If no conditions are met, the workflow will complete successfully without running tests
# to satisfy Branch Protection rules.

on:
  workflow_dispatch: # Activate this workflow manually
  push:
    branches:
      - main
      # release branches have the form v1.9.x
      - "v[0-9].*[0-9].x"
    # when we push, we do not need to satisfy Branch Protection rules, so we can ignore PRs that just change docs
    paths-ignore:
      - 'docs/**'
      - 'docs-website/**'
  pull_request:
    types:
      - opened
      - reopened
      - synchronize

env:
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  CORE_AZURE_CS_ENDPOINT: ${{ secrets.CORE_AZURE_CS_ENDPOINT }}
  CORE_AZURE_CS_API_KEY: ${{ secrets.CORE_AZURE_CS_API_KEY }}
  AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
  AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }}
  PYTHON_VERSION: "3.10"
  HATCH_VERSION: "1.16.5"

jobs:
  check-if-changed:
  # This job checks if the relevant files have been changed.
  # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level.
  # This ensures the "Mark tests as completed" job always runs, which is required by Branch Protection rules.
    name: Check if changed
    runs-on: ubuntu-slim
    permissions:
      pull-requests: read
    # Specifying outputs is not needed to make the job work, but only to comply with actionlint
    outputs:
      changes: ${{ steps.changes.outputs.changes }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      - name: Check for changed code
        id: changes
        uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
        with:
          filters: |
            changes:
              - "haystack/**/*.py"
              - "test/**/*.py"
              - "pyproject.toml"
              - ".github/utils/*.py"
              - "scripts/*.py"

  format:
    needs: check-if-changed
    # Run tests if: manual trigger, push to main/release, or relevant files changed
    if: |
      github.event_name == 'workflow_dispatch' ||
      github.event_name == 'push' ||
      (needs.check-if-changed.outputs.changes == 'true')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Ruff - check format and linting
        run: hatch run fmt-check

      - name: Check presence of license header
        run: docker run --rm -v "$(pwd):/github/workspace" ghcr.io/korandoru/hawkeye check

  check-imports:
    needs: format
    runs-on: ubuntu-slim
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        run: pip install hatch==${{ env.HATCH_VERSION }}

      - name: Check imports
        run: hatch run python .github/utils/check_imports.py

  unit-tests:
    name: Unit / ${{ matrix.os }}
    needs: format
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-latest
          - windows-latest
          - macos-latest
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - name: Run
        run: hatch run test:unit

      - uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        id: cache
        if: matrix.os == 'macos-latest'
        with:
          path: ${{ steps.hatch.outputs.env }}
          key: ${{ runner.os }}-${{ github.sha }}

      - name: Coveralls
        # We upload only coverage for ubuntu as handling both os
        # complicates the workflow too much for little to no gain
        if: matrix.os == 'ubuntu-latest'
        uses: coverallsapp/github-action@5cbfd81b66ca5d10c19b062c04de0199c215fb6e # v2.3.7
        continue-on-error: true
        with:
          path-to-lcov: coverage.xml

  mypy:
    needs: unit-tests
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          # With the default value of 1, there are corner cases where tj-actions/changed-files
          # fails with a `no merge base` error
          fetch-depth: 0
      - name: Get changed files
        id: files
        uses: tj-actions/changed-files@22103cc46bda19c2b464ffe86db46df6922fd323 # v47.0.5
        with:
          files: |
            **/*.py
            pyproject.toml
          files_ignore: |
            test/**
            .github/**
            scripts/**
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        if: steps.files.outputs.any_changed == 'true'
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        if: steps.files.outputs.any_changed == 'true'
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - name: Mypy
        if: steps.files.outputs.any_changed == 'true'
        run: |
          mkdir .mypy_cache
          hatch run test:types

  integration-tests-linux:
    name: Integration / ubuntu-latest
    needs: unit-tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"


      - name: Run
        run: hatch run test:integration-only-fast

  integration-tests-macos:
    name: Integration / macos-latest
    needs: unit-tests
    runs-on: macos-latest
    timeout-minutes: 30
    env:
      HAYSTACK_MPS_ENABLED: false

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
        id: cache
        with:
          path: ${{ steps.hatch.outputs.env }}
          key: ${{ runner.os }}-${{ github.sha }}


      - name: Run
        run: hatch run test:integration-only-fast

  integration-tests-windows:
    name: Integration / windows-latest
    needs: unit-tests
    runs-on: windows-latest
    timeout-minutes: 30
    env:
      HAYSTACK_XPU_ENABLED: false

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: "${{ env.PYTHON_VERSION }}"

      - name: Install Hatch
        id: hatch
        shell: bash
        run: |
          pip install hatch==${{ env.HATCH_VERSION }}
          echo "env=$(hatch env find test)" >> "$GITHUB_OUTPUT"

      - name: Run
        run: hatch run test:integration-only-fast

  notify-slack-on-failure:
    if: failure() && github.ref_name == 'main'
    needs:
      - check-imports
      - mypy
      - integration-tests-linux
      - integration-tests-macos
      - integration-tests-windows
    runs-on: ubuntu-slim
    steps:
      - uses: deepset-ai/notify-slack-action@3cda73b77a148f16f703274198e7771340cf862b # v1
        with:
          slack-webhook-url: ${{ secrets.SLACK_WEBHOOK_URL_NOTIFICATIONS }}

  tests-completed:
    # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules.
    name: Mark tests as completed
    runs-on: ubuntu-slim
    if: ${{ always() && !cancelled() }}
    needs:
      - check-imports
      - mypy
      - integration-tests-linux
      - integration-tests-macos
      - integration-tests-windows

    steps:
    - name: Mark tests as completed
      run: |
        if [ "${{ needs.check-imports.result }}" = "failure" ] ||
           [ "${{ needs.mypy.result }}" = "failure" ] ||
           [ "${{ needs.integration-tests-linux.result }}" = "failure" ] ||
           [ "${{ needs.integration-tests-macos.result }}" = "failure" ] ||
           [ "${{ needs.integration-tests-windows.result }}" = "failure" ]; then
          echo "Tests failed!"
          exit 1
        else
          echo "Tests completed!"
        fi

name: Github workflows linter

on:
  pull_request:
    paths:
      - ".github/workflows/**"

jobs:
  lint-workflows:
    runs-on: ubuntu-slim
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0
        with:
          go-version: ">=1.24.0"

      - name: Install actionlint
        run: go install github.com/rhysd/actionlint/cmd/actionlint@latest

      - name: Run actionlint
        env:
          SHELLCHECK_OPTS: --exclude=SC2102
        run: actionlint