lancedb/lancedb

name: Cargo Publish

on:
  push:
    tags-ignore:
      # We don't publish pre-releases for Rust. Crates.io is just a source
      # distribution, so we don't need to publish pre-releases.
      - "v*-beta*"
      - "*-v*" # for example, python-vX.Y.Z

env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  # Up-to-date compilers needed for fp16kernels.
  CC: gcc-12
  CXX: g++-12

jobs:
  build:
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
    timeout-minutes: 30
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - uses: rust-lang/crates-io-auth-action@v1
        id: auth
      - name: Publish the package
        run: |
          cargo publish -p lancedb --all-features --token ${{ steps.auth.outputs.token }}
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [build]
    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}

name: Codex Fix CI

on:
  workflow_dispatch:
    inputs:
      workflow_run_url:
        description: "Failing CI workflow run URL (e.g., https://github.com/lancedb/lancedb/actions/runs/12345678)"
        required: true
        type: string
      branch:
        description: "Branch to fix (e.g., main, release/v2.0, or feature-branch)"
        required: true
        type: string
      guidelines:
        description: "Additional guidelines for the fix (optional)"
        required: false
        type: string

permissions:
  contents: write
  pull-requests: write
  actions: read

jobs:
  fix-ci:
    runs-on: warp-ubuntu-latest-x64-4x
    timeout-minutes: 60
    env:
      CC: clang
      CXX: clang++
    steps:
      - name: Show inputs
        run: |
          echo "workflow_run_url = ${{ inputs.workflow_run_url }}"
          echo "branch = ${{ inputs.branch }}"
          echo "guidelines = ${{ inputs.guidelines }}"

      - name: Checkout Repo
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.branch }}
          fetch-depth: 0
          persist-credentials: true

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install Codex CLI
        run: npm install -g @openai/codex

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: stable
          components: clippy, rustfmt

      - uses: Swatinem/rust-cache@v2

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler libssl-dev

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: |
          pip install maturin ruff pytest pyarrow pandas polars

      - name: Set up Java
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: '11'
          cache: maven

      - name: Install Node.js dependencies for TypeScript bindings
        run: |
          cd nodejs
          npm ci

      - name: Configure git user
        run: |
          git config user.name "lancedb automation"
          git config user.email "robot@lancedb.com"

      - name: Run Codex to fix CI failure
        env:
          WORKFLOW_RUN_URL: ${{ inputs.workflow_run_url }}
          BRANCH: ${{ inputs.branch }}
          GUIDELINES: ${{ inputs.guidelines }}
          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
        run: |
          set -euo pipefail

          cat <<EOF >/tmp/codex-prompt.txt
          You are running inside the lancedb repository on a GitHub Actions runner. Your task is to fix a CI failure.

          Input parameters:
          - Failing workflow run URL: ${WORKFLOW_RUN_URL}
          - Branch to fix: ${BRANCH}
          - Additional guidelines: ${GUIDELINES:-"None provided"}

          Follow these steps exactly:

          1. Extract the run ID from the workflow URL. The URL format is https://github.com/lancedb/lancedb/actions/runs/<run_id>.

          2. Use "gh run view <run_id> --json jobs,conclusion,name" to get information about the failed run.

          3. Identify which jobs failed. For each failed job, use "gh run view <run_id> --job <job_id> --log-failed" to get the failure logs.

          4. Analyze the failure logs to understand what went wrong. Common failures include:
             - Compilation errors
             - Test failures
             - Clippy warnings treated as errors
             - Formatting issues
             - Dependency issues

          5. Based on the analysis, fix the issues in the codebase:
             - For compilation errors: Fix the code that doesn't compile
             - For test failures: Fix the failing tests or the code they test
             - For clippy warnings: Apply the suggested fixes
             - For formatting issues: Run "cargo fmt --all"
             - For other issues: Apply appropriate fixes

          6. After making fixes, verify them locally:
             - Run "cargo fmt --all" to ensure formatting is correct
             - Run "cargo clippy --workspace --tests --all-features -- -D warnings" to check for issues
             - Run ONLY the specific failing tests to confirm they pass now:
               - For Rust test failures: Run the specific test with "cargo test -p <crate> <test_name>"
               - For Python test failures: Build with "cd python && maturin develop" then run "pytest <specific_test_file>::<test_name>"
               - For Java test failures: Run "cd java && mvn test -Dtest=<TestClass>#<testMethod>"
               - For TypeScript test failures: Run "cd nodejs && npm run build && npm test -- --testNamePattern='<test_name>'"
               - Do NOT run the full test suite - only run the tests that were failing

          7. If the additional guidelines are provided, follow them as well.

          8. Inspect "git status --short" and "git diff" to review your changes.

          9. Create a fix branch: "git checkout -b codex/fix-ci-<run_id>".

          10. Stage all changes with "git add -A" and commit with message "fix: resolve CI failures from run <run_id>".

          11. Push the branch: "git push origin codex/fix-ci-<run_id>". If the remote branch exists, delete it first with "gh api -X DELETE repos/lancedb/lancedb/git/refs/heads/codex/fix-ci-<run_id>" then push. Do NOT use "git push --force" or "git push -f".

          12. Create a pull request targeting "${BRANCH}":
              - Title: "ci: <short summary describing the fix>" (e.g., "ci: fix clippy warnings in lancedb" or "ci: resolve test flakiness in vector search")
              - First, write the PR body to /tmp/pr-body.md using a heredoc (cat <<'PREOF' > /tmp/pr-body.md). The body should include:
                - Link to the failing workflow run
                - Summary of what failed
                - Description of the fixes applied
              - Then run "gh pr create --base ${BRANCH} --body-file /tmp/pr-body.md".

          13. Display the new PR URL, "git status --short", and a summary of what was fixed.

          Constraints:
          - Use bash commands for all operations.
          - Do not merge the PR.
          - Do not modify GitHub workflow files unless they are the cause of the failure.
          - If any command fails, diagnose and attempt to fix the issue instead of aborting immediately.
          - If you cannot fix the issue automatically, create the PR anyway with a clear explanation of what you tried and what remains to be fixed.
          - env "GH_TOKEN" is available, use "gh" tools for GitHub-related operations.
          EOF

          printenv OPENAI_API_KEY | codex login --with-api-key
          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"

name: Codex Update Lance Dependency

on:
  workflow_call:
    inputs:
      tag:
        description: "Tag name from Lance"
        required: true
        type: string
  workflow_dispatch:
    inputs:
      tag:
        description: "Tag name from Lance"
        required: true
        type: string

permissions:
  contents: write
  pull-requests: write
  actions: read

jobs:
  update:
    runs-on: ubuntu-latest
    steps:
      - name: Show inputs
        run: |
          echo "tag = ${{ inputs.tag }}"

      - name: Checkout Repo LanceDB
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          persist-credentials: true

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: Install Codex CLI
        run: npm install -g @openai/codex

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: stable
          components: clippy, rustfmt

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y protobuf-compiler libssl-dev

      - name: Install cargo-info
        run: cargo install cargo-info

      - name: Install Python dependencies
        run: python3 -m pip install --upgrade pip packaging

      - name: Configure git user
        run: |
          git config user.name "lancedb automation"
          git config user.email "robot@lancedb.com"

      - name: Run Codex to update Lance dependency
        env:
          TAG: ${{ inputs.tag }}
          GITHUB_TOKEN: ${{ secrets.ROBOT_TOKEN }}
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
          OPENAI_API_KEY: ${{ secrets.CODEX_TOKEN }}
        run: |
          set -euo pipefail
          VERSION="${TAG#refs/tags/}"
          VERSION="${VERSION#v}"
          BRANCH_NAME="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"

          # Use "chore" for beta/rc versions, "feat" for stable releases
          if [[ "${VERSION}" == *beta* ]] || [[ "${VERSION}" == *rc* ]]; then
            COMMIT_TYPE="chore"
          else
            COMMIT_TYPE="feat"
          fi

          cat <<EOF >/tmp/codex-prompt.txt
          You are running inside the lancedb repository on a GitHub Actions runner. Update the Lance dependency to version ${VERSION} and prepare a pull request for maintainers to review.

          Follow these steps exactly:
          1. Use script "ci/set_lance_version.py" to update Lance Rust dependencies. The script already refreshes Cargo metadata, so allow it to finish even if it takes time.
          2. Update the Java lance-core dependency version in "java/pom.xml": change the "<lance-core.version>...</lance-core.version>" property to "${VERSION}".
          3. Run "cargo clippy --workspace --tests --all-features -- -D warnings". If diagnostics appear, fix them yourself and rerun clippy until it exits cleanly. Do not skip any warnings.
          4. After clippy succeeds, run "cargo fmt --all" to format the workspace.
          5. Ensure the repository is clean except for intentional changes. Inspect "git status --short" and "git diff" to confirm the dependency update and any required fixes.
          6. Create and switch to a new branch named "${BRANCH_NAME}" (replace any duplicated hyphens if necessary).
          7. Stage all relevant files with "git add -A". Commit using the message "${COMMIT_TYPE}: update lance dependency to v${VERSION}".
          8. Push the branch to origin. If the remote branch already exists, delete it first with "gh api -X DELETE repos/lancedb/lancedb/git/refs/heads/${BRANCH_NAME}" then push with "git push origin ${BRANCH_NAME}". Do NOT use "git push --force" or "git push -f".
          9. env "GH_TOKEN" is available, use "gh" tools for github related operations like creating pull request.
          10. Create a pull request targeting "main" with title "${COMMIT_TYPE}: update lance dependency to v${VERSION}". First, write the PR body to /tmp/pr-body.md using a heredoc (cat <<'EOF' > /tmp/pr-body.md). The body should summarize the dependency bump, clippy/fmt verification, and link the triggering tag (${TAG}). Then run "gh pr create --body-file /tmp/pr-body.md".
          11. After creating the PR, display the PR URL, "git status --short", and a concise summary of the commands run and their results.

          Constraints:
          - Use bash commands; avoid modifying GitHub workflow files other than through the scripted task above.
          - Do not merge the PR.
          - If any command fails, diagnose and fix the issue instead of aborting.
          EOF

          printenv OPENAI_API_KEY | codex login --with-api-key
          codex --config shell_environment_policy.ignore_default_excludes=true exec --dangerously-bypass-approvals-and-sandbox "$(cat /tmp/codex-prompt.txt)"

      - name: Trigger sophon dependency update
        env:
          TAG: ${{ inputs.tag }}
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
        run: |
          set -euo pipefail
          VERSION="${TAG#refs/tags/}"
          VERSION="${VERSION#v}"
          LANCEDB_BRANCH="codex/update-lance-${VERSION//[^a-zA-Z0-9]/-}"

          echo "Triggering sophon workflow with:"
          echo "  lance_ref: ${TAG#refs/tags/}"
          echo "  lancedb_ref: ${LANCEDB_BRANCH}"

          gh workflow run codex-bump-lancedb-lance.yml \
            --repo lancedb/sophon \
            -f lance_ref="${TAG#refs/tags/}" \
            -f lancedb_ref="${LANCEDB_BRANCH}"

      - name: Show latest sophon workflow run
        env:
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
        run: |
          set -euo pipefail
          echo "Latest sophon workflow run:"
          gh run list --repo lancedb/sophon --workflow codex-bump-lancedb-lance.yml --limit 1 --json databaseId,url,displayTitle

name: PR Checks

on:
  pull_request_target:
    types: [opened, edited, synchronize, reopened]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  labeler:
    permissions:
      pull-requests: write
    name: Label PR
    runs-on: ubuntu-latest
    steps:
      - uses: srvaroa/labeler@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  commitlint:
    permissions:
      pull-requests: write
    name: Verify PR title / description conforms to semantic-release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-node@v4
        with:
          node-version: "18"
      # These rules are disabled because Github will always ensure there
      # is a blank line between the title and the body and Github will
      # word wrap the description field to ensure a reasonable max line
      # length.
      - run: npm install @commitlint/config-conventional
      - run: >
          echo 'module.exports = {
            "rules": {
              "body-max-line-length": [0, "always", Infinity],
              "footer-max-line-length": [0, "always", Infinity],
              "body-leading-blank": [0, "always"]
            }
          }' > .commitlintrc.js
      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
        env:
          COMMIT_MSG: >
            ${{ github.event.pull_request.title }}

            ${{ github.event.pull_request.body }}
      - if: failure()
        uses: actions/github-script@v7
        with:
          script: |
            const message = `**ACTION NEEDED**
              
              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.

              The PR title and description are used as the merge commit message.\
              Please update your PR title and description to match the specification.

              For details on the error please inspect the "PR Title Check" action.
              `
            // Get list of current comments
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number
            });
            // Check if this job already commented
            for (const comment of comments) {
              if (comment.body === message) {
                return // Already commented
              }
            }
            // Post the comment about Conventional Commits
            github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: message
            })
            core.setFailed(message)

name: Deploy docs to Pages

on:
  push:
    branches: ["main"]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  pages: write
  id-token: write

# Allow one concurrent deployment
concurrency:
  group: "pages"
  cancel-in-progress: true

env:
  # This reduces the disk space needed for the build
  RUSTFLAGS: "-C debuginfo=0"
  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
  # CI builds are faster with incremental disabled.
  CARGO_INCREMENTAL: "0"
  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"

jobs:
  # Single deploy job since we're just deploying
  build:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependencies needed for ubuntu
        run: |
          sudo apt install -y protobuf-compiler libssl-dev
          rustup update && rustup default
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
          cache: "pip"
          cache-dependency-path: "docs/requirements.txt"
      - uses: Swatinem/rust-cache@v2
      - name: Build Python
        working-directory: python
        run: |
          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .
          python -m pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -r ../docs/requirements.txt
      - name: Set up node
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
          cache-dependency-path: docs/package-lock.json
      - name: Install node dependencies
        working-directory: nodejs
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Build docs
        working-directory: docs
        run: |
          PYTHONPATH=. mkdocs build
      - name: Setup Pages
        uses: actions/configure-pages@v5
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v3
        with:
          path: "docs/site"
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build Java LanceDB Core

on:
  push:
    branches:
      - main
    paths:
      - java/**
      - .github/workflows/java.yml
  pull_request:
    paths:
      - java/**
      - .github/workflows/java.yml

jobs:
  build-java:
    runs-on: ubuntu-24.04
    name: Build
    defaults:
      run:
        working-directory: ./java
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up Java 17
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 17
          cache: "maven"
      - name: Java Style Check
        run: ./mvnw checkstyle:check
      - name: Build and install
        run: ./mvnw clean install

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build and publish Java packages
on:
  push:
    tags:
      - "v*"
  pull_request:
    paths:
      - .github/workflows/java-publish.yml

jobs:
  publish:
    name: Build and Publish
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    defaults:
      run:
        working-directory: ./java
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up Java 8
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 8
          cache: "maven"
          server-id: ossrh
          server-username: SONATYPE_USER
          server-password: SONATYPE_TOKEN
          gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
          gpg-passphrase: ${{ secrets.GPG_PASSPHRASE }}
      - name: Set git config
        run: |
          git config --global user.email "dev+gha@lancedb.com"
          git config --global user.name "LanceDB Github Runner"
      - name: Dry run
        if: github.event_name == 'pull_request'
        run: |
          ./mvnw --batch-mode -DskipTests package -pl lancedb-core -am
      - name: Publish
        if: startsWith(github.ref, 'refs/tags/v')
        run: |
          echo "use-agent" >> ~/.gnupg/gpg.conf
          echo "pinentry-mode loopback" >> ~/.gnupg/gpg.conf
          export GPG_TTY=$(tty)
          ./mvnw --batch-mode -DskipTests -DpushChanges=false -Dgpg.passphrase=${{ secrets.GPG_PASSPHRASE }} deploy -pl lancedb-core -am -P deploy-to-ossrh
        env:
          SONATYPE_USER: ${{ secrets.SONATYPE_USER }}
          SONATYPE_TOKEN: ${{ secrets.SONATYPE_TOKEN }}

  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [publish]
    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}

name: Lance Release Timer

on:
  schedule:
    - cron: "*/10 * * * *"
  workflow_dispatch:

permissions:
  contents: read
  actions: write

concurrency:
  group: lance-release-timer
  cancel-in-progress: false

jobs:
  trigger-update:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Check for new Lance tag
        id: check
        env:
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
        run: |
          python3 ci/check_lance_release.py --github-output "$GITHUB_OUTPUT"

      - name: Look for existing PR
        if: steps.check.outputs.needs_update == 'true'
        id: pr
        env:
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
        run: |
          set -euo pipefail
          TITLE="chore: update lance dependency to v${{ steps.check.outputs.latest_version }}"
          COUNT=$(gh pr list --search "\"$TITLE\" in:title" --state open --limit 1 --json number --jq 'length')
          if [ "$COUNT" -gt 0 ]; then
            echo "Open PR already exists for $TITLE"
            echo "pr_exists=true" >> "$GITHUB_OUTPUT"
          else
            echo "No existing PR for $TITLE"
            echo "pr_exists=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Trigger codex update workflow
        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
        env:
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
        run: |
          set -euo pipefail
          TAG=${{ steps.check.outputs.latest_tag }}
          gh workflow run codex-update-lance-dependency.yml -f tag=refs/tags/$TAG

      - name: Show latest codex workflow run
        if: steps.check.outputs.needs_update == 'true' && steps.pr.outputs.pr_exists != 'true'
        env:
          GH_TOKEN: ${{ secrets.ROBOT_TOKEN }}
        run: |
          set -euo pipefail
          gh run list --workflow codex-update-lance-dependency.yml --limit 1 --json databaseId,url,displayTitle

name: Check license headers
on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - rust/**
      - python/**
      - nodejs/**
      - java/**
      - .github/workflows/license-header-check.yml
jobs:
  check-licenses:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4
      - name: Install license-header-checker
        working-directory: /tmp
        run: |
            curl -s https://raw.githubusercontent.com/lluissm/license-header-checker/master/install.sh | bash
            mv /tmp/bin/license-header-checker /usr/local/bin/
      - name: Check license headers (rust)
        run: license-header-checker -a -v ./rust/license_header.txt ./ rs && [[ -z `git status -s` ]]
      - name: Check license headers (python)
        run: license-header-checker -a -v ./python/license_header.txt python py && [[ -z `git status -s` ]]
      - name: Check license headers (typescript)
        run: license-header-checker -a -v ./nodejs/license_header.txt nodejs ts && [[ -z `git status -s` ]]
      - name: Check license headers (java)
        run: license-header-checker -a -v ./nodejs/license_header.txt java java && [[ -z `git status -s` ]]

name: Create release commit

# This workflow increments versions, tags the version, and pushes it.
# When a tag is pushed, another workflow is triggered that creates a GH release
# and uploads the binaries. This workflow is only for creating the tag.

# This script will enforce that a minor version is incremented if there are any
# breaking changes since the last minor increment. However, it isn't able to
# differentiate between breaking changes in Node versus Python. If you wish to
# bypass this check, you can manually increment the version and push the tag.
on:
  workflow_dispatch:
    inputs:
      dry_run:
        description: 'Dry run (create the local commit/tags but do not push it)'
        required: true
        default: false
        type: boolean
      type:
        description: 'What kind of release is this?'
        required: true
        default: 'preview'
        type: choice
        options:
          - preview
          - stable
      python:
        description: 'Make a Python release'
        required: true
        default: true
        type: boolean
      other:
        description: 'Make a Node/Rust/Java release'
        required: true
        default: true
        type: boolean
      bump-minor:
        description: 'Bump minor version'
        required: true
        default: false
        type: boolean

jobs:
  make-release:
    # Creates tag and GH release. The GH release will trigger the build and release jobs.
    runs-on: ubuntu-24.04
    permissions:
      contents: write
    steps:
      - name: Output Inputs
        run: echo "${{ toJSON(github.event.inputs) }}"
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
          # It's important we use our token here, as the default token will NOT
          # trigger any workflows watching for new tags. See:
          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
      - name: Validate Lance dependency is at stable version
        if: ${{ inputs.type == 'stable' }}
        run: python ci/validate_stable_lance.py
      - name: Set git configs for bumpversion
        shell: bash
        run: |
          git config user.name 'Lance Release'
          git config user.email 'lance-dev@lancedb.com'
      - name: Bump Python version
        if: ${{ inputs.python }}
        working-directory: python
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Need to get the commit before bumping the version, so we can
          # determine if there are breaking changes in the next step as well.
          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV

          pip install bump-my-version PyGithub packaging
          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
      - name: Bump Node/Rust version
        if: ${{ inputs.other }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pip install bump-my-version PyGithub packaging
          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
          bash ci/update_lockfiles.sh --amend
      - name: Push new version tag
        if: ${{ !inputs.dry_run }}
        uses: ad-m/github-push-action@master
        with:
          # Need to use PAT here too to trigger next workflow. See comment above.
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
          branch: ${{ github.ref }}
          tags: true

name: NodeJS (NAPI)

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - Cargo.toml
      - Cargo.lock
      - nodejs/**
      - rust/**
      - docs/src/js/**
      - .github/workflows/nodejs.yml
      - docker-compose.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  RUST_BACKTRACE: "1"

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    env:
      # Need up-to-date compilers for kernels
      CC: gcc-12
      CXX: g++-12
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v4
      with:
        node-version: 20
        cache: 'npm'
        cache-dependency-path: nodejs/package-lock.json
    - uses: actions-rust-lang/setup-rust-toolchain@v1
      with:
        components: rustfmt, clippy
    - name: Install dependencies
      run: |
        sudo apt update
        sudo apt install -y protobuf-compiler libssl-dev
    - uses: Swatinem/rust-cache@v2
    - name: Format Rust
      run: cargo fmt --all -- --check
    - name: Lint Rust
      run: cargo clippy --profile ci --all --all-features -- -D warnings
    - name: Lint Typescript
      run: |
        npm ci
        npm run lint-ci
    - name: Lint examples
      working-directory: nodejs/examples
      run: npm ci && npm run lint-ci
  linux:
    name: Linux (NodeJS ${{ matrix.node-version }})
    timeout-minutes: 30
    strategy:
      matrix:
        node-version: [ "18", "20" ]
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v4
      name: Setup Node.js 20 for build
      with:
        # @napi-rs/cli v3 requires Node >= 20.12 (via @inquirer/prompts@8).
        # Build always on Node 20; tests run on the matrix version below.
        node-version: 20
        cache: 'npm'
        cache-dependency-path: nodejs/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        sudo apt update
        sudo apt install -y protobuf-compiler libssl-dev
    - name: Build
      run: |
        npm ci --include=optional
        npm run build:debug -- --profile ci
    - uses: actions/setup-node@v4
      name: Setup Node.js ${{ matrix.node-version }} for test
      with:
        node-version: ${{ matrix.node-version }}
    - name: Compile TypeScript
      run: npm run tsc
    - name: Setup localstack
      working-directory: .
      run: docker compose up --detach --wait
    - name: Test
      env:
        S3_TEST: "1"
      run: npm run test
    - name: Setup examples
      working-directory: nodejs/examples
      run: npm ci
    - name: Test examples
      working-directory: ./
      env:
        OPENAI_API_KEY: test
        OPENAI_BASE_URL: http://0.0.0.0:8000
      run: |
        python ci/mock_openai.py &
        cd nodejs/examples
        npm test
    - name: Check docs
      run: |
        # We run this as part of the job because the binary needs to be built
        # first to export the types of the native code.
        set -e
        npm ci
        npm run docs
        if ! git diff --exit-code -- ../ ':(exclude)Cargo.lock'; then
          echo "Docs need to be updated"
          echo "Run 'npm run docs', fix any warnings, and commit the changes."
          exit 1
        fi
  macos:
    timeout-minutes: 30
    runs-on: "macos-14"
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v4
      with:
        node-version: 20
        cache: 'npm'
        cache-dependency-path: nodejs/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        brew install protobuf
    - name: Build
      run: |
        npm ci --include=optional
        npm run build:debug -- --profile ci
        npm run tsc
    - name: Test
      run: |
        npm run test

name: NPM Publish

env:
  MACOSX_DEPLOYMENT_TARGET: '10.13'
  CARGO_INCREMENTAL: '0'

permissions:
  contents: write
  id-token: write

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/npm-publish.yml
      - Cargo.toml # Change in dependency frequently breaks builds
      - Cargo.lock

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  gh-release:
    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Extract version
        id: extract_version
        env:
          GITHUB_REF: ${{ github.ref }}
        run: |
          set -e
          echo "Extracting tag and version from $GITHUB_REF"
          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
            VERSION=${BASH_REMATCH[1]}
            TAG=v$VERSION
            echo "tag=$TAG" >> $GITHUB_OUTPUT
            echo "version=$VERSION" >> $GITHUB_OUTPUT
          else
            echo "Failed to extract version from $GITHUB_REF"
            exit 1
          fi
          echo "Extracted version $VERSION from $GITHUB_REF"
          if [[ $VERSION =~ beta ]]; then
            echo "This is a beta release"

            # Get last release (that is not this one)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | python ci/semver_sort.py v \
              | tail -n 1)
          else
            echo "This is a stable release"
            # Get last stable tag (ignore betas)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | grep -v beta \
              | python ci/semver_sort.py v \
              | tail -n 1)
          fi
          echo "Found from tag $FROM_TAG"
          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
      - name: Create Release Notes
        id: release_notes
        uses: mikepenz/release-changelog-builder-action@v4
        with:
          configuration: .github/release_notes.json
          toTag: ${{ steps.extract_version.outputs.tag }}
          fromTag: ${{ steps.extract_version.outputs.from_tag }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create GH release
        uses: softprops/action-gh-release@v2
        with:
          prerelease: ${{ contains('beta', github.ref) }}
          tag_name: ${{ steps.extract_version.outputs.tag }}
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: false
          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
          body: ${{ steps.release_notes.outputs.changelog }}

  build-lancedb:
    strategy:
      fail-fast: false
      matrix:
        settings:
          - target: aarch64-apple-darwin
            host: macos-latest
            features: fp16kernels
            pre_build: brew install protobuf
          - target: x86_64-pc-windows-msvc
            host: windows-latest
            features: ","
            pre_build: |-
              choco install --no-progress protoc ninja nasm
              tail -n 1000 /c/ProgramData/chocolatey/logs/chocolatey.log
              # There is an issue where choco doesn't add nasm to the path
              export PATH="$PATH:/c/Program Files/NASM"
              nasm -v
          - target: aarch64-pc-windows-msvc
            host: windows-latest
            features: ","
            pre_build: |-
                choco install --no-progress protoc
                rustup target add aarch64-pc-windows-msvc
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
            features: fp16kernels
            # https://github.com/napi-rs/napi-rs/blob/main/debian.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              # The base image (manylinux2014-cross) sets TARGET_CC to the old
              # GCC 4.8 cross-compiler. aws-lc-sys checks TARGET_CC before CC,
              # so it picks up GCC even though the napi-rs image sets CC=clang.
              # Override to use the image's clang-18 which supports -fuse-ld=lld.
              export TARGET_CC=clang TARGET_CXX=clang++
          - target: x86_64-unknown-linux-musl
            # This one seems to need some extra memory
            host: ubuntu-2404-8x-x64
            features: fp16kernels
            pre_build: |-
              set -e &&
              sudo apt-get update &&
              sudo apt-get install -y protobuf-compiler pkg-config &&
              rustup target add x86_64-unknown-linux-musl &&
              export EXTRA_ARGS="-x"
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-x64
            # https://github.com/napi-rs/napi-rs/blob/main/debian-aarch64.Dockerfile
            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            features: "fp16kernels"
            pre_build: |-
              set -e &&
              apt-get update &&
              apt-get install -y protobuf-compiler pkg-config &&
              export TARGET_CC=clang TARGET_CXX=clang++ &&
              # The manylinux2014 sysroot has glibc 2.17 headers which lack
              # AT_HWCAP2 (added in Linux 3.17). Define it for aws-lc-sys.
              export CFLAGS="$CFLAGS -DAT_HWCAP2=26" &&
              rustup target add aarch64-unknown-linux-gnu
          - target: aarch64-unknown-linux-musl
            host: ubuntu-2404-8x-x64
            features: ","
            pre_build: |-
              set -e &&
              sudo apt-get update &&
              sudo apt-get install -y protobuf-compiler &&
              rustup target add aarch64-unknown-linux-musl &&
              export EXTRA_ARGS="-x"
    name: build - ${{ matrix.settings.target }}
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        if: ${{ !matrix.settings.docker }}
        with:
          node-version: 20
          cache: npm
          cache-dependency-path: nodejs/package-lock.json
      - name: Install
        uses: dtolnay/rust-toolchain@stable
        if: ${{ !matrix.settings.docker }}
        with:
          toolchain: stable
          targets: ${{ matrix.settings.target }}
      - name: Cache cargo
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            .cargo-cache
            target/
          key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }}
      - name: Install dependencies
        run: npm ci
      - name: Install Zig
        uses: mlugg/setup-zig@v2
        if: ${{ contains(matrix.settings.target, 'musl') }}
        with:
          version: 0.14.1
      - name: Install cargo-zigbuild
        uses: taiki-e/install-action@v2
        if: ${{ contains(matrix.settings.target, 'musl') }}
        with:
          tool: cargo-zigbuild
      - name: Build in docker
        uses: addnab/docker-run-action@v3
        if: ${{ matrix.settings.docker }}
        with:
          image: ${{ matrix.settings.docker }}
          options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \
                    -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \
                    -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \
                    -v ${{ github.workspace }}:/build -w /build/nodejs"
          run: |
            set -e
            ${{ matrix.settings.pre_build }}
            npx napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              --output-dir dist/
      - name: Build
        run: |
          ${{ matrix.settings.pre_build }}
          npx napi build --platform --release \
              --features ${{ matrix.settings.features }} \
              --target ${{ matrix.settings.target }} \
              --dts ../lancedb/native.d.ts \
              --js ../lancedb/native.js \
              --strip \
              $EXTRA_ARGS \
              --output-dir dist/
        if: ${{ !matrix.settings.docker }}
        shell: bash
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/*.node
          if-no-files-found: error
      # The generic files are the same in all distros so we just pick
      # one to do the upload.
      - name: Make generic artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        run: npm run tsc
      - name: Upload Generic Artifacts
        if: ${{ matrix.settings.target == 'aarch64-apple-darwin' }}
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-dist
          path: |
            nodejs/dist/*
            !nodejs/dist/*.node
  test-lancedb:
    name: "Test: ${{ matrix.settings.target }} - node@${{ matrix.node }}"
    needs:
      - build-lancedb
    strategy:
      fail-fast: false
      matrix:
        settings:
          # TODO: Get tests passing on Windows (failing from test tmpdir issue)
          # - host: windows-latest
          #   target: x86_64-pc-windows-msvc
          - host: macos-latest
            target: aarch64-apple-darwin
          - target: x86_64-unknown-linux-gnu
            host: ubuntu-latest
          - target: aarch64-unknown-linux-gnu
            host: ubuntu-2404-8x-arm64
        node:
          - '20'
    runs-on: ${{ matrix.settings.host }}
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
      - uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node }}
          cache: npm
          cache-dependency-path: nodejs/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: lancedb-${{ matrix.settings.target }}
          path: nodejs/dist/
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
          # run-id: 13982782871
      - name: List packages
        run: ls -R dist
      - name: Move built files
        run: cp dist/native.d.ts dist/native.js dist/*.node lancedb/
      - name: Test bindings
        run: npm test
  publish:
    name: Publish
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    needs:
      - test-lancedb
    steps:
      - uses: actions/checkout@v4
      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: nodejs/package-lock.json
          registry-url: "https://registry.npmjs.org"
      - name: Install dependencies
        run: npm ci
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - uses: actions/download-artifact@v4
        name: Download arch-specific binaries
        with:
          pattern: lancedb-*
          path: nodejs/nodejs-artifacts
          merge-multiple: true
          # For testing purposes:
          # run-id: 13982782871
          # github-token: ${{ secrets.GITHUB_TOKEN }} # token with actions:read permissions on target repo
      - name: Display structure of downloaded files
        run: find dist && find nodejs-artifacts
      - name: Move artifacts
        run: npx napi artifacts -d nodejs-artifacts
      - name: List packages
        run: find npm
      - name: Publish
        env:
          DRY_RUN: ${{ !startsWith(github.ref, 'refs/tags/v') }}
        run: |
          npm config set provenance true
          ARGS="--access public"
          if [[ $DRY_RUN == "true" ]]; then
            ARGS="$ARGS --dry-run"
          fi
          VERSION=$(node -p "require('./package.json').version")
          if [[ $VERSION == *-* ]]; then
            ARGS="$ARGS --tag preview"
          fi
          npm publish $ARGS
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [build-lancedb, test-lancedb, publish]
    if: always() && failure() && startsWith(github.ref, 'refs/tags/v')
    permissions:
      contents: read
      issues: write
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}

name: PyPI Publish

on:
  push:
    tags:
      - 'python-v*'
  pull_request:
    # This should trigger a dry run (we skip the final publish step)
    paths:
      - .github/workflows/pypi-publish.yml
      - Cargo.toml # Change in dependency frequently breaks builds
      - Cargo.lock

env:
  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"

jobs:
  linux:
    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
    timeout-minutes: 60
    strategy:
      matrix:
        config:
          - platform: x86_64
            manylinux: "2_17"
            extra_args: ""
            runner: ubuntu-22.04
          - platform: x86_64
            manylinux: "2_28"
            extra_args: "--features fp16kernels"
            runner: ubuntu-22.04
          - platform: aarch64
            manylinux: "2_17"
            extra_args: ""
            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
            runner: ubuntu-2404-8x-arm64
          - platform: aarch64
            manylinux: "2_28"
            extra_args: "--features fp16kernels"
            runner: ubuntu-2404-8x-arm64
    runs-on: ${{ matrix.config.runner }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
      - uses: ./.github/workflows/build_linux_wheel
        with:
          python-minor-version: 10
          args: "--release --strip ${{ matrix.config.extra_args }}"
          arm-build: ${{ matrix.config.platform == 'aarch64' }}
          manylinux: ${{ matrix.config.manylinux }}
      - uses: ./.github/workflows/upload_wheel
        if: startsWith(github.ref, 'refs/tags/python-v')
        with:
          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
          fury_token: ${{ secrets.FURY_TOKEN }}
  mac:
    timeout-minutes: 90
    runs-on: ${{ matrix.config.runner }}
    strategy:
      matrix:
        config:
          - target: aarch64-apple-darwin
            runner: warp-macos-14-arm64-6x
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.15
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - uses: ./.github/workflows/build_mac_wheel
        with:
          python-minor-version: 10
          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
      - uses: ./.github/workflows/upload_wheel
        if: startsWith(github.ref, 'refs/tags/python-v')
        with:
          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
          fury_token: ${{ secrets.FURY_TOKEN }}
  windows:
    timeout-minutes: 60
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - uses: ./.github/workflows/build_windows_wheel
        with:
          python-minor-version: 10
          args: "--release --strip"
          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
      - uses: ./.github/workflows/upload_wheel
        if: startsWith(github.ref, 'refs/tags/python-v')
        with:
          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
          fury_token: ${{ secrets.FURY_TOKEN }}
  gh-release:
    if: startsWith(github.ref, 'refs/tags/python-v')
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Extract version
        id: extract_version
        env:
          GITHUB_REF: ${{ github.ref }}
        run: |
          set -e
          echo "Extracting tag and version from $GITHUB_REF"
          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
            VERSION=${BASH_REMATCH[1]}
            TAG=python-v$VERSION
            echo "tag=$TAG" >> $GITHUB_OUTPUT
            echo "version=$VERSION" >> $GITHUB_OUTPUT
          else
            echo "Failed to extract version from $GITHUB_REF"
            exit 1
          fi
          echo "Extracted version $VERSION from $GITHUB_REF"
          if [[ $VERSION =~ beta ]]; then
            echo "This is a beta release"

            # Get last release (that is not this one)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^python-v \
              | grep -vF "$TAG" \
              | python ci/semver_sort.py python-v \
              | tail -n 1)
          else
            echo "This is a stable release"
            # Get last stable tag (ignore betas)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^python-v \
              | grep -vF "$TAG" \
              | grep -v beta \
              | python ci/semver_sort.py python-v \
              | tail -n 1)
          fi
          echo "Found from tag $FROM_TAG"
          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
      - name: Create Python Release Notes
        id: python_release_notes
        uses: mikepenz/release-changelog-builder-action@v4
        with:
          configuration: .github/release_notes.json
          toTag: ${{ steps.extract_version.outputs.tag }}
          fromTag: ${{ steps.extract_version.outputs.from_tag }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create Python GH release
        uses: softprops/action-gh-release@v2
        with:
          prerelease: ${{ contains('beta', github.ref) }}
          tag_name: ${{ steps.extract_version.outputs.tag }}
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: false
          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
          body: ${{ steps.python_release_notes.outputs.changelog }}
  report-failure:
    name: Report Workflow Failure
    runs-on: ubuntu-latest
    needs: [linux, mac, windows]
    permissions:
      contents: read
      issues: write
    if: always() && failure() && startsWith(github.ref, 'refs/tags/python-v')
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/create-failure-issue
        with:
          job-results: ${{ toJSON(needs) }}
          workflow-name: ${{ github.workflow }}

name: Python

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - Cargo.toml
      - Cargo.lock
      - python/**
      - rust/**
      - .github/workflows/python.yml
      - .github/workflows/build_linux_wheel/**
      - .github/workflows/build_mac_wheel/**
      - .github/workflows/build_windows_wheel/**
      - .github/workflows/run_tests/**

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Color output for pytest is off by default.
  PYTEST_ADDOPTS: "--color=yes"
  FORCE_COLOR: "1"
  PIP_EXTRA_INDEX_URL: "https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/"
  RUST_BACKTRACE: "1"

jobs:
  lint:
    name: "Lint"
    timeout-minutes: 30
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - name: Install ruff
        run: |
          pip install ruff==0.9.9
      - name: Format check
        run: ruff format --check .
      - name: Lint
        run: ruff check .

  type-check:
    name: "Type Check"
    timeout-minutes: 60
    runs-on: ubuntu-2404-8x-x64
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - name: Install protobuf compiler
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
          pip install toml
      - name: Install dependencies
        run: |
          python ../ci/parse_requirements.py pyproject.toml --extras dev,tests,embeddings > requirements.txt
          pip install -r requirements.txt
      - name: Run pyright
        run: pyright

  doctest:
    name: "Doctest"
    timeout-minutes: 60
    runs-on: ubuntu-2404-8x-x64
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
          cache: "pip"
      - name: Install protobuf
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Install
        run: |
          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
          pip install tantivy
          pip install mlx
      - name: Doctest
        run: pytest --doctest-modules python/lancedb
  linux:
    name: "Linux: python-3.${{ matrix.python-minor-version }}"
    timeout-minutes: 30
    strategy:
      matrix:
        python-minor-version: ["10", "13"]
    runs-on: "ubuntu-24.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Install protobuf
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: 3.${{ matrix.python-minor-version }}
      - uses: ./.github/workflows/build_linux_wheel
        with:
          args: --profile ci
      - uses: ./.github/workflows/run_tests
        with:
          integration: true
      - name: Test without pylance or pandas
        run: |
          pip uninstall -y pylance pandas
          pytest -vv python/tests/test_table.py
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  platform:
    name: "Mac"
    timeout-minutes: 30
    runs-on: macos-14
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - uses: ./.github/workflows/build_mac_wheel
        with:
          args: --profile ci
      - uses: ./.github/workflows/run_tests
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  windows:
    name: "Windows: ${{ matrix.config.name }}"
    timeout-minutes: 60
    strategy:
      matrix:
        config:
          - name: x86
            runner: windows-latest
    runs-on: "${{ matrix.config.runner }}"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - uses: ./.github/workflows/build_windows_wheel
        with:
          args: --profile ci
      - uses: ./.github/workflows/run_tests
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  pydantic1x:
    timeout-minutes: 30
    runs-on: "ubuntu-24.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.10"
      - name: Install lancedb
        run: |
          pip install "pydantic<2"
          pip install pyarrow==16
          pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
          pip install tantivy
      - name: Run tests
        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests

name: Rust

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - Cargo.toml
      - Cargo.lock
      - rust/**
      - .github/workflows/rust.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: "1"

jobs:
  lint:
    timeout-minutes: 30
    runs-on: ubuntu-24.04
    defaults:
      run:
        shell: bash
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          components: rustfmt, clippy
      - uses: Swatinem/rust-cache@v2
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Run format
        run: cargo fmt --all -- --check
      - name: Run clippy
        run: cargo clippy --profile ci --workspace --tests --all-features -- -D warnings
      - name: Run clippy (without remote feature)
        run: cargo clippy --profile ci --workspace --tests -- -D warnings

  build-no-lock:
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    env:
      # Need up-to-date compilers for kernels
      CC: clang
      CXX: clang++
    steps:
      - uses: actions/checkout@v4
      # Building without a lock file often requires the latest Rust version since downstream
      # dependencies may have updated their minimum Rust version.
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: "stable"
      # Remove cargo.lock to force a fresh build
      - name: Remove Cargo.lock
        run: rm -f Cargo.lock
      - uses: rui314/setup-mold@v1
      - uses: Swatinem/rust-cache@v2
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Build all
        run: |
          cargo build --profile ci --benches --all-features --tests

  linux:
    timeout-minutes: 30
    # To build all features, we need more disk space than is available
    # on the free OSS github runner. This is mostly due to the the
    # sentence-transformers feature.
    runs-on: ubuntu-2404-4x-x64
    defaults:
      run:
        shell: bash
        working-directory: rust
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - uses: Swatinem/rust-cache@v2
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - uses: rui314/setup-mold@v1
      - name: Make Swap
        run: |
          sudo fallocate -l 16G /swapfile
          sudo chmod 600 /swapfile
          sudo mkswap /swapfile
          sudo swapon /swapfile
      - name: Build
        run: cargo build --profile ci --all-features --tests --locked --examples
      - name: Run feature tests
        run: CARGO_ARGS="--profile ci" make -C ./lancedb feature-tests
      - name: Run examples
        run: cargo run  --profile ci --example simple --locked
      - name: Run remote tests
        # Running this requires access to secrets, so skip if this is
        # a PR from a fork.
        if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
        run: CARGO_ARGS="--profile ci" make -C ./lancedb remote-tests

  macos:
    timeout-minutes: 30
    strategy:
      matrix:
        mac-runner: ["macos-14", "macos-15"]
    runs-on: "${{ matrix.mac-runner }}"
    defaults:
      run:
        shell: bash
        working-directory: rust
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: CPU features
        run: sysctl -a | grep cpu
      - uses: Swatinem/rust-cache@v2
      - name: Install dependencies
        run: brew install protobuf
      - name: Run tests
        run: |
          # Don't run the s3 integration tests since docker isn't available
          # on this image.
          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps \
            | jq -r '.packages[] | .features | keys | .[]' \
            | grep -v s3-test | sort | uniq | paste -s -d "," -`
          cargo test --profile ci --features $ALL_FEATURES --locked

  windows:
    runs-on: windows-2022
    strategy:
      matrix:
        target:
          - x86_64-pc-windows-msvc
          - aarch64-pc-windows-msvc
    defaults:
      run:
        working-directory: rust/lancedb
    steps:
      - uses: actions/checkout@v4
      - name: Set target
        run: rustup target add ${{ matrix.target }}
      - uses: Swatinem/rust-cache@v2
      - name: Install Protoc v21.12
        run: choco install --no-progress protoc
      - name: Build
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }}
      - name: Run tests
        # Can only run tests when target matches host
        if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }}
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo test --profile ci --features aws,remote --locked

  msrv:
    # Check the minimum supported Rust version
    name: MSRV Check - Rust v${{ matrix.msrv }}
    runs-on: ubuntu-24.04
    strategy:
      matrix:
        msrv: ["1.91.0"] # This should match up with rust-version in Cargo.toml
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Install ${{ matrix.msrv }}
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.msrv }}
      - uses: Swatinem/rust-cache@v2
      - name: Downgrade  dependencies
        # These packages have newer requirements for MSRV
        run: |
          cargo update -p aws-sdk-bedrockruntime --precise 1.77.0
          cargo update -p aws-sdk-dynamodb --precise 1.68.0
          cargo update -p aws-config --precise 1.6.0
          cargo update -p aws-sdk-kms --precise 1.63.0
          cargo update -p aws-sdk-s3 --precise 1.79.0
          cargo update -p aws-sdk-sso --precise 1.62.0
          cargo update -p aws-sdk-ssooidc --precise 1.63.0
          cargo update -p aws-sdk-sts --precise 1.63.0
          cargo update -p home --precise 0.5.9
      - name: cargo +${{ matrix.msrv }} check
        env:
          RUSTUP_TOOLCHAIN: ${{ matrix.msrv }}
        run: cargo check --profile ci --workspace --tests --benches --all-features

name: Update package-lock.json

on:
  workflow_dispatch:

jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock
        with:
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}

name: Update NodeJs package-lock.json

on:
  workflow_dispatch:

jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock_nodejs
        with:
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}