Mintplex-Labs/anything-llm

# This GitHub action is for publishing of the primary image for AnythingLLM
# It will publish a linux/amd64 and linux/arm64 image at the same time
# This file should ONLY BY USED FOR `master` BRANCH. 
# TODO: GitHub now has an ubuntu-24.04-arm64 runner, but we still need
# to use QEMU to build the arm64 image because Chromium is not available for Linux arm64
# so builds will still fail, or fail much more often. Its inconsistent and frustrating.
name: Publish AnythingLLM Primary Docker image (amd64/arm64)

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    branches: ['master'] # master branch only. Do not modify.
    paths-ignore:
      - '**.md'
      - '.gitmodules'
      - 'cloud-deployments/**/*'
      - 'images/**/*'
      - '.vscode/**/*'
      - '**/.env.example'
      - '.github/ISSUE_TEMPLATE/**/*'
      - '.devcontainer/**/*'
      - 'embed/**/*' # Embed is submodule
      - 'browser-extension/**/*' # Chrome extension is submodule
      - 'server/utils/agents/aibitat/example/**/*' # Do not push new image for local dev testing of new aibitat images.
      - 'extras/**/*' # Extra is just for news and other local content.

jobs:
  push_multi_platform_to_registries:
    name: Push Docker multi-platform image to multiple registries
    runs-on: ubuntu-22.04-arm
    permissions:
      packages: write
      contents: read
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4

      - name: Check if DockerHub build needed
        shell: bash
        run: |
          # Check if the secret for USERNAME is set (don't even check for the password)
          if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then
            echo "DockerHub build not needed"
            echo "enabled=false" >> $GITHUB_OUTPUT
          else
            echo "DockerHub build needed"
            echo "enabled=true" >> $GITHUB_OUTPUT
          fi
        id: dockerhub

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: v0.22.0
      
      - name: Log in to Docker Hub
        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
        # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR
        if: steps.dockerhub.outputs.enabled == 'true' 
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      
      - name: Log in to the Container registry
        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
        with:
          images: |
            ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }}
            ghcr.io/${{ github.repository }}
          tags: |
            type=raw,value=latest,enable={{is_default_branch}}
            type=ref,event=branch
            type=ref,event=tag
            type=ref,event=pr

      - name: Build and push multi-platform Docker image
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
      
      # For Docker scout there are some intermediary reported CVEs which exists outside
      # of execution content or are unreachable by an attacker but exist in image.
      # We create VEX files for these so they don't show in scout summary. 
      - name: Collect known and verified CVE exceptions
        id: cve-list
        run: |
          # Collect CVEs from filenames in vex folder
          CVE_NAMES=""
          for file in ./docker/vex/*.vex.json; do
            [ -e "$file" ] || continue
            filename=$(basename "$file")
            stripped_filename=${filename%.vex.json}
            CVE_NAMES+=" $stripped_filename"
          done
          echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT
        shell: bash

      # About VEX attestations https://docs.docker.com/scout/explore/exceptions/
      # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications
      - name: Add VEX attestations
        env:
          CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }}
        run: |
          echo $CVE_EXCEPTIONS
          curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s --
          for cve in $CVE_EXCEPTIONS; do
            for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do
              echo "Attaching VEX exception $cve to $tag"
              docker scout attestation add \
              --file "./docker/vex/$cve.vex.json" \
              --predicate-type https://openvex.dev/ns/v0.2.0 \
              $tag
            done
          done
        shell: bash

name: Publish AnythingLLM Docker image on Release (amd64 & arm64)

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  release:
    types: [published]

jobs:
  push_multi_platform_to_registries:
    name: Push Docker multi-platform image to multiple registries
    runs-on: ubuntu-22.04-arm
    permissions:
      packages: write
      contents: read
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4

      - name: Check if DockerHub build needed
        shell: bash
        run: |
          # Check if the secret for USERNAME is set (don't even check for the password)
          if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then
            echo "DockerHub build not needed"
            echo "enabled=false" >> $GITHUB_OUTPUT
          else
            echo "DockerHub build needed"
            echo "enabled=true" >> $GITHUB_OUTPUT
          fi
        id: dockerhub

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: v0.22.0
      
      - name: Log in to Docker Hub
        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
        # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR
        if: steps.dockerhub.outputs.enabled == 'true' 
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      
      - name: Log in to the Container registry
        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
        with:
          images: |
            ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }}
            ghcr.io/${{ github.repository }}
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Build and push multi-platform Docker image
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          platforms: linux/amd64,linux/arm64
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
      
      # For Docker scout there are some intermediary reported CVEs which exists outside
      # of execution content or are unreachable by an attacker but exist in image.
      # We create VEX files for these so they don't show in scout summary. 
      - name: Collect known and verified CVE exceptions
        id: cve-list
        run: |
          # Collect CVEs from filenames in vex folder
          CVE_NAMES=""
          for file in ./docker/vex/*.vex.json; do
            [ -e "$file" ] || continue
            filename=$(basename "$file")
            stripped_filename=${filename%.vex.json}
            CVE_NAMES+=" $stripped_filename"
          done
          echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT
        shell: bash

      # About VEX attestations https://docs.docker.com/scout/explore/exceptions/
      # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications
      - name: Add VEX attestations
        env:
          CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }}
        run: |
          echo $CVE_EXCEPTIONS
          curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s --
          for cve in $CVE_EXCEPTIONS; do
            for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do
              echo "Attaching VEX exception $cve to $tag"
              docker scout attestation add \
              --file "./docker/vex/$cve.vex.json" \
              --predicate-type https://openvex.dev/ns/v0.2.0 \
              $tag
            done
          done
        shell: bash

# Builds a QA GHCR image for a PR when the "PR: Ready for QA" label is present.
# Triggers on:
#   - "PR: Ready for QA" label added to a PR
#   - New commits pushed to a PR that already has the label will trigger a new build
name: Build QA GHCR Image

on:
  pull_request:
    types: [labeled, synchronize]
    paths-ignore:
      - "**.md"
      - ".gitmodules"
      - "cloud-deployments/**/*"
      - "images/**/*"
      - ".vscode/**/*"
      - "**/.env.example"
      - ".github/ISSUE_TEMPLATE/**/*"
      - ".devcontainer/**/*"
      - "embed/**/*"
      - "browser-extension/**/*"
      - "extras/**/*"

concurrency:
  group: qa-build-pr-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  build:
    name: Build and push QA image for PR
    runs-on: ubuntu-22.04-arm
    # Run when labeled with "PR: Ready for QA"
    if: >-
      ${{ contains(github.event.pull_request.labels.*.name, 'PR: Ready for QA') }}
    permissions:
      packages: write
      contents: read
      pull-requests: write
    steps:
      - name: Check out the repo
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
        with:
          version: v0.22.0

      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set lowercase repository owner
        run: echo "REPO_OWNER_LC=${GITHUB_REPOSITORY_OWNER,,}" >> $GITHUB_ENV

      - name: Build and push
        uses: docker/build-push-action@v6
        with:
          context: .
          file: ./docker/Dockerfile
          push: true
          sbom: true
          provenance: mode=max
          platforms: linux/arm64
          tags: ghcr.io/${{ env.REPO_OWNER_LC }}/${{ github.event.repository.name }}:pr-${{ github.event.pull_request.number }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

# This GitHub action is for checking the versions of the packages in the project.
# Any package that is present in both the `server` and `collector` package.json file
# is checked to ensure that they are the same version.
name: Check package versions

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "server/package.json"
      - "collector/package.json"

jobs:
  run-script:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'

      - name: Run verifyPackageVersions.mjs script
        run: |
          cd extras/scripts
          node verifyPackageVersions.mjs

      - name: Fail job on error
        if: failure()
        run: exit 1

# This GitHub action is for validation of all languages which translations are offered for
# in the locales folder in `frontend/src`. All languages are compared to the EN translation
# schema since that is the fallback language setting. This workflow will run on all PRs that
# modify any files in the translation directory
name: Verify translations files

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "frontend/src/locales/**.js"

jobs:
  run-script:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'

      - name: Run verifyTranslations.mjs script
        run: |
          cd frontend/src/locales
          node verifyTranslations.mjs

      - name: Fail job on error
        if: failure()
        run: exit 1

# Cleans up the GHCR image tag when the PR is closed or the "PR: Ready for QA" label is removed.
name: Cleanup QA GHCR Image

on:
  pull_request:
    types: [closed, unlabeled]
  workflow_dispatch:
    inputs:
      pr_number:
        description: 'PR number to clean up (e.g., 123)'
        required: true

jobs:
  cleanup-manual:
    name: Delete QA GHCR image tag (manual)
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch'
    permissions:
      packages: write
    steps:
      - name: Delete PR tag from GHCR
        env:
          GH_TOKEN: ${{ secrets.ALLM_RW_PACKAGES }}
          PR_NUMBER: ${{ inputs.pr_number }}
        run: |
          # Must use lowercase - packages are published with lowercase owner
          ORG_LC="${GITHUB_REPOSITORY_OWNER,,}"
          REPO_LC="${GITHUB_REPOSITORY#*/}"
          REPO_LC="${REPO_LC,,}"
          
          echo "Looking for tag: pr-${PR_NUMBER}"
          echo "Package: /orgs/${ORG_LC}/packages/container/${REPO_LC}/versions"
          
          VERSION_ID=$(gh api \
            -H "Accept: application/vnd.github+json" \
            --paginate \
            "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions" \
            --jq ".[] | select(.metadata.container.tags[] == \"pr-${PR_NUMBER}\") | .id")
          
          if [ -n "$VERSION_ID" ]; then
            echo "Deleting package version $VERSION_ID (tag: pr-${PR_NUMBER})"
            gh api \
              --method DELETE \
              -H "Accept: application/vnd.github+json" \
              "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions/$VERSION_ID"
          else
            echo "No package found with tag pr-${PR_NUMBER}, skipping cleanup"
          fi

  cleanup-auto:
    name: Delete QA GHCR image tag (auto)
    runs-on: ubuntu-latest
    if: >-
      (github.event.action == 'closed' && contains(github.event.pull_request.labels.*.name, 'PR: Ready for QA')) ||
      (github.event.action == 'unlabeled' && github.event.label.name == 'PR: Ready for QA')
    permissions:
      packages: write
    steps:
      - name: Delete PR tag from GHCR
        env:
          GH_TOKEN: ${{ secrets.ALLM_RW_PACKAGES }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          # Must use lowercase - packages are published with lowercase owner
          ORG_LC="${GITHUB_REPOSITORY_OWNER,,}"
          REPO_LC="${GITHUB_REPOSITORY#*/}"
          REPO_LC="${REPO_LC,,}"
          
          VERSION_ID=$(gh api \
            -H "Accept: application/vnd.github+json" \
            --paginate \
            "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions" \
            --jq ".[] | select(.metadata.container.tags[] == \"pr-${PR_NUMBER}\") | .id" \
            2>/dev/null || true)
          
          if [ -n "$VERSION_ID" ]; then
            echo "Deleting package version $VERSION_ID (tag: pr-${PR_NUMBER})"
            gh api \
              --method DELETE \
              -H "Accept: application/vnd.github+json" \
              "/orgs/${ORG_LC}/packages/container/${REPO_LC}/versions/$VERSION_ID"
          else
            echo "No package found with tag pr-${PR_NUMBER}, skipping cleanup"
          fi

name: Lint

concurrency:
  group: lint-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "server/**/*.js"
      - "server/eslint.config.mjs"
      - "collector/**/*.js"
      - "collector/eslint.config.mjs"
      - "frontend/src/**/*.js"
      - "frontend/src/**/*.jsx"
      - "frontend/eslint.config.js"

jobs:
  lint:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: "18"

      - name: Cache server dependencies
        uses: actions/cache@v4
        with:
          path: server/node_modules
          key: ${{ runner.os }}-yarn-server-${{ hashFiles('server/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-server-

      - name: Cache frontend dependencies
        uses: actions/cache@v4
        with:
          path: frontend/node_modules
          key: ${{ runner.os }}-yarn-frontend-${{ hashFiles('frontend/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-frontend-

      - name: Cache collector dependencies
        uses: actions/cache@v4
        with:
          path: collector/node_modules
          key: ${{ runner.os }}-yarn-collector-${{ hashFiles('collector/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-collector-

      - name: Install server dependencies
        run: cd server && yarn install --frozen-lockfile

      - name: Install frontend dependencies
        run: cd frontend && yarn install --frozen-lockfile

      - name: Install collector dependencies
        run: cd collector && yarn install --frozen-lockfile
        env:
          PUPPETEER_SKIP_DOWNLOAD: "true"
          SHARP_IGNORE_GLOBAL_LIBVIPS: "true"

      - name: Lint server
        run: cd server && yarn lint:check

      - name: Lint frontend
        run: cd frontend && yarn lint:check

      - name: Lint collector
        run: cd collector && yarn lint:check

name: Run backend tests

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "server/**.js"
      - "collector/**.js"

jobs:
  run-script:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Set up Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'

      - name: Cache root dependencies
        uses: actions/cache@v3
        with:
          path: |
            node_modules
            ~/.cache/yarn
          key: ${{ runner.os }}-yarn-root-${{ hashFiles('**/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-root-

      - name: Cache server dependencies
        uses: actions/cache@v3
        with:
          path: |
            server/node_modules
            ~/.cache/yarn
          key: ${{ runner.os }}-yarn-server-${{ hashFiles('server/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-server-

      - name: Cache collector dependencies
        uses: actions/cache@v3
        with:
          path: |
            collector/node_modules
            ~/.cache/yarn
          key: ${{ runner.os }}-yarn-collector-${{ hashFiles('collector/yarn.lock') }}
          restore-keys: |
            ${{ runner.os }}-yarn-collector-

      - name: Install root dependencies
        if: steps.cache-root.outputs.cache-hit != 'true'
        run: yarn install --frozen-lockfile

      - name: Install server dependencies
        if: steps.cache-server.outputs.cache-hit != 'true'
        run: cd server && yarn install --frozen-lockfile

      - name: Install collector dependencies
        if: steps.cache-collector.outputs.cache-hit != 'true'
        run: cd collector && yarn install --frozen-lockfile
        env:
          PUPPETEER_SKIP_DOWNLOAD: "true"
          SHARP_IGNORE_GLOBAL_LIBVIPS: "true"

      - name: Setup environment and Prisma
        run: yarn setup:envs && yarn prisma:setup

      - name: Run test suites
        run: yarn test

      - name: Fail job on error
        if: failure()
        run: exit 1

name: Generate Sponsors README

on:
  schedule:
    - cron: "0 12 * * 3" # Run every Wednesday at 12:00 PM UTC

permissions:
  contents: write
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout 🛎️
        uses: actions/checkout@v2

      - name: Generate All Sponsors README
        id: generate-all-sponsors
        uses: JamesIves/github-sponsors-readme-action@v1
        with:
          token: ${{ secrets.SPONSOR_PAT }}
          file: 'README.md'
          organization: true
          active-only: false
          marker: 'all-sponsors'

      - name: Commit and Push 🚀
        uses: stefanzweifel/git-auto-commit-action@v5
        id: auto-commit-action
        with:
          commit_message: 'Update Sponsors README'
          file_pattern: 'README.md'

      - name: Generate PR if changes detected
        uses: peter-evans/create-pull-request@v7
        if: steps.auto-commit-action.outputs.files_changed == 'true'
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          title: 'Update Sponsors README'
          branch: 'chore/update-sponsors'
          base: 'master'
          draft: false
          reviewers: 'timothycarambat'
          assignees: 'timothycarambat'
          maintainer-can-modify: true