milvus-io/milvus

name: all-contributors

on:
    schedule:
    # * is a special character in YAML so you have to quote this string
    #         ┌───────────── minute (0 - 59)
    #         │  ┌───────────── hour (0 - 23)
    #         │  │   ┌───────────── day of the month (1 - 31)
    #         │  │   │ ┌───────────── month (1 - 12 or JAN-DEC)
    #         │  │   │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #         │  │   │ │ │
    #         │  │   │ │ │
    #         │  │   │ │ │
    - cron:  '0 0,12 * * *'

jobs:
  contributor:
    runs-on: ubuntu-latest
    if: github.repository == 'milvus-io/milvus'
    steps:
      - name: checkout code
        uses: actions/checkout@v2
        with:
         token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
    
      - name: log path
        run: |
          pwd
          ls
          
      - name: Update README.md
        uses: milvus-io/hero-bot@dco-enabled
        with:
          # Required
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          repos: 'milvus-io/milvus,bootcamp,community,docs,milvus-helm,milvus-sdk-go,milvus-sdk-java,milvus-sdk-node,milvus.io,pymilvus,pymilvus-orm'
          targetFile: './README.md'
          # Optional
          isAscend: True
          width: '30px'
          customUserConfig: 'milvus-io/milvus/.contributors'
          workingDir: ${{ github.workspace }}
          
      - name: Update README_CN.md
        uses: milvus-io/hero-bot@dco-enabled
        with:
          # Required
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          repos: 'milvus-io/milvus,bootcamp,community,docs,milvus-helm,milvus-sdk-go,milvus-sdk-java,milvus-sdk-node,milvus.io,pymilvus,pymilvus-orm'
          targetFile: './README_CN.md'
          # Optional
          isAscend: True
          width: '30px'
          customUserConfig: 'milvus-io/milvus/.contributors'
          workingDir: ${{ github.workspace }}
    
      - name: commit code
        run: |
          pwd
          git config --system user.email "sre-ci-robot@zilliz.com"
          git config --system user.name "sre-ci-robot"
          git add -u
          git diff-index --cached --quiet HEAD || (git commit -s -m 'Update all contributors' && git push)

name: Bump Milvus Version

on:
  workflow_dispatch:
    inputs:
      imageTag:
        description: "the milvus image tag"
        required: true
        type: string

jobs:
  update-knowhere-commits:
    name: Bump Milvus Version for release
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Update milvus image tag
        # continue-on-error: true
        shell: bash
        run: |
          sed -i 's/\(milvusdb\/milvus:\)[^[:space:]]*/\1${{ inputs.imageTag }}/' deployments/docker/standalone/docker-compose.yml
          sed -i 's/\(milvusdb\/milvus:\)[^[:space:]]*/\1${{ inputs.imageTag }}/' deployments/binary/README.md
          sed -i 's/\(milvusdb\/milvus:\)[^[:space:]]*-gpu/\1${{ inputs.imageTag }}-gpu/' deployments/docker/gpu/standalone/docker-compose.yml
          sed -i 's/\(milvusdb\/milvus:\)[^[:space:]]*/\1${{ inputs.imageTag }}/' deployments/docker/cluster-distributed-deployment/inventory.ini
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git diff
          git add deployments
          git commit -m "Bump milvus version to ${{ inputs.imageTag }}"
      - name: Create Pull Request
        id: cpr
        continue-on-error: true
        uses: peter-evans/create-pull-request@v3
        with:
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          author: sre-ci-robot <sre-ci-robot@users.noreply.github.com>
          signoff: true
          branch: bump_milvus_commit_${{ github.sha }}
          delete-branch: true
          title: '[automated] Bump milvus version to ${{ inputs.imageTag }}'
          body: |
            Bump milvus version to ${{ inputs.imageTag }}
            Signed-off-by: sre-ci-robot sre-ci-robot@users.noreply.github.com
      - name: Check outputs
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

name: Bump Standalone Embed Milvus Version

on:
  workflow_dispatch:
    inputs:
      imageTag:
        description: "the milvus image tag"
        required: true
        type: string

jobs:
  update-knowhere-commits:
    name: Bump Milvus Version for release
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Update milvus image tag
        # continue-on-error: true
        shell: bash
        run: |
          sed -i 's/\(milvusdb\/milvus:\)[^[:space:]]*/\1${{ inputs.imageTag }}/' scripts/standalone_embed.sh
          sed -i 's/\(milvusdb\/milvus:\)[^[:space:]]*/\1${{ inputs.imageTag }}/' scripts/standalone_embed.bat
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git add scripts
          git status
          git commit -m "Bump milvus version to ${{ inputs.imageTag }}"
      - name: Create Pull Request
        id: cpr
        continue-on-error: true
        uses: peter-evans/create-pull-request@v3
        with:
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          author: sre-ci-robot <sre-ci-robot@users.noreply.github.com>
          signoff: true
          branch: bump_milvus_commit_${{ github.sha }}
          delete-branch: true
          title: '[automated] Bump milvus version to ${{ inputs.imageTag }}'
          body: |
            Bump milvus version to ${{ inputs.imageTag }}
            Signed-off-by: sre-ci-robot sre-ci-robot@users.noreply.github.com
      - name: Check outputs
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

name: Add Comment for issue

on:
  issues:
    types: [opened]

jobs:
  check_issue_title:
    name: Check issue
    runs-on: ubuntu-latest
    env:
      TITLE_PASSED: "T"
      ISSUE_TITLE: ${{ github.event.issue.title }}
    permissions:
      issues: write
    timeout-minutes: 20
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Check Issue
        shell: bash
        run: |
          echo "Issue title: ${ISSUE_TITLE//\"/\\\"}"

          cat >> check_title.py << EOF
          import re
          import sys
          check_str = sys.argv[1]
          pattern = re.compile(r'[\u4e00-\u9fa5]')
          match = pattern.search(check_str)
          if match:
              print("TITLE_PASSED=F")
          else:
              print("TITLE_PASSED=T")
          EOF
          
          python3 check_title.py "${ISSUE_TITLE//\"/\\\"}" >> "$GITHUB_ENV"
          cat $GITHUB_ENV

      - name: Check env
        shell: bash
        run: |
          echo ${{ env.TITLE_PASSED }}
      - name: Add comment
        if: ${{ env.TITLE_PASSED == 'F'}}
        uses: peter-evans/create-or-update-comment@5f728c3dae25f329afbe34ee4d08eef25569d79f
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            The title and description of this issue contains Chinese. Please use English to describe your issue.

name: Claude Code

on:
  issue_comment:
    types: [created]
  pull_request_review_comment:
    types: [created]
  issues:
    types: [opened, assigned]
  pull_request_review:
    types: [submitted]

jobs:
  claude:
    if: |
      (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||
      (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
      (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
      issues: write
      id-token: write
      actions: read # Required for Claude to read CI results on PRs
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Run Claude Code
        id: claude
        uses: anthropics/claude-code-action@v1
        with:
          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          github_token: ${{ secrets.GITHUB_TOKEN }}

          # This is an optional setting that allows Claude to read CI results on PRs
          additional_permissions: |
            actions: read

          # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.
          # prompt: 'Update the pull request description to include a summary of changes.'

          # Optional: Add claude_args to customize behavior and configuration
          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
          # or https://code.claude.com/docs/en/cli-reference for available options
          # claude_args: '--allowed-tools Bash(gh pr:*)'

name: Claude Code Review

on:
  pull_request_target:
    types: [opened, synchronize, ready_for_review, reopened]
    # Optional: Only run on specific file changes
    # paths:
    #   - "src/**/*.ts"
    #   - "src/**/*.tsx"
    #   - "src/**/*.js"
    #   - "src/**/*.jsx"

jobs:
  claude-review:
    # Optional: Filter by PR author
    # if: |
    #   github.event.pull_request.user.login == 'external-contributor' ||
    #   github.event.pull_request.user.login == 'new-developer' ||
    #   github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'

    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
      issues: write
      id-token: write

    steps:
      - name: Checkout PR code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 1

      - name: Run Claude Code Review
        id: claude-review
        uses: anthropics/claude-code-action@v1
        with:
          claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
          github_token: ${{ secrets.GITHUB_TOKEN }}
          plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'
          plugins: 'code-review@claude-code-plugins'
          prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
          # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md
          # or https://code.claude.com/docs/en/cli-reference for available options

name: Code Checker
# TODO: do not trigger action for some document file update

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    branches:
      - master
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'scripts/**'
      - 'internal/**'
      - 'pkg/**'
      - 'client/**'
      - 'cmd/**'
      - 'build/**'
      - 'tests/integration/**'
      - 'tests/go_client/**'
      - '.github/workflows/code-checker.yaml'
      - '.env'
      - docker-compose.yml
      - Makefile
      - '!**.md'
      - '!build/ci/jenkins/**'
      # FIXME(wxyu): not need to run code check, update the ci-passed rules and remove these two lines
      - go.mod
      - go.sum
      - .golangci.yml
      - rules.go

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  ubuntu:
    name: Code Checker AMD64 Ubuntu 22.04
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      fail-fast: false
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Checkout
        uses: actions/checkout@v2
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'ubuntu22.04'
      - name: Code Check
        env:
          OS_NAME: 'ubuntu22.04'
        run: |
          ./build/builder.sh /bin/bash -c "git config --global --add safe.directory /go/src/github.com/milvus-io/milvus && make check-proto-product && make verifiers"
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'ubuntu22.04'

  amazonlinux:
    if: ${{ false }} # skip for now
    name: Code Checker Amazonlinux 2023
    # Run in amazonlinux docker
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Checkout
        uses: actions/checkout@v2
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'amazonlinux2023'
      - name: Code Check
        run: |
          sed -i 's/ubuntu22.04/amazonlinux2023/g' .env
          ./build/builder.sh /bin/bash -c "make install"
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'amazonlinux2023'

  rockylinux:
    if: ${{ false }} # skip for now
    name: Code Checker rockylinux9
    # Run in rockylinux docker
    runs-on: ubuntu-latest
    timeout-minutes: 180
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Checkout
        uses: actions/checkout@v2
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'rockylinux9'
      - name: Code Check
        run: |
          sed -i 's/ubuntu22.04/rockylinux9/g' .env
          ./build/builder.sh /bin/bash -c "make install"
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'rockylinux9'

name: Daily Release
on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    #         ┌───────────── minute (0 - 59)
    #         │ ┌───────────── hour (0 - 23)
    #         │ │  ┌───────────── day of the month (1 - 31)
    #         │ │  │ ┌───────────── month (1 - 12 or JAN-DEC)
    #         │ │  │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #         │ │  │ │ │
    #         │ │  │ │ │
    #         │ │  │ │ │
    - cron:  '0 18 * * *'

jobs:
  nightly:
    name: Run Daily Release
    if: github.repository == 'milvus-io/milvus'
    runs-on: ubuntu-latest
    env:
      IMAGE_REPO: "milvusdb"
      DEV: "milvus"
      DAILY: "daily-build"
      TAG_PREFIX: "master-"
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
      with:
        fetch-depth: '0'

    - name: Get the latest of Milvus dev image tag
      shell: bash
      working-directory: scripts
      run: echo "tag=$(./docker_image_find_tag.sh -n ${IMAGE_REPO}/${DEV} -t ${TAG_PREFIX}latest -f ${TAG_PREFIX} -F -L -q)" >> $GITHUB_ENV


    - name: Pull latest milvus image with tag prefix master-
      run: |
        docker pull "${IMAGE_REPO}/${DEV}:${{ env.tag }}"
        docker tag "${IMAGE_REPO}/${DEV}:${{ env.tag }}" "${IMAGE_REPO}/${DAILY}:${{ env.tag }}"

    - name: Log in to Docker Hub
      uses: docker/login-action@v1
      with:
        username: ${{ secrets.DOCKERHUB_USER }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    - name: Push Docker image
      run: |
        docker push "${IMAGE_REPO}/${DAILY}:${{ env.tag }}"

    - name: Set release build
      shell: bash
      run: |
        tag=${{ env.tag }}
        IFS=- read branch date sha <<< "$tag"
        echo "build=$date" >> $GITHUB_ENV
        echo "sha=$(git rev-parse $sha)" >> $GITHUB_ENV

    - name: Create a daily release
      uses: actions/create-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        body: "Image: ${{ env.IMAGE_REPO}}/${{ env.DAILY }}:${{ env.tag }}"
        prerelease: true
        tag_name: "v2.0.0-testing-${{ env.build }}"
        release_name: "milvus-2.0.0-testing-${{ env.build }}"
        commitish: "${{ env.sha }}"

# via https://github.com/zed-industries/zed/blob/main/.github/workflows/delete_comments.yml
name: Delete Mediafire Comments

on:
  issue_comment:
    types: [created]

permissions:
  issues: write

jobs:
  delete_comment:
    runs-on: ubuntu-latest
    steps:
      - name: Check for specific strings in comment
        id: check_comment
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
        with:
          script: |
            const comment = context.payload.comment.body;
            const triggerStrings = ['www.mediafire.com'];
            return triggerStrings.some(triggerString => comment.includes(triggerString));

      - name: Delete comment if it contains any of the specific strings
        if: steps.check_comment.outputs.result == 'true'
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
        with:
          script: |
            const commentId = context.payload.comment.id;
            await github.rest.issues.deleteComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: commentId
            });

name: Deploy Test

on:
  workflow_dispatch:
    inputs:
      old_image_repo:
        description: The image repository name to use for the deploy test
        required: true
        default: 'milvusdb/milvus'

      old_image_tag:
        description: The old image tag to use for the deploy test
        required: true
        default: 'v2.1.0'

      previous_release_version:
        description: The previous release version to use for the deploy test
        required: true
        default: 'v2.1.0'

      new_image_repo:
        description: The image repository name to use for the deploy test
        required: true
        default: 'milvusdb/milvus'

      new_image_tag:
        description: The new image tag to use for the deploy test
        required: true
        default: 'master-latest'

  schedule:
    # * is a special character in YAML so you have to quote this string
    #         ┌───────────── minute (0 - 59)
    #         │  ┌───────────── hour (0 - 23)
    #         │  │ ┌───────────── day of the month (1 - 31)
    #         │  │ │ ┌───────────── month (1 - 12 or JAN-DEC)
    #         │  │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #         │  │ │ │ │
    #         │  │ │ │ │
    #         │  │ │ │ │
    - cron: "30 20 * * *"

jobs:

  test-docker-compose:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        mode: [standalone, cluster]
        task: [reinstall, upgrade]

    steps:
      - name: Set env param
        env:
          DEFAULT_OLD_IMAGE_REPO: "milvusdb/milvus"
          DEFAULT_OLD_IMAGE_TAG: "latest"
          DEFAULT_PREVIOUS_RELEASE_VERSION: "v2.1.0"
          DEFAULT_NEW_IMAGE_REPO: "milvusdb/milvus"
          DEFAULT_NEW_IMAGE_TAG: "master-latest"
        run: |
          echo "OLD_IMAGE_REPO=${{ github.event.inputs.old_image_repo || env.DEFAULT_OLD_IMAGE_REPO }}" >> $GITHUB_ENV
          echo "OLD_IMAGE_TAG=${{ github.event.inputs.old_image_tag || env.DEFAULT_OLD_IMAGE_TAG }}" >> $GITHUB_ENV
          echo "PREVIOUS_RELEASE_VERSION=${{ github.event.inputs.previous_release_version || env.DEFAULT_PREVIOUS_RELEASE_VERSION }}" >> $GITHUB_ENV
          echo "NEW_IMAGE_REPO=${{ github.event.inputs.new_image_repo || env.DEFAULT_NEW_IMAGE_REPO }}" >> $GITHUB_ENV
          echo "NEW_IMAGE_TAG=${{ github.event.inputs.new_image_tag || env.DEFAULT_NEW_IMAGE_TAG }}" >> $GITHUB_ENV

      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client
        run: |
          pip install -r requirements.txt --trusted-host https://test.pypi.org
          sudo systemctl restart docker
          sleep 30s
      - name: Download dataset
        shell: bash
        working-directory: tests/python_client/assets/ann_hdf5
        run: |
          echo "Downloading dataset..."
          bash download.sh

      - name: First Milvus deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          source ../utils.sh
          if [ ${{ matrix.task }} == "reinstall" ]; then
            wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/${{ matrix.mode }}/docker-compose.yml -O docker-compose.yml;
            replace_image_tag ${{ env.NEW_IMAGE_REPO }} ${{ env.NEW_IMAGE_TAG }};

          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            wget https://github.com/milvus-io/milvus/releases/download/${{ env.PREVIOUS_RELEASE_VERSION }}/milvus-${{ matrix.mode }}-docker-compose.yml -O docker-compose.yml;
            replace_image_tag ${{ env.OLD_IMAGE_REPO }} ${{ env.OLD_IMAGE_TAG }};
          fi
          docker compose up -d
          bash ../check_healthy.sh
          docker compose ps -a
          sleep 10s
      - name: Run first test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          python scripts/first_recall_test.py
          if [ ${{ matrix.task }} == "reinstall" ]; then
            python3 scripts/action_before_reinstall.py
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            python3 scripts/action_before_upgrade.py
          fi
          
      - name: Milvus Idle Time
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          sleep 60s
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          docker compose ps -a || true
          mkdir -p logs/first_deploy
          bash ../../../scripts/export_log_docker.sh ./logs/first_deploy || echo "export logs failed"
      - name: Second Milvus deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          source ../utils.sh
          if [ ${{ matrix.task }} == "reinstall" ]; then
            docker compose restart
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            wget https://raw.githubusercontent.com/milvus-io/milvus/master/deployments/docker/${{ matrix.mode }}/docker-compose.yml -O docker-compose.yml;
            replace_image_tag ${{ env.NEW_IMAGE_REPO }} ${{ env.NEW_IMAGE_TAG }};
            docker compose up -d;
          fi
          bash ../check_healthy.sh
          docker compose ps -a

          echo "sleep 120s for the second deployment to be ready"
          sleep 120s
      - name: Run second test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          python scripts/second_recall_test.py
          if [ ${{ matrix.task }} == "reinstall" ]; then
            python3 scripts/action_after_reinstall.py
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            python3 scripts/action_after_upgrade.py
          fi
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          docker compose ps -a || true
          mkdir -p logs/second_deploy
          bash ../../../scripts/export_log_docker.sh ./logs/second_deploy || echo "export logs failed"

      - name: Restart docker
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          echo "restart docker service"
          sudo systemctl restart docker
          sleep 20s
          docker compose up -d
          bash ../check_healthy.sh
          docker compose ps -a

          echo "sleep 120s for the deployment to be ready after docker restart"
          sleep 120s


      - name: Run third test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          python scripts/second_recall_test.py
          if [ ${{ matrix.task }} == "reinstall" ]; then
            python3 scripts/action_after_reinstall.py
          fi
          if [ ${{ matrix.task }} == "upgrade" ]; then
            python3 scripts/action_after_upgrade.py
          fi
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy/${{ matrix.mode }}
        run: |
          docker compose ps -a || true
          mkdir -p logs/second_deploy
          bash ../../../scripts/export_log_docker.sh ./logs/third_deploy || echo "export logs failed"      
      
      - name: 'Send mail'
        if: ${{ failure() }}
        uses: dawidd6/action-send-mail@v3
        with:
          server_address: ${{ secrets.EMAIL_SERVICE_NAME }}
          server_port: 465
          username: ${{ secrets.TEST_EMAIL_USERNAME }}
          password: ${{ secrets.TEST_EMAIL_PASSWORD  }}
          subject: Deploy Test
          body: "test ${{ matrix.mode }} ${{ matrix.task }} failed \n You can view it at https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}"
          to: ${{ secrets.QA_EMAIL_ADDRESS }}
          from: GitHub Actions

      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: docker-compose-logs-${{ matrix.mode }}-${{ matrix.task }}
          path: tests/python_client/deploy/${{ matrix.mode }}/logs

  test-helm-install:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        mq_type: [pulsar, kafka]
        mode: [standalone,cluster]
        task: [reinstall,upgrade]
        exclude:
          - mq_type: kafka
            task: upgrade

    steps:

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0

      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version

      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client
        run: |
          pip install -r requirements.txt --trusted-host https://test.pypi.org

      - name: Modify chart value config
        timeout-minutes: 1
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          yq -i ".kafka.enabled = false" cluster-values.yaml
          yq -i ".pulsar.enabled = false" cluster-values.yaml
          yq -i ".kafka.enabled = false" cluster-values-second.yaml
          yq -i ".pulsar.enabled = false" cluster-values-second.yaml
          yq -i ".${{ matrix.mq_type }}.enabled = true" cluster-values.yaml
          yq -i ".${{ matrix.mq_type }}.enabled = true" cluster-values-second.yaml
          if [ ${{ matrix.mq_type }} == "kafka" ]; then
              yq -i ".kafka.enabled = true" standalone-values.yaml;
          fi

      - name: First Milvus Deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update

          # if the task is reinstall, install milvus with latest image in repo milvusdb/milvus
          # for cluster mode
          if [ ${{ matrix.task }} == "reinstall" ] && [ ${{ matrix.mode }} == "cluster" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus -f cluster-values.yaml;
          fi
          # for standalone mode
          if [ ${{ matrix.task }} == "reinstall" ] && [ ${{ matrix.mode }} == "standalone" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus -f standalone-values.yaml;
          fi

          # if the task is upgrade, install milvus with latest rc image in repo milvusdb/milvus
          if [ ${{ matrix.task }} == "upgrade" ] && [ ${{ matrix.mode }} == "cluster" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus --set image.all.repository=milvusdb/milvus --set image.all.tag=latest --set etcd.image.repository=milvusdb/etcd --set etcd.image.tag=3.5.0-r6 -f cluster-values.yaml;
          fi
          if [ ${{ matrix.task }} == "upgrade" ] && [ ${{ matrix.mode }} == "standalone" ]; then
              echo "task: ${{ matrix.task }} mode: ${{ matrix.mode }}";
              helm install --wait --timeout 720s deploy-testing milvus/milvus --set image.all.repository=milvusdb/milvus --set image.all.tag=latest --set etcd.image.repository=milvusdb/etcd --set etcd.image.tag=3.5.0-r6 -f standalone-values.yaml;
          fi

          kubectl get pods
          sleep 20s
          kubectl get pods
          kubectl port-forward service/deploy-testing-milvus 19530 >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530

      - name: Run first test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          # first test
          if [ ${{ matrix.task }} == "reinstall" ]; then python scripts/action_before_reinstall.py; fi
          if [ ${{ matrix.task }} == "upgrade" ]; then python scripts/action_before_upgrade.py; fi
      - name: Milvus Idle Time
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          sleep 60s
      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          kubectl get pod
          # export k8s log for milvus
          bash ../../scripts/export_log_k8s.sh default deploy-testing k8s_logs/first_deploy
      - name: Restart Milvus
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          # uninstall milvus
          if [ ${{ matrix.mode }} == "standalone" ];
          then
            kubectl delete pod -l app.kubernetes.io/instance=deploy-testing --grace-period=0 --force;
            kubectl delete pod -l release=deploy-testing --grace-period=0 --force;
          else
            helm uninstall deploy-testing
          fi

      - name: Seconde Milvus Deployment
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          if [ ${{ matrix.mode }} == "cluster" ]; then helm install --wait --timeout 720s deploy-testing milvus/milvus -f cluster-values-second.yaml; fi
          if [ ${{ matrix.mode }} == "standalone" ]; then helm upgrade --wait --timeout 720s deploy-testing milvus/milvus -f standalone-values.yaml; fi
          kubectl get pods
          sleep 20s
          kubectl get pods
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/deploy-testing-milvus 19530 >/dev/null 2>&1 &
          sleep 120s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530

      - name: Run second test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          # second test
          if [ ${{ matrix.task }} == "reinstall" ]; then python scripts/action_after_reinstall.py; fi
          if [ ${{ matrix.task }} == "upgrade" ]; then python scripts/action_after_upgrade.py; fi

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          kubectl get pod
          # export k8s log for milvus
          bash ../../scripts/export_log_k8s.sh default deploy-testing k8s_logs/second_deploy

      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: helm-log-${{ matrix.mq_type }}-${{ matrix.mode }}-${{ matrix.task }}
          path: tests/python_client/deploy/k8s_logs

name: IO Latency Chaos Test

on:
  workflow_dispatch:
jobs:
  
  test-io-latency-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        pod: [pulsar, etcd, minio]
    
    steps:

      - name: Set env param
        run: |
          echo "RELEASE=test-${{ matrix.pod }}-io-latency" >> $GITHUB_ENV

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_on: error
          shell: bash
          command: |
            pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org
            pip install --upgrade protobuf
      
      - name: Deploy Chaos Mesh
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          kubectl get po -n chaos-testing
      
      - name: Deploy Milvus
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          echo "latest tag:"
          bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml  -n=chaos-testing; fi
          if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f standalone-values.yaml -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          python scripts/hello_milvus.py
      
      - name: Chaos Test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/io_latency\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_io_latency.yaml\'/g" constants.py
          sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
          cat constants.py
          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

      - name: Result Analysis
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos/reports
        run: |
          echo "result analysis"
          cat ${{ env.RELEASE }}.log || echo "no log file"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get iochaos -n chaos-testing
          kubectl get pod -n chaos-testing
          # wait all pod to be ready
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530
 
          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test

      - name: Deploy Milvus Again If Previous E2E Test Failed
        timeout-minutes: 15
        if: ${{ failure() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          kubectl config set-context --current --namespace=chaos-testing
          bash scripts/uninstall_milvus.sh ${{ env.RELEASE }}
          if [ ${{ matrix.pod }} != "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml  -n=chaos-testing; fi
          if [ ${{ matrix.pod }} == "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set cluster.enabled=false --set etcd.replicaCount=1 --set minio.mode=standalone --set pulsar.enabled=false -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          python scripts/hello_milvus.py              
      
      - name: Data Consist Test
        timeout-minutes: 5
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "data consist chaos test failed"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing       
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9  
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/data-consist-test
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon k8s_logs/chaos-mesh-daemon
      
      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: |
            tests/python_client/chaos/k8s_logs
            tests/python_client/chaos/reports

name: Jenkins Checker
# Lint Jenkinsfile  and related groovy files


on:
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/ci/jenkins/**.groovy'
      - 'ci/jenkins/**.groovy'
      - '.github/workflows/jenkins-checker.yaml'
jobs:
  check-jenkinsfile:
    name: Jenkinsfile Checker
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Validate Jenkinsfile
        shell: bash
        run: |
          function validate(){
            local file_path=${1:-Jenkinsfile}
            local jenkins_url=${2:-"https://jenkins.milvus.io:18080/"}
       
            JENKINS_CRUMB=`curl "${jenkins_url}/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,\":\",//crumb)"`
            response=$(curl --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 40  -X POST -H $JENKINS_CRUMB -F "jenkinsfile=<${file_path}" ${jenkins_url}/pipeline-model-converter/validate)
            
            if [[ ${response} =~ "Error"  ]]
            then
                echo " ${response}"
                echo "Validate ${file_path} failed !"
                
                exit 1  
            fi
          }
          function validate_path(){
            local path=${1}
            local jenkins_url=${2}

            for file in ${path}
            do
                if [ -f "$file"  ] 
                then 
                    # echo "$file"
                    file_name=$(basename "$file")
                    if echo "${file_name}" | grep -q -E '\.groovy$'
                    then 
                        # echo "Validate groovy file ${file_name}"
                        validate $file ${jenkins_url}
                    elif [[ "${file_name}" == "Jenkinsfile" ]]
                    then
                        # echo "Validate Jenkinsfile"
                        validate $file ${jenkins_url} 
                    fi
                fi
            done
          }
          validate_path "ci/jenkins/*" "https://jenkins.milvus.io:18080/"
    

name: License Checker

on:
  workflow_dispatch:
jobs:
  check-license:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Check License Header
        uses: apache/skywalking-eyes@main
        with:
          log: info
          config: .github/.licenserc.yaml

name: Mac Code Checker

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    branches:
      - master
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'scripts/**'
      - 'internal/**'
      - 'pkg/**'
      - 'client/**'
      - 'cmd/**'
      - 'build/**'
      - 'tests/integration/**'
      - 'tests/go_client/**'
      - '.github/workflows/mac.yaml'
      - '.env'
      - docker-compose.yml
      - Makefile
      - '!**.md'
      - '!build/ci/jenkins/**'
      # FIXME(wxyu): not need to run code check, update the ci-passed rules and remove these two lines
      - go.mod
      - go.sum
  schedule:
    # Runs at 17:00 UTC every day, for refreshing cache every day
    - cron: '0 17 * * *'
  workflow_dispatch:
    inputs:
      branch:
        description: 'Branch to run the workflow on'
        required: true
        default: 'master'
        type: string

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  mac:
    name: Code Checker MacOS
    runs-on: macos-15
    timeout-minutes: 480
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          ref: ${{ github.event.inputs.branch || github.ref }}
      - name: Setup Python environment
        uses: actions/setup-python@v4
        with:
          python-version: '<3.12'
      - name: Install CMake 3.31.8
        run: |
          cmake --version || true
          which cmake || true
          if ! brew list cmake &>/dev/null; then
            brew uninstall cmake -f
          fi
          curl -LO https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-macos-universal.tar.gz
          tar -zxf cmake-3.31.8-macos-universal.tar.gz -C /tmp/
          sudo mv /tmp/cmake-3.31.8-macos-universal/CMake.app /Applications/
          sudo rm -f /usr/local/bin/cmake /usr/local/bin/cpack /usr/local/bin/ctest
          sudo rm -f /opt/homebrew/bin/cmake /opt/homebrew/bin/cpack /opt/homebrew/bin/ctest
          echo -e "\n=== Creating symlinks for CMake ==="
          sudo ln -s /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake
          sudo ln -s /Applications/CMake.app/Contents/bin/cpack /usr/local/bin/cpack
          sudo ln -s /Applications/CMake.app/Contents/bin/ctest /usr/local/bin/ctest
          CMAKE_MODULE_DIR="/Applications/CMake.app/Contents/share/cmake-3.31"
          sudo ln -s "$CMAKE_MODULE_DIR" /usr/local/share/cmake-3.31
          export PATH="/usr/local/bin:$PATH"          
          cmake --version
          which cmake
      - name: Setup Go environment
        uses: actions/setup-go@v4
        with:
          go-version: '1.25.8'
          cache: false
      - name: Download Caches
        uses: ./.github/actions/macos-cache-restore
      - name: Code Check
        env:
          CCACHE_DIR: /var/tmp/ccache
          CCACHE_COMPILERCHECK: content
          CCACHE_COMPRESS: 1
          CCACHE_COMPRESSLEVEL: 9
          CCACHE_MAXSIZE: 1G
        run: |
          if [[ ! -d "/var/tmp/ccache" ]];then
            mkdir -p /var/tmp/ccache
          fi
          brew install libomp ninja openblas ccache pkg-config
          BREW_PREFIX=$(brew --prefix)
          # aws-sdk-cpp 1.11.750 had a packaging bug where Paginator.h was
          # not installed (missing CMake install rule for pagination headers).
          # Homebrew backported the fix with "rebuild 1", but the CI runner's
          # formula may be stale.  Run brew update first so we get the
          # patched bottle.
          brew update
          brew install aws-sdk-cpp || brew upgrade aws-sdk-cpp
          # Install llvm@17 explicitly as a stable version
          brew install llvm@17
          if [[ ! -d "${BREW_PREFIX}/opt/llvm" ]]; then
            ln -s ${BREW_PREFIX}/opt/llvm@17 ${BREW_PREFIX}/opt/llvm
          fi
          pip3 install conan==1.66.0
          make milvus
      - name: Upload Cmake log
        uses: actions/upload-artifact@v4
        if: ${{ failure() }}
        with:
          name: cmake-log
          path: cmake_build/CMakeFiles/*.log
      - name: Save Caches
        uses: ./.github/actions/macos-cache-save
        if: github.event_name != 'pull_request'

name: Build and test

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'scripts/**'
      - 'internal/**'
      - 'client/**'
      - 'pkg/**'
      - 'cmd/**'
      - 'build/**'
      - 'tests/integration/**' # run integration test
      - '.github/workflows/main.yaml'
      - '.env'
      - docker-compose.yml
      - Makefile
      - go.mod
      - '!**.md'
      - '!build/ci/jenkins/**'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'scripts/**'
      - 'internal/**'
      - 'pkg/**'
      - 'client/**'
      - 'cmd/**'
      - 'build/**'
      - 'tests/integration/**' # run integration test
      - '.github/workflows/main.yaml'
      - '.env'
      - docker-compose.yml
      - Makefile
      - go.mod
      - '!**.md'
      - '!build/ci/jenkins/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  Build:
    name: Build and test AMD64 Ubuntu 22.04
    runs-on: ubuntu-latest
    steps:
      - name: 'Setup $HOME'
        # hot fix
        run: |
            # Check if $HOME is not set
            if [ -z "$HOME" ]; then
                echo '$HOME was no set'
                echo "HOME=/home/zilliz-user" >> $GITHUB_ENV
            fi
            echo "HOME variable is:$HOME"
            echo "GITHUB_ENV variable is:$GITHUB_ENV"
      - name: Setup mold
        uses: rui314/setup-mold@v1
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: 'Check Changed files'
        id: changed-files-cpp
        uses: tj-actions/changed-files@v46
        with:
          since_last_remote_commit: 'true'
          files: |
            **/*.cpp
            **/*.cc
            **/*.c
            **/*.h
            **/*.hpp
            **/*.CMakeLists.txt
            **/conanfile.*
      - name: 'Setup Use USE_ASAN'
        if: steps.changed-files-cpp.outputs.any_changed == 'true'
        run: |
          echo "useasan=OFF" >> $GITHUB_ENV
          echo "Setup USE_ASAN to true since cpp file(s) changed"
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'ubuntu22.04'
          kind: 'cpp'
      - name: Build
        run: |
          ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} mode=RelWithDebInfo build-cpp && chmod -R a+rwx /home/milvus/.conan"
      - run: |
          zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/**" -x ".git/**"
      - name: Archive code
        uses: actions/upload-artifact@v4
        with:
          name: code
          path: code.zip
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'ubuntu22.04'
          kind: 'cpp'

  UT-Cpp:
    # skip the UT-Cpp job in github workflow, it run in jenkins now see: UT-CPP.groovy
    if: false 
    name: UT for Cpp
    needs: Build
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Download code
        uses: actions/download-artifact@v4.1.3
        with:
          name: code
      - run: |
          unzip code.zip
          rm code.zip
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'ubuntu22.04'
          kind: 'cpp'
      - name: Start Service
        shell: bash
        run: |
          docker compose up -d azurite gcpnative
      - name: UT
        run: |
          chmod +x build/builder.sh
          chmod +x scripts/*
          chmod +x internal/core/output/unittest/*
          ./build/builder.sh /bin/bash -c ./scripts/run_cpp_codecov.sh
      - name: Archive result
        uses: actions/upload-artifact@v4
        with:
          name: cpp-result
          path: |
            ./go_coverage.txt
            ./lcov_output.info
            *.info
            *.out
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'ubuntu22.04'
          kind: 'cpp'
  UT-Go:
    name: UT for Go
    needs: Build
    runs-on: ubuntu-latest
    timeout-minutes: 150
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Download code
        uses: actions/download-artifact@v4.1.3
        with:
          name: code
      - run: |
          unzip code.zip
          rm code.zip
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'ubuntu22.04'
          kind: 'go'
      - name: Start Service
        shell: bash
        run: |
          docker compose up -d pulsar etcd minio azurite gcpnative
      - name: UT
        run: |
          chmod +x build/builder.sh
          chmod +x scripts/run_go_codecov.sh
          ./build/builder.sh /bin/bash -c "make codecov-go-without-build"
      - name: Archive result
        uses: actions/upload-artifact@v4
        with:
          name: go-result
          path: |
            ./go_coverage.txt
            ./lcov_output.info
            *.info
            *.out
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'ubuntu22.04'
          kind: 'go'

  integration-test:
    name: Integration Test
    needs: Build
    runs-on: ubuntu-latest
    timeout-minutes: 150
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 20480
          swap-size-mb: 1024
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
      - name: Download code
        uses: actions/download-artifact@v4.1.3
        with:
          name: code
      - run: |
          unzip code.zip
          rm code.zip
      - name: Download Caches
        uses: ./.github/actions/cache-restore
        with:
          os: 'ubuntu22.04'
          kind: 'go'
      - name: Start Service
        shell: bash
        run: |
          docker compose up -d pulsar etcd minio
      - name: IntegrationTest
        run: |
          chmod +x build/builder.sh
          chmod +x scripts/run_intergration_test.sh
          ./build/builder.sh /bin/bash -c "make build-go && make integration-test" 
      - name: Archive result
        uses: actions/upload-artifact@v4
        with:
          name: it-result
          path: |
            ./it_coverage.txt
            *.info
            *.out
      - name: Save Caches
        uses: ./.github/actions/cache-save
        if: github.event_name != 'pull_request'
        with:
          os: 'ubuntu22.04'
          kind: 'go'

  codecov:
    name: Upload Code Coverage
    needs: [UT-Go, integration-test]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
      - name: Checkout
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Download Go code coverage results
        uses: actions/download-artifact@v4.1.3
        with:
          name: go-result
      - name: Download Integration Test coverage results
        uses: actions/download-artifact@v4.1.3
        with:
          name: it-result
      - name: Display structure of code coverage results
        run: |
          ls -lah
      - name: Upload coverage to Codecov
        if: ${{ github.repository == 'milvus-io/milvus' }}
        uses: codecov/codecov-action@v4
        id: upload_cov
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./go_coverage.txt,./it_coverage.txt
          name: ubuntu-20.04-unittests
          fail_ci_if_error: true
          disable_safe_directory: true
          verbose: true
      - name: Retry Upload coverage to Codecov
        if: ${{ failure()  && github.repository == 'milvus-io/milvus' }}
        uses: codecov/codecov-action@v4
        id: retry_upload_cov
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./go_coverage.txt,./it_coverage.txt
          name: ubuntu-20.04-unittests
          fail_ci_if_error: true
          disable_safe_directory: true
          verbose: true

name: Markdown Links Check
on:
  workflow_dispatch:
  schedule:
    - cron: "30 20 * * *"

jobs:
  check-links:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@master
      - uses: gaurav-nelson/github-action-markdown-link-check@v1
        # checks all markdown files from /docs including all subfolders
        with:
          use-quiet-mode: 'yes'
          use-verbose-mode: 'yes'
          folder-path: 'docs/'
      - uses: actions/checkout@master
      - uses: gaurav-nelson/github-action-markdown-link-check@v1
        # checks all markdown files from root but ignores subfolders
        with:
          use-quiet-mode: 'yes'
          use-verbose-mode: 'yes'
          max-depth: 0

name: Memory Stress Chaos Test

on:
  workflow_dispatch:
jobs:
  
  test-memory-stress-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        pod: [datanode, indexnode, proxy, pulsar, querynode, etcd, minio]
    
    steps:

      - name: Set env param
        run: |
          echo "RELEASE=test-${{ matrix.pod }}-memory-stress" >> $GITHUB_ENV

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_on: error
          shell: bash
          command: |
            pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org
            pip install --upgrade protobuf
      
      - name: Deploy Chaos Mesh
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          kubectl get po -n chaos-testing
      
      - name: Deploy Milvus
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          echo "latest tag:"
          bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml  -n=chaos-testing; fi
          if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f standalone-values.yaml -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          python scripts/hello_milvus.py
      
      - name: Chaos Test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/mem_stress\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_mem_stress.yaml\'/g" constants.py
          sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
          cat constants.py
          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

      - name: Result Analysis
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos/reports
        run: |
          echo "result analysis"
          cat ${{ env.RELEASE }}.log || echo "no log file"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get networkchaos -n chaos-testing
          kubectl get pod -n chaos-testing
          # wait all pod to be ready
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test

      - name: Deploy Milvus Again If Previous E2E Test Failed
        timeout-minutes: 15
        if: ${{ failure() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          kubectl config set-context --current --namespace=chaos-testing
          bash scripts/uninstall_milvus.sh ${{ env.RELEASE }}
          if [ ${{ matrix.pod }} != "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml  -n=chaos-testing; fi
          if [ ${{ matrix.pod }} == "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set cluster.enabled=false --set etcd.replicaCount=1 --set minio.mode=standalone --set pulsar.enabled=false -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          python scripts/hello_milvus.py              
      
      - name: Data Consist Test
        timeout-minutes: 5
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "data consist chaos test failed"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing       
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9  
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/data-consist-test
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon k8s_logs/chaos-mesh-daemon
      
      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: |
            tests/python_client/chaos/k8s_logs
            tests/python_client/chaos/reports

name: Network Latency Chaos Test

on:
  workflow_dispatch:
jobs:
  
  test-network-latency-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        pod: [datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio]
    
    steps:

      - name: Set env param
        run: |
          echo "RELEASE=test-${{ matrix.pod }}-network-latency" >> $GITHUB_ENV

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_on: error
          shell: bash
          command: |
            pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org
            pip install --upgrade protobuf
      
      - name: Deploy Chaos Mesh
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          kubectl get po -n chaos-testing
      
      - name: Deploy Milvus
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          echo "latest tag:"
          bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml  -n=chaos-testing; fi
          if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f standalone-values.yaml -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          python scripts/hello_milvus.py
      
      - name: Chaos Test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/network_latency\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_network_latency.yaml\'/g" constants.py
          sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
          cat constants.py
          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

      - name: Result Analysis
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos/reports
        run: |
          echo "result analysis"
          cat ${{ env.RELEASE }}.log || echo "no log file"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get networkchaos -n chaos-testing
          kubectl get pod -n chaos-testing
          # wait all pod to be ready
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test

      - name: Deploy Milvus Again If Previous E2E Test Failed
        timeout-minutes: 15
        if: ${{ failure() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          kubectl config set-context --current --namespace=chaos-testing
          bash scripts/uninstall_milvus.sh ${{ env.RELEASE }}
          if [ ${{ matrix.pod }} != "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus -f cluster-values.yaml  -n=chaos-testing; fi
          if [ ${{ matrix.pod }} == "standalone" ]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set cluster.enabled=false --set etcd.replicaCount=1 --set minio.mode=standalone --set pulsar.enabled=false -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          python scripts/hello_milvus.py              
      
      - name: Data Consist Test
        timeout-minutes: 5
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "data consist chaos test failed"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing       
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9  
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/data-consist-test
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon k8s_logs/chaos-mesh-daemon
      
      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: |
            tests/python_client/chaos/k8s_logs
            tests/python_client/chaos/reports

name: Network Partition Chaos Test

on:
  workflow_dispatch:

jobs:
  
  test-network-partition-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        pod: [datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio]
    
    steps:

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pip install -r ../requirements.txt
          pip install --upgrade protobuf      

      - name: Deploy Chaos Mesh
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install chaos-mesh chaos-mesh/chaos-mesh -n=chaos-testing --version v2.0.1 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          # wait for all pod running 
          sleep 60s
          kubectl get po -n chaos-testing

      - name: Deploy Milvus
        shell: bash
        run: |
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          # install milvus with latest image
          helm install --wait --timeout 360s chaos-testing milvus/milvus --set cluster.enabled=true --set image.all.repository=milvusdb/milvus --set image.all.tag=master-latest -n=chaos-testing
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/chaos-testing-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
      - name: Chaos Test
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/network_partition\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_network_partition.yaml\'/g" constants.py
          cat constants.py
          pytest -s -v test_chaos.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-testing
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon
      
      - name: Upload logs
        if: ${{ always() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: tests/python_client/chaos/k8s_logs     

name: Cluster N node Chaos Test

on:
  workflow_dispatch:
    inputs:
      image_tag:
        description: The image tag to use for the chaos test
        required: true
        default: 'master-latest'
      image_repo:
        description: The image repo to use for the chaos test
        required: true
        default: 'milvusdb/milvus'    
  schedule:
    - cron: "30 19 * * *"
jobs:
  
  test-cluster-n-node-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        chaos_type: [pod_failure, pod_kill]
        pod: [querynode, datanode, indexnode, proxy]
    
    steps:
      - name: Set env param
        env:
          DEFAULT_IMAGE_TAG: master-latest
          DEFAULT_IMAGE_REPO: milvusdb/milvus        
        run: |
          chaos_type=${{ matrix.chaos_type }}
          release="test"-${{ matrix.pod }}-${chaos_type/_/-}
          echo "RELEASE=$release" >> $GITHUB_ENV
          echo "IMAGE_REPO=${{ github.event.inputs.image_repo || env.DEFAULT_IMAGE_REPO}}" >> $GITHUB_ENV
          echo "IMAGE_TAG=${{ github.event.inputs.image_tag || env.DEFAULT_IMAGE_TAG}}" >> $GITHUB_ENV
      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_on: error
          shell: bash
          command: |
            pip install -r tests/python_client/requirements.txt
      
      - name: Deploy Chaos Mesh
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          sleep 60s
          kubectl get po -n chaos-testing
      
      - name: Deploy Milvus
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          echo "latest tag:"
          bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q
          declare -A pod_map=( ["querynode"]="queryNode" ["indexnode"]="indexNode" ["datanode"]="dataNode" ["proxy"]="proxy")
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} --set ${pod_map[${{ matrix.pod }}]}.replicas=2 -f cluster-values.yaml  -n=chaos-testing
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530
          # check whether milvus server is healthy
          pytest -s -v ../testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python scripts/hello_milvus.py --host 127.0.0.1

      - name: Chaos Test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${{ matrix.chaos_type }}\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_${{ matrix.chaos_type }}.yaml\'/g" constants.py
          sed -i "s/CHAOS_DURATION =.*/CHAOS_DURATION = 80/g" constants.py
          sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
          cat constants.py
          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

      - name: Result Analysis
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos/reports
        run: |
          echo "result analysis"
          cat ${{ env.RELEASE }}.log || echo "no log file"

      - name: Milvus E2E Test
        timeout-minutes: 10
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test

      - name: Data Consist Test
        timeout-minutes: 5
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v test_chaos_data_consist.py --host 127.0.0.1 --log-cli-level=INFO || echo "data consist chaos test failed"

      - name: Milvus E2E Test
        timeout-minutes: 5
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          python chaos/scripts/hello_milvus.py --host 127.0.0.1

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/data-consist-test
          bash ../../scripts/export_log_k8s.sh chaos-testing chaos-daemon
      
      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}-${{ matrix.chaos_type }}
          path: tests/python_client/chaos/k8s_logs

name: Pod Kill Chaos Test

on:
  workflow_dispatch:
    inputs:
      image_tag:
        description: The image tag to use for the chaos test
        required: true
        default: 'master-latest'
      image_repo:
        description: The image repo to use for the chaos test
        required: true
        default: 'milvusdb/milvus'        
  schedule:
    - cron: "30 18 * * *"
jobs:
  
  test-pod-kill-chaos:
   
    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        pod: [allstandalone, allcluster, standalone, datacoord, datanode, indexcoord, indexnode, proxy, pulsar, querycoord, querynode, rootcoord, etcd, minio]
    
    steps:

      - name: Set env param
        env:
          DEFAULT_IMAGE_TAG: master-latest
          DEFAULT_IMAGE_REPO: milvusdb/milvus
        run: |
          echo "RELEASE=test-${{ matrix.pod }}-pod-kill" >> $GITHUB_ENV
          echo "IMAGE_REPO=${{ github.event.inputs.image_repo || env.DEFAULT_IMAGE_REPO}}" >> $GITHUB_ENV
          echo "IMAGE_TAG=${{ github.event.inputs.image_tag || env.DEFAULT_IMAGE_TAG}}" >> $GITHUB_ENV

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0
      
      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version
      
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_on: error
          shell: bash
          command: |
            pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org
      
      - name: Deploy Chaos Mesh
        timeout-minutes: 2
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          kubectl get po -n chaos-testing
      
      - name: Deploy Milvus
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          echo "latest tag:"
          bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f cluster-values.yaml  -n=chaos-testing; fi
          if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f standalone-values.yaml -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530

      - name: Run e2e test before chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client
        run: |
          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no

      - name: Run data presistence test before chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no      

      - name: Chaos Test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py
          sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
          cat constants.py
          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

      - name: Result Analysis
        timeout-minutes: 1
        shell: bash
        working-directory: tests/python_client/chaos/reports
        run: |
          echo "result analysis"
          cat ${{ env.RELEASE }}.log || echo "no log file"

      - name: Wait all pods ready
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          # wait all pod to be ready
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530

      - name: Run e2e test after chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client
        run: |

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no

      - name: Run data presistence test after chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no    

      - name: Verify all collections after chaos
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          
          pytest -s -v testcases/test_get_collections.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          sleep 2s
          pytest -s -v testcases/test_all_collections_after_chaos.py --host 127.0.0.1 -n 4 --log-cli-level=INFO --capture=no

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test
      
      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: |
            tests/python_client/chaos/k8s_logs
            tests/python_client/chaos/reports

name: Pod Kill Chaos Test For Kafka Version

on:
  workflow_dispatch:
    inputs:
      image_tag:
        description: The image tag to use for the chaos test
        required: true
        default: 'master-latest'
      image_repo:
        description: The image repo to use for the chaos test
        required: true
        default: 'milvusdb/milvus'
  schedule:
    - cron: "30 17 * * *"
jobs:

  test-pod-kill-chaos:

    runs-on: ubuntu-latest
    timeout-minutes: 40
    strategy:
      fail-fast: false
      matrix:
        pod: [allstandalone, allcluster, standalone, datacoord, datanode, indexcoord, indexnode, proxy, kafka, querycoord, querynode, rootcoord, etcd, minio]

    steps:

      - name: Set env param
        env:
          DEFAULT_IMAGE_TAG: master-latest
          DEFAULT_IMAGE_REPO: milvusdb/milvus
        run: |
          echo "RELEASE=test-${{ matrix.pod }}-pod-kill" >> $GITHUB_ENV
          echo "IMAGE_REPO=${{ github.event.inputs.image_repo || env.DEFAULT_IMAGE_REPO}}" >> $GITHUB_ENV
          echo "IMAGE_TAG=${{ github.event.inputs.image_tag || env.DEFAULT_IMAGE_TAG}}" >> $GITHUB_ENV

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0

      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version

      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        uses: nick-invision/retry@v2
        with:
          timeout_minutes: 5
          max_attempts: 3
          retry_on: error
          shell: bash
          command: |
            pip install -r tests/python_client/requirements.txt --trusted-host https://test.pypi.org

      - name: Deploy Chaos Mesh
        timeout-minutes: 2
        shell: bash
        run: |
          helm repo add chaos-mesh https://charts.chaos-mesh.org
          helm search repo chaos-mesh
          kubectl create ns chaos-testing
          helm install --wait --timeout 360s chaos-mesh chaos-mesh/chaos-mesh --namespace=chaos-testing --version v2.0.3 --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock
          kubectl get po -n chaos-testing

      - name: Deploy Milvus
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          echo "latest tag:"
          bash ../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus -t master-latest -f master- -F -L -q
          helm repo add milvus https://zilliztech.github.io/milvus-helm
          helm repo update
          if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set pulsar.enabled=false --set kafka.enabled=true --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f cluster-values.yaml  -n=chaos-testing; fi
          if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set pulsar.enabled=false --set kafka.enabled=true --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f standalone-values.yaml -n=chaos-testing; fi
          kubectl get pods -n chaos-testing
          sleep 20s
          kubectl get pods -n chaos-testing
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530

      - name: Run e2e test before chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client
        run: |
          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no

      - name: Run data presistence test before chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no      

      - name: Chaos Test
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          # replace chaos object
          sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/pod_kill\/'/g" constants.py
          sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${{ matrix.pod }}_pod_kill.yaml\'/g" constants.py
          sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${{ env.RELEASE }}\'/g" constants.py
          cat constants.py
          timeout 14m pytest -s -v test_chaos.py --host 127.0.0.1 --log-cli-level=INFO --capture=no || echo "chaos test failed"

      - name: Result Analysis
        timeout-minutes: 1
        shell: bash
        working-directory: tests/python_client/chaos/reports
        run: |
          echo "result analysis"
          cat ${{ env.RELEASE }}.log || echo "no log file"

      - name: Wait all pods ready
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client
        run: |
          kubectl get pod -n chaos-testing
          # wait all pod to be ready
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl wait --for=condition=Ready pod -l release=${{ env.RELEASE }} -n chaos-testing --timeout=360s
          kubectl get pod -n chaos-testing
          ps aux|grep forward|grep -v grep|awk '{print $2}'|xargs kill -9
          kubectl port-forward service/${{ env.RELEASE }}-milvus 19530 -n chaos-testing >/dev/null 2>&1 &

          sleep 20s
          nc -vz 127.0.0.1 19530

      - name: Run e2e test after chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client
        run: |

          pytest -s -v testcases/test_e2e.py --host 127.0.0.1 --log-cli-level=INFO --capture=no

      - name: Run data presistence test after chaos
        timeout-minutes: 5
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          pytest -s -v testcases/test_data_persistence.py --host 127.0.0.1 --log-cli-level=INFO --capture=no    

      - name: Verify all collections after chaos
        timeout-minutes: 15
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          
          pytest -s -v testcases/test_get_collections.py --host 127.0.0.1 --log-cli-level=INFO --capture=no
          sleep 2s
          pytest -s -v testcases/test_all_collections_after_chaos.py --host 127.0.0.1 -n 4 --log-cli-level=INFO --capture=no

      - name: Export logs
        if: ${{ always() }}
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          #in this step, verify whether pod has been killed by pod's age
          kubectl get po -n chaos-testing
          # export k8s log for chaos mesh and milvus
          bash ../../scripts/export_log_k8s.sh chaos-testing ${{ env.RELEASE }} k8s_logs/chaos-test
      
      - name: Upload logs
        if: ${{ ! success() }}
        uses: actions/upload-artifact@v2
        with:
          name: logs-${{ matrix.pod }}
          path: |
            tests/python_client/chaos/k8s_logs
            tests/python_client/chaos/reports

name: Publish Builder
# TODO: do not trigger action for some document file update

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/builder/cpu/**'
      - '.github/workflows/publish-builder.yaml'
      - '!**.md'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/builder/cpu/**'
      - '.github/workflows/publish-builder.yaml'
      - '!**.md'
  workflow_dispatch:
    inputs:
      push_to_dockerhub:
        description: 'Push to Docker Hub'
        required: true
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  publish-builder:
    name: ${{ matrix.arch }} ${{ matrix.os }}
    runs-on: ubuntu-latest
    timeout-minutes: 500
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu24.04, ubuntu22.04, amazonlinux2023, rockylinux9]
    env:
      OS_NAME: ${{ matrix.os }}
      IMAGE_ARCH: ${{ matrix.arch }}
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        if: ${{ ! startsWith(runner.name, 'self') }} # skip this step if it is self-hosted runner
        with:
          root-reserve-mb: 10240
          overprovision-lvm: 'true'
          swap-size-mb: 2048
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          remove-codeql: 'true'
          remove-docker-images: 'true'
      - name: Checkout
        uses: actions/checkout@v2
      - name: Get version from system time after release step
        id: extracter
        run: |
          echo "::set-output name=version::$(date +%Y%m%d)"
          echo "::set-output name=sha_short::$(git rev-parse --short=7 HEAD)"
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            milvusdb/milvus-env
          tags: |
            type=raw,enable=true,value=${{ matrix.os }}-{{date 'YYYYMMDD'}}-{{sha}}
            type=raw,enable=true,value=${{ matrix.os }}-latest
      # - name: Setup upterm session
      #   uses: lhotari/action-upterm@v1
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        if: github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch' || inputs.push_to_dockerhub)
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch' || inputs.push_to_dockerhub) }}
          tags: ${{ steps.meta.outputs.tags }}
          file: build/docker/builder/cpu/${{ matrix.os }}/Dockerfile
      - name: Bump Builder Version
        uses: ./.github/actions/bump-builder-version
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu22.04'
        with:
          tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}"
          type: cpu
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}

name: Publish Gpu Builder
# TODO: do not trigger action for some document file update

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/builder/gpu/**'
      - '.github/workflows/publish-gpu-builder.yaml'
      - '!**.md'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/builder/gpu/**'
      - '.github/workflows/publish-gpu-builder.yaml'
      - '!**.md'
  workflow_dispatch:
    inputs:
      RunnerHost:
        description: "Use this label for self-host runner"
        required: true
        type: string
        default: "ubuntu-latest"
      push_to_dockerhub:
        description: 'Push to Docker Hub'
        required: true
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  publish-gpu-builder:
    runs-on: ubuntu-latest
    timeout-minutes: 500
    env:
      OS: ubuntu22.04
    steps:
      - name: Maximize build space
        uses: easimon/maximize-build-space@master
        with:
          root-reserve-mb: 10240
          overprovision-lvm: 'true'
          swap-size-mb: 2048
          remove-dotnet: 'true'
          remove-android: 'true'
          remove-haskell: 'true'
          remove-codeql: 'true'
          remove-docker-images: 'true'
      - name: Checkout
        uses: actions/checkout@v2
      - name: Get version from system time after release step
        id: extracter
        run: |
          echo "version=$(date +%Y%m%d)" >> $GITHUB_OUTPUT
          echo "sha_short=$(git rev-parse --short=7 HEAD)" >> $GITHUB_OUTPUT
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: |
            milvusdb/milvus-env
          tags: |
            type=raw,enable=true,value=gpu-${{ env.OS }}-{{date 'YYYYMMDD'}}-{{sha}}
            type=raw,enable=true,value=gpu-${{ env.OS }}-latest
      # - name: Setup upterm session
      #   uses: lhotari/action-upterm@v1
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        if: github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch' || inputs.push_to_dockerhub)
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USER }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build and push
        uses: docker/build-push-action@v5
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: ${{ github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch' || inputs.push_to_dockerhub) }}
          tags: ${{ steps.meta.outputs.tags }}
          file: build/docker/builder/gpu/${{ env.OS }}/Dockerfile
      - name: Bump Builder Version
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && env.OS == 'ubuntu22.04'
        uses: ./.github/actions/bump-builder-version
        with:
          tag: "${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}"
          type: gpu
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}

name: Publish KRTE Images
# TODO: do not trigger action for some document file update

# This workflow is triggered on pushes or pull request to the repository.
on:
  workflow_dispatch:
  push:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/krte/**'
      - '.github/workflows/publish-krte-images.yaml'
      - '!**.md'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'build/docker/krte/**'
      - '.github/workflows/publish-krte-images.yaml'
      - '!**.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  publish-krte-images:
    name: KRTE
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Get version from system time after release step
        id: extracter
        run: |
          echo "::set-output name=version::$(date +%Y%m%d)"
          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
      - name: Docker Build
        shell: bash
        working-directory: build/docker/krte
        run: |
          export TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}
          docker build --build-arg IMAGE_ARG="milvusdb/krte:$TAG" --build-arg GO_VERSION=1.15.8 -t "milvusdb/krte:$TAG" .
      - name: Docker Push
        if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' )&& github.repository == 'milvus-io/milvus'
        continue-on-error: true
        shell: bash
        run: |
          docker login -u ${{ secrets.DOCKERHUB_USER }} \
                       -p ${{ secrets.DOCKERHUB_TOKEN }}
          export TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}
          docker push "milvusdb/krte:$TAG"
          echo "Push krte image Succeeded"
      - name: Update KRTE Image Changes
        if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && github.repository == 'milvus-io/milvus'
        continue-on-error: true
        shell: bash
        working-directory: build/ci/jenkins/pod
        run: |
          export TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}
          sed -i "s#krte:.*#krte:${TAG}#g" ./build/ci/jenkins/pod/rte.yaml
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git add rte.yaml
          git commit -m "Update KRTE image changes"
      - name: Create Pull Request
        id: cpr
        if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && github.repository == 'milvus-io/milvus'
        continue-on-error: true
        uses: peter-evans/create-pull-request@v3
        with:
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
          signoff: false
          branch: update_krte_${{ github.sha }}
          delete-branch: true
          title: '[automated] Update KRTE image changes'
          body: |
            Update KRTE image changes

            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
      - name: Check outputs
        if: success() && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && github.repository == 'milvus-io/milvus'
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

name: Publish Test Images
# TODO: do not trigger action for some document file update

# This workflow is triggered on pushes or pull request to the repository.
on:
  push:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'tests/docker/Dockerfile'
      - 'tests/python_client/requirements.txt'
      - '.github/workflows/publish-test-images.yaml'
      - '!**.md'
  pull_request:
    # file paths to consider in the event. Optional; defaults to all.
    paths:
      - 'tests/docker/Dockerfile'
      - 'tests/python_client/requirements.txt'
      - '.github/workflows/publish-test-images.yaml'
      - '!**.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  publish-pytest-images:
    name: PyTest
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Get version from system time after release step
        id: extracter
        run: |
          echo "::set-output name=version::$(date +%Y%m%d)"
          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
      - name: Docker Pull
        shell: bash
        working-directory: tests/docker
        run: |
          docker compose pull pytest
      - name: Docker Build
        shell: bash
        working-directory: tests/docker
        run: |
          IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} docker compose build pytest
          export LATEST_IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}
          IMAGE_TAG=latest docker compose build pytest
      - name: Docker Push
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
        continue-on-error: true
        shell: bash
        working-directory: tests/docker
        run: |
          docker login -u ${{ secrets.DOCKERHUB_USER }} \
                       -p ${{ secrets.DOCKERHUB_TOKEN }}
          IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} docker compose push pytest
          IMAGE_TAG=latest docker compose push pytest
          echo "Push pytest image Succeeded"
      - name: Update Pytest Image Changes
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
        continue-on-error: true
        shell: bash
        working-directory: .
        run: |
          sed -i "s#^IMAGE_TAG=.*#IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./tests/docker/.env
          sed -i "s#^LATEST_IMAGE_TAG=.*#LATEST_IMAGE_TAG=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./tests/docker/.env
          sed -i "s#pytest:.*#pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./ci/jenkins/pod/rte.yaml
          sed -i "s#pytest:.*#pytest:${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" ./ci/jenkins/pod/e2e.yaml
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git add ./tests/docker/.env
          git add ./ci/jenkins/pod/rte.yaml
          git add ./ci/jenkins/pod/e2e.yaml
          git commit -m "Update Pytest image changes"
      - name: Create Pull Request
        id: cpr
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
        continue-on-error: true
        uses: peter-evans/create-pull-request@v3
        with:
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
          signoff: true
          branch: update_pytest_${{ github.sha }}
          delete-branch: true
          title: '[automated] Update Pytest image changes'
          body: |
            Update Pytest image changes
            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
      - name: Check outputs
        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

name: Rerun Failure Checks
on:
  issue_comment:
    types: [created]
jobs:
  rerun-checks:
    if:  "github.event_name == 'issue_comment'&& startsWith(github.event.comment.body, 'rerun ut')"
    runs-on: ubuntu-latest
    steps:
      - name: Is Organization Member
        shell: bash
        if:  "github.event_name == 'issue_comment'"
        id: is_organization_member
        run: |
          response_code=$(curl -sIL -w %{http_code} -H "Authorization: token ${{ secrets.RERUN_BOT_TOKEN }}" -H "Accept: application/vnd.github.v3+json" \
          -o /dev/null https://api.github.com/orgs/${GITHUB_REPOSITORY_OWNER}/members/${{ github.event.sender.login }})
          echo " response code is ${response_code} "
          if [[ ${response_code} == '204' ]];then
            echo "::set-output name=is-member::true"
          elif [[ ${response_code} == '404' ]]; then 
            echo "::set-output name=is-member::false"
          else
            echo "Please check if the repository secret RERUN_BOT_TOKEN  still exists and have the permission to read organization membership."
            exit 1
          fi
      - name: Create Comment for Non-Org Member
        if: "github.event_name == 'issue_comment' && steps.is_organization_member.outputs.is-member == 'false'"
        uses: actions-cool/issues-helper@v2.5.0       
        with:
          actions: 'create-comment'
          token: ${{ secrets.GITHUB_TOKEN }}
          issue-number: ${{ github.event.issue.number }}
          body: |
            Hello ${{ github.event.sender.login }}, you are not in the organization, so you do not have the permission to rerun the workflow, please contact `@milvus-io/milvus-maintainers` for help.
      - uses: actions/checkout@v2
      - name: Rerun Failure Checks
        if: "github.event_name == 'issue_comment' && steps.is_organization_member.outputs.is-member == 'true'"
        uses: zymap/bot@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # replace here to your token
        with:
          repo_owner: milvus-io  # replace here to your repo owner
          repo_name: milvus     # replace here to your repo name
          rerun_cmd: rerun ut
          comment: ${{ github.event.comment.body }}

 

name: SIMD Compatibility Test

on:
  workflow_dispatch:
  schedule:
    - cron: "30 18 * * 0"

jobs:
  test-simd-compatibility:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        simd_type: ["sse4_2", "avx", "avx2", "avx512"]
    steps:

      - name: Creating kind cluster
        uses: helm/kind-action@v1.2.0

      - name: Print cluster information
        run: |
          kubectl config view
          kubectl cluster-info
          kubectl get nodes
          kubectl get pods -n kube-system
          helm version
          kubectl version

      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependency
        shell: bash
        working-directory: tests/python_client/deploy
        run: |
          pip install -r requirements.txt
          pip install --upgrade protobuf

      - name: install milvus operator and milvus cluster
        shell: bash
        working-directory: tests/python_client/customize/template
        run: |
          kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.3/cert-manager.yaml
          sleep 30s
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=360s
          kubectl get pods -A

          #install milvus operator
          kubectl apply -f https://raw.githubusercontent.com/milvus-io/milvus-operator/main/deploy/manifests/deployment.yaml
          sleep 30s
          kubectl wait --for=condition=Ready pod --all -n milvus-operator --timeout=360s
          kubectl get pods -A
          #install milvus cluster

          # change simdType
          sed -i "s/simdType.*/simdType\: ${{ matrix.simd_type }}/g" minimum.yaml
          kubectl apply -f minimum.yaml
          sleep 60s
          # wait all pod running
          kubectl get pods -A
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release-minio --timeout=360s
          kubectl get pods -A
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release-etcd --timeout=360s
          kubectl get pods -A
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release-pulsar --timeout=360s
          kubectl get pods -A
          sleep 60s
          kubectl get pods -A
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-release --timeout=360s
          sleep 30s
          kubectl get pods -A

          # port-forward
          kubectl port-forward service/my-release-milvus 19530 >/dev/null 2>&1 &
          sleep 20s
          # check whether port-forward success
          nc -vz 127.0.0.1 19530

      - name: Run E2E test
        shell: bash
        working-directory: tests/python_client/chaos
        run: |
          python scripts/hello_milvus.py

name: Update Knowhere Commit

on:
  workflow_dispatch:
  schedule:
    # * is a special character in YAML so you have to quote this string
    #         ┌───────────── minute (0 - 59)
    #         │ ┌───────────── hour (0 - 23)
    #         │ │  ┌───────────── day of the month (1 - 31)
    #         │ │  │ ┌───────────── month (1 - 12 or JAN-DEC)
    #         │ │  │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #         │ │  │ │ │
    #         │ │  │ │ │
    #         │ │  │ │ │
    - cron:  '0 15 * * *'


jobs:
  update-knowhere-commits:
    name: update-knowhere-commit
    runs-on: ubuntu-latest
    env:
      OS_NAME: ubuntu20.04
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Install GitHub CLI
        run: |
          sudo apt update
          sudo apt install -y gh
      - name: Close Previous PRs
        run: |
          REPO=${{ github.repository }}

          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

          # Set up GitHub CLI authentication non-interactively
          gh auth login --with-token <<< "${{ secrets.ALL_CONTRIBUTORS_TOKEN }}"
            
          BRANCH_PREFIX="update_knowhere_commit_"
            
          prs=$(gh pr list --repo "$REPO" --state open --json number,headRefName --jq ".[] | select(.headRefName | startswith(\"$BRANCH_PREFIX\")) | .number")
            
          for pr in $prs; do
              echo "Closing old PR #$pr"
              gh pr close $pr --repo "$REPO"
          done
        env:
          GH_TOKEN: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
      - name: get knowhere latest commit
        id: get-knowhere-latest-commit
        run: | 
          cd ..
          rm -fr Knowhere
          git clone https://github.com/zilliztech/Knowhere.git
          cd Knowhere
          export commit=$(git rev-parse --short HEAD)
          echo $commit
          echo "::set-output name=knowhere-commit::$commit"
          cd ../milvus
      - name: Update Commit Changes
        continue-on-error: true
        shell: bash
        run: |
          sed -i "0,/(\ KNOWHERE_VERSION/ s#( KNOWHERE_VERSION.*#( KNOWHERE_VERSION ${{ steps.get-knowhere-latest-commit.outputs.knowhere-commit }} )#g" internal/core/thirdparty/knowhere/CMakeLists.txt
          head -n 17 internal/core/thirdparty/knowhere/CMakeLists.txt
          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"
          git add internal/core/thirdparty/knowhere/CMakeLists.txt
          git commit -m "Update knowhere commit"
      - name: Create Pull Request
        id: cpr
        continue-on-error: true
        uses: peter-evans/create-pull-request@v3
        with:
          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
          author: sre-ci-robot <sre-ci-robot@users.noreply.github.com>
          signoff: true
          branch: update_knowhere_commit_${{ github.sha }}
          delete-branch: true
          title: '[automated] Update Knowhere Commit'
          body: |
            Update Knowhere Commit
            Signed-off-by: sre-ci-robot sre-ci-robot@users.noreply.github.com
      - name: Check outputs
        run: |
          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
      - name: Send Message To Feishu
        env:
          ACTIONS_FEISHU_TAG: 'v1.3.1'
          INPUT_WEBHOOK: "${{ secrets.FEISHU_WEBHOOK_KNOWHERE_COMMIT_BOT }}"
          INPUT_MESSAGE_TYPE: text
          INPUT_CONTENT: "Pr url by rebot: ${{ steps.cpr.outputs.pull-request-url }}"
        run: |
          wget -q https://github.com/xiachufang/actions-feishu/releases/download/${{ env.ACTIONS_FEISHU_TAG }}/linux-amd64-actions-feishu.tar.gz
          tar zxf linux-amd64-actions-feishu.tar.gz feishu
          ./feishu

name: Weekly Release
on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    #         ┌───────────── minute (0 - 59)
    #         │ ┌───────────── hour (0 - 23)
    #         │ │  ┌───────────── day of the month (1 - 31)
    #         │ │  │ ┌───────────── month (1 - 12 or JAN-DEC)
    #         │ │  │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    #         │ │  │ │ │
    #         │ │  │ │ │
    #         │ │  │ │ │
    - cron:  '0 0  * * 2,5'

jobs:
  nightly:
    name: Run Weekly Release
    if: github.repository == 'milvus-io/milvus'
    runs-on: ubuntu-latest
    env:
      IMAGE_REPO: "milvusdb"
      DEV: "milvus"
      WEEKLY: "weekly-build"
      TAG_PREFIX: "master-"
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
      with:
        fetch-depth: '0'

    - name: Get the latest of Milvus dev image tag
      shell: bash
      working-directory: scripts
      run: echo "tag=$(./docker_image_find_tag.sh -n ${IMAGE_REPO}/${DEV} -t ${TAG_PREFIX}latest -f ${TAG_PREFIX} -F -L -q)" >> $GITHUB_ENV


    - name: Pull latest milvus image with tag prefix master-
      run: |
        docker pull "${IMAGE_REPO}/${DEV}:${{ env.tag }}"
        docker tag "${IMAGE_REPO}/${DEV}:${{ env.tag }}" "${IMAGE_REPO}/${WEEKLY}:${{ env.tag }}"

    - name: Log in to Docker Hub
      uses: docker/login-action@v1
      with:
        username: ${{ secrets.DOCKERHUB_USER }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    - name: Push Docker image
      run: |
        docker push "${IMAGE_REPO}/${WEEKLY}:${{ env.tag }}"

    - name: Set release build
      shell: bash
      run: |
        tag=${{ env.tag }}
        IFS=- read branch date sha <<< "$tag"
        echo "build=$date" >> $GITHUB_ENV
        echo "sha=$(git rev-parse $sha)" >> $GITHUB_ENV

    - name: Create a weekly release
      uses: actions/create-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        body: "Image: ${{ env.IMAGE_REPO}}/${{ env.WEEKLY }}:${{ env.tag }}"
        prerelease: true
        tag_name: "v2.2-testing-${{ env.build }}"
        release_name: "milvus-2.2-testing-${{ env.build }}"
        commitish: "${{ env.sha }}"