name: Server (sanitize)

on:
  workflow_dispatch: # allows manual triggering
    inputs:
      sha:
        description: 'Commit SHA1 to build'
        required: false
        type: string
      slow_tests:
        description: 'Run slow tests'
        required: true
        type: boolean
  push:
    branches:
      - master
    paths: [
      '.github/workflows/server-sanitize.yml',
      '**/CMakeLists.txt',
      '**/Makefile',
      '**/*.h',
      '**/*.hpp',
      '**/*.c',
      '**/*.cpp',
      'tools/server/**.*'
    ]

  pull_request:
    types: [opened, synchronize, reopened]
    paths: [
      '.github/workflows/server-sanitize.yml'
    ]

env:
  # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
  HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
  LLAMA_ARG_LOG_COLORS: 1
  LLAMA_ARG_LOG_PREFIX: 1
  LLAMA_ARG_LOG_TIMESTAMPS: 1
  LLAMA_ARG_LOG_VERBOSITY: 10

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  server:
    runs-on: hf-jobs-cpu-upgrade

    strategy:
      matrix:
        sanitizer: [ADDRESS, UNDEFINED] # THREAD is very slow
        build_type: [RelWithDebInfo]
      fail-fast: false

    steps:
      - name: Clone
        id: checkout
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
          ref: ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha || github.head_ref || github.ref_name }}

      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y build-essential cmake python3-full

      - name: ccache
        uses: ggml-org/ccache-action@v1.2.24
        with:
          restore: false
          save: false

      - name: ccache-buckets-restore
        uses: ./.github/actions/ccache-buckets
        with:
          key: server-sanitize-${{ matrix.sanitizer }}
          folder: llama.cpp
          hf_bucket: ggml-org/cache

      - name: Build
        id: cmake_build
        run: |
          cmake -B build \
            -DLLAMA_BUILD_BORINGSSL=ON \
            -DGGML_SCHED_NO_REALLOC=ON \
            -DGGML_SANITIZE_ADDRESS=${{ matrix.sanitizer == 'ADDRESS' }} \
            -DGGML_SANITIZE_THREAD=${{ matrix.sanitizer == 'THREAD' }} \
            -DGGML_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }} \
            -DLLAMA_SANITIZE_ADDRESS=${{ matrix.sanitizer == 'ADDRESS' }} \
            -DLLAMA_SANITIZE_THREAD=${{ matrix.sanitizer == 'THREAD' }} \
            -DLLAMA_SANITIZE_UNDEFINED=${{ matrix.sanitizer == 'UNDEFINED' }}
          cmake --build build --config ${{ matrix.build_type }} -j $(nproc) --target llama-server

      - name: ccache-buckets-save
        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
        uses: ./.github/actions/ccache-buckets
        env:
          HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
        with:
          key: server-sanitize-${{ matrix.sanitizer }}
          folder: llama.cpp
          evict-old-files: 1d
          hf_bucket: ggml-org/cache
          save: true

      - name: Install Python dependencies
        run: |
          python3 -m venv .venv
          .venv/bin/pip install -r tools/server/tests/requirements.txt

      - name: Tests
        id: server_integration_tests
        if: ${{ (!matrix.disabled_on_pr || !github.event.pull_request) }}
        run: |
          source .venv/bin/activate
          cd tools/server/tests
          PYTEST_WORKERS=1 ./tests.sh

      - name: Slow tests
        id: server_integration_tests_slow
        if: ${{ (github.event.schedule || github.event.inputs.slow_tests == 'true') && matrix.build_type == 'Release' }}
        run: |
          source .venv/bin/activate
          cd tools/server/tests
          PYTEST_WORKERS=1 SLOW_TESTS=1 ./tests.sh