From c8ccc08c9a5d0f7d0a73dc7ba4f53d9cfa414a8e Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:21:37 +0800
Subject: [PATCH 01/20] Delete .github/workflows/build-and-release.yaml

---
 .github/workflows/build-and-release.yaml | 144 -----------------------
 1 file changed, 144 deletions(-)
 delete mode 100644 .github/workflows/build-and-release.yaml

diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml
deleted file mode 100644
index 7307c85ab..000000000
--- a/.github/workflows/build-and-release.yaml
+++ /dev/null
@@ -1,144 +0,0 @@
-name: Build Release
-
-on: workflow_dispatch
-
-permissions:
-  contents: write
-
-jobs:
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, windows-2019, macos-13]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-
-      # Used to host cibuildwheel
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          RUST_LOG=trace python -m uv pip install -e .[all] --verbose
-        shell: bash
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        env:
-          RUST_LOG: trace        
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: cmd
-
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
-        env:
-          # disable repair
-          CIBW_REPAIR_WHEEL_COMMAND: ""
-        with:
-          package-dir: .
-          output-dir: wheelhouse
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.os }}
-          path: ./wheelhouse/*.whl
-
-  build_wheels_arm64:
-    name: Build arm64 wheels
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-        with:
-          platforms: linux/arm64
-
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
-        env:
-          CIBW_SKIP: "*musllinux* pp*"
-          CIBW_REPAIR_WHEEL_COMMAND: ""
-          CIBW_ARCHS: "aarch64"
-          CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*"
-        with:
-          output-dir: wheelhouse
-
-      - name: Upload wheels as artifacts
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels_arm64
-          path: ./wheelhouse/*.whl
-
-  build_sdist:
-    name: Build source distribution
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          RUST_LOG=trace python -m uv pip install -e .[all] --verbose
-          python -m uv pip install build
-        shell: bash
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        env:
-          RUST_LOG: trace        
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-          python -m uv pip install build
-        shell: cmd
-
-      - name: Build source distribution
-        run: |
-          python -m build --sdist
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: sdist
-          path: ./dist/*.tar.gz
-
-  release:
-    name: Release
-    needs: [build_wheels, build_wheels_arm64, build_sdist]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          merge-multiple: true
-          path: dist
-
-      - uses: softprops/action-gh-release@v2
-        with:
-          files: dist/*
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 6b53054802a2f746d1a5c083eab2452fa505b458 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:21:48 +0800
Subject: [PATCH 02/20] Delete .github/workflows/build-docker.yaml

---
 .github/workflows/build-docker.yaml | 50 -----------------------------
 1 file changed, 50 deletions(-)
 delete mode 100644 .github/workflows/build-docker.yaml

diff --git a/.github/workflows/build-docker.yaml b/.github/workflows/build-docker.yaml
deleted file mode 100644
index b5c7346db..000000000
--- a/.github/workflows/build-docker.yaml
+++ /dev/null
@@ -1,50 +0,0 @@
-name: Build Docker
-
-on: workflow_dispatch
-
-permissions:
-  contents: write
-  packages: write
-
-jobs:
-  docker:
-    name: Build and push Docker image
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3 
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Build and push
-        id: docker_build
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: "docker/simple/Dockerfile"
-          push: ${{ startsWith(github.ref, 'refs/tags/') }}
-          pull: true
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ghcr.io/abetlen/llama-cpp-python:latest
-            ghcr.io/abetlen/llama-cpp-python:${{ github.ref_name }}
-          build-args: |
-            BUILDKIT_INLINE_CACHE=1
-
-      - name: Publish to GitHub Tag
-        if: steps.docker_build.outputs.digest && startsWith(github.ref, 'refs/tags/')
-        run: |
-          echo "Docker image published for tag: ${{ github.ref_name }}"

From dc7475949b6fe8beb0163ca3d6d8b56c206bece7 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:21:57 +0800
Subject: [PATCH 03/20] Delete .github/workflows/build-wheels-metal.yaml

---
 .github/workflows/build-wheels-metal.yaml | 79 -----------------------
 1 file changed, 79 deletions(-)
 delete mode 100644 .github/workflows/build-wheels-metal.yaml

diff --git a/.github/workflows/build-wheels-metal.yaml b/.github/workflows/build-wheels-metal.yaml
deleted file mode 100644
index 9b97bf2f5..000000000
--- a/.github/workflows/build-wheels-metal.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
-name: Build Wheels (Metal)
-
-on: workflow_dispatch
-
-permissions:
-  contents: write
-
-jobs:
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [macos-13, macos-14, macos-15]
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-
-      # Used to host cibuildwheel
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-          cache: 'pip'
-          
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          RUST_LOG=trace python -m uv pip install -e .[all] --verbose
-        shell: bash
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        env:
-          RUST_LOG: trace        
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: cmd
-
-      - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
-        env:
-          # disable repair
-          CIBW_REPAIR_WHEEL_COMMAND: ""
-          CIBW_ARCHS: "arm64"
-          CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DGGML_METAL=on"
-          CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*"
-        with:
-          package-dir: .
-          output-dir: wheelhouse2
-
-      - uses: actions/upload-artifact@v4
-        with:
-          name: wheels-mac_${{ matrix.os }}
-          path: ./wheelhouse2/*.whl
-
-  release:
-    name: Release
-    needs: [build_wheels]
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          merge-multiple: true
-          path: dist2
-          
-      - uses: softprops/action-gh-release@v2
-        with:
-          files: dist2/*
-          # set release name to <tag>-metal
-          tag_name: ${{ github.ref_name }}-metal
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 600a2918c4d7b00406e5e871dfe89c4167d8db7c Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:22:05 +0800
Subject: [PATCH 04/20] Delete
 .github/workflows/generate-index-from-release.yaml

---
 .../generate-index-from-release.yaml          | 57 -------------------
 1 file changed, 57 deletions(-)
 delete mode 100644 .github/workflows/generate-index-from-release.yaml

diff --git a/.github/workflows/generate-index-from-release.yaml b/.github/workflows/generate-index-from-release.yaml
deleted file mode 100644
index 255ee67d6..000000000
--- a/.github/workflows/generate-index-from-release.yaml
+++ /dev/null
@@ -1,57 +0,0 @@
-name: Wheels Index
-
-on:
-  # Trigger on new release
-  workflow_run:
-    workflows: ["Release", "Build Wheels (CUDA)", "Build Wheels (Metal)"]
-    types:
-      - completed
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-
-# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
-# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
-concurrency:
-  group: "pages"
-  cancel-in-progress: false
-
-jobs:
-  # Single deploy job since we're just deploying
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Setup Pages
-        uses: actions/configure-pages@v5
-      - name: Build
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          ./scripts/get-releases.sh
-          ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
-          ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$'
-          ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'
-          ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$'
-          ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
-          # ./scripts/releases-to-pep-503.sh index/whl/cu125 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
-          # ./scripts/releases-to-pep-503.sh index/whl/cu126 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
-          ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$'
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          # Upload entire repository
-          path: 'index'
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v4

From 3e30a6c6986ed05c05520ce9527943e81986ae67 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:22:14 +0800
Subject: [PATCH 05/20] Delete .github/workflows/publish-to-test.yaml

---
 .github/workflows/publish-to-test.yaml | 62 --------------------------
 1 file changed, 62 deletions(-)
 delete mode 100644 .github/workflows/publish-to-test.yaml

diff --git a/.github/workflows/publish-to-test.yaml b/.github/workflows/publish-to-test.yaml
deleted file mode 100644
index de3ae42aa..000000000
--- a/.github/workflows/publish-to-test.yaml
+++ /dev/null
@@ -1,62 +0,0 @@
-# Based on: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
-
-name: Publish to TestPyPI
-
-on:
-  workflow_dispatch:
-    inputs:
-      dev_version:
-        description: 'Dev version N'
-        required: true
-
-
-jobs:
-  build-n-publish:
-    name: Build and publish
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: "recursive"
-        
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.11"
-        cache: 'pip'
-        
-    - name: Append Dev Version to __version__
-      run: |
-        DEV_VERSION=${{ github.event.inputs.dev_version }}
-        CURRENT_VERSION=$(awk -F= '/__version__ =/ {print $2}' llama_cpp/__init__.py | tr -d ' "')
-        NEW_VERSION="${CURRENT_VERSION}.dev${DEV_VERSION}"
-        sed -i 's/__version__ = \".*\"/__version__ = \"'"${NEW_VERSION}"'\"/' llama_cpp/__init__.py
-        
-    - name: Install dependencies (Linux/MacOS)
-      if: runner.os != 'Windows'
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install uv
-        RUST_LOG=trace python -m uv pip install -e .[all] --verbose
-      shell: bash
-
-    - name: Install dependencies (Windows)
-      if: runner.os == 'Windows'
-      env:
-        RUST_LOG: trace       
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install uv
-        python -m uv pip install -e .[all] --verbose
-      shell: cmd
-        
-    - name: Build source distribution
-      run: |
-        python -m build --sdist
-        
-    - name: Publish to Test PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-        repository-url: https://test.pypi.org/legacy/

From 723504a41eda541eb1007585ce80addf465370c5 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:22:23 +0800
Subject: [PATCH 06/20] Delete .github/workflows/publish.yaml

---
 .github/workflows/publish.yaml | 51 ----------------------------------
 1 file changed, 51 deletions(-)
 delete mode 100644 .github/workflows/publish.yaml

diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
deleted file mode 100644
index bb76f5394..000000000
--- a/.github/workflows/publish.yaml
+++ /dev/null
@@ -1,51 +0,0 @@
-name: Publish to PyPI
-
-# Based on: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
-
-on: workflow_dispatch
-
-jobs:
-  build-n-publish:
-    name: Build and publish
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: "recursive"
-
-    - name: Set up Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.9"
-
-    - name: Install dependencies (Linux/MacOS)
-      if: runner.os != 'Windows'
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install uv
-        RUST_LOG=trace python -m uv pip install -e .[all] --verbose
-        python -m uv pip install build
-      shell: bash
-
-    - name: Install dependencies (Windows)
-      if: runner.os == 'Windows'
-      env:
-        RUST_LOG: trace
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install uv
-        python -m uv pip install -e .[all] --verbose
-        python -m uv pip install build
-      shell: cmd
-
-    - name: Build source distribution
-      run: |
-        python -m build --sdist
-
-    - name: Publish distribution to PyPI
-      # TODO: move to tag based releases
-      # if: startsWith(github.ref, 'refs/tags')
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        password: ${{ secrets.PYPI_API_TOKEN }}

From 90a2e59ec9015dbf70f3d5fe3e1f036d4dd5cd3b Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:22:32 +0800
Subject: [PATCH 07/20] Delete .github/workflows/test-pypi.yaml

---
 .github/workflows/test-pypi.yaml | 112 -------------------------------
 1 file changed, 112 deletions(-)
 delete mode 100644 .github/workflows/test-pypi.yaml

diff --git a/.github/workflows/test-pypi.yaml b/.github/workflows/test-pypi.yaml
deleted file mode 100644
index 335033bba..000000000
--- a/.github/workflows/test-pypi.yaml
+++ /dev/null
@@ -1,112 +0,0 @@
-name: Tests for PyPI package
-
-on: workflow_dispatch
-
-jobs:
-  build-linux:
-
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
-    steps:
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose 
-        shell: bash
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        env:
-          RUST_LOG: trace           
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install llama-cpp-python[all] --verbose 
-        shell: cmd
-          
-      - name: Test with pytest
-        run: |
-          python -c "import llama_cpp"
-
-  build-windows:
-
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
-    steps:
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-          
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose 
-        shell: bash
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        env:
-          RUST_LOG: trace          
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install llama-cpp-python[all] --verbose 
-        shell: cmd
-          
-      - name: Test with pytest
-        run: |
-          python -c "import llama_cpp"
-
-  build-macos:
-
-    runs-on: macos-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
-    steps:
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'   
-
-      - name: Install dependencies (Linux/MacOS)
-        if: runner.os != 'Windows'
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          RUST_LOG=trace python -m uv pip install llama-cpp-python[all] --verbose 
-        shell: bash
-
-      - name: Install dependencies (Windows)
-        if: runner.os == 'Windows'
-        env:
-          RUST_LOG: trace  
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install llama-cpp-python[all] --verbose 
-        shell: cmd
-          
-      - name: Test with pytest
-        run: |
-          python -c "import llama_cpp"

From 2bf1cdc2d3f9e8f910ff83fbe75d6ecae714c6a3 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:22:41 +0800
Subject: [PATCH 08/20] Delete .github/workflows/test.yaml

---
 .github/workflows/test.yaml | 171 ------------------------------------
 1 file changed, 171 deletions(-)
 delete mode 100644 .github/workflows/test.yaml

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
deleted file mode 100644
index 95f6e5a27..000000000
--- a/.github/workflows/test.yaml
+++ /dev/null
@@ -1,171 +0,0 @@
-name: Tests
-on:
-  pull_request:
-    branches:
-      - main
-  push:
-    branches:
-      - main
-
-env:
-  REPO_ID: Qwen/Qwen2-0.5B-Instruct-GGUF
-  MODEL_FILE: qwen2-0_5b-instruct-q8_0.gguf
-
-jobs:
-  download-model:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-      - name: Install huggingface-hub
-        run: pip install huggingface-hub
-      - name: Download model
-        run: huggingface-cli download ${{ env.REPO_ID }} ${{ env.MODEL_FILE }}
-      - name: Cache model
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/huggingface/hub
-          key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
-
-  build-linux:
-    needs: download-model
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-          
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-      - name: Restore model cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/huggingface/hub
-          key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
-      - name: Install dependencies (Linux/MacOS)
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: bash
-      - name: Test with pytest
-        run: |
-          python -m pytest
-
-  build-windows:
-    needs: download-model
-    runs-on: windows-latest
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-          
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-
-      - name: Restore model cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/huggingface/hub
-          key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
-
-      - name: Install dependencies (Windows)
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install uv
-          python -m uv pip install -e .[all] --verbose
-        shell: cmd
-          
-      - name: Test with pytest
-        run: |
-          python -m pytest
-
-  build-macos:
-    needs: download-model
-    runs-on: macos-13
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-          
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: 'pip'
-
-      - name: System Info
-        run: |
-          uname -a
-          sysctl -n machdep.cpu.brand_string
-          python3 -c "import platform; print(platform.machine(), platform.architecture())"
-
-      - name: Restore model cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/huggingface/hub
-          key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
-          
-      - name: Install dependencies (Linux/MacOS)
-        run: |
-          python3 -m pip install --upgrade pip
-          python3 -m pip install uv
-          python3 -m uv pip install -e .[all] --verbose
-          CMAKE_ARGS="-DLLAMA_METAL=off" python3 -m uv pip install .[all] --verbose
-        shell: bash
-
-      - name: Test with pytest
-        run: |
-          python3 -m pytest
-
-  build-macos-metal:
-    needs: download-model
-    runs-on: macos-13
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: "recursive"
-          
-      - name: Set up Python 3.9
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.9"
-
-      - name: System Info
-        run: |
-          uname -a
-          sysctl -n machdep.cpu.brand_string
-          python3 -c "import platform; print(platform.machine(), platform.architecture())"
-
-      - name: Restore model cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/huggingface/hub
-          key: ${{ runner.os }}-model-${{ env.REPO_ID }}-${{ env.MODEL_FILE }}
-
-      - name: Install dependencies
-        run: |
-          python3 -m pip install --upgrade pip
-          CMAKE_ARGS="-DLLAMA_METAL=on" python3 -m pip install .[all] --verbose
-        shell: bash
-
-      - name: Test with pytest
-        run: |
-          python3 -m pytest

From 621772b18c1f47b68139cba4ebef38a9b5c9aeb8 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:23:27 +0800
Subject: [PATCH 09/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 745b2e602..751fb01f9 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -8,7 +8,7 @@ permissions:
 jobs:
   define_matrix:
     name: Define Build Matrix
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     defaults:
@@ -20,9 +20,9 @@ jobs:
         id: set-matrix
         run: |
           $matrix = @{
-              'os' = @('ubuntu-latest', 'windows-2019')
-              'pyver' = @("3.9", "3.10", "3.11", "3.12")
-              'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1") #, "12.5.1", "12.6.1")
+              'os' = @('windows-latest')
+              'pyver' = @("3.12")
+              'cuda' = @("12.5")
               'releasetag' = @("basic")
           }
 

From c23a8443ea511b1188f91ee418161c8a7e32bdfa Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:29:30 +0800
Subject: [PATCH 10/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 751fb01f9..5659505d1 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -8,7 +8,7 @@ permissions:
 jobs:
   define_matrix:
     name: Define Build Matrix
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     defaults:

From a355c993f2409f7b9a922b063a82fa32927792b4 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:31:51 +0800
Subject: [PATCH 11/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 33 ++++++++++++++++--------
 1 file changed, 22 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 5659505d1..47d57c114 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -8,7 +8,7 @@ permissions:
 jobs:
   define_matrix:
     name: Define Build Matrix
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     defaults:
@@ -43,12 +43,33 @@ jobs:
       AVXVER: ${{ matrix.releasetag }}
 
     steps:
+      - name: Debug Before Adding MSBuild
+        run: |
+          Write-Output "Current PATH:"
+          Write-Output $env:PATH
+
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
         with:
           vs-version: '[16.11,16.12)'
 
+      - name: Debug After Adding MSBuild
+        run: |
+          Write-Output "Updated PATH:"
+          Write-Output $env:PATH
+          Get-ChildItem "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin"
+
+      - name: Explicitly Add MSBuild to PATH
+        if: runner.os == 'Windows'
+        run: |
+          $msbuildPath = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin"
+          if (-Not (Test-Path $msbuildPath)) {
+            throw "MSBuild path not found: $msbuildPath"
+          }
+          $env:PATH += ";$msbuildPath"
+          Write-Output "##[add-path]$env:PATH"
+
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
@@ -114,23 +135,13 @@ jobs:
           $env:VERBOSE = '1'
           $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all'
           $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS"
-          # if ($env:AVXVER -eq 'AVX') {
           $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
-          # }
-          # if ($env:AVXVER -eq 'AVX512') {
-          #  $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on'
-          # }
-          # if ($env:AVXVER -eq 'basic') {
-          #  $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
-          # }
           python -m build --wheel
-          # write the build tag to the output
           Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV
 
       - uses: softprops/action-gh-release@v2
         with:
           files: dist/*
-          # Set tag_name to <tag>-cu<cuda_version>
           tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 1b0515ed632b901a7983bcfe4b3483e24fbf9f5a Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:37:27 +0800
Subject: [PATCH 12/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 37 +++++++++---------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 47d57c114..ccdfd5e0c 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -8,7 +8,7 @@ permissions:
 jobs:
   define_matrix:
     name: Define Build Matrix
-    runs-on: windows-latest
+    runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     defaults:
@@ -20,9 +20,9 @@ jobs:
         id: set-matrix
         run: |
           $matrix = @{
-              'os' = @('windows-latest')
+              'os' = @('ubuntu-latest', 'windows-2019')
               'pyver' = @("3.12")
-              'cuda' = @("12.5")
+              'cuda' = @("12.5.1")
               'releasetag' = @("basic")
           }
 
@@ -43,33 +43,12 @@ jobs:
       AVXVER: ${{ matrix.releasetag }}
 
     steps:
-      - name: Debug Before Adding MSBuild
-        run: |
-          Write-Output "Current PATH:"
-          Write-Output $env:PATH
-
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
         with:
           vs-version: '[16.11,16.12)'
 
-      - name: Debug After Adding MSBuild
-        run: |
-          Write-Output "Updated PATH:"
-          Write-Output $env:PATH
-          Get-ChildItem "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin"
-
-      - name: Explicitly Add MSBuild to PATH
-        if: runner.os == 'Windows'
-        run: |
-          $msbuildPath = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin"
-          if (-Not (Test-Path $msbuildPath)) {
-            throw "MSBuild path not found: $msbuildPath"
-          }
-          $env:PATH += ";$msbuildPath"
-          Write-Output "##[add-path]$env:PATH"
-
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
@@ -135,13 +114,23 @@ jobs:
           $env:VERBOSE = '1'
           $env:CMAKE_ARGS = '-DGGML_CUDA=on -DCMAKE_CUDA_ARCHITECTURES=all'
           $env:CMAKE_ARGS = "-DGGML_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS"
+          # if ($env:AVXVER -eq 'AVX') {
           $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
+          # }
+          # if ($env:AVXVER -eq 'AVX512') {
+          #  $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX512=on'
+          # }
+          # if ($env:AVXVER -eq 'basic') {
+          #  $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DGGML_AVX=off -DGGML_AVX2=off -DGGML_FMA=off -DGGML_F16C=off'
+          # }
           python -m build --wheel
+          # write the build tag to the output
           Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV
 
       - uses: softprops/action-gh-release@v2
         with:
           files: dist/*
+          # Set tag_name to <tag>-cu<cuda_version>
           tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 10ddc16da553117eb25e690e46e2b04e816680a2 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 10:44:27 +0800
Subject: [PATCH 13/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index ccdfd5e0c..8605d6583 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -20,7 +20,7 @@ jobs:
         id: set-matrix
         run: |
           $matrix = @{
-              'os' = @('ubuntu-latest', 'windows-2019')
+              'os' = @('windows-2022')
               'pyver' = @("3.12")
               'cuda' = @("12.5.1")
               'releasetag' = @("basic")
@@ -46,8 +46,6 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
-        with:
-          vs-version: '[16.11,16.12)'
 
       - uses: actions/checkout@v4
         with:

From 2be2479d9f6b9176c945078acae41a93b775fc4d Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 15:40:43 +0800
Subject: [PATCH 14/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 8605d6583..3a9cb5a54 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -46,7 +46,9 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
-
+        with:
+          vs-version: '[17.13]'
+  
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"

From cf20b7735654d7da80fff5f15e6e3ae4494a8fdc Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 15:45:00 +0800
Subject: [PATCH 15/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 3a9cb5a54..f84a77e00 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -47,7 +47,7 @@ jobs:
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
         with:
-          vs-version: '[17.13]'
+          vs-version: '[17.13.35710.127]'
   
       - uses: actions/checkout@v4
         with:

From 2fb4bb03bf167adf94d0e41634d600196a83b20e Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 15:51:35 +0800
Subject: [PATCH 16/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index f84a77e00..e11af49bd 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -20,7 +20,7 @@ jobs:
         id: set-matrix
         run: |
           $matrix = @{
-              'os' = @('windows-2022')
+              'os' = @('windows-2019')
               'pyver' = @("3.12")
               'cuda' = @("12.5.1")
               'releasetag' = @("basic")
@@ -46,8 +46,6 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
-        with:
-          vs-version: '[17.13.35710.127]'
   
       - uses: actions/checkout@v4
         with:

From 28f066022dc085c9fd12f5ca73a9ce75f090d87f Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 16:02:05 +0800
Subject: [PATCH 17/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index e11af49bd..b24922ebb 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -46,6 +46,9 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
+        with:
+          vs-version: '[16.4,16.5)'
+
   
       - uses: actions/checkout@v4
         with:

From ce74e6114be2bb815a7d4a9344cdcb239329911b Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 16:02:54 +0800
Subject: [PATCH 18/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index b24922ebb..fdbd97726 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -8,7 +8,7 @@ permissions:
 jobs:
   define_matrix:
     name: Define Build Matrix
-    runs-on: ubuntu-latest
+    runs-on: windows-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     defaults:
@@ -46,8 +46,6 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
-        with:
-          vs-version: '[16.4,16.5)'
 
   
       - uses: actions/checkout@v4

From 3e2278abe21b00121d1cead1044000a5b0380fb9 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 16:03:26 +0800
Subject: [PATCH 19/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index fdbd97726..478a8c9da 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -46,8 +46,9 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
+        with:
+          vs-version: '[16.4,16.5)'
 
-  
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"

From 3ac5d778a3608d9b9730c13a16a36dd2176a6511 Mon Sep 17 00:00:00 2001
From: allenchen121 <82314446+allenchen121@users.noreply.github.com>
Date: Wed, 5 Mar 2025 16:06:40 +0800
Subject: [PATCH 20/20] Update build-wheels-cuda.yaml

---
 .github/workflows/build-wheels-cuda.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/build-wheels-cuda.yaml b/.github/workflows/build-wheels-cuda.yaml
index 478a8c9da..7c196a4cc 100644
--- a/.github/workflows/build-wheels-cuda.yaml
+++ b/.github/workflows/build-wheels-cuda.yaml
@@ -46,8 +46,6 @@ jobs:
       - name: Add MSBuild to PATH
         if: runner.os == 'Windows'
         uses: microsoft/setup-msbuild@v2
-        with:
-          vs-version: '[16.4,16.5)'
 
       - uses: actions/checkout@v4
         with: