llama-server followups (#16353)
* llama-server followups Misc fixes for #16031 - Add back dropped ROCm build flag for multi-GPU support on windows - Fix amdhip64_*.dll version detection for "latest" selection - Fix embeddings API for consistent normalize behavior with prior versions * ci: set up for automated llama.cpp update testing * reduce batch for fa-disabled, and constrained vram * mlx: fix v3 load bug on m5 Imagegen was incorrectly loading v3 first. This DRYs out the loading code so imagegen gets the same new v4/v3 selection logic. * fix reload bug on embedding models * bump version * steer user how to enable iGPU when disabled
This commit is contained in:
Daniel Hiltgen
@@ -0,0 +1,596 @@
|
||||
name: test-llamacpp-update
|
||||
|
||||
# PR validation artifacts from this workflow are intentionally unsigned and not
|
||||
# notarized. They are for llama.cpp update testing only and must not be
|
||||
# published as release artifacts.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'LLAMA_CPP_VERSION'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CGO_CFLAGS: '-O3'
|
||||
CGO_CXXFLAGS: '-O3'
|
||||
|
||||
jobs:
|
||||
setup-environment:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
GOFLAGS: ${{ steps.goflags.outputs.GOFLAGS }}
|
||||
VERSION: ${{ steps.goflags.outputs.VERSION }}
|
||||
vendorsha: ${{ steps.goflags.outputs.vendorsha }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Set environment
|
||||
id: goflags
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
VERSION="0.0.0-llamacpp-${GITHUB_SHA::7}"
|
||||
{
|
||||
echo "GOFLAGS='-ldflags=-w -s \"-X=github.com/ollama/ollama/version.Version=${VERSION}\" \"-X=github.com/ollama/ollama/server.mode=release\"'"
|
||||
echo "VERSION=${VERSION}"
|
||||
echo "vendorsha=$(cat LLAMA_CPP_VERSION)-$(cat MLX_VERSION)-$(cat MLX_C_VERSION)"
|
||||
} >>"${GITHUB_OUTPUT}"
|
||||
|
||||
darwin-build:
|
||||
runs-on: macos-26-xlarge
|
||||
needs: setup-environment
|
||||
env:
|
||||
GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
|
||||
VERSION: ${{ needs.setup-environment.outputs.VERSION }}
|
||||
CGO_CFLAGS: '-mmacosx-version-min=14.0 -O3'
|
||||
CGO_CXXFLAGS: '-mmacosx-version-min=14.0 -O3'
|
||||
CGO_LDFLAGS: '-mmacosx-version-min=14.0 -O3'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
LLAMA_CPP_VERSION
|
||||
MLX_VERSION
|
||||
MLX_C_VERSION
|
||||
- name: Build unsigned Darwin runtime
|
||||
run: ./scripts/build_darwin.sh build package
|
||||
- name: Log build results
|
||||
run: ls -l dist/
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-darwin.tgz
|
||||
path: dist/ollama-darwin.tgz
|
||||
compression-level: 0
|
||||
|
||||
# Build payload export stages independently and combine the exported
|
||||
# filesystem artifacts below. This preserves parallelism without Docker
|
||||
# registry credentials or oversized GitHub layer caches.
|
||||
linux-payloads:
|
||||
runs-on: ${{ matrix.arch == 'arm64' && 'linux-arm64' || 'linux' }}
|
||||
needs: setup-environment
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- arch: amd64
|
||||
target: publish-llama-server-cpu
|
||||
payload: cpu
|
||||
- arch: amd64
|
||||
target: publish-llama-server-cuda_v12
|
||||
payload: cuda_v12
|
||||
- arch: amd64
|
||||
target: publish-llama-server-cuda_v13
|
||||
payload: cuda_v13
|
||||
- arch: amd64
|
||||
target: publish-llama-server-rocm_v7_2
|
||||
payload: rocm_v7_2
|
||||
- arch: amd64
|
||||
target: publish-llama-server-vulkan
|
||||
payload: vulkan
|
||||
- arch: arm64
|
||||
target: publish-llama-server-cpu
|
||||
payload: cpu
|
||||
- arch: arm64
|
||||
target: publish-llama-server-cuda_v12
|
||||
payload: cuda_v12
|
||||
- arch: arm64
|
||||
target: publish-llama-server-cuda_v13
|
||||
payload: cuda_v13
|
||||
- arch: arm64
|
||||
target: publish-llama-server-cuda_jetpack5
|
||||
payload: cuda_jetpack5
|
||||
- arch: arm64
|
||||
target: publish-llama-server-cuda_jetpack6
|
||||
payload: cuda_jetpack6
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/${{ matrix.arch }}
|
||||
target: ${{ matrix.target }}
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
GOFLAGS=${{ needs.setup-environment.outputs.GOFLAGS }}
|
||||
CGO_CFLAGS=${{ env.CGO_CFLAGS }}
|
||||
CGO_CXXFLAGS=${{ env.CGO_CXXFLAGS }}
|
||||
APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu
|
||||
APT_PORTS_MIRROR=http://azure.ports.ubuntu.com/ubuntu-ports
|
||||
outputs: type=local,dest=${{ runner.temp }}/payload
|
||||
- name: Pack Linux payload
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tar -C "${{ runner.temp }}/payload" -cf - . | zstd -9 -T0 >"${{ runner.temp }}/linux-payload-${{ matrix.arch }}-${{ matrix.payload }}.tar.zst"
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: linux-payload-${{ matrix.arch }}-${{ matrix.payload }}
|
||||
path: ${{ runner.temp }}/linux-payload-${{ matrix.arch }}-${{ matrix.payload }}.tar.zst
|
||||
compression-level: 0
|
||||
|
||||
linux-go:
|
||||
runs-on: ${{ matrix.arch == 'arm64' && 'linux-arm64' || 'linux' }}
|
||||
needs: setup-environment
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch: [amd64, arm64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
- uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/${{ matrix.arch }}
|
||||
target: publish-go
|
||||
provenance: false
|
||||
sbom: false
|
||||
build-args: |
|
||||
GOFLAGS=${{ needs.setup-environment.outputs.GOFLAGS }}
|
||||
CGO_CFLAGS=${{ env.CGO_CFLAGS }}
|
||||
CGO_CXXFLAGS=${{ env.CGO_CXXFLAGS }}
|
||||
APT_MIRROR=http://azure.archive.ubuntu.com/ubuntu
|
||||
APT_PORTS_MIRROR=http://azure.ports.ubuntu.com/ubuntu-ports
|
||||
outputs: type=local,dest=${{ runner.temp }}/payload
|
||||
- name: Pack Linux Go payload
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tar -C "${{ runner.temp }}/payload" -cf - . | zstd -9 -T0 >"${{ runner.temp }}/linux-payload-${{ matrix.arch }}-go.tar.zst"
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: linux-payload-${{ matrix.arch }}-go
|
||||
path: ${{ runner.temp }}/linux-payload-${{ matrix.arch }}-go.tar.zst
|
||||
compression-level: 0
|
||||
|
||||
# MLX payloads are intentionally excluded from this workflow; the Dockerfile
|
||||
# still exposes publish-mlx for a separate MLX-specific workflow.
|
||||
linux-bundles:
|
||||
runs-on: ${{ matrix.arch == 'arm64' && 'linux-arm64' || 'linux' }}
|
||||
needs: [linux-payloads, linux-go]
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
arch: [amd64, arm64]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: linux-payload-${{ matrix.arch }}-*
|
||||
path: ${{ runner.temp }}/payloads
|
||||
merge-multiple: true
|
||||
- name: Assemble Linux payload tree
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
src="${{ runner.temp }}/payloads"
|
||||
arch="${{ matrix.arch }}"
|
||||
dest="dist/linux-${arch}"
|
||||
copy_payload() {
|
||||
local name="$1"
|
||||
local payload="${src}/linux-payload-${arch}-${name}.tar.zst"
|
||||
if [ ! -f "${payload}" ]; then
|
||||
echo "missing payload ${payload}"
|
||||
exit 1
|
||||
fi
|
||||
zstd -d <"${payload}" | tar -C "${dest}" -xf -
|
||||
}
|
||||
|
||||
mkdir -p "${dest}"
|
||||
copy_payload go
|
||||
copy_payload cpu
|
||||
copy_payload cuda_v12
|
||||
copy_payload cuda_v13
|
||||
if [ "${arch}" = "amd64" ]; then
|
||||
copy_payload vulkan
|
||||
copy_payload rocm_v7_2
|
||||
else
|
||||
copy_payload cuda_jetpack5
|
||||
copy_payload cuda_jetpack6
|
||||
fi
|
||||
|
||||
./scripts/deduplicate_cuda_libs.sh "${dest}"
|
||||
- name: Verify Linux build payloads
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
base="dist/linux-${{ matrix.arch }}"
|
||||
for payload in \
|
||||
"${base}/bin/ollama" \
|
||||
"${base}/lib/ollama/llama-server"
|
||||
do
|
||||
[ -f "${payload}" ] || { echo "missing ${payload}"; exit 1; }
|
||||
done
|
||||
- name: Create archive input lists
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for COMPONENT in bin/* lib/ollama/*; do
|
||||
case "${COMPONENT}" in
|
||||
bin/ollama*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}.tar.in ;;
|
||||
lib/ollama/*.so*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}.tar.in ;;
|
||||
lib/ollama/llama-server*|lib/ollama/llama-quantize*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}.tar.in ;;
|
||||
lib/ollama/cuda_v*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}.tar.in ;;
|
||||
lib/ollama/vulkan*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}.tar.in ;;
|
||||
lib/ollama/mlx*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}-mlx.tar.in ;;
|
||||
lib/ollama/include*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}-mlx.tar.in ;;
|
||||
lib/ollama/cuda_jetpack5) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}-jetpack5.tar.in ;;
|
||||
lib/ollama/cuda_jetpack6) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}-jetpack6.tar.in ;;
|
||||
lib/ollama/rocm_v*) echo "${COMPONENT}" >>ollama-linux-${{ matrix.arch }}-rocm.tar.in ;;
|
||||
esac
|
||||
done
|
||||
working-directory: dist/linux-${{ matrix.arch }}
|
||||
- name: Log archive input lists
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for ARCHIVE in dist/linux-${{ matrix.arch }}/*.tar.in; do
|
||||
echo "${ARCHIVE}"
|
||||
cat "${ARCHIVE}"
|
||||
done
|
||||
- name: Create Linux archives
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for ARCHIVE in dist/linux-${{ matrix.arch }}/*.tar.in; do
|
||||
tar c -C dist/linux-${{ matrix.arch }} -T "${ARCHIVE}" --owner 0 --group 0 | zstd -19 -T0 >"$(basename "${ARCHIVE//.*/}.tar.zst")" &
|
||||
done
|
||||
wait
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-linux-${{ matrix.arch }}.tar.zst
|
||||
path: ollama-linux-${{ matrix.arch }}.tar.zst
|
||||
compression-level: 0
|
||||
- if: matrix.arch == 'amd64'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-linux-amd64-rocm.tar.zst
|
||||
path: ollama-linux-amd64-rocm.tar.zst
|
||||
compression-level: 0
|
||||
- if: matrix.arch == 'arm64'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-linux-arm64-jetpack5.tar.zst
|
||||
path: ollama-linux-arm64-jetpack5.tar.zst
|
||||
compression-level: 0
|
||||
- if: matrix.arch == 'arm64'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-linux-arm64-jetpack6.tar.zst
|
||||
path: ollama-linux-arm64-jetpack6.tar.zst
|
||||
compression-level: 0
|
||||
|
||||
windows-depends:
|
||||
needs: setup-environment
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows]
|
||||
arch: [amd64]
|
||||
preset: ['CPU']
|
||||
build-steps: ['cpu cpuArm64']
|
||||
include:
|
||||
- os: windows
|
||||
arch: amd64
|
||||
preset: 'CUDA 12'
|
||||
build-steps: cuda12
|
||||
install: https://developer.download.nvidia.com/compute/cuda/12.8.0/local_installers/cuda_12.8.0_571.96_windows.exe
|
||||
cuda-components:
|
||||
- '"cudart"'
|
||||
- '"nvcc"'
|
||||
- '"cublas"'
|
||||
- '"cublas_dev"'
|
||||
cuda-version: '12.8'
|
||||
- os: windows
|
||||
arch: amd64
|
||||
preset: 'CUDA 13'
|
||||
build-steps: cuda13
|
||||
install: https://developer.download.nvidia.com/compute/cuda/13.0.0/local_installers/cuda_13.0.0_windows.exe
|
||||
cuda-components:
|
||||
- '"cudart"'
|
||||
- '"nvcc"'
|
||||
- '"cublas"'
|
||||
- '"cublas_dev"'
|
||||
- '"crt"'
|
||||
- '"nvvm"'
|
||||
- '"nvptxcompiler"'
|
||||
cuda-version: '13.0'
|
||||
- os: windows
|
||||
arch: amd64
|
||||
preset: 'ROCm 7'
|
||||
build-steps: rocm7
|
||||
install: https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-26.Q1-Win11-For-HIP.exe
|
||||
rocm-version: '7.1'
|
||||
- os: windows
|
||||
arch: amd64
|
||||
preset: Vulkan
|
||||
build-steps: vulkan
|
||||
install: https://sdk.lunarg.com/sdk/download/1.4.321.1/windows/vulkansdk-windows-X64-1.4.321.1.exe
|
||||
runs-on: ${{ matrix.arch == 'arm64' && format('{0}-{1}', matrix.os, matrix.arch) || matrix.os }}
|
||||
env:
|
||||
GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
|
||||
VERSION: ${{ needs.setup-environment.outputs.VERSION }}
|
||||
steps:
|
||||
- name: Install system dependencies
|
||||
run: |
|
||||
choco install -y --no-progress ccache ninja
|
||||
if (Get-Command ccache -ErrorAction SilentlyContinue) {
|
||||
ccache -o cache_dir=${{ github.workspace }}\.ccache
|
||||
}
|
||||
- if: matrix.preset == 'CPU'
|
||||
name: Install Windows ARM64 cross compiler
|
||||
run: |
|
||||
Invoke-WebRequest -Uri "https://github.com/mstorsjo/llvm-mingw/releases/download/20240619/llvm-mingw-20240619-ucrt-x86_64.zip" -OutFile "${{ runner.temp }}\llvm-mingw-ucrt.zip"
|
||||
Expand-Archive -Path ${{ runner.temp }}\llvm-mingw-ucrt.zip -DestinationPath "C:\Program Files\"
|
||||
$installPath=(Resolve-Path -Path "C:\Program Files\llvm-mingw-*-ucrt-x86_64").path
|
||||
if (!(Test-Path "$installPath\bin\aarch64-w64-mingw32-gcc.exe")) {
|
||||
throw "llvm-mingw x86_64 package is missing the aarch64 cross compiler"
|
||||
}
|
||||
- if: startsWith(matrix.preset, 'CUDA ') || startsWith(matrix.preset, 'ROCm ') || startsWith(matrix.preset, 'Vulkan')
|
||||
id: cache-install
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: |
|
||||
C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
|
||||
C:\Program Files\AMD\ROCm
|
||||
C:\VulkanSDK
|
||||
key: ${{ matrix.install }}
|
||||
- if: startsWith(matrix.preset, 'CUDA ')
|
||||
name: Install CUDA ${{ matrix.cuda-version }}
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
|
||||
Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
|
||||
$subpackages = @(${{ join(matrix.cuda-components, ', ') }}) | Foreach-Object {"${_}_${{ matrix.cuda-version }}"}
|
||||
Start-Process -FilePath .\install.exe -ArgumentList (@("-s") + $subpackages) -NoNewWindow -Wait
|
||||
}
|
||||
|
||||
$cudaPath = (Resolve-Path "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*").path
|
||||
echo "$cudaPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
- if: startsWith(matrix.preset, 'ROCm')
|
||||
name: Install ROCm ${{ matrix.rocm-version }}
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
|
||||
Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
|
||||
Start-Process -FilePath .\install.exe -ArgumentList '-install' -NoNewWindow -Wait
|
||||
}
|
||||
|
||||
$hipPath = (Resolve-Path "C:\Program Files\AMD\ROCm\*").path
|
||||
echo "$hipPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "CC=$hipPath\bin\clang.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
echo "CXX=$hipPath\bin\clang++.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
echo "HIPCXX=$hipPath\bin\clang++.exe" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
echo "HIP_PLATFORM=amd" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
echo "CMAKE_PREFIX_PATH=$hipPath" | Out-File -FilePath $env:GITHUB_ENV -Append
|
||||
- if: matrix.preset == 'Vulkan'
|
||||
name: Install Vulkan
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ("${{ steps.cache-install.outputs.cache-hit }}" -ne 'true') {
|
||||
Invoke-WebRequest -Uri "${{ matrix.install }}" -OutFile "install.exe"
|
||||
Start-Process -FilePath .\install.exe -ArgumentList "-c","--am","--al","in" -NoNewWindow -Wait
|
||||
}
|
||||
|
||||
$vulkanPath = (Resolve-Path "C:\VulkanSDK\*").path
|
||||
$vulkanRuntime = Join-Path $vulkanPath "Helpers\VulkanRT.exe"
|
||||
if (Test-Path $vulkanRuntime) {
|
||||
Start-Process -FilePath $vulkanRuntime -ArgumentList "/s" -NoNewWindow -Wait
|
||||
}
|
||||
echo "$vulkanPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
echo "VULKAN_SDK=$vulkanPath" >> $env:GITHUB_ENV
|
||||
- if: ${{ !cancelled() && matrix.preset != 'CPU' && steps.cache-install.outputs.cache-hit != 'true' }}
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: |
|
||||
C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA
|
||||
C:\Program Files\AMD\ROCm
|
||||
C:\VulkanSDK
|
||||
key: ${{ matrix.install }}
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/cache@v4
|
||||
with:
|
||||
path: ${{ github.workspace }}\.ccache
|
||||
key: ccache-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.preset }}-${{ needs.setup-environment.outputs.vendorsha }}
|
||||
- name: Build Windows dependencies
|
||||
run: |
|
||||
Import-Module 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\Tools\Microsoft.VisualStudio.DevShell.dll'
|
||||
Enter-VsDevShell -VsInstallPath 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise' -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -no_logo'
|
||||
$steps = "${{ matrix.build-steps }}".Split(' ', [System.StringSplitOptions]::RemoveEmptyEntries)
|
||||
./scripts/build_windows.ps1 @steps
|
||||
env:
|
||||
CMAKE_GENERATOR: Ninja
|
||||
- name: Log build results
|
||||
run: |
|
||||
gci -path .\dist -Recurse -File | ForEach-Object { get-filehash -path $_.FullName -Algorithm SHA256 } | format-list
|
||||
- if: matrix.preset == 'CPU'
|
||||
name: Verify Windows CPU payloads
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for payload in \
|
||||
dist/windows-amd64/lib/ollama/llama-server.exe \
|
||||
dist/windows-arm64/lib/ollama/llama-server.exe
|
||||
do
|
||||
[ -f "$payload" ] || { echo "missing $payload"; exit 1; }
|
||||
done
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: depends-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.preset }}
|
||||
path: dist\*
|
||||
compression-level: 0
|
||||
|
||||
windows-build:
|
||||
runs-on: windows
|
||||
needs: setup-environment
|
||||
env:
|
||||
GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
|
||||
VERSION: ${{ needs.setup-environment.outputs.VERSION }}
|
||||
steps:
|
||||
- name: Install clang and gcc-compat
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
Set-ExecutionPolicy Bypass -Scope Process -Force
|
||||
Invoke-WebRequest -Uri "https://github.com/mstorsjo/llvm-mingw/releases/download/20240619/llvm-mingw-20240619-ucrt-x86_64.zip" -OutFile "${{ runner.temp }}\llvm-mingw-ucrt.zip"
|
||||
Expand-Archive -Path ${{ runner.temp }}\llvm-mingw-ucrt.zip -DestinationPath "C:\Program Files\"
|
||||
$installPath=(Resolve-Path -Path "C:\Program Files\llvm-mingw-*-ucrt-x86_64").path
|
||||
echo "$installPath\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
|
||||
if (!(Test-Path "$installPath\bin\aarch64-w64-mingw32-gcc.exe")) {
|
||||
throw "llvm-mingw x86_64 package is missing the aarch64 cross compiler"
|
||||
}
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
LLAMA_CPP_VERSION
|
||||
MLX_VERSION
|
||||
MLX_C_VERSION
|
||||
- name: Verify gcc is actually clang
|
||||
run: |
|
||||
$ErrorActionPreference='Continue'
|
||||
$version=& gcc -v 2>&1
|
||||
$version=$version -join "`n"
|
||||
echo "gcc is $version"
|
||||
if ($version -notmatch 'clang') {
|
||||
echo "ERROR: GCC must be clang for proper utf16 handling"
|
||||
exit 1
|
||||
}
|
||||
$ErrorActionPreference='Stop'
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "20"
|
||||
- name: Build Windows binaries and app launchers
|
||||
run: ./scripts/build_windows.ps1 ollama ollamaArm64 app appArm64
|
||||
- name: Verify Windows build payloads
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for payload in \
|
||||
dist/windows-amd64/ollama.exe \
|
||||
dist/windows-arm64/ollama.exe
|
||||
do
|
||||
[ -f "$payload" ] || { echo "missing $payload"; exit 1; }
|
||||
done
|
||||
- name: Log build results
|
||||
run: |
|
||||
gci -path .\dist -Recurse -File | ForEach-Object { get-filehash -path $_.FullName -Algorithm SHA256 } | format-list
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: build-windows-amd64
|
||||
path: dist\*
|
||||
compression-level: 0
|
||||
|
||||
windows-package:
|
||||
runs-on: windows
|
||||
needs: [setup-environment, windows-build, windows-depends]
|
||||
env:
|
||||
GOFLAGS: ${{ needs.setup-environment.outputs.GOFLAGS }}
|
||||
VERSION: ${{ needs.setup-environment.outputs.VERSION }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache-dependency-path: |
|
||||
go.sum
|
||||
LLAMA_CPP_VERSION
|
||||
MLX_VERSION
|
||||
MLX_C_VERSION
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: depends-windows*
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: build-windows*
|
||||
path: dist
|
||||
merge-multiple: true
|
||||
- name: Copy unsigned install script
|
||||
run: Copy-Item -Path .\scripts\install.ps1 -Destination .\dist\install.ps1 -ErrorAction Stop
|
||||
- name: Log dist contents after download
|
||||
run: gci -path .\dist -recurse
|
||||
- name: Verify Windows package inputs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for payload in \
|
||||
dist/windows-amd64/ollama.exe \
|
||||
dist/windows-amd64/lib/ollama/llama-server.exe \
|
||||
dist/windows-arm64/ollama.exe \
|
||||
dist/windows-arm64/lib/ollama/llama-server.exe
|
||||
do
|
||||
[ -f "$payload" ] || { echo "missing $payload"; exit 1; }
|
||||
done
|
||||
- name: Build unsigned Windows installer and zips
|
||||
run: ./scripts/build_windows.ps1 deps installer zip
|
||||
- name: Log contents after build
|
||||
run: |
|
||||
gci -path .\dist -Recurse -File | ForEach-Object { get-filehash -path $_.FullName -Algorithm SHA256 } | format-list
|
||||
- name: Verify Windows package outputs
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for payload in \
|
||||
dist/ollama-windows-amd64.zip \
|
||||
dist/ollama-windows-arm64.zip \
|
||||
dist/ollama-windows-amd64-rocm.zip \
|
||||
dist/OllamaSetup.exe \
|
||||
dist/install.ps1
|
||||
do
|
||||
[ -f "$payload" ] || { echo "missing $payload"; exit 1; }
|
||||
done
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-windows-amd64.zip
|
||||
path: dist/ollama-windows-amd64.zip
|
||||
compression-level: 0
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-windows-arm64.zip
|
||||
path: dist/ollama-windows-arm64.zip
|
||||
compression-level: 0
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ollama-windows-amd64-rocm.zip
|
||||
path: dist/ollama-windows-amd64-rocm.zip
|
||||
compression-level: 0
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: OllamaSetup.exe
|
||||
path: dist/OllamaSetup.exe
|
||||
compression-level: 0
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: install.ps1
|
||||
path: dist/install.ps1
|
||||
compression-level: 0
|
||||
+27
@@ -96,6 +96,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
[ -e "$lib" ] && cp -a "$lib" dist/lib/ollama/ || true; \
|
||||
done
|
||||
|
||||
FROM scratch AS publish-llama-server-cpu
|
||||
COPY --from=llama-server-cpu dist/lib/ollama /lib/ollama/
|
||||
|
||||
FROM cuda-12-deps AS llama-server-cuda_v12
|
||||
COPY LLAMA_CPP_VERSION .
|
||||
COPY llama/server llama/server
|
||||
@@ -105,6 +108,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake --build build/llama-server-cuda_v12 -- -l $(nproc) \
|
||||
&& cmake --install build/llama-server-cuda_v12 --component llama-server --strip
|
||||
|
||||
FROM scratch AS publish-llama-server-cuda_v12
|
||||
COPY --from=llama-server-cuda_v12 dist/lib/ollama /lib/ollama/
|
||||
|
||||
FROM cuda-13-deps AS llama-server-cuda_v13
|
||||
COPY LLAMA_CPP_VERSION .
|
||||
COPY llama/server llama/server
|
||||
@@ -114,6 +120,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake --build build/llama-server-cuda_v13 -- -l $(nproc) \
|
||||
&& cmake --install build/llama-server-cuda_v13 --component llama-server --strip
|
||||
|
||||
FROM scratch AS publish-llama-server-cuda_v13
|
||||
COPY --from=llama-server-cuda_v13 dist/lib/ollama /lib/ollama/
|
||||
|
||||
FROM rocm-7-deps AS llama-server-rocm_v7_2
|
||||
ENV CC=clang CXX=clang++
|
||||
COPY LLAMA_CPP_VERSION .
|
||||
@@ -125,6 +134,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake --install build/llama-server-rocm_v7_2 --component llama-server --strip
|
||||
RUN rm -f dist/lib/ollama/rocm_v7_2/rocblas/library/*gfx90[06]*
|
||||
|
||||
FROM scratch AS publish-llama-server-rocm_v7_2
|
||||
COPY --from=llama-server-rocm_v7_2 dist/lib/ollama /lib/ollama/
|
||||
|
||||
FROM vulkan-deps AS llama-server-vulkan
|
||||
COPY LLAMA_CPP_VERSION .
|
||||
COPY llama/server llama/server
|
||||
@@ -134,6 +146,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake --build build/llama-server-vulkan -- -l $(nproc) \
|
||||
&& cmake --install build/llama-server-vulkan --component llama-server --strip
|
||||
|
||||
FROM scratch AS publish-llama-server-vulkan
|
||||
COPY --from=llama-server-vulkan dist/lib/ollama /lib/ollama/
|
||||
|
||||
#
|
||||
# JetPack stages — self-contained with their own base images
|
||||
#
|
||||
@@ -155,6 +170,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake --build build/llama-server-cuda_jetpack5 -- -l $(nproc) \
|
||||
&& cmake --install build/llama-server-cuda_jetpack5 --component llama-server --strip
|
||||
|
||||
FROM scratch AS publish-llama-server-cuda_jetpack5
|
||||
COPY --from=jetpack-5 dist/lib/ollama /lib/ollama/
|
||||
|
||||
FROM --platform=linux/arm64 nvcr.io/nvidia/l4t-jetpack:${JETPACK6VERSION} AS jetpack-6
|
||||
ARG CMAKEVERSION
|
||||
ARG NINJAVERSION
|
||||
@@ -172,6 +190,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake --build build/llama-server-cuda_jetpack6 -- -l $(nproc) \
|
||||
&& cmake --install build/llama-server-cuda_jetpack6 --component llama-server --strip
|
||||
|
||||
FROM scratch AS publish-llama-server-cuda_jetpack6
|
||||
COPY --from=jetpack-6 dist/lib/ollama /lib/ollama/
|
||||
|
||||
#
|
||||
# MLX stage
|
||||
#
|
||||
@@ -210,6 +231,9 @@ RUN --mount=type=cache,target=/root/.ccache \
|
||||
&& cmake -S . -B build/mlx_cuda_v13 -DOLLAMA_MLX_BACKENDS=cuda_v13 -DBLAS_INCLUDE_DIRS=/usr/include/openblas -DLAPACK_INCLUDE_DIRS=/usr/include/openblas -DCMAKE_CUDA_FLAGS="-t ${OLLAMA_MLX_NVCC_THREADS}" ${MLX_CUDA_RAM_MB:+-DMLX_CUDA_RAM_MB=${MLX_CUDA_RAM_MB}} -DOLLAMA_PAYLOAD_INSTALL_PREFIX=/go/src/github.com/ollama/ollama/dist \
|
||||
&& cmake --build build/mlx_cuda_v13 --target ollama-mlx-cuda_v13 -- -l $(nproc) ${OLLAMA_MLX_BUILD_JOBS:+-j ${OLLAMA_MLX_BUILD_JOBS}}
|
||||
|
||||
FROM scratch AS publish-mlx
|
||||
COPY --from=mlx /go/src/github.com/ollama/ollama/dist/lib/ollama /lib/ollama/
|
||||
|
||||
#
|
||||
# Go build
|
||||
#
|
||||
@@ -230,6 +254,9 @@ ENV CGO_CXXFLAGS="${CGO_CXXFLAGS}"
|
||||
RUN --mount=type=cache,target=/root/.cache/go-build \
|
||||
go build -trimpath -buildmode=pie -o /bin/ollama .
|
||||
|
||||
FROM scratch AS publish-go
|
||||
COPY --from=build /bin/ollama /bin/ollama
|
||||
|
||||
#
|
||||
# Assembly stages — combine llama-server variants + GPU runtime libs
|
||||
#
|
||||
|
||||
+1
-1
@@ -1 +1 @@
|
||||
b9409
|
||||
b9452
|
||||
|
||||
@@ -575,6 +575,7 @@ if(OLLAMA_HAVE_LLAMA_SERVER)
|
||||
ollama_append_cache_arg_if_set(_rocm_args AMDGPU_TARGETS)
|
||||
ollama_append_cache_arg_if_set(_rocm_args CMAKE_HIP_ARCHITECTURES)
|
||||
ollama_append_cache_arg_if_set(_rocm_args CMAKE_HIP_FLAGS)
|
||||
ollama_append_cache_arg_if_set(_rocm_args GGML_CUDA_NO_PEER_COPY)
|
||||
ollama_append_cache_arg_if_set(_rocm_args CMAKE_PREFIX_PATH)
|
||||
ollama_add_llama_server_build(${_backend}
|
||||
PRESET ${_rocm_preset}
|
||||
|
||||
+1
-1
@@ -388,7 +388,7 @@ func filterIntegratedGPUs(devices []ml.DeviceInfo) []ml.DeviceInfo {
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Info("dropping integrated GPU",
|
||||
slog.Info("dropping integrated GPU; to enable, set OLLAMA_IGPU_ENABLE=1",
|
||||
"id", device.ID,
|
||||
"library", device.Library,
|
||||
"compute", device.Compute(),
|
||||
|
||||
@@ -96,6 +96,9 @@ if(GGML_HIP AND OLLAMA_RUNNER_DIR MATCHES "^rocm_v")
|
||||
ollama_set_cache_default(CMAKE_HIP_FLAGS STRING
|
||||
"-parallel-jobs=4" "HIP compiler flags")
|
||||
if(WIN32)
|
||||
# Windows ROCm split-load needs peer copies disabled for correctness.
|
||||
ollama_set_cache_default(GGML_CUDA_NO_PEER_COPY BOOL ON
|
||||
"Disable direct peer device copies")
|
||||
# HIP on Windows currently emits attributes and deprecated pragma
|
||||
# warnings from ROCm headers. Keep the workaround local to Windows ROCm
|
||||
# so it can be removed when the ROCm toolchain no longer needs it.
|
||||
|
||||
+49
-9
@@ -141,9 +141,10 @@ type llamaServerRunner struct {
|
||||
// used to map DeviceIDs to device names for VRAMByGPU lookups.
|
||||
gpus []ml.DeviceInfo
|
||||
|
||||
ggml *ggml.GGML
|
||||
totalLayers uint64 // maximum offloadable model layers
|
||||
loadStart time.Time
|
||||
ggml *ggml.GGML
|
||||
totalLayers uint64 // maximum offloadable model layers
|
||||
loadStart time.Time
|
||||
rawEmbeddings bool
|
||||
|
||||
sem *semaphore.Weighted
|
||||
|
||||
@@ -559,20 +560,32 @@ func appendBatchArgs(params []string, opts api.Options, embedding bool, numParal
|
||||
return params
|
||||
}
|
||||
|
||||
func appendFlashAttentionArgs(params []string, gpus []ml.DeviceInfo) []string {
|
||||
// LlamaServerFlashAttention resolves the flash-attention mode passed to llama-server.
|
||||
func LlamaServerFlashAttention(gpus []ml.DeviceInfo) ml.FlashAttentionType {
|
||||
enabled := envconfig.FlashAttention(false)
|
||||
userSet := enabled == envconfig.FlashAttention(true)
|
||||
if userSet {
|
||||
if enabled {
|
||||
return append(params, "--flash-attn", "on")
|
||||
return ml.FlashAttentionEnabled
|
||||
}
|
||||
return append(params, "--flash-attn", "off")
|
||||
return ml.FlashAttentionDisabled
|
||||
}
|
||||
|
||||
if !ml.FlashAttentionSupported(gpus) {
|
||||
return append(params, "--flash-attn", "off")
|
||||
return ml.FlashAttentionDisabled
|
||||
}
|
||||
return ml.FlashAttentionAuto
|
||||
}
|
||||
|
||||
func appendFlashAttentionArgs(params []string, gpus []ml.DeviceInfo) []string {
|
||||
switch LlamaServerFlashAttention(gpus) {
|
||||
case ml.FlashAttentionEnabled:
|
||||
return append(params, "--flash-attn", "on")
|
||||
case ml.FlashAttentionDisabled:
|
||||
return append(params, "--flash-attn", "off")
|
||||
default:
|
||||
return append(params, "--flash-attn", "auto")
|
||||
}
|
||||
return append(params, "--flash-attn", "auto")
|
||||
}
|
||||
|
||||
func appendMainGPUArgs(params []string, opts api.Options) []string {
|
||||
@@ -778,6 +791,7 @@ func NewLlamaServerRunner(
|
||||
gpus: gpus,
|
||||
ggml: f,
|
||||
totalLayers: f.KV().BlockCount() + 1,
|
||||
rawEmbeddings: legacyEmbeddingsWereRaw(f.KV()),
|
||||
sem: semaphore.NewWeighted(int64(numParallel)),
|
||||
launch: launch,
|
||||
output: memWriter,
|
||||
@@ -801,6 +815,28 @@ func cloneStringMap(src map[string]string) map[string]string {
|
||||
return dst
|
||||
}
|
||||
|
||||
func legacyEmbeddingsWereRaw(kv ggml.KV) bool {
|
||||
arch := kv.Architecture()
|
||||
if _, ok := kv[fmt.Sprintf("%s.pooling_type", arch)]; !ok {
|
||||
return false
|
||||
}
|
||||
|
||||
// Legacy /api/embeddings returned runner output, so preserve only old raw embed paths.
|
||||
switch arch {
|
||||
case "bert":
|
||||
if kv.String("tokenizer.ggml.model", "bert") != "bert" {
|
||||
return true
|
||||
}
|
||||
return !kv.Bool("normalize_embeddings", true)
|
||||
case "nomic-bert", "nomic-bert-moe":
|
||||
return !kv.Bool("normalize_embeddings", false)
|
||||
case "gemma3", "gemma-embedding", "qwen3":
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (s *llamaServerRunner) startProcess() error {
|
||||
cmd, port, err := startLlamaServer(s.launch, s.output)
|
||||
if err != nil {
|
||||
@@ -2034,7 +2070,11 @@ func (s *llamaServerRunner) Embedding(ctx context.Context, input string) ([]floa
|
||||
|
||||
// Use "input" field (not "content") to get the OAI-compatible response format
|
||||
// which includes tokens_evaluated for prompt token counting
|
||||
data, err := json.Marshal(map[string]string{"input": input})
|
||||
req := map[string]any{"input": input}
|
||||
if s.rawEmbeddings {
|
||||
req["embd_normalize"] = -1
|
||||
}
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("error marshaling embed data: %w", err)
|
||||
}
|
||||
|
||||
@@ -1298,6 +1298,47 @@ func TestLlamaServerEmbedding(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyEmbeddingsWereRaw(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
kv ggml.KV
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "bert t5 raw like bge-m3",
|
||||
kv: ggml.KV{
|
||||
"general.architecture": "bert",
|
||||
"bert.pooling_type": uint32(1),
|
||||
"tokenizer.ggml.model": "t5",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "nomic bert default raw",
|
||||
kv: ggml.KV{
|
||||
"general.architecture": "nomic-bert",
|
||||
"nomic-bert.pooling_type": uint32(1),
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "qwen3 remains normalized",
|
||||
kv: ggml.KV{
|
||||
"general.architecture": "qwen3",
|
||||
"qwen3.pooling_type": uint32(1),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := legacyEmbeddingsWereRaw(tt.kv); got != tt.want {
|
||||
t.Fatalf("legacyEmbeddingsWereRaw() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLlamaServerEmbeddingFallbackFormat(t *testing.T) {
|
||||
// Fallback: non-OAI array format (from "content" field)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unsafe"
|
||||
|
||||
@@ -42,6 +43,11 @@ type windowsFileVersion struct {
|
||||
ls uint32
|
||||
}
|
||||
|
||||
type windowsVersionTranslation struct {
|
||||
language uint16
|
||||
codePage uint16
|
||||
}
|
||||
|
||||
func WindowsROCmRuntimeDLLPath(libDirs []string) (string, error) {
|
||||
choice, err := windowsROCmRuntimeDLLChoice(libDirs)
|
||||
if err != nil {
|
||||
@@ -255,6 +261,10 @@ func readWindowsFileVersion(path string) (windowsFileVersion, bool) {
|
||||
return windowsFileVersion{}, false
|
||||
}
|
||||
|
||||
if version, ok := readWindowsStringFileVersion(versionInfo); ok {
|
||||
return version, true
|
||||
}
|
||||
|
||||
var fixedInfo *windows.VS_FIXEDFILEINFO
|
||||
var fixedInfoLen uint32
|
||||
if err := windows.VerQueryValue(unsafe.Pointer(&versionInfo[0]), `\`, unsafe.Pointer(&fixedInfo), &fixedInfoLen); err != nil {
|
||||
@@ -266,6 +276,76 @@ func readWindowsFileVersion(path string) (windowsFileVersion, bool) {
|
||||
return windowsFileVersion{ms: fixedInfo.FileVersionMS, ls: fixedInfo.FileVersionLS}, true
|
||||
}
|
||||
|
||||
func readWindowsStringFileVersion(versionInfo []byte) (windowsFileVersion, bool) {
|
||||
translations := windowsVersionTranslations(versionInfo)
|
||||
if len(translations) == 0 {
|
||||
translations = []windowsVersionTranslation{{language: 0x0409, codePage: 0x04b0}}
|
||||
}
|
||||
|
||||
for _, translation := range translations {
|
||||
for _, name := range []string{"FileVersion", "ProductVersion"} {
|
||||
value, ok := windowsVersionString(versionInfo, translation, name)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
version, ok := parseWindowsFileVersionString(value)
|
||||
if ok {
|
||||
return version, true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return windowsFileVersion{}, false
|
||||
}
|
||||
|
||||
func windowsVersionTranslations(versionInfo []byte) []windowsVersionTranslation {
|
||||
var translations *windowsVersionTranslation
|
||||
var translationsLen uint32
|
||||
if err := windows.VerQueryValue(unsafe.Pointer(&versionInfo[0]), `\VarFileInfo\Translation`, unsafe.Pointer(&translations), &translationsLen); err != nil {
|
||||
return nil
|
||||
}
|
||||
if translations == nil || translationsLen < uint32(unsafe.Sizeof(windowsVersionTranslation{})) {
|
||||
return nil
|
||||
}
|
||||
return unsafe.Slice(translations, int(translationsLen)/int(unsafe.Sizeof(windowsVersionTranslation{})))
|
||||
}
|
||||
|
||||
func windowsVersionString(versionInfo []byte, translation windowsVersionTranslation, name string) (string, bool) {
|
||||
subBlock := fmt.Sprintf(`\StringFileInfo\%04x%04x\%s`, translation.language, translation.codePage, name)
|
||||
var value *uint16
|
||||
var valueLen uint32
|
||||
if err := windows.VerQueryValue(unsafe.Pointer(&versionInfo[0]), subBlock, unsafe.Pointer(&value), &valueLen); err != nil {
|
||||
return "", false
|
||||
}
|
||||
if value == nil || valueLen == 0 {
|
||||
return "", false
|
||||
}
|
||||
return windows.UTF16PtrToString(value), true
|
||||
}
|
||||
|
||||
func parseWindowsFileVersionString(s string) (windowsFileVersion, bool) {
|
||||
parts := strings.FieldsFunc(s, func(r rune) bool {
|
||||
return r < '0' || r > '9'
|
||||
})
|
||||
if len(parts) < 2 {
|
||||
return windowsFileVersion{}, false
|
||||
}
|
||||
|
||||
var version [4]uint16
|
||||
for i := 0; i < len(version) && i < len(parts); i++ {
|
||||
part, err := strconv.ParseUint(parts[i], 10, 16)
|
||||
if err != nil {
|
||||
return windowsFileVersion{}, false
|
||||
}
|
||||
version[i] = uint16(part)
|
||||
}
|
||||
|
||||
return windowsFileVersion{
|
||||
ms: uint32(version[0])<<16 | uint32(version[1]),
|
||||
ls: uint32(version[2])<<16 | uint32(version[3]),
|
||||
}, true
|
||||
}
|
||||
|
||||
func (v windowsFileVersion) Compare(other windowsFileVersion) int {
|
||||
if v.ms < other.ms {
|
||||
return -1
|
||||
|
||||
+20
-6
@@ -21,7 +21,7 @@ set -e
|
||||
|
||||
status() { echo >&2 ">>> $@"; }
|
||||
usage() {
|
||||
echo "usage: $(basename $0) [build app [sign]]"
|
||||
echo "usage: $(basename $0) [build package app sign]"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -130,7 +130,7 @@ _merge_darwin_payload() {
|
||||
done
|
||||
}
|
||||
|
||||
_sign_darwin() {
|
||||
_prepare_darwin_runtime() {
|
||||
status "Creating universal binary..."
|
||||
mkdir -p dist/darwin
|
||||
lipo -create -output dist/darwin/ollama dist/darwin-amd64/ollama dist/darwin-arm64/ollama
|
||||
@@ -146,7 +146,23 @@ _sign_darwin() {
|
||||
lipo dist/darwin/llama-quantize -verify_arch x86_64 arm64
|
||||
|
||||
_merge_darwin_payload
|
||||
}
|
||||
|
||||
_create_darwin_runtime_tarball() {
|
||||
status "Creating universal tarball..."
|
||||
rm -f dist/ollama-darwin.tar dist/ollama-darwin.tgz
|
||||
tar -cf dist/ollama-darwin.tar --strip-components 2 dist/darwin/ollama dist/darwin/llama-server dist/darwin/llama-quantize
|
||||
tar -rf dist/ollama-darwin.tar --strip-components 4 dist/darwin/lib/ollama
|
||||
gzip -9vc <dist/ollama-darwin.tar >dist/ollama-darwin.tgz
|
||||
}
|
||||
|
||||
_package_darwin_runtime() {
|
||||
_prepare_darwin_runtime
|
||||
_create_darwin_runtime_tarball
|
||||
}
|
||||
|
||||
_sign_darwin() {
|
||||
_prepare_darwin_runtime
|
||||
if [ -n "$APPLE_IDENTITY" ]; then
|
||||
for F in dist/darwin/ollama dist/darwin/llama-server dist/darwin/llama-quantize dist/darwin/lib/ollama/* dist/darwin/lib/ollama/mlx_metal_v*/*; do
|
||||
[ -f "$F" ] && [ ! -L "$F" ] || continue
|
||||
@@ -160,10 +176,7 @@ _sign_darwin() {
|
||||
rm -f "$TEMP"
|
||||
fi
|
||||
|
||||
status "Creating universal tarball..."
|
||||
tar -cf dist/ollama-darwin.tar --strip-components 2 dist/darwin/ollama dist/darwin/llama-server dist/darwin/llama-quantize
|
||||
tar -rf dist/ollama-darwin.tar --strip-components 4 dist/darwin/lib/ollama
|
||||
gzip -9vc <dist/ollama-darwin.tar >dist/ollama-darwin.tgz
|
||||
_create_darwin_runtime_tarball
|
||||
}
|
||||
|
||||
_build_macapp() {
|
||||
@@ -282,6 +295,7 @@ fi
|
||||
for CMD in "$@"; do
|
||||
case $CMD in
|
||||
build) _build_darwin ;;
|
||||
package) _package_darwin_runtime ;;
|
||||
sign) _sign_darwin ;;
|
||||
app) _build_macapp ;;
|
||||
*) usage ;;
|
||||
|
||||
+59
-13
@@ -143,10 +143,20 @@ func resolveContextShift(shift *bool, numCtx int) bool {
|
||||
}
|
||||
|
||||
func effectiveModelContext(numCtx int, f *ggml.GGML) int {
|
||||
if f != nil {
|
||||
if trainCtx := int(f.KV().ContextLength()); trainCtx > 0 && numCtx > trainCtx {
|
||||
return trainCtx
|
||||
}
|
||||
return effectiveContext(numCtx, modelTrainContext(f))
|
||||
}
|
||||
|
||||
func modelTrainContext(f *ggml.GGML) int {
|
||||
if f == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return int(f.KV().ContextLength())
|
||||
}
|
||||
|
||||
func effectiveContext(numCtx, trainCtx int) int {
|
||||
if trainCtx > 0 && numCtx > trainCtx {
|
||||
return trainCtx
|
||||
}
|
||||
|
||||
return numCtx
|
||||
@@ -522,7 +532,8 @@ func (s *Scheduler) load(req *LlmRequest, systemInfo ml.SystemInfo, gpus []ml.De
|
||||
predicted := llm.PredictServerVRAM(req.model.ModelPath, f, predictedCtx)
|
||||
loadGpus, launchOpts = selectLlamaServerPlacement(systemInfo, gpus, predicted, req.opts)
|
||||
availableForBatch, _, _ := availableMemoryForPlacement(systemInfo, loadGpus, launchOpts)
|
||||
req.applyAutomaticGenerationBatch(completion, predictedCtx, predicted, availableForBatch)
|
||||
flashAttention := llm.LlamaServerFlashAttention(loadGpus)
|
||||
req.applyAutomaticGenerationBatch(completion, predictedCtx, predicted, availableForBatch, flashAttention, loadGpus)
|
||||
launchOpts.NumBatch = req.opts.NumBatch
|
||||
predictedForLoad := predicted + generationBatchSurchargeForCompletion(completion, launchOpts.NumBatch)
|
||||
|
||||
@@ -676,8 +687,10 @@ iGPUScan:
|
||||
}
|
||||
|
||||
totalSize, vramSize := llama.MemorySize()
|
||||
if effectiveNumCtx := llama.ContextLength(); req.numCtxAuto && effectiveNumCtx > 0 {
|
||||
trainContext := modelTrainContext(f)
|
||||
if effectiveNumCtx := llama.ContextLength(); req.model.ModelPath != "" && effectiveNumCtx > 0 {
|
||||
req.opts.NumCtx = effectiveNumCtx
|
||||
req.contextShift = resolveContextShift(req.shift, effectiveNumCtx)
|
||||
}
|
||||
runner := &runnerRef{
|
||||
model: req.model,
|
||||
@@ -697,6 +710,7 @@ iGPUScan:
|
||||
numBatchAuto: req.numBatchAuto,
|
||||
useMMapAuto: req.useMMapAuto,
|
||||
contextShift: req.contextShift,
|
||||
trainContext: trainContext,
|
||||
}
|
||||
runner.numParallel = numParallel
|
||||
runner.refMu.Lock() // hold lock until running or aborted
|
||||
@@ -763,7 +777,7 @@ func (req *LlmRequest) reduceAutoNumCtxForLoadOOM(f *ggml.GGML, numParallel int,
|
||||
predictedCtx := effectiveLlamaServerContext(req.opts.NumCtx, f, numParallel)
|
||||
predictedVRAM := llm.PredictServerVRAM(req.model.ModelPath, f, predictedCtx)
|
||||
available, _, _ := availableMemoryForPlacement(systemInfo, gpus, launchOpts)
|
||||
req.applyAutomaticGenerationBatch(completion, predictedCtx, predictedVRAM, available)
|
||||
req.applyAutomaticGenerationBatch(completion, predictedCtx, predictedVRAM, available, llm.LlamaServerFlashAttention(gpus), gpus)
|
||||
newNumBatch = req.opts.NumBatch
|
||||
return oldNumCtx, effectiveNumCtx, newNumCtx, oldNumBatch, newNumBatch, true
|
||||
}
|
||||
@@ -781,20 +795,21 @@ func effectiveLlamaServerContext(numCtx int, f *ggml.GGML, numParallel int) int
|
||||
}
|
||||
|
||||
const (
|
||||
llamaServerGenerationBatchDefault = 512
|
||||
llamaServerGenerationBatchMedium = 1024
|
||||
llamaServerGenerationBatchLarge = 2048
|
||||
llamaServerGenerationBatchDefault = 512
|
||||
llamaServerGenerationBatchConstrained = 256
|
||||
llamaServerGenerationBatchMedium = 1024
|
||||
llamaServerGenerationBatchLarge = 2048
|
||||
|
||||
llamaServerGenerationBatchMediumHeadroomPercent = 75
|
||||
llamaServerGenerationBatchLargeHeadroomPercent = 60
|
||||
)
|
||||
|
||||
func (req *LlmRequest) applyAutomaticGenerationBatch(completion bool, effectiveCtx int, predictedVRAM, availableMemory uint64) {
|
||||
func (req *LlmRequest) applyAutomaticGenerationBatch(completion bool, effectiveCtx int, predictedVRAM, availableMemory uint64, flashAttention ml.FlashAttentionType, gpus []ml.DeviceInfo) {
|
||||
if !completion || !req.numBatchAuto {
|
||||
return
|
||||
}
|
||||
|
||||
req.opts.NumBatch = automaticGenerationBatch(effectiveCtx, predictedVRAM, availableMemory)
|
||||
req.opts.NumBatch = automaticGenerationBatch(effectiveCtx, predictedVRAM, availableMemory, flashAttention, gpus)
|
||||
}
|
||||
|
||||
func generationBatchSurchargeForCompletion(completion bool, batch int) uint64 {
|
||||
@@ -804,7 +819,14 @@ func generationBatchSurchargeForCompletion(completion bool, batch int) uint64 {
|
||||
return generationBatchSurcharge(batch)
|
||||
}
|
||||
|
||||
func automaticGenerationBatch(effectiveCtx int, predictedVRAM, availableMemory uint64) int {
|
||||
func automaticGenerationBatch(effectiveCtx int, predictedVRAM, availableMemory uint64, flashAttention ml.FlashAttentionType, gpus []ml.DeviceInfo) int {
|
||||
if flashAttention == ml.FlashAttentionDisabled && hasCUDADevice(gpus) {
|
||||
if constrainedCUDAWithoutFlashAttention(effectiveCtx, gpus) {
|
||||
return llamaServerGenerationBatchConstrained
|
||||
}
|
||||
return llamaServerGenerationBatchDefault
|
||||
}
|
||||
|
||||
batch := generationBatchForContext(effectiveCtx)
|
||||
for batch > llamaServerGenerationBatchDefault && !generationBatchFits(batch, predictedVRAM, availableMemory) {
|
||||
batch = nextLowerGenerationBatch(batch)
|
||||
@@ -812,6 +834,28 @@ func automaticGenerationBatch(effectiveCtx int, predictedVRAM, availableMemory u
|
||||
return batch
|
||||
}
|
||||
|
||||
func hasCUDADevice(gpus []ml.DeviceInfo) bool {
|
||||
return slices.ContainsFunc(gpus, func(gpu ml.DeviceInfo) bool {
|
||||
return gpu.Library == "CUDA"
|
||||
})
|
||||
}
|
||||
|
||||
func constrainedCUDAWithoutFlashAttention(effectiveCtx int, gpus []ml.DeviceInfo) bool {
|
||||
if effectiveCtx <= 4096 {
|
||||
return false
|
||||
}
|
||||
return slices.ContainsFunc(gpus, func(gpu ml.DeviceInfo) bool {
|
||||
if gpu.Library != "CUDA" {
|
||||
return false
|
||||
}
|
||||
memory := gpu.FreeMemory
|
||||
if memory == 0 || (gpu.TotalMemory > 0 && gpu.TotalMemory < memory) {
|
||||
memory = gpu.TotalMemory
|
||||
}
|
||||
return memory > 0 && memory <= 8*format.GibiByte
|
||||
})
|
||||
}
|
||||
|
||||
func generationBatchForContext(effectiveCtx int) int {
|
||||
switch {
|
||||
case effectiveCtx > 32768:
|
||||
@@ -1316,6 +1360,7 @@ type runnerRef struct {
|
||||
numBatchAuto bool
|
||||
useMMapAuto bool
|
||||
contextShift bool
|
||||
trainContext int
|
||||
*api.Options
|
||||
}
|
||||
|
||||
@@ -1357,6 +1402,7 @@ func (runner *runnerRef) needsReload(ctx context.Context, req *LlmRequest) bool
|
||||
// Don't reload runner if num_gpu=-1 was provided
|
||||
optsExisting := runner.Options.Runner
|
||||
optsNew := req.opts.Runner
|
||||
optsNew.NumCtx = effectiveContext(optsNew.NumCtx, runner.trainContext)
|
||||
if runner.numCtxAuto && req.numCtxAuto {
|
||||
optsNew.NumCtx = optsExisting.NumCtx
|
||||
}
|
||||
|
||||
+57
-3
@@ -151,7 +151,7 @@ func TestSchedLoadStoresEffectiveContextLength(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchedLoadPreservesExplicitContextLength(t *testing.T) {
|
||||
func TestSchedLoadStoresEffectiveExplicitContextLength(t *testing.T) {
|
||||
ctx, done := context.WithTimeout(t.Context(), 500*time.Millisecond)
|
||||
defer done()
|
||||
|
||||
@@ -167,7 +167,7 @@ func TestSchedLoadPreservesExplicitContextLength(t *testing.T) {
|
||||
case err := <-scenario.req.errCh:
|
||||
require.NoError(t, err)
|
||||
case runner := <-scenario.req.successCh:
|
||||
require.Equal(t, 262144, runner.Options.NumCtx)
|
||||
require.Equal(t, 131072, runner.Options.NumCtx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -981,6 +981,34 @@ func TestSchedNeedsReloadUsesEffectiveAutomaticContextShift(t *testing.T) {
|
||||
require.True(t, runner.needsReload(ctx, req))
|
||||
}
|
||||
|
||||
func TestSchedNeedsReloadUsesEffectiveExplicitContext(t *testing.T) {
|
||||
ctx, done := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer done()
|
||||
|
||||
llm := &mockLlm{vramByGPU: map[ml.DeviceID]uint64{}}
|
||||
opts := api.DefaultOptions()
|
||||
opts.NumCtx = 2048
|
||||
model := &Model{ModelPath: "model.gguf"}
|
||||
runner := &runnerRef{
|
||||
model: model,
|
||||
Options: &opts,
|
||||
llama: llm,
|
||||
numParallel: 1,
|
||||
contextShift: true,
|
||||
trainContext: 2048,
|
||||
}
|
||||
req := &LlmRequest{
|
||||
model: model,
|
||||
opts: api.DefaultOptions(),
|
||||
}
|
||||
req.opts.NumCtx = 262144
|
||||
|
||||
require.False(t, runner.needsReload(ctx, req))
|
||||
|
||||
req.opts.NumCtx = 1024
|
||||
require.True(t, runner.needsReload(ctx, req))
|
||||
}
|
||||
|
||||
func TestSchedNeedsReloadIgnoresAutomaticNumBatchDerivation(t *testing.T) {
|
||||
ctx, done := context.WithTimeout(t.Context(), 100*time.Millisecond)
|
||||
defer done()
|
||||
@@ -1050,16 +1078,20 @@ func TestAutomaticGenerationBatch(t *testing.T) {
|
||||
effectiveCtx int
|
||||
predicted uint64
|
||||
available uint64
|
||||
flash ml.FlashAttentionType
|
||||
gpus []ml.DeviceInfo
|
||||
want int
|
||||
}{
|
||||
{
|
||||
name: "small context keeps default",
|
||||
effectiveCtx: 4096,
|
||||
flash: ml.FlashAttentionAuto,
|
||||
want: 512,
|
||||
},
|
||||
{
|
||||
name: "medium context uses 1024 with unknown memory",
|
||||
effectiveCtx: 32768,
|
||||
flash: ml.FlashAttentionAuto,
|
||||
want: 1024,
|
||||
},
|
||||
{
|
||||
@@ -1067,6 +1099,7 @@ func TestAutomaticGenerationBatch(t *testing.T) {
|
||||
effectiveCtx: 131072,
|
||||
predicted: 8 * format.GibiByte,
|
||||
available: 14 * format.GibiByte,
|
||||
flash: ml.FlashAttentionAuto,
|
||||
want: 2048,
|
||||
},
|
||||
{
|
||||
@@ -1074,6 +1107,7 @@ func TestAutomaticGenerationBatch(t *testing.T) {
|
||||
effectiveCtx: 131072,
|
||||
predicted: 9 * format.GibiByte,
|
||||
available: 14 * format.GibiByte,
|
||||
flash: ml.FlashAttentionAuto,
|
||||
want: 1024,
|
||||
},
|
||||
{
|
||||
@@ -1081,6 +1115,7 @@ func TestAutomaticGenerationBatch(t *testing.T) {
|
||||
effectiveCtx: 131072,
|
||||
predicted: 8 * format.GibiByte,
|
||||
available: 11 * format.GibiByte,
|
||||
flash: ml.FlashAttentionAuto,
|
||||
want: 1024,
|
||||
},
|
||||
{
|
||||
@@ -1088,13 +1123,32 @@ func TestAutomaticGenerationBatch(t *testing.T) {
|
||||
effectiveCtx: 32768,
|
||||
predicted: 8500 * format.MebiByte,
|
||||
available: 11 * format.GibiByte,
|
||||
flash: ml.FlashAttentionAuto,
|
||||
want: 512,
|
||||
},
|
||||
{
|
||||
name: "flash attention disabled suppresses promotion",
|
||||
effectiveCtx: 131072,
|
||||
predicted: 8 * format.GibiByte,
|
||||
available: 14 * format.GibiByte,
|
||||
flash: ml.FlashAttentionDisabled,
|
||||
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 14 * format.GibiByte}},
|
||||
want: 512,
|
||||
},
|
||||
{
|
||||
name: "constrained CUDA without flash attention uses smaller batch",
|
||||
effectiveCtx: 131072,
|
||||
predicted: 3 * format.GibiByte,
|
||||
available: 6 * format.GibiByte,
|
||||
flash: ml.FlashAttentionDisabled,
|
||||
gpus: []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, FreeMemory: 6 * format.GibiByte}},
|
||||
want: 256,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, automaticGenerationBatch(tt.effectiveCtx, tt.predicted, tt.available))
|
||||
require.Equal(t, tt.want, automaticGenerationBatch(tt.effectiveCtx, tt.predicted, tt.available, tt.flash, tt.gpus))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,13 +74,12 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
// Check if MLX initialized successfully
|
||||
if !mlx.IsMLXAvailable() {
|
||||
log.Fatalf("MLX initialization failed: %v", mlx.GetMLXInitError())
|
||||
if err := mlx.InitMLX(); err != nil {
|
||||
log.Fatalf("MLX initialization failed: %v", err)
|
||||
}
|
||||
|
||||
// Restore strict error handling now that we know MLX is working.
|
||||
// During init(), a safe handler prevented exit(-1) on GPU errors.
|
||||
// During InitMLX(), a safe handler prevented exit(-1) on GPU errors.
|
||||
mlx.RestoreDefaultErrorHandler()
|
||||
|
||||
// CPU profiling
|
||||
|
||||
+18
-121
@@ -47,14 +47,14 @@ import "C"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
mlxrunnermlx "github.com/ollama/ollama/x/mlxrunner/mlx"
|
||||
)
|
||||
|
||||
// Dtype represents MLX data types
|
||||
@@ -1707,99 +1707,6 @@ var (
|
||||
mlxInitError error
|
||||
)
|
||||
|
||||
// mlxLibName returns the platform-specific shared library filename.
|
||||
func mlxLibName() string {
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
return "mlxc.dll"
|
||||
case "darwin":
|
||||
return "libmlxc.dylib"
|
||||
default:
|
||||
return "libmlxc.so"
|
||||
}
|
||||
}
|
||||
|
||||
func findMLXLibraryInDir(dir, libName string) string {
|
||||
if dir == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
candidate := filepath.Join(dir, libName)
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate
|
||||
}
|
||||
|
||||
if mlxDirs, err := filepath.Glob(filepath.Join(dir, "mlx*")); err == nil {
|
||||
for _, mlxDir := range mlxDirs {
|
||||
candidate = filepath.Join(mlxDir, libName)
|
||||
if _, err := os.Stat(candidate); err == nil {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// findMLXLibrary searches for the MLX shared library in standard locations.
|
||||
// Returns the path to the library, or empty string if not found.
|
||||
func findMLXLibrary() string {
|
||||
libName := mlxLibName()
|
||||
|
||||
// 1. OLLAMA_LIBRARY_PATH — check each dir and mlx_* subdirs
|
||||
if paths, ok := os.LookupEnv("OLLAMA_LIBRARY_PATH"); ok {
|
||||
for _, dir := range filepath.SplitList(paths) {
|
||||
if candidate := findMLXLibraryInDir(dir, libName); candidate != "" {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Executable directory and lib/ollama/mlx* subdirs
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
if eval, err := filepath.EvalSymlinks(exe); err == nil {
|
||||
exe = eval
|
||||
}
|
||||
exeDir := filepath.Dir(exe)
|
||||
|
||||
// Check exe dir directly (macOS copies dylib here)
|
||||
if candidate := findMLXLibraryInDir(exeDir, libName); candidate != "" {
|
||||
return candidate
|
||||
}
|
||||
|
||||
// Check exe_dir/lib/ollama/mlx* subdirectories
|
||||
// and exe_dir/../lib/ollama/mlx* (standard bin/lib sibling layout)
|
||||
for _, libOllamaDir := range []string{
|
||||
filepath.Join(exeDir, "lib", "ollama"),
|
||||
filepath.Join(exeDir, "..", "lib", "ollama"),
|
||||
} {
|
||||
if candidate := findMLXLibraryInDir(libOllamaDir, libName); candidate != "" {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Build directory (for tests run from repo root)
|
||||
if cwd, err := os.Getwd(); err == nil {
|
||||
for _, dir := range []string{
|
||||
filepath.Join(cwd, "build", "lib", "ollama"),
|
||||
filepath.Join(cwd, "dist", runtime.GOOS+"-"+runtime.GOARCH, "lib", "ollama"),
|
||||
filepath.Join(cwd, "dist", runtime.GOOS+"_"+runtime.GOARCH, "lib", "ollama"),
|
||||
} {
|
||||
if candidate := findMLXLibraryInDir(dir, libName); candidate != "" {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
if runtime.GOOS == "darwin" {
|
||||
if candidate := findMLXLibraryInDir(filepath.Join(cwd, "dist", "darwin"), libName); candidate != "" {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// InitMLX initializes the MLX library by dynamically loading libmlxc.
|
||||
// This must be called before using any MLX functions.
|
||||
// Returns an error if the library cannot be loaded.
|
||||
@@ -1808,10 +1715,9 @@ func InitMLX() error {
|
||||
return mlxInitError
|
||||
}
|
||||
|
||||
// Search for the library using Go path discovery
|
||||
libPath := findMLXLibrary()
|
||||
if libPath == "" {
|
||||
mlxInitError = fmt.Errorf("failed to initialize MLX: %s not found", mlxLibName())
|
||||
libPath, err := mlxrunnermlx.LoadedLibraryPath()
|
||||
if err != nil {
|
||||
mlxInitError = fmt.Errorf("failed to initialize MLX: %w", err)
|
||||
return mlxInitError
|
||||
}
|
||||
|
||||
@@ -1832,27 +1738,6 @@ func InitMLX() error {
|
||||
|
||||
mlxInitialized = true
|
||||
mlxInitError = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsMLXAvailable returns whether MLX was successfully initialized
|
||||
func IsMLXAvailable() bool {
|
||||
return mlxInitialized && mlxInitError == nil
|
||||
}
|
||||
|
||||
// GetMLXInitError returns any error that occurred during MLX initialization
|
||||
func GetMLXInitError() error {
|
||||
return mlxInitError
|
||||
}
|
||||
|
||||
func init() {
|
||||
// Initialize MLX dynamic library first
|
||||
if err := InitMLX(); err != nil {
|
||||
// Don't panic in init - let the caller handle the error
|
||||
// Store the error for later retrieval
|
||||
mlxInitError = err
|
||||
return
|
||||
}
|
||||
|
||||
// Enter safe mode: replace the default exit(-1) error handler with one
|
||||
// that logs and stores errors. This prevents a GPU init failure from
|
||||
@@ -1870,8 +1755,20 @@ func init() {
|
||||
msg := C.GoString(C.mlx_get_init_error())
|
||||
mlxInitError = fmt.Errorf("MLX GPU init failed: %s", msg)
|
||||
mlxInitialized = false
|
||||
return
|
||||
return mlxInitError
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsMLXAvailable returns whether MLX was successfully initialized
|
||||
func IsMLXAvailable() bool {
|
||||
return mlxInitialized && mlxInitError == nil
|
||||
}
|
||||
|
||||
// GetMLXInitError returns any error that occurred during MLX initialization
|
||||
func GetMLXInitError() error {
|
||||
return mlxInitError
|
||||
}
|
||||
|
||||
// RestoreDefaultErrorHandler restores the default MLX error handler (exit on error).
|
||||
|
||||
@@ -35,6 +35,17 @@ func CheckInit() error {
|
||||
return initError
|
||||
}
|
||||
|
||||
// LoadedLibraryPath returns the MLX dynamic library path selected by this package.
|
||||
func LoadedLibraryPath() (string, error) {
|
||||
if initLoadedPath != "" {
|
||||
return initLoadedPath, nil
|
||||
}
|
||||
if initError != nil {
|
||||
return "", initError
|
||||
}
|
||||
return "", fmt.Errorf("MLX dynamic library not loaded")
|
||||
}
|
||||
|
||||
// tryLoadFromDir searches a directory for the mlxc shared library and loads it.
|
||||
func tryLoadFromDir(dir string) bool {
|
||||
// On Windows, MSVC produces mlxc.dll (no lib prefix)
|
||||
|
||||
Reference in New Issue
Block a user