Mirror_Repos/ollama_ollama
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CUDA: require driver 550 or newer for v12 (#16895)

Browse Source 
Our cuda_v12 build requires nvcc fatbin compression, which in turn requires driver 550 or newer.  This change filters incompatible CUDA devices based on the runtime and driver version.  This allows users to build from source with older toolkits to support older drivers.

Fixes #16449
This commit is contained in:
Daniel Hiltgen
2026-06-25 08:46:00 -07:00
committed by GitHub
parent d9075caf1a
commit f22ec2ec49
4 changed files with 195 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
discover/cuda_compat.go
+48 -7
View File
@@ -7,34 +7,66 @@ import (
"github.com/ollama/ollama/ml"
)
const (
cudaV12RuntimeMajor = 12
minFatbinCompressionCUDARuntimeMinor = 4
minFatbinCompressionNVIDIADriverMajor = 550
minLegacyComputeJITCUDARuntimeMinor = 8
// Older CUDA compute targets need newer drivers when they are JITed from PTX.
minLegacyComputeJITNVIDIADriverMajor = 570
)
func filterOldCUDADriver(_ context.Context, devices []ml.DeviceInfo) []ml.DeviceInfo {
oldCUDA := func(dev ml.DeviceInfo) bool {
return dev.Library == "CUDA" && dev.ComputeMajor > 0 && dev.ComputeMajor < 7
}
needsCheck := false
hasCUDA := false
for _, dev := range devices {
if oldCUDA(dev) {
needsCheck = true
if dev.Library == "CUDA" {
hasCUDA = true
break
}
}
if !needsCheck {
if !hasCUDA {
return devices
}
driver := nvidiaDriverMajorFromDevices(devices)
if driver == 0 {
slog.Warn("could not verify NVIDIA driver compatibility for an older NVIDIA GPU")
slog.Warn("could not verify NVIDIA driver compatibility for CUDA")
return devices
}
if driver >= 570 {
// Match the driver floor to the CUDA runtime we are about to load, so source
// builds with older CUDA runtimes can still run on matching older drivers.
runtimeMajor, runtimeMinor, hasRuntime := cudaRuntimeVersionFromDevices(devices)
runtimeMayUseCompressedFatbins := hasRuntime &&
runtimeMajor == cudaV12RuntimeMajor &&
runtimeMinor >= minFatbinCompressionCUDARuntimeMinor
// CUDA v12.8+ source builds are expected to either use Ollama's PTX packaging
// for older compute targets or be built against a matching local driver/toolkit.
runtimeMayJITLegacyCompute := hasRuntime &&
runtimeMajor == cudaV12RuntimeMajor &&
runtimeMinor >= minLegacyComputeJITCUDARuntimeMinor
if driver >= minLegacyComputeJITNVIDIADriverMajor || (!runtimeMayUseCompressedFatbins && !runtimeMayJITLegacyCompute) {
return devices
}
filtered := devices[:0]
for _, dev := range devices {
if oldCUDA(dev) {
if dev.Library != "CUDA" {
filtered = append(filtered, dev)
continue
}
if runtimeMayUseCompressedFatbins && driver < minFatbinCompressionNVIDIADriverMajor {
slog.Warn("NVIDIA driver too old",
"device", dev.Description, "compute", dev.Compute(), "driver", driver, "required_driver", "550 or newer")
continue
}
if runtimeMayJITLegacyCompute && oldCUDA(dev) {
slog.Warn("NVIDIA driver too old",
"device", dev.Description, "compute", dev.Compute(), "driver", driver, "required_driver", "570 or newer")
continue
@@ -52,3 +84,12 @@ func nvidiaDriverMajorFromDevices(devices []ml.DeviceInfo) int {
}
return 0
}
func cudaRuntimeVersionFromDevices(devices []ml.DeviceInfo) (int, int, bool) {
for _, dev := range devices {
if dev.Library == "CUDA" {
return cudaRuntimeVersion(dev.LibraryPath)
}
}
return 0, 0, false
}
discover/cuda_compat_test.go
+145
View File
@@ -0,0 +1,145 @@
package discover
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/ollama/ollama/ml"
)
func TestFilterOldCUDADriver(t *testing.T) {
tests := []struct {
name string
runtimeLib string
devices []ml.DeviceInfo
wantIDs []string
}{
{
name: "driver before CUDA compression support filters all CUDA devices",
runtimeLib: "libcudart.so.12.8.90",
devices: []ml.DeviceInfo{
{
DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"},
Description: "NVIDIA V100",
ComputeMajor: 7,
ComputeMinor: 0,
NVIDIADriverMajor: 535,
},
{
DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"},
Description: "NVIDIA L4",
ComputeMajor: 8,
ComputeMinor: 9,
NVIDIADriverMajor: 535,
},
{
DeviceID: ml.DeviceID{ID: "GPU-2", Library: "Vulkan"},
},
},
wantIDs: []string{"GPU-2"},
},
{
name: "driver before legacy compute support filters only older CUDA devices",
runtimeLib: "libcudart.so.12.8.90",
devices: []ml.DeviceInfo{
{
DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"},
Description: "NVIDIA GTX 1080 Ti",
ComputeMajor: 6,
ComputeMinor: 1,
NVIDIADriverMajor: 565,
},
{
DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"},
Description: "NVIDIA RTX 6000",
ComputeMajor: 7,
ComputeMinor: 5,
NVIDIADriverMajor: 565,
},
},
wantIDs: []string{"GPU-1"},
},
{
name: "driver with full support keeps all CUDA devices",
runtimeLib: "libcudart.so.12.8.90",
devices: []ml.DeviceInfo{
{
DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"},
Description: "NVIDIA GTX 1080 Ti",
ComputeMajor: 6,
ComputeMinor: 1,
NVIDIADriverMajor: 570,
},
{
DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"},
Description: "NVIDIA RTX 6000",
ComputeMajor: 7,
ComputeMinor: 5,
NVIDIADriverMajor: 570,
},
},
wantIDs: []string{"GPU-0", "GPU-1"},
},
{
name: "source build with older CUDA runtime keeps devices",
runtimeLib: "libcudart.so.12.2.140",
devices: []ml.DeviceInfo{
{
DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"},
Description: "NVIDIA V100",
ComputeMajor: 7,
ComputeMinor: 0,
NVIDIADriverMajor: 535,
},
{
DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"},
Description: "NVIDIA GTX 1080 Ti",
ComputeMajor: 6,
ComputeMinor: 1,
NVIDIADriverMajor: 535,
},
},
wantIDs: []string{"GPU-0", "GPU-1"},
},
{
name: "unknown driver keeps devices",
devices: []ml.DeviceInfo{
{
DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"},
Description: "NVIDIA V100",
ComputeMajor: 7,
ComputeMinor: 0,
},
},
wantIDs: []string{"GPU-0"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.runtimeLib != "" {
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, tt.runtimeLib), nil, 0o644); err != nil {
t.Fatal(err)
}
for i := range tt.devices {
if tt.devices[i].Library == "CUDA" {
tt.devices[i].LibraryPath = []string{dir}
}
}
}
got := filterOldCUDADriver(context.Background(), tt.devices)
if len(got) != len(tt.wantIDs) {
t.Fatalf("got %d devices, want %d: %#v", len(got), len(tt.wantIDs), got)
}
for i, wantID := range tt.wantIDs {
if got[i].ID != wantID {
t.Fatalf("device %d ID = %q, want %q", i, got[i].ID, wantID)
}
}
})
}
}
docs/gpu.mdx
+1 -1
View File
@@ -3,7 +3,7 @@ title: Hardware support
---
## Nvidia
Ollama supports Nvidia GPUs with compute capability 5.0+ and driver version 531 and newer.
Ollama supports Nvidia GPUs with compute capability 5.0+ and driver version 550 and newer.
Nvidia GPUs with compute capability 5.0 through 6.2 require driver version 570 or newer.
Check your compute compatibility to see if your card is supported:
docs/windows.mdx
+1 -1
View File
@@ -11,7 +11,7 @@ terminal application. As usual the Ollama [API](/api) will be served on
## System Requirements
- Windows 10 22H2 or newer, Home or Pro
- NVIDIA 452.39 or newer Drivers if you have an NVIDIA card
- NVIDIA 551.61 or newer Drivers if you have an NVIDIA card
- AMD ROCm v7 / HIP7-capable driver stack for ROCm acceleration, or a Vulkan-capable AMD Radeon driver for Vulkan acceleration
Ollama uses unicode characters for progress indication, which may render as unknown squares in some older terminal fonts in Windows 10. If you see this, try changing your terminal font settings.