Mirror_Repos/ollama_ollama
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llama: enable FA on CUDA CC 6.x GPUs (#16994)

Browse Source 
Recent upstream Pascal kernel fixes let us compile native SM60/SM61 kernels again instead of relying on PTX JIT, so allow Flash Attention auto at runtime for CC 6.x devices.

Fixes #16591

Fixes #16754
This commit is contained in:
Daniel Hiltgen
2026-07-02 17:11:39 -07:00
committed by GitHub
parent e436db25ff
commit dba1e27fa8
6 changed files with 19 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/development.md
+2 -2
View File
@@ -51,10 +51,10 @@ cmake -B build . -DOLLAMA_LLAMA_BACKENDS=cuda_v13 -DCMAKE_CUDA_ARCHITECTURES=nat
cmake -B build . -DOLLAMA_LLAMA_BACKENDS=rocm_v7_2 -DCMAKE_HIP_ARCHITECTURES=gfx1100 cmake -B build . -DOLLAMA_LLAMA_BACKENDS=rocm_v7_2 -DCMAKE_HIP_ARCHITECTURES=gfx1100
``` ```
You can tune GGML build options by setting `GGML_*` values during configure. For example, to build CUDA v12 for Pascal without flash attention kernels: You can tune GGML build options by setting `GGML_*` values during configure. For example, to disable CUDA flash attention kernels for local debugging:
```shell ```shell
cmake -B build . -DOLLAMA_LLAMA_BACKENDS=cuda_v12 -DCMAKE_CUDA_ARCHITECTURES=61 -DGGML_CUDA_FA=OFF cmake -B build . -DOLLAMA_LLAMA_BACKENDS=cuda_v12 -DGGML_CUDA_FA=OFF
``` ```
## macOS (Apple Silicon) ## macOS (Apple Silicon)
docs/faq.mdx
+1 -1
View File
@@ -343,7 +343,7 @@ When loading a new model, Ollama evaluates the required VRAM for the model again
## How can I enable Flash Attention? ## How can I enable Flash Attention?
Flash Attention is a feature of most modern models that can significantly reduce memory usage as the context size grows. To enable Flash Attention, set the `OLLAMA_FLASH_ATTENTION` environment variable to `1` when starting the Ollama server. Flash Attention is a feature of most modern models that can significantly reduce memory usage as the context size grows. Ollama uses Flash Attention automatically when the selected backend and devices support it. To force Flash Attention on, set `OLLAMA_FLASH_ATTENTION=1` when starting the Ollama server. To disable it, set `OLLAMA_FLASH_ATTENTION=0`.
## How can I set the quantization type for the K/V cache? ## How can I set the quantization type for the K/V cache?
llama/server/CMakePresets.json
+1 -1
View File
@@ -68,7 +68,7 @@
"inherits": ["llama_cuda_v12_base"], "inherits": ["llama_cuda_v12_base"],
"binaryDir": "${sourceDir}/../../build/llama-server-cuda_v12", "binaryDir": "${sourceDir}/../../build/llama-server-cuda_v12",
"cacheVariables": { "cacheVariables": {
"CMAKE_CUDA_ARCHITECTURES": "50-virtual;52-virtual;60-virtual;61-virtual;70;75;80;86;89;90;90a;120" "CMAKE_CUDA_ARCHITECTURES": "50-virtual;52-virtual;60;61;70;75;80;86;89;90;90a;120"
} }
}, },
{ {
llm/llama_server_test.go
+1 -1
View File
@@ -1955,7 +1955,7 @@ func TestAppendFlashAttentionArgs(t *testing.T) {
supportedGPU := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 13, ComputeMajor: 8, ComputeMinor: 9}} supportedGPU := []ml.DeviceInfo{{DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 13, ComputeMajor: 8, ComputeMinor: 9}}
oldGPU := []ml.DeviceInfo{ oldGPU := []ml.DeviceInfo{
{DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 8, ComputeMinor: 9}, {DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 8, ComputeMinor: 9},
{DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}, {DeviceID: ml.DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0},
} }
tests := []struct { tests := []struct {
ml/device.go
+1 -1
View File
@@ -585,7 +585,7 @@ func FlashAttentionSupported(l []DeviceInfo) bool {
func cudaFlashAttentionSupported(gpu DeviceInfo) bool { func cudaFlashAttentionSupported(gpu DeviceInfo) bool {
if gpu.Library != "CUDA" || if gpu.Library != "CUDA" ||
gpu.ComputeMajor < 7 || gpu.ComputeMajor < 6 ||
(gpu.ComputeMajor == 7 && gpu.ComputeMinor == 2) { (gpu.ComputeMajor == 7 && gpu.ComputeMinor == 2) {
return false return false
} }
ml/device_test.go
+13 -2
View File
@@ -186,8 +186,19 @@ func TestFlashAttentionSupported(t *testing.T) {
gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0}}, gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0}},
}, },
{ {
name: "cuda compute 6.2 unsupported", name: "cuda compute 6.0 supported",
gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 0}},
want: true,
},
{
name: "cuda compute 6.1 supported",
gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 1}},
want: true,
},
{
name: "cuda compute 6.2 supported",
gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}}, gpus: []DeviceInfo{{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}},
want: true,
}, },
{ {
name: "cuda compute 7.2 unsupported", name: "cuda compute 7.2 unsupported",
@@ -216,7 +227,7 @@ func TestFlashAttentionSupported(t *testing.T) {
name: "mixed cuda unsupported", name: "mixed cuda unsupported",
gpus: []DeviceInfo{ gpus: []DeviceInfo{
{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 8, ComputeMinor: 9}, {DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 8, ComputeMinor: 9},
{DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 6, ComputeMinor: 2}, {DeviceID: DeviceID{Library: "CUDA"}, DriverMajor: 12, ComputeMajor: 5, ComputeMinor: 0},
}, },
}, },
{ {