From f22ec2ec49dba452122fed36573de1f2784868de Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Thu, 25 Jun 2026 08:46:00 -0700 Subject: [PATCH] CUDA: require driver 550 or newer for v12 (#16895) Our cuda_v12 build requires nvcc fatbin compression, which in turn requires driver 550 or newer. This change filters incompatible CUDA devices based on the runtime and driver version. This allows users to build from source with older toolkits to support older drivers. Fixes #16449 --- discover/cuda_compat.go | 55 +++++++++++-- discover/cuda_compat_test.go | 145 +++++++++++++++++++++++++++++++++++ docs/gpu.mdx | 2 +- docs/windows.mdx | 2 +- 4 files changed, 195 insertions(+), 9 deletions(-) create mode 100644 discover/cuda_compat_test.go diff --git a/discover/cuda_compat.go b/discover/cuda_compat.go index 2dd16f7c..47ad1a22 100644 --- a/discover/cuda_compat.go +++ b/discover/cuda_compat.go @@ -7,34 +7,66 @@ import ( "github.com/ollama/ollama/ml" ) +const ( + cudaV12RuntimeMajor = 12 + + minFatbinCompressionCUDARuntimeMinor = 4 + minFatbinCompressionNVIDIADriverMajor = 550 + + minLegacyComputeJITCUDARuntimeMinor = 8 + // Older CUDA compute targets need newer drivers when they are JITed from PTX. + minLegacyComputeJITNVIDIADriverMajor = 570 +) + func filterOldCUDADriver(_ context.Context, devices []ml.DeviceInfo) []ml.DeviceInfo { oldCUDA := func(dev ml.DeviceInfo) bool { return dev.Library == "CUDA" && dev.ComputeMajor > 0 && dev.ComputeMajor < 7 } - needsCheck := false + hasCUDA := false for _, dev := range devices { - if oldCUDA(dev) { - needsCheck = true + if dev.Library == "CUDA" { + hasCUDA = true break } } - if !needsCheck { + if !hasCUDA { return devices } driver := nvidiaDriverMajorFromDevices(devices) if driver == 0 { - slog.Warn("could not verify NVIDIA driver compatibility for an older NVIDIA GPU") + slog.Warn("could not verify NVIDIA driver compatibility for CUDA") return devices } - if driver >= 570 { + + // Match the driver floor to the CUDA runtime we are about to load, so source + // builds with older CUDA runtimes can still run on matching older drivers. + runtimeMajor, runtimeMinor, hasRuntime := cudaRuntimeVersionFromDevices(devices) + runtimeMayUseCompressedFatbins := hasRuntime && + runtimeMajor == cudaV12RuntimeMajor && + runtimeMinor >= minFatbinCompressionCUDARuntimeMinor + // CUDA v12.8+ source builds are expected to either use Ollama's PTX packaging + // for older compute targets or be built against a matching local driver/toolkit. + runtimeMayJITLegacyCompute := hasRuntime && + runtimeMajor == cudaV12RuntimeMajor && + runtimeMinor >= minLegacyComputeJITCUDARuntimeMinor + if driver >= minLegacyComputeJITNVIDIADriverMajor || (!runtimeMayUseCompressedFatbins && !runtimeMayJITLegacyCompute) { return devices } filtered := devices[:0] for _, dev := range devices { - if oldCUDA(dev) { + if dev.Library != "CUDA" { + filtered = append(filtered, dev) + continue + } + if runtimeMayUseCompressedFatbins && driver < minFatbinCompressionNVIDIADriverMajor { + slog.Warn("NVIDIA driver too old", + "device", dev.Description, "compute", dev.Compute(), "driver", driver, "required_driver", "550 or newer") + continue + } + if runtimeMayJITLegacyCompute && oldCUDA(dev) { slog.Warn("NVIDIA driver too old", "device", dev.Description, "compute", dev.Compute(), "driver", driver, "required_driver", "570 or newer") continue @@ -52,3 +84,12 @@ func nvidiaDriverMajorFromDevices(devices []ml.DeviceInfo) int { } return 0 } + +func cudaRuntimeVersionFromDevices(devices []ml.DeviceInfo) (int, int, bool) { + for _, dev := range devices { + if dev.Library == "CUDA" { + return cudaRuntimeVersion(dev.LibraryPath) + } + } + return 0, 0, false +} diff --git a/discover/cuda_compat_test.go b/discover/cuda_compat_test.go new file mode 100644 index 00000000..77475a3a --- /dev/null +++ b/discover/cuda_compat_test.go @@ -0,0 +1,145 @@ +package discover + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/ollama/ollama/ml" +) + +func TestFilterOldCUDADriver(t *testing.T) { + tests := []struct { + name string + runtimeLib string + devices []ml.DeviceInfo + wantIDs []string + }{ + { + name: "driver before CUDA compression support filters all CUDA devices", + runtimeLib: "libcudart.so.12.8.90", + devices: []ml.DeviceInfo{ + { + DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"}, + Description: "NVIDIA V100", + ComputeMajor: 7, + ComputeMinor: 0, + NVIDIADriverMajor: 535, + }, + { + DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"}, + Description: "NVIDIA L4", + ComputeMajor: 8, + ComputeMinor: 9, + NVIDIADriverMajor: 535, + }, + { + DeviceID: ml.DeviceID{ID: "GPU-2", Library: "Vulkan"}, + }, + }, + wantIDs: []string{"GPU-2"}, + }, + { + name: "driver before legacy compute support filters only older CUDA devices", + runtimeLib: "libcudart.so.12.8.90", + devices: []ml.DeviceInfo{ + { + DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"}, + Description: "NVIDIA GTX 1080 Ti", + ComputeMajor: 6, + ComputeMinor: 1, + NVIDIADriverMajor: 565, + }, + { + DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"}, + Description: "NVIDIA RTX 6000", + ComputeMajor: 7, + ComputeMinor: 5, + NVIDIADriverMajor: 565, + }, + }, + wantIDs: []string{"GPU-1"}, + }, + { + name: "driver with full support keeps all CUDA devices", + runtimeLib: "libcudart.so.12.8.90", + devices: []ml.DeviceInfo{ + { + DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"}, + Description: "NVIDIA GTX 1080 Ti", + ComputeMajor: 6, + ComputeMinor: 1, + NVIDIADriverMajor: 570, + }, + { + DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"}, + Description: "NVIDIA RTX 6000", + ComputeMajor: 7, + ComputeMinor: 5, + NVIDIADriverMajor: 570, + }, + }, + wantIDs: []string{"GPU-0", "GPU-1"}, + }, + { + name: "source build with older CUDA runtime keeps devices", + runtimeLib: "libcudart.so.12.2.140", + devices: []ml.DeviceInfo{ + { + DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"}, + Description: "NVIDIA V100", + ComputeMajor: 7, + ComputeMinor: 0, + NVIDIADriverMajor: 535, + }, + { + DeviceID: ml.DeviceID{ID: "GPU-1", Library: "CUDA"}, + Description: "NVIDIA GTX 1080 Ti", + ComputeMajor: 6, + ComputeMinor: 1, + NVIDIADriverMajor: 535, + }, + }, + wantIDs: []string{"GPU-0", "GPU-1"}, + }, + { + name: "unknown driver keeps devices", + devices: []ml.DeviceInfo{ + { + DeviceID: ml.DeviceID{ID: "GPU-0", Library: "CUDA"}, + Description: "NVIDIA V100", + ComputeMajor: 7, + ComputeMinor: 0, + }, + }, + wantIDs: []string{"GPU-0"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.runtimeLib != "" { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, tt.runtimeLib), nil, 0o644); err != nil { + t.Fatal(err) + } + for i := range tt.devices { + if tt.devices[i].Library == "CUDA" { + tt.devices[i].LibraryPath = []string{dir} + } + } + } + + got := filterOldCUDADriver(context.Background(), tt.devices) + if len(got) != len(tt.wantIDs) { + t.Fatalf("got %d devices, want %d: %#v", len(got), len(tt.wantIDs), got) + } + for i, wantID := range tt.wantIDs { + if got[i].ID != wantID { + t.Fatalf("device %d ID = %q, want %q", i, got[i].ID, wantID) + } + } + }) + } +} diff --git a/docs/gpu.mdx b/docs/gpu.mdx index 1394fc78..9658283a 100644 --- a/docs/gpu.mdx +++ b/docs/gpu.mdx @@ -3,7 +3,7 @@ title: Hardware support --- ## Nvidia -Ollama supports Nvidia GPUs with compute capability 5.0+ and driver version 531 and newer. +Ollama supports Nvidia GPUs with compute capability 5.0+ and driver version 550 and newer. Nvidia GPUs with compute capability 5.0 through 6.2 require driver version 570 or newer. Check your compute compatibility to see if your card is supported: diff --git a/docs/windows.mdx b/docs/windows.mdx index 297febd5..4e52bf1c 100644 --- a/docs/windows.mdx +++ b/docs/windows.mdx @@ -11,7 +11,7 @@ terminal application. As usual the Ollama [API](/api) will be served on ## System Requirements - Windows 10 22H2 or newer, Home or Pro -- NVIDIA 452.39 or newer Drivers if you have an NVIDIA card +- NVIDIA 551.61 or newer Drivers if you have an NVIDIA card - AMD ROCm v7 / HIP7-capable driver stack for ROCm acceleration, or a Vulkan-capable AMD Radeon driver for Vulkan acceleration Ollama uses unicode characters for progress indication, which may render as unknown squares in some older terminal fonts in Windows 10. If you see this, try changing your terminal font settings.