cudaPackages: overhaul of how we package cuda packages

There are many different versions of the `cudatoolkit` and related
cuda packages, and it can be tricky to ensure they remain compatible.

- `cudaPackages` is now a package set with `cudatoolkit`, `cudnn`, `cutensor`, `nccl`, as well as `cudatoolkit` split into smaller packages ("redist");
- expressions should now use `cudaPackages` as parameter instead of the individual cuda packages;
- `makeScope` is now used, so it is possible to use `.overrideScope'` to set e.g. a different `cudnn` version;
- `release-cuda.nix` is introduced to easily evaluate cuda packages using hydra.

doc/languages-frameworks/cuda.section.md

@@ -0,0 +1,34 @@
+# CUDA {#cuda}
+
+CUDA-only packages are stored in the `cudaPackages` packages set. This set
+includes the `cudatoolkit`, portions of the toolkit in separate derivations,
+`cudnn`, `cutensor` and `nccl`.
+
+A package set is available for each CUDA version, so for example
+`cudaPackages_11_6`. Within each set is a matching version of the above listed
+packages. Additionally, other versions of the packages that are packaged and
+compatible are available as well. For example, there can be a
+`cudaPackages.cudnn_8_3_2` package.
+
+To use one or more CUDA packages in an expression, give the expression a `cudaPackages` parameter, and in case CUDA is optional
+```nix
+cudaSupport ? false
+cudaPackages ? {}
+```
+
+When using `callPackage`, you can choose to pass in a different variant, e.g.
+when a different version of the toolkit suffices
+```nix
+mypkg = callPackage { cudaPackages = cudaPackages_11_5; }
+```
+
+If another version of say `cudnn` or `cutensor` is needed, you can override the
+package set to make it the default. This guarantees you get a consistent package
+set.
+```nix
+mypkg = let
+  cudaPackages = cudaPackages_11_5.overrideScope' (final: prev {
+    cudnn = prev.cudnn_8_3_2;
+  }});
+in callPackage { inherit cudaPackages; };
+```

doc/languages-frameworks/index.xml

@@ -11,6 +11,7 @@
  <xi:include href="bower.section.xml" />
  <xi:include href="coq.section.xml" />
  <xi:include href="crystal.section.xml" />
+ <xi:include href="cuda.section.xml" />
  <xi:include href="dhall.section.xml" />
  <xi:include href="dotnet.section.xml" />
  <xi:include href="emscripten.section.xml" />

nixos/modules/services/misc/ethminer.nix

@@ -85,7 +85,7 @@ in
   config = mkIf cfg.enable {
 
     systemd.services.ethminer = {
-      path = optional (cfg.toolkit == "cuda") [ pkgs.cudatoolkit ];
+      path = optional (cfg.toolkit == "cuda") [ pkgs.cudaPackages.cudatoolkit ];
       description = "ethminer ethereum mining service";
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];

pkgs/applications/misc/blender/default.nix

@@ -5,7 +5,7 @@
 , openvdb, libXxf86vm, tbb, alembic
 , zlib, zstd, fftw, opensubdiv, freetype, jemalloc, ocl-icd, addOpenGLRunpath
 , jackaudioSupport ? false, libjack2
-, cudaSupport ? config.cudaSupport or false, cudatoolkit_11
+, cudaSupport ? config.cudaSupport or false, cudaPackages ? {}
 , colladaSupport ? true, opencollada
 , spaceNavSupport ? stdenv.isLinux, libspnav
 , makeWrapper
@@ -63,7 +63,7 @@ stdenv.mkDerivation rec {
       llvmPackages.openmp SDL Cocoa CoreGraphics ForceFeedback OpenAL OpenGL
     ])
     ++ optional jackaudioSupport libjack2
-    ++ optional cudaSupport cudatoolkit_11
+    ++ optional cudaSupport cudaPackages.cudatoolkit
     ++ optional colladaSupport opencollada
     ++ optional spaceNavSupport libspnav;
   pythonPath = with python310Packages; [ numpy requests ];

pkgs/applications/misc/firestarter/default.nix

@@ -8,11 +8,13 @@
 , glibc
 , git
 , pkg-config
-, cudatoolkit
+, cudaPackages ? {}
 , withCuda ? false
 }:
 
 let
+  inherit (cudaPackages) cudatoolkit;
+
   hwloc = stdenv.mkDerivation rec {
     pname = "hwloc";
     version = "2.2.0";

pkgs/applications/science/math/caffe/default.nix

@@ -13,13 +13,17 @@
 , Accelerate, CoreGraphics, CoreVideo
 , lmdbSupport ? true, lmdb
 , leveldbSupport ? true, leveldb, snappy
-, cudaSupport ? config.cudaSupport or false, cudatoolkit
-, cudnnSupport ? cudaSupport, cudnn ? null
-, ncclSupport ? false, nccl ? null
+, cudaSupport ? config.cudaSupport or false, cudaPackages ? {}
+, cudnnSupport ? cudaSupport
+, ncclSupport ? false
 , pythonSupport ? false, python ? null, numpy ? null
 , substituteAll
 }:
 
+let
+  inherit (cudaPackages) cudatoolkit cudnn nccl;
+in
+
 assert leveldbSupport -> (leveldb != null && snappy != null);
 assert cudnnSupport -> cudaSupport;
 assert ncclSupport -> cudaSupport;

pkgs/applications/science/math/cntk/default.nix

@@ -2,10 +2,14 @@
 , fetchpatch
 , openblas, blas, lapack, opencv3, libzip, boost, protobuf, mpi
 , onebitSGDSupport ? false
-, cudaSupport ? false, addOpenGLRunpath, cudatoolkit, nvidia_x11
-, cudnnSupport ? cudaSupport, cudnn
+, cudaSupport ? false, cudaPackages ? {}, addOpenGLRunpath, cudatoolkit, nvidia_x11
+, cudnnSupport ? cudaSupport
 }:
 
+let
+  inherit (cudaPackages) cudatoolkit cudnn;
+in
+
 assert cudnnSupport -> cudaSupport;
 assert blas.implementation == "openblas" && lapack.implementation == "openblas";
 

pkgs/applications/science/math/mxnet/default.nix

@@ -1,10 +1,14 @@
 { config, stdenv, lib, fetchurl, fetchpatch, bash, cmake
 , opencv3, gtest, blas, gomp, llvmPackages, perl
-, cudaSupport ? config.cudaSupport or false, cudatoolkit, nvidia_x11
-, cudnnSupport ? cudaSupport, cudnn
+, cudaSupport ? config.cudaSupport or false, cudaPackages ? {}, nvidia_x11
+, cudnnSupport ? cudaSupport
 , cudaCapabilities ? [ "3.7" "5.0" "6.0" "7.0" "7.5" "8.0" "8.6" ]
 }:
 
+let
+  inherit (cudaPackages) cudatoolkit cudnn;
+in
+
 assert cudnnSupport -> cudaSupport;
 
 stdenv.mkDerivation rec {

pkgs/development/compilers/cudatoolkit/auto-add-opengl-runpath-hook.sh

@@ -0,0 +1,14 @@
+# Run autoOpenGLRunpath on all files
+echo "Sourcing auto-add-opengl-runpath-hook"
+
+autoAddOpenGLRunpathPhase  () {
+    # TODO: support multiple outputs
+    for file in $(find ${out,lib,bin} -type f); do
+        addOpenGLRunpath $file
+    done
+}
+
+if [ -z "${dontUseAutoAddOpenGLRunpath-}" ]; then
+    echo "Using autoAddOpenGLRunpathPhase"
+    postFixupHooks+=(autoAddOpenGLRunpathPhase)
+fi

pkgs/development/compilers/cudatoolkit/common.nix

@@ -254,6 +254,7 @@ stdenv.mkDerivation rec {
   '';
   passthru = {
     cc = gcc;
+    majorMinorVersion = lib.versions.majorMinor version;
     majorVersion = lib.versions.majorMinor version;
   };
 

pkgs/development/compilers/cudatoolkit/extension.nix

@@ -0,0 +1,15 @@
+final: prev: let
+  ### Cuda Toolkit
+
+  # Function to build the class cudatoolkit package
+  buildCudaToolkitPackage = final.callPackage ./common.nix;
+
+  # Version info for the classic cudatoolkit packages that contain everything that is in redist.
+  cudatoolkitVersions = final.lib.importTOML ./versions.toml;
+
+  ### Add classic cudatoolkit package
+  cudatoolkit = buildCudaToolkitPackage ((attrs: attrs // { gcc = prev.pkgs.${attrs.gcc}; }) cudatoolkitVersions.${final.cudaVersion});
+
+in {
+  inherit cudatoolkit;
+}

pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix

@@ -0,0 +1,51 @@
+{ lib
+, stdenv
+, fetchurl
+, autoPatchelfHook
+, autoAddOpenGLRunpathHook
+}:
+
+pname:
+attrs:
+
+let
+  arch = "linux-x86_64";
+in stdenv.mkDerivation {
+  inherit pname;
+  inherit (attrs) version;
+
+  src = assert (lib.hasAttr arch attrs); fetchurl {
+    url = "https://developer.download.nvidia.com/compute/cuda/redist/${attrs.${arch}.relative_path}";
+    inherit (attrs.${arch}) sha256;
+  };
+
+  nativeBuildInputs = [
+    autoPatchelfHook
+    # This hook will make sure libcuda can be found
+    # in typically /lib/opengl-driver by adding that
+    # directory to the rpath of all ELF binaries.
+    # Check e.g. with `patchelf --print-rpath path/to/my/binary
+    autoAddOpenGLRunpathHook
+  ];
+
+  buildInputs = [
+    stdenv.cc.cc.lib
+  ];
+
+  dontBuild = true;
+
+  # TODO: choose whether to install static/dynamic libs
+  installPhase = ''
+    runHook preInstall
+    rm LICENSE
+    mkdir -p $out
+    mv * $out
+    runHook postInstall
+  '';
+
+  meta = {
+    description = attrs.name;
+    license = lib.licenses.unfree;
+    platforms = lib.optionals (lib.hasAttr arch attrs) [ "x86_64-linux" ];
+  };
+}

pkgs/development/compilers/cudatoolkit/redist/extension.nix

@@ -0,0 +1,32 @@
+final: prev: let
+
+  inherit (final) callPackage;
+  inherit (prev) cudaVersion lib pkgs;
+
+  ### Cuda Toolkit Redist
+
+  # Manifest files for redist cudatoolkit. These can be found at
+  # https://developer.download.nvidia.com/compute/cuda/redist/
+  cudaToolkitRedistManifests = {
+    "11.4" = ./manifests/redistrib_11.4.4.json;
+    "11.5" = ./manifests/redistrib_11.5.2.json;
+    "11.6" = ./manifests/redistrib_11.6.2.json;
+  };
+
+  # Function to build a single cudatoolkit redist package
+  buildCudaToolkitRedistPackage = callPackage ./build-cuda-redist-package.nix { };
+
+  # Function that builds all cudatoolkit redist packages given a cuda version and manifest file
+  buildCudaToolkitRedistPackages = { version, manifest }: let
+    attrs = lib.filterAttrs (key: value: key != "release_date") (lib.importJSON manifest);
+  in lib.mapAttrs buildCudaToolkitRedistPackage attrs;
+
+  redistExists = cudaToolkitRedistManifests ? "${cudaVersion}";
+
+  # All cudatoolkit redist packages for the current cuda version
+  cudaToolkitRedistPackages = if
+    lib.hasAttr cudaVersion cudaToolkitRedistManifests
+  then buildCudaToolkitRedistPackages { version = cudaVersion; manifest = cudaToolkitRedistManifests.${cudaVersion}; }
+  else {};
+
+in cudaToolkitRedistPackages