From 0ce6406573fba73b249b8e0aa72e1ed9ee1827b5 Mon Sep 17 00:00:00 2001 From: Dennis Gosnell Date: Thu, 9 Feb 2023 21:20:53 +0900 Subject: [PATCH 1/5] Make sure all nix result files are in gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 81465f4..1e980a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Nix result files -/result +result +result-* # Haskell .gitignore dist From 4268d783453f853f70615f87c83b06caa1e8746e Mon Sep 17 00:00:00 2001 From: Dennis Gosnell Date: Thu, 9 Feb 2023 21:29:04 +0900 Subject: [PATCH 2/5] Add newPkgSet and newPkgSetDevShell output attributes. --- .../stacklock2nix/cabal2nixArgsForPkg.nix | 31 ++++ nix/build-support/stacklock2nix/default.nix | 134 +++++++++++++++++- .../stacklock2nix/suggestedOverlay.nix | 8 +- 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/nix/build-support/stacklock2nix/cabal2nixArgsForPkg.nix b/nix/build-support/stacklock2nix/cabal2nixArgsForPkg.nix index a7ce78f..fe9496f 100644 --- a/nix/build-support/stacklock2nix/cabal2nixArgsForPkg.nix +++ b/nix/build-support/stacklock2nix/cabal2nixArgsForPkg.nix @@ -36,24 +36,55 @@ # for testing, but this is not necessary to build the Haskell library. # # Please feel free to send PRs adding necessary overrides here. +# +# Make sure to keep this list in alphabetical order. cabal2nixArgsOverrides { "gi-cairo" = ver: { cairo = pkgs.cairo; }; + "gi-gdk" = ver: { gtk3 = pkgs.gtk3; }; + "gi-gio" = ver: { glib = pkgs.glib; }; + "gi-glib" = ver: { glib = pkgs.glib; }; + "gi-gtk" = ver: { gtk3 = pkgs.gtk3; }; + "gi-gmodule" = ver: { gmodule = null; }; + "gi-gobject" = ver: { glib = pkgs.glib; }; + "gi-harfbuzz" = ver: { harfbuzz-gobject = null; }; + "gi-pango" = ver: { cairo = pkgs.cairo; pango = pkgs.pango; }; + "gi-vte" = ver: { vte_291 = pkgs.vte; }; + "glib" = ver: { glib = pkgs.glib; }; + "haskell-gi" = ver: { glib = pkgs.glib; gobject-introspection = pkgs.gobject-introspection; }; + "haskell-gi-base" = ver: { glib = pkgs.glib; }; + + # The PSQueue and fingertree-psqueue packages are used in benchmarks, but they are not on Stackage. + "psqueues" = ver: { fingertree-psqueue = null; PSQueue = null; }; + "saltine" = ver: { libsodium = pkgs.libsodium; }; + "secp256k1-haskell" = ver: { secp256k1 = pkgs.secp256k1; }; + "splitmix" = ver: { testu01 = null; }; + + "test-framework" = ver: { libxml = pkgs.libxml; }; + "termonad" = ver: { vte_291 = pkgs.vte; gtk3 = pkgs.gtk3; pcre2 = pkgs.pcre2;}; + + # unordered-containers uses the Haskell package nothunks in its test-suite, + # but nothunks is not in Stackage. We disable the tests for unordered-containers + # in the suggestedOverlay.nix file, but callCabal2Nix is called on it before + # suggestedOverlay.nix is applied. So here we need to just pass in null for + # the nothunks dependency, since it won't end up being used. + "unordered-containers" = ver: { nothunks = null; }; + "zlib" = ver: { zlib = pkgs.zlib; }; } diff --git a/nix/build-support/stacklock2nix/default.nix b/nix/build-support/stacklock2nix/default.nix index 0e16c7a..25b8439 100644 --- a/nix/build-support/stacklock2nix/default.nix +++ b/nix/build-support/stacklock2nix/default.nix @@ -5,6 +5,7 @@ , lib , runCommand , stdenv +, path }@topargs: { # The path to your stack.yaml file. @@ -77,6 +78,10 @@ # This is not used if `baseHaskellPkgSet` is `null`. all-cabal-hashes ? null , callPackage ? topargs.callPackage +, # Path to Nixpkgs. + # + # nixpkgsPath :: Path + nixpkgsPath ? topargs.path }: # The stack.yaml path can be computed from the stack.yaml.lock path, or @@ -372,7 +377,7 @@ let # This roughly returns an overlay that looks like the following: # ``` # { lens = callHackage "lens" "5.0.1" {}; - # conduit = callHackage "conduit" "5.0.1" {}; + # conduit = callHackage "conduit" "1.3.4" {}; # } # ``` haskPkgLocksToOverlay = haskPkgLocks: hfinal: hprev: @@ -635,6 +640,15 @@ let inherit all-cabal-hashes; }); + devShellForPkgSet = packageSet: + if packageSet == null then + null + else + packageSet.shellFor { + packages = localPkgsSelector; + nativeBuildInputs = additionalDevShellNativeBuildInputs packageSet; + }; + # A development shell created by passing all your local packages (from # `localPkgsSelector`) to `pkgSet.shellFor`. # @@ -642,15 +656,119 @@ let # # Note that this derivation is specifically meant to be passed to `nix # develop` or `nix-shell`. - devShell = - if pkgSet == null then + devShell = devShellForPkgSet pkgSet; + + # An Nixpkgs Haskell overlay that has GHC boot packages set to `null`. This + # is used as an initial overlay when creating a brand new package set. + newPkgSetCompilerConfig = self: super: { + # TODO: Should llvmPackages be enabled here? + # llvmPackages = pkgs.lib.dontRecurseIntoAttrs self.ghc.llvmPackages; + + # Disable GHC core libraries. + array = null; + base = null; + binary = null; + bytestring = null; + Cabal = null; + containers = null; + deepseq = null; + directory = null; + exceptions = null; + filepath = null; + ghc-bignum = null; + ghc-boot = null; + ghc-boot-th = null; + ghc-compact = null; + ghc-heap = null; + ghc-prim = null; + ghci = null; + haskeline = null; + hpc = null; + integer-gmp = null; + libiserv = null; + mtl = null; + parsec = null; + pretty = null; + process = null; + rts = null; + stm = null; + template-haskell = null; + + # GHC only builds terminfo if it is a native compiler + # terminfo = if pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform then null else self.terminfo_0_4_1_5; + terminfo = null; + + text = null; + time = null; + transformers = null; + unix = null; + + # GHC only bundles the xhtml library if haddock is enabled, check if this is + # still the case when updating: https://gitlab.haskell.org/ghc/ghc/-/blob/0198841877f6f04269d6050892b98b5c3807ce4c/ghc.mk#L463 + # xhtml = if self.ghc.hasHaddock or true then null else self.xhtml_3000_2_2_1; + xhtml = null; + }; + + # This is similar to `pkgSet`. + # + # While `pkgSet` is `baseHaskellPkgSet` overridden with overlays from your + # stack.yaml.lock file, `newPkgSet` is a completely new Nixpkgs Haskell + # package set. It _only_ contains packages defined in the `stack.yaml` file. + # + # newPkgSet :: HaskellPkgSet + # + # `newPkgSet` will contain local packages. For instance, if you have a local + # package called `my-haskell-pkg`: + # + # Example: `newPkgSet.my-haskell-pkg` + # + # `newPkgSet` will also contain packages in your stack.yaml resolver. For + # instance: + # + # Example: `newPkgSet.lens` + # + # `pkgSet` will _not_ contain packages from the underlying Haskell package + # set. For instance, `termonad` is not available in Stackage, so it is not + # available in `newPkgSet`. + newPkgSet = + if baseHaskellPkgSet == null then null else - pkgSet.shellFor { - packages = localPkgsSelector; - nativeBuildInputs = additionalDevShellNativeBuildInputs pkgSet; - }; + let + haskPkgSet = callPackage (nixpkgsPath + "/pkgs/development/haskell-modules") { + haskellLib = haskell.lib.compose; + + # TODO: Is it okay to use a completely different package set as the + # base package set like this? + buildHaskellPackages = baseHaskellPkgSet; + + ghc = baseHaskellPkgSet.ghc; + + compilerConfig = newPkgSetCompilerConfig; + + initialPackages = _: _: {}; + + overrides = lib.composeManyExtensions [ + # It is not possible to put these overlays into the + # `initialPackages` argument, because they use functions like + # `callHackage` and `callCabal2nix`, which appear to not be + # available when `initialPackages` gets evaluated. + combinedOverlay + additionalHaskellPkgSetOverrides + ]; + + nonHackagePackages = _: _: {}; + configurationCommon = _: _: _: {}; + configurationNix = _: _: _: {}; + configurationArm = _: _: _: {}; + configurationDarwin = _: _: _: {}; + }; + in haskPkgSet; + # Same as `devShell`, but based on `newPkgSet`. + # + # newPkgSetDevShell :: Drv + newPkgSetDevShell = devShellForPkgSet newPkgSet; in { inherit @@ -662,6 +780,8 @@ in localPkgsSelector pkgSet devShell + newPkgSet + newPkgSetDevShell ; # These are a bunch of internal attributes, used for testing. diff --git a/nix/build-support/stacklock2nix/suggestedOverlay.nix b/nix/build-support/stacklock2nix/suggestedOverlay.nix index ab56d85..f9ae3e5 100644 --- a/nix/build-support/stacklock2nix/suggestedOverlay.nix +++ b/nix/build-support/stacklock2nix/suggestedOverlay.nix @@ -28,6 +28,8 @@ # ``` # # These will be maintained on a best-effort basis. Again, please send PRs. +# +# Make sure to keep this list in alphabetical order. hfinal: hprev: with haskell.lib.compose; { @@ -155,9 +157,6 @@ hfinal: hprev: with haskell.lib.compose; { nanospec = dontCheck hprev.nanospec; - # test suite doesn't build - nothunks = dontCheck hprev.nothunks; - # circular dependency in tests options = dontCheck hprev.options; @@ -213,6 +212,9 @@ hfinal: hprev: with haskell.lib.compose; { # tests don't support musl unix-time = dontCheck hprev.unix-time; + # Test suite requires the nothunks library, which isn't on stackage. + unordered-containers = dontCheck hprev.unordered-containers; + vector = dontCheck hprev.vector; # test suite uses phantom js From e9af3b818722d7742c89aaaa0b2cb27bd1c08c7f Mon Sep 17 00:00:00 2001 From: Dennis Gosnell Date: Thu, 9 Feb 2023 21:24:51 +0900 Subject: [PATCH 3/5] Add tests for the newPkgSet feature --- test/default.nix | 4 ++++ test/nixpkgs.nix | 30 ++++++++++++++++++++++++ test/test-new-package-set.nix | 43 +++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 test/default.nix create mode 100644 test/nixpkgs.nix create mode 100644 test/test-new-package-set.nix diff --git a/test/default.nix b/test/default.nix new file mode 100644 index 0000000..e3ea774 --- /dev/null +++ b/test/default.nix @@ -0,0 +1,4 @@ + +{...}: + +(import ./nixpkgs.nix {}).stacklock2nix-tests diff --git a/test/nixpkgs.nix b/test/nixpkgs.nix new file mode 100644 index 0000000..8ef4b6a --- /dev/null +++ b/test/nixpkgs.nix @@ -0,0 +1,30 @@ + +{...}: + +let + flake-lock = builtins.fromJSON (builtins.readFile ../my-example-haskell-lib-advanced/flake.lock); + + # Use the Nixpkgs that the my-example-haskell-lib-advanced is pinned to. + nixpkgs-src = builtins.fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/${flake-lock.nodes.nixpkgs.locked.rev}.tar.gz"; + sha256 = flake-lock.nodes.nixpkgs.locked.narHash; + }; + + overlays = [ + # stacklock2nix overlay + (import ../nix/overlay.nix) + + # tests + (final: prev: { + stacklock2nix-tests = + (final.callPackage ./test-new-package-set.nix {}) ++ [ + # new tests go here + ]; + }) + ]; + + pkgs = import nixpkgs-src { inherit overlays; }; + +in + +pkgs diff --git a/test/test-new-package-set.nix b/test/test-new-package-set.nix new file mode 100644 index 0000000..be9ffb2 --- /dev/null +++ b/test/test-new-package-set.nix @@ -0,0 +1,43 @@ + +{ stacklock2nix +, haskell +, cabal-install +, fetchurl +}: + +let + hasklib = haskell.lib.compose; + + stacklock = stacklock2nix { + stackYaml = ../my-example-haskell-lib-advanced/stack.yaml; + baseHaskellPkgSet = haskell.packages.ghc924; + additionalHaskellPkgSetOverrides = hfinal: hprev: { + # The servant-cassava.cabal file is malformed on GitHub: + # https://github.com/haskell-servant/servant-cassava/pull/29 + servant-cassava = + hasklib.overrideCabal + { editedCabalFile = null; revision = null; } + hprev.servant-cassava; + + amazonka = hasklib.dontCheck hprev.amazonka; + amazonka-core = hasklib.dontCheck hprev.amazonka-core; + amazonka-sso = hasklib.dontCheck hprev.amazonka-sso; + amazonka-sts = hasklib.dontCheck hprev.amazonka-sts; + }; + cabal2nixArgsOverrides = args: args // { + amazonka-sso = ver: { amazonka-test = null; }; + amazonka-sts = ver: { amazonka-test = null; }; + }; + additionalDevShellNativeBuildInputs = stacklockHaskellPkgSet: [ + cabal-install + ]; + all-cabal-hashes = fetchurl { + name = "all-cabal-hashes"; + url = "https://github.com/commercialhaskell/all-cabal-hashes/archive/9ab160f48cb535719783bc43c0fbf33e6d52fa99.tar.gz"; + sha256 = "sha256-QC07T3MEm9LIMRpxIq3Pnqul60r7FpAdope6S62sEX8="; + }; + }; +in +[ stacklock.newPkgSet.my-example-haskell-app + stacklock.newPkgSetDevShell +] From 2b3c2eef8b10852fff08b2220f647a360bb8a71d Mon Sep 17 00:00:00 2001 From: Dennis Gosnell Date: Thu, 9 Feb 2023 22:44:06 +0900 Subject: [PATCH 4/5] Add tests to CI --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6bd71b8..835e7ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,10 @@ jobs: run: "nix develop -L --command cabal test all" working-directory: "./my-example-haskell-lib-advanced/" + - name: "Tests" + run: "nix-build" + working-directory: "./test/" + nix-commit-from-flake-lock: name: nix / ubuntu-latest / using stacklock2nix commit from flake.lock runs-on: ubuntu-latest From 867cd13a20167efb6b45af0188cc8843f0c56911 Mon Sep 17 00:00:00 2001 From: Dennis Gosnell Date: Fri, 10 Feb 2023 08:41:19 +0900 Subject: [PATCH 5/5] Add changelog entry for v1.1.0 --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c386d35..92508f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,32 @@ +## 1.1.0 + +* Added two new attributes to the attribute set returned from a call to + `stacklock2nix`: `newPkgSet` and `newPkgSetDevShell`. These two values are + similar to the existing `pkgSet` and `devShell` attributes. Whereas + `pkgSet` and `devShell` take the `baseHaskellPkgSet` argument and overlay + it with package overrides created from your `stack.yaml` file, `newPkgSet` + and `newPkgSetDevShell` are a completely new package set, containing _only_ + packages from your `stack.yaml`. + + The effect of this is that `pkgSet` will contain packages that are in + Nixpkgs, but not in Stackage. For instance, when using `pkgSet`, you + should be able to access the package + [`pkgSet.termonad`](https://hackage.haskell.org/package/termonad) because + it is available on Hackage (and in Nixpkgs), even though it is not in any + Stackage resolver. + + However, `newPkgSet` will only contain packages in your `stack.yaml` file. + For instance, you'll never be able to access `newPkgSet.termonad` or + `newPkgSet.spago`, because they will likely never be available on Stackage. + + In general, in your own projects, should you use `pkgSet` or `newPkgSet`? + + For building your own projects, most of the time `pkgSet` and `newPkgSet` + should be similar. `newPkgSet` may be slightly safer, since there is + almost no chance you accidentally use a Haskell package outside of your + `stack.yaml`. `pkgSet` may be slightly more convenient depending on what + you're trying to do. + ## 1.0.0 * This is the 1.0 release of `stacklock2nix`. I've tested `stacklock2nix` on