From 1971f92eedbef1993c05cb25d60713e718f67ccc Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 1 Oct 2018 11:36:51 +0800 Subject: [PATCH 1/3] [nix] Split overlays into their own files This is a pure refactoring of the nix logic. The primary goal is to modularize the overlays and subsequently make rebasing easier by removing these lines from the `default.nix`. --- default.nix | 112 ++++------------------------------ nix/overlays/benchmark.nix | 11 ++++ nix/overlays/debug.nix | 7 +++ nix/overlays/dont-check.nix | 5 ++ nix/overlays/faster-build.nix | 5 ++ nix/overlays/metric.nix | 5 ++ nix/overlays/required.nix | 62 +++++++++++++++++++ 7 files changed, 108 insertions(+), 99 deletions(-) create mode 100644 nix/overlays/benchmark.nix create mode 100644 nix/overlays/debug.nix create mode 100644 nix/overlays/dont-check.nix create mode 100644 nix/overlays/faster-build.nix create mode 100644 nix/overlays/metric.nix create mode 100644 nix/overlays/required.nix diff --git a/default.nix b/default.nix index ba9601a3e08..2a8534a0ef4 100644 --- a/default.nix +++ b/default.nix @@ -29,111 +29,25 @@ with pkgs.lib; with pkgs.haskell.lib; let - justStaticExecutablesGitRev = import ./scripts/set-git-rev { - inherit pkgs gitrev; - inherit (cardanoPkgs) ghc; - }; - addRealTimeTestLogs = drv: overrideCabal drv (attrs: { - testTarget = "--show-details=streaming"; + # the GHC we are using + # at some point use: pkgs.haskell.compiler.ghc843; + ghc = overrideDerivation pkgs.haskell.compiler.ghc822 (drv: { + patches = drv.patches ++ [ ./ghc-8.0.2-darwin-rec-link.patch ]; }); - requiredOverlay = self: super: { - inherit pkgs; - srcroot = ./.; - cardano-sl-core = overrideCabal super.cardano-sl-core (drv: { - configureFlags = (drv.configureFlags or []) ++ [ - "-f-asserts" - ]; - }); - cardano-sl = overrideCabal super.cardano-sl (drv: { - # production full nodes shouldn't use wallet as it means different constants - configureFlags = (drv.configureFlags or []) ++ [ - "-f-asserts" - ]; - passthru = { - inherit enableProfiling; - }; - }); - cardano-sl-wallet-static = justStaticExecutablesGitRev self.cardano-sl-wallet; - cardano-sl-client = addRealTimeTestLogs super.cardano-sl-client; - cardano-sl-generator = addRealTimeTestLogs super.cardano-sl-generator; - cardano-sl-networking = addRealTimeTestLogs super.cardano-sl-networking; - cardano-sl-auxx-static = justStaticExecutablesGitRev self.cardano-sl-auxx; - cardano-sl-wallet-new-static = justStaticExecutablesGitRev self.cardano-sl-wallet-new; - cardano-sl-node-static = justStaticExecutablesGitRev self.cardano-sl-node; - cardano-sl-explorer-static = justStaticExecutablesGitRev self.cardano-sl-explorer; - cardano-report-server-static = justStaticExecutablesGitRev self.cardano-report-server; - cardano-sl-faucet-static = justStaticExecutablesGitRev self.cardano-sl-faucet; - cardano-sl-tools-static = justStaticExecutablesGitRev self.cardano-sl-tools; - # Undo configuration-nix.nix change to hardcode security binary on darwin - # This is needed for macOS binary not to fail during update system (using http-client-tls) - # Instead, now the binary is just looked up in $PATH as it should be installed on any macOS - x509-system = overrideDerivation super.x509-system (drv: { - postPatch = ":"; - }); - - # TODO: get rid of pthreads option once cryptonite 0.25 is released - # DEVOPS-393: https://github.com/haskell-crypto/cryptonite/issues/193 - cryptonite = appendPatch (appendConfigureFlag super.cryptonite "--ghc-option=-optl-pthread") ./pkgs/cryptonite-segfault-blake.patch; - - # Due to https://github.com/input-output-hk/stack2nix/issues/56 - hfsevents = self.callPackage ./pkgs/hfsevents.nix { inherit (pkgs.darwin.apple_sdk.frameworks) Cocoa CoreServices; }; - mkDerivation = args: super.mkDerivation (args // { - enableLibraryProfiling = enableProfiling; - enableExecutableProfiling = enableProfiling; - # Static linking for everything to work around - # https://ghc.haskell.org/trac/ghc/ticket/14444 - # This will be the default in nixpkgs since - # https://github.com/NixOS/nixpkgs/issues/29011 - enableSharedExecutables = false; - } // optionalAttrs (args ? src) { - src = localLib.cleanSourceTree args.src; - }); - }; - benchmarkOverlay = self: super: { - mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { - # Enables building but not running of benchmarks for all - # cardano-sl packages when enableBenchmarks argument is true. - doBenchmark = true; - configureFlags = (args.configureFlags or []) ++ ["--enable-benchmarks"]; - } // optionalAttrs (localLib.isBenchmark args) { - # Provide a dummy installPhase for benchmark packages. - installPhase = "mkdir -p $out"; - }); - }; - - debugOverlay = self: super: { - mkDerivation = args: super.mkDerivation (args // { - # TODO: DEVOPS-355 - dontStrip = true; - configureFlags = (args.configureFlags or []) ++ [ "--ghc-options=-g --disable-executable-stripping --disable-library-stripping" "--profiling-detail=toplevel-functions"]; - }); - }; - + # Overlay logic for *haskell* packages. + requiredOverlay = import ./nix/overlays/required.nix pkgs localLib enableProfiling gitrev ghc; + benchmarkOverlay = import ./nix/overlays/benchmark.nix pkgs localLib; + debugOverlay = import ./nix/overlays/debug.nix pkgs; # Disabling optimization for cardano-sl packages will # return a build ~20% faster (measured in DEVOPS-1032). - fasterBuildOverlay = self: super: { - mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { - configureFlags = (args.configureFlags or []) ++ [ "--ghc-options=-O0" ]; - }); - }; - - dontCheckOverlay = self: super: { - mkDerivation = args: super.mkDerivation (args // { - doCheck = false; - }); - }; - - metricOverlay = self: super: { - mkDerivation = args: super.mkDerivation (args // { - enablePhaseMetrics = true; - }); - }; + fasterBuildOverlay = import ./nix/overlays/faster-build.nix pkgs localLib; + dontCheckOverlay = import ./nix/overlays/dont-check.nix pkgs; + metricOverlay = import ./nix/overlays/metric.nix pkgs; + # This will yield a set of haskell packages, based on the given compiler. cardanoPkgsBase = ((import ./pkgs { inherit pkgs; }).override { - ghc = overrideDerivation pkgs.haskell.compiler.ghc822 (drv: { - patches = drv.patches ++ [ ./ghc-8.0.2-darwin-rec-link.patch ]; - }); + inherit ghc; }); activeOverlays = [ requiredOverlay ] diff --git a/nix/overlays/benchmark.nix b/nix/overlays/benchmark.nix new file mode 100644 index 00000000000..35c68fedf38 --- /dev/null +++ b/nix/overlays/benchmark.nix @@ -0,0 +1,11 @@ +pkgs: localLib: self: super: with pkgs.lib; { + mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { + # Enables building but not running of benchmarks for all + # cardano-sl packages when enableBenchmarks argument is true. + doBenchmark = true; + configureFlags = (args.configureFlags or []) ++ ["--enable-benchmarks"]; + } // optionalAttrs (localLib.isBenchmark args) { + # Provide a dummy installPhase for benchmark packages. + installPhase = "mkdir -p $out"; + }); + } diff --git a/nix/overlays/debug.nix b/nix/overlays/debug.nix new file mode 100644 index 00000000000..24182eb620b --- /dev/null +++ b/nix/overlays/debug.nix @@ -0,0 +1,7 @@ +pkgs: self: super: { + mkDerivation = args: super.mkDerivation (args // { + # TODO: DEVOPS-355 + dontStrip = true; + configureFlags = (args.configureFlags or []) ++ [ "--ghc-options=-g --disable-executable-stripping --disable-library-stripping" "--profiling-detail=toplevel-functions"]; + }); + } diff --git a/nix/overlays/dont-check.nix b/nix/overlays/dont-check.nix new file mode 100644 index 00000000000..017a22a1ab4 --- /dev/null +++ b/nix/overlays/dont-check.nix @@ -0,0 +1,5 @@ +pkgs: self: super: { + mkDerivation = args: super.mkDerivation (args // { + doCheck = false; + }); + } diff --git a/nix/overlays/faster-build.nix b/nix/overlays/faster-build.nix new file mode 100644 index 00000000000..611d54100e9 --- /dev/null +++ b/nix/overlays/faster-build.nix @@ -0,0 +1,5 @@ +pkgs: localLib: self: super: { + mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { + configureFlags = (args.configureFlags or []) ++ [ "--ghc-options=-O0" ]; + }); + } diff --git a/nix/overlays/metric.nix b/nix/overlays/metric.nix new file mode 100644 index 00000000000..675ab32b06c --- /dev/null +++ b/nix/overlays/metric.nix @@ -0,0 +1,5 @@ +pkgs: self: super: { + mkDerivation = args: super.mkDerivation (args // { + enablePhaseMetrics = true; + }); + } diff --git a/nix/overlays/required.nix b/nix/overlays/required.nix new file mode 100644 index 00000000000..9a889b4f90f --- /dev/null +++ b/nix/overlays/required.nix @@ -0,0 +1,62 @@ +pkgs: localLib: enableProfiling: gitrev: ghc: +self: super: +with pkgs.haskell.lib; with pkgs.lib; +let + justStaticExecutablesGitRev = import ../../scripts/set-git-rev { + inherit pkgs gitrev ghc; + }; + addRealTimeTestLogs = drv: overrideCabal drv (attrs: { + testTarget = "--show-details=streaming"; + }); +in { + srcroot = ./.; + cardano-sl-core = overrideCabal super.cardano-sl-core (drv: { + configureFlags = (drv.configureFlags or []) ++ [ + "-f-asserts" + ]; + }); + cardano-sl = overrideCabal super.cardano-sl (drv: { + # production full nodes shouldn't use wallet as it means different constants + configureFlags = (drv.configureFlags or []) ++ [ + "-f-asserts" + ]; + passthru = { + inherit enableProfiling; + }; + }); + cardano-sl-wallet-static = justStaticExecutablesGitRev super.cardano-sl-wallet; + cardano-sl-client = addRealTimeTestLogs super.cardano-sl-client; + cardano-sl-generator = addRealTimeTestLogs super.cardano-sl-generator; + cardano-sl-networking = addRealTimeTestLogs super.cardano-sl-networking; + cardano-sl-auxx-static = justStaticExecutablesGitRev super.cardano-sl-auxx; + cardano-sl-wallet-new-static = justStaticExecutablesGitRev super.cardano-sl-wallet-new; + cardano-sl-node-static = justStaticExecutablesGitRev self.cardano-sl-node; + cardano-sl-explorer-static = justStaticExecutablesGitRev self.cardano-sl-explorer; + cardano-report-server-static = justStaticExecutablesGitRev self.cardano-report-server; + cardano-sl-faucet-static = justStaticExecutablesGitRev self.cardano-sl-faucet; + cardano-sl-tools-static = justStaticExecutablesGitRev super.cardano-sl-tools; + # Undo configuration-nix.nix change to hardcode security binary on darwin + # This is needed for macOS binary not to fail during update system (using http-client-tls) + # Instead, now the binary is just looked up in $PATH as it should be installed on any macOS + x509-system = overrideDerivation super.x509-system (drv: { + postPatch = ":"; + }); + + # TODO: get rid of pthreads option once cryptonite 0.25 is released + # DEVOPS-393: https://github.com/haskell-crypto/cryptonite/issues/193 + cryptonite = appendPatch (appendConfigureFlag super.cryptonite "--ghc-option=-optl-pthread") ../../pkgs/cryptonite-segfault-blake.patch; + + # Due to https://github.com/input-output-hk/stack2nix/issues/56 + hfsevents = self.callPackage ../../pkgs/hfsevents.nix { inherit (pkgs.darwin.apple_sdk.frameworks) Cocoa CoreServices; }; + mkDerivation = args: super.mkDerivation (args // { + enableLibraryProfiling = enableProfiling; + enableExecutableProfiling = enableProfiling; + # Static linking for everything to work around + # https://ghc.haskell.org/trac/ghc/ticket/14444 + # This will be the default in nixpkgs since + # https://github.com/NixOS/nixpkgs/issues/29011 + enableSharedExecutables = false; + } // optionalAttrs (args ? src) { + src = localLib.cleanSourceTree args.src; + }); + } From 2d0604ada14954154a5720980bb77e858087cc99 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Tue, 2 Oct 2018 13:57:22 +1000 Subject: [PATCH 2/3] nix overlays: Fix build and use nixpkgs style - Provide pkgs as top-level attribute so that it can be shared. - Fix build of set-git-rev - Use the overlay function arguments style of where arguments other than self and super are named. --- default.nix | 17 ++++++------- nix/overlays/benchmark.nix | 10 +++++--- nix/overlays/debug.nix | 4 ++- nix/overlays/dont-check.nix | 4 ++- nix/overlays/faster-build.nix | 7 +++++- nix/overlays/metric.nix | 4 ++- nix/overlays/required.nix | 47 +++++++++++++++++++++++------------ 7 files changed, 61 insertions(+), 32 deletions(-) diff --git a/default.nix b/default.nix index 2a8534a0ef4..a70b63318ef 100644 --- a/default.nix +++ b/default.nix @@ -26,7 +26,6 @@ in }: with pkgs.lib; -with pkgs.haskell.lib; let # the GHC we are using @@ -36,14 +35,13 @@ let }); # Overlay logic for *haskell* packages. - requiredOverlay = import ./nix/overlays/required.nix pkgs localLib enableProfiling gitrev ghc; - benchmarkOverlay = import ./nix/overlays/benchmark.nix pkgs localLib; - debugOverlay = import ./nix/overlays/debug.nix pkgs; - # Disabling optimization for cardano-sl packages will - # return a build ~20% faster (measured in DEVOPS-1032). - fasterBuildOverlay = import ./nix/overlays/faster-build.nix pkgs localLib; - dontCheckOverlay = import ./nix/overlays/dont-check.nix pkgs; - metricOverlay = import ./nix/overlays/metric.nix pkgs; + requiredOverlay = import ./nix/overlays/required.nix { inherit pkgs localLib enableProfiling gitrev; + inherit (cardanoPkgs) ghc; }; + benchmarkOverlay = import ./nix/overlays/benchmark.nix { inherit pkgs localLib; }; + debugOverlay = import ./nix/overlays/debug.nix { inherit pkgs; }; + fasterBuildOverlay = import ./nix/overlays/faster-build.nix { inherit pkgs localLib; }; + dontCheckOverlay = import ./nix/overlays/dont-check.nix { inherit pkgs; }; + metricOverlay = import ./nix/overlays/metric.nix { inherit pkgs; }; # This will yield a set of haskell packages, based on the given compiler. cardanoPkgsBase = ((import ./pkgs { inherit pkgs; }).override { @@ -64,6 +62,7 @@ let in args: pkgs.callPackage ./scripts/launch/connect-to-cluster (args // { inherit gitrev useStackBinaries; } // walletConfig ); other = rec { + inherit pkgs; testlist = innerClosePropagation [] [ cardanoPkgs.cardano-sl ]; walletIntegrationTests = pkgs.callPackage ./scripts/test/wallet/integration { inherit gitrev useStackBinaries; }; validateJson = pkgs.callPackage ./tools/src/validate-json {}; diff --git a/nix/overlays/benchmark.nix b/nix/overlays/benchmark.nix index 35c68fedf38..08d20fcbbe8 100644 --- a/nix/overlays/benchmark.nix +++ b/nix/overlays/benchmark.nix @@ -1,10 +1,14 @@ -pkgs: localLib: self: super: with pkgs.lib; { - mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { +{ pkgs, localLib }: + +with localLib; + +self: super: { + mkDerivation = args: super.mkDerivation (args // optionalAttrs (isCardanoSL args.pname) { # Enables building but not running of benchmarks for all # cardano-sl packages when enableBenchmarks argument is true. doBenchmark = true; configureFlags = (args.configureFlags or []) ++ ["--enable-benchmarks"]; - } // optionalAttrs (localLib.isBenchmark args) { + } // optionalAttrs (isBenchmark args) { # Provide a dummy installPhase for benchmark packages. installPhase = "mkdir -p $out"; }); diff --git a/nix/overlays/debug.nix b/nix/overlays/debug.nix index 24182eb620b..6d96e3f7186 100644 --- a/nix/overlays/debug.nix +++ b/nix/overlays/debug.nix @@ -1,4 +1,6 @@ -pkgs: self: super: { +{ pkgs }: + +self: super: { mkDerivation = args: super.mkDerivation (args // { # TODO: DEVOPS-355 dontStrip = true; diff --git a/nix/overlays/dont-check.nix b/nix/overlays/dont-check.nix index 017a22a1ab4..8d7a0a79cb4 100644 --- a/nix/overlays/dont-check.nix +++ b/nix/overlays/dont-check.nix @@ -1,4 +1,6 @@ -pkgs: self: super: { +{ pkgs }: + +self: super: { mkDerivation = args: super.mkDerivation (args // { doCheck = false; }); diff --git a/nix/overlays/faster-build.nix b/nix/overlays/faster-build.nix index 611d54100e9..53c6cb354ef 100644 --- a/nix/overlays/faster-build.nix +++ b/nix/overlays/faster-build.nix @@ -1,4 +1,9 @@ -pkgs: localLib: self: super: { +# Disabling optimization for cardano-sl packages will +# return a build ~20% faster (measured in DEVOPS-1032). + +{ pkgs, localLib }: + +self: super: { mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { configureFlags = (args.configureFlags or []) ++ [ "--ghc-options=-O0" ]; }); diff --git a/nix/overlays/metric.nix b/nix/overlays/metric.nix index 675ab32b06c..2053070e9ab 100644 --- a/nix/overlays/metric.nix +++ b/nix/overlays/metric.nix @@ -1,4 +1,6 @@ -pkgs: self: super: { +{ pkgs }: + +self: super: { mkDerivation = args: super.mkDerivation (args // { enablePhaseMetrics = true; }); diff --git a/nix/overlays/required.nix b/nix/overlays/required.nix index 9a889b4f90f..93869bc2aca 100644 --- a/nix/overlays/required.nix +++ b/nix/overlays/required.nix @@ -1,6 +1,8 @@ -pkgs: localLib: enableProfiling: gitrev: ghc: -self: super: -with pkgs.haskell.lib; with pkgs.lib; +{ pkgs, localLib, enableProfiling, gitrev, ghc }: + +with pkgs.haskell.lib; +with localLib; + let justStaticExecutablesGitRev = import ../../scripts/set-git-rev { inherit pkgs gitrev ghc; @@ -8,8 +10,13 @@ let addRealTimeTestLogs = drv: overrideCabal drv (attrs: { testTarget = "--show-details=streaming"; }); -in { - srcroot = ./.; +in + +self: super: { + + ######################################################################## + # Overides of Cardano SL packages + cardano-sl-core = overrideCabal super.cardano-sl-core (drv: { configureFlags = (drv.configureFlags or []) ++ [ "-f-asserts" @@ -35,19 +42,10 @@ in { cardano-report-server-static = justStaticExecutablesGitRev self.cardano-report-server; cardano-sl-faucet-static = justStaticExecutablesGitRev self.cardano-sl-faucet; cardano-sl-tools-static = justStaticExecutablesGitRev super.cardano-sl-tools; - # Undo configuration-nix.nix change to hardcode security binary on darwin - # This is needed for macOS binary not to fail during update system (using http-client-tls) - # Instead, now the binary is just looked up in $PATH as it should be installed on any macOS - x509-system = overrideDerivation super.x509-system (drv: { - postPatch = ":"; - }); - # TODO: get rid of pthreads option once cryptonite 0.25 is released - # DEVOPS-393: https://github.com/haskell-crypto/cryptonite/issues/193 - cryptonite = appendPatch (appendConfigureFlag super.cryptonite "--ghc-option=-optl-pthread") ../../pkgs/cryptonite-segfault-blake.patch; + ######################################################################## + # The base Haskell package builder - # Due to https://github.com/input-output-hk/stack2nix/issues/56 - hfsevents = self.callPackage ../../pkgs/hfsevents.nix { inherit (pkgs.darwin.apple_sdk.frameworks) Cocoa CoreServices; }; mkDerivation = args: super.mkDerivation (args // { enableLibraryProfiling = enableProfiling; enableExecutableProfiling = enableProfiling; @@ -59,4 +57,21 @@ in { } // optionalAttrs (args ? src) { src = localLib.cleanSourceTree args.src; }); + + ######################################################################## + # Configuration of overrides for other dependencies + + # Undo configuration-nix.nix change to hardcode security binary on darwin + # This is needed for macOS binary not to fail during update system (using http-client-tls) + # Instead, now the binary is just looked up in $PATH as it should be installed on any macOS + x509-system = overrideDerivation super.x509-system (drv: { + postPatch = ":"; + }); + + # TODO: get rid of pthreads option once cryptonite 0.25 is released + # DEVOPS-393: https://github.com/haskell-crypto/cryptonite/issues/193 + cryptonite = appendPatch (appendConfigureFlag super.cryptonite "--ghc-option=-optl-pthread") ../../pkgs/cryptonite-segfault-blake.patch; + + # Due to https://github.com/input-output-hk/stack2nix/issues/56 + hfsevents = self.callPackage ../../pkgs/hfsevents.nix { inherit (pkgs.darwin.apple_sdk.frameworks) Cocoa CoreServices; }; } From acde183af1676cfca1c7c68d6df29326f9804907 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Tue, 2 Oct 2018 18:24:58 +1000 Subject: [PATCH 3/3] fixup! nix overlays: Fix build and use nixpkgs style --- nix/overlays/faster-build.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nix/overlays/faster-build.nix b/nix/overlays/faster-build.nix index 53c6cb354ef..b92bb4b8902 100644 --- a/nix/overlays/faster-build.nix +++ b/nix/overlays/faster-build.nix @@ -3,8 +3,10 @@ { pkgs, localLib }: +with localLib; + self: super: { - mkDerivation = args: super.mkDerivation (args // optionalAttrs (localLib.isCardanoSL args.pname) { + mkDerivation = args: super.mkDerivation (args // optionalAttrs (isCardanoSL args.pname) { configureFlags = (args.configureFlags or []) ++ [ "--ghc-options=-O0" ]; }); }