From 45a77bca39ed42e31d07ca61f0552e888db57500 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Mon, 18 Jun 2018 09:11:55 +0800 Subject: [PATCH 1/5] [DEVOPS-908] nix: Make sure CI builds all targets It now builds all tests and benchmarks, and runs all tests. Issue was brought up in FP Complete's Cardano Report (12 April 2018). --- default.nix | 27 ++++++++++++++++----------- lib.nix | 2 ++ release.nix | 1 + scripts/ci/ci.sh | 4 ++-- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/default.nix b/default.nix index 0953b489326..19d7edcdaf3 100644 --- a/default.nix +++ b/default.nix @@ -47,11 +47,6 @@ let kill $TAILPID ''; }); - # Enables building but not running of benchmarks when - # enableBenchmarks argument is true. - buildWithBenchmarks = drv: if enableBenchmarks - then doBenchmark (appendConfigureFlag drv "--enable-benchmarks") - else drv; cardanoPkgs = ((import ./pkgs { inherit pkgs; }).override { ghc = overrideDerivation pkgs.haskell.compiler.ghc822 (drv: { @@ -65,7 +60,7 @@ let ]; }); - cardano-sl = overrideCabal (buildWithBenchmarks super.cardano-sl) (drv: { + cardano-sl = overrideCabal super.cardano-sl (drv: { # production full nodes shouldn't use wallet as it means different constants configureFlags = (drv.configureFlags or []) ++ [ "-f-asserts" @@ -77,16 +72,13 @@ let }; }); - cardano-sl-networking = buildWithBenchmarks super.cardano-sl-networking; - cardano-sl-block-bench = buildWithBenchmarks super.cardano-sl-block-bench; - cardano-sl-explorer = buildWithBenchmarks super.cardano-sl-explorer; cardano-sl-wallet-static = justStaticExecutables super.cardano-sl-wallet; cardano-sl-client = addRealTimeTestLogs super.cardano-sl-client; cardano-sl-generator = addRealTimeTestLogs super.cardano-sl-generator; # cardano-sl-auxx = addGitRev (justStaticExecutables super.cardano-sl-auxx); cardano-sl-auxx = addGitRev (justStaticExecutables super.cardano-sl-auxx); cardano-sl-node = addGitRev super.cardano-sl-node; - cardano-sl-wallet-new = addGitRev (justStaticExecutables (buildWithBenchmarks super.cardano-sl-wallet-new)); + cardano-sl-wallet-new = addGitRev (justStaticExecutables super.cardano-sl-wallet-new); cardano-sl-tools = addGitRev (justStaticExecutables (overrideCabal super.cardano-sl-tools (drv: { # waiting on load-command size fix in dyld doCheck = ! pkgs.stdenv.isDarwin; @@ -118,7 +110,15 @@ let # This will be the default in nixpkgs since # https://github.com/NixOS/nixpkgs/issues/29011 enableSharedExecutables = false; - } // optionalAttrs (args ? src) { + } // optionalAttrs (enableBenchmarks && localLib.isCardanoSL args.pname) ({ + # Enables building but not running of benchmarks for all + # cardano-sl packages when enableBenchmarks argument is true. + doBenchmark = true; + configureFlags = (args.configureFlags or []) ++ ["--enable-benchmarks"]; + } // optionalAttrs (localLib.isBenchmark args) { + # Provide a dummy installPhase for benchmark packages. + installPhase = "mkdir -p $out"; + }) // optionalAttrs (args ? src) { src = let cleanSourceFilter = with pkgs.stdenv; name: type: let baseName = baseNameOf (toString name); in ! ( @@ -167,6 +167,11 @@ let inherit system config gitrev pkgs; cardano-sl-explorer = cardanoPkgs.cardano-sl-explorer-static; }); + all-cardano-sl = pkgs.buildEnv { + name = "all-cardano-sl"; + paths = attrValues (filterAttrs (name: drv: localLib.isCardanoSL name) cardanoPkgs); + ignoreCollisions = true; + }; mkDocker = { environment, connectArgs ? {} }: import ./docker.nix { inherit environment connect gitrev pkgs connectArgs; }; stack2nix = import (pkgs.fetchFromGitHub { owner = "avieth"; diff --git a/lib.nix b/lib.nix index d58eb9ccc83..f6ae2f91e45 100644 --- a/lib.nix +++ b/lib.nix @@ -16,4 +16,6 @@ let lib = pkgs.lib; in lib // (rec { inherit fetchNixPkgs; + isCardanoSL = lib.hasPrefix "cardano-sl"; + isBenchmark = args: !((args.isExecutable or false) || (args.isLibrary or true)); }) diff --git a/release.nix b/release.nix index 8f09c4a7be5..f41e7807c43 100644 --- a/release.nix +++ b/release.nix @@ -37,6 +37,7 @@ let cardano-sl-tools = supportedSystems; cardano-sl-wallet = supportedSystems; cardano-sl-wallet-new = supportedSystems; + all-cardano-sl = supportedSystems; cardano-sl-explorer-static = [ "x86_64-linux" ]; cardano-sl-explorer-frontend = [ "x86_64-linux" ]; cardano-report-server-static = [ "x86_64-linux" ]; diff --git a/scripts/ci/ci.sh b/scripts/ci/ci.sh index 566d4d42646..9adc6b98264 100755 --- a/scripts/ci/ci.sh +++ b/scripts/ci/ci.sh @@ -13,11 +13,11 @@ if [[ ("$OS_NAME" == "linux") && ("$BUILDKITE_BRANCH" == "master") ]]; fi export with_haddock -targets="cardano-sl cardano-sl-auxx cardano-sl-tools cardano-sl-wallet cardano-sl-wallet-new daedalus-bridge" +targets="all-cardano-sl daedalus-bridge" # There are no macOS explorer devs atm and it's only deployed on linux if [[ "$OS_NAME" == "linux" ]]; then - targets="$targets cardano-sl-explorer-static cardano-sl-explorer-frontend" + targets="$targets cardano-sl-explorer-frontend" fi # TODO: CSL-1133: Add test coverage to CI. To be reenabled when build times From cdd5e2c3a834291cfd9e6679ec3a9b868a6fc97c Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Mon, 18 Jun 2018 23:58:07 +0800 Subject: [PATCH 2/5] [DEVOPS-908] node-ipc -> cardano-sl-node-ipc So that this package gets built as part of all-cardano-sl. --- node-ipc/{node-ipc.cabal => cardano-sl-node-ipc.cabal} | 2 +- stack.yaml | 2 +- wallet-new/cardano-sl-wallet-new.cabal | 2 +- wallet/cardano-sl-wallet.cabal | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename node-ipc/{node-ipc.cabal => cardano-sl-node-ipc.cabal} (94%) diff --git a/node-ipc/node-ipc.cabal b/node-ipc/cardano-sl-node-ipc.cabal similarity index 94% rename from node-ipc/node-ipc.cabal rename to node-ipc/cardano-sl-node-ipc.cabal index 1cb04af37fe..d75bcdfe881 100644 --- a/node-ipc/node-ipc.cabal +++ b/node-ipc/cardano-sl-node-ipc.cabal @@ -1,4 +1,4 @@ -name: node-ipc +name: cardano-sl-node-ipc version: 1.3.0 license: MIT license-file: LICENSE diff --git a/stack.yaml b/stack.yaml index fc1f30d8c8f..ddf228a3150 100644 --- a/stack.yaml +++ b/stack.yaml @@ -253,4 +253,4 @@ ghc-options: cardano-sl-util-test: -Werror -fwarn-redundant-constraints cardano-sl-wallet: -Werror -fwarn-redundant-constraints cardano-sl-wallet-new: -Werror -fwarn-redundant-constraints - node-ipc: -Werror -fwarn-redundant-constraints + cardano-sl-node-ipc: -Werror -fwarn-redundant-constraints diff --git a/wallet-new/cardano-sl-wallet-new.cabal b/wallet-new/cardano-sl-wallet-new.cabal index 10dfb1b0482..c7a424bd810 100755 --- a/wallet-new/cardano-sl-wallet-new.cabal +++ b/wallet-new/cardano-sl-wallet-new.cabal @@ -128,6 +128,7 @@ library , cardano-sl-db , cardano-sl-infra , cardano-sl-networking + , cardano-sl-node-ipc , cardano-sl-ssc , cardano-sl-txp , cardano-sl-update @@ -153,7 +154,6 @@ library , mtl , neat-interpolation , network-transport - , node-ipc , optparse-applicative , QuickCheck , reflection diff --git a/wallet/cardano-sl-wallet.cabal b/wallet/cardano-sl-wallet.cabal index f08a0197ff3..0e0e7cbb23f 100644 --- a/wallet/cardano-sl-wallet.cabal +++ b/wallet/cardano-sl-wallet.cabal @@ -129,6 +129,7 @@ library , cardano-sl-ssc , cardano-sl-txp , cardano-sl-networking + , cardano-sl-node-ipc , cardano-sl-update , cardano-sl-util , containers @@ -148,7 +149,6 @@ library , memory , monad-control , mtl - , node-ipc , quickcheck-instances , random , reflection From 94ad270e2ab5ee272850ad25cd2a02823f453aa9 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Mon, 18 Jun 2018 11:48:52 +0800 Subject: [PATCH 3/5] [DEVOPS-908] utils: Don't fail the build if git is unavailable The build shouldn't depend on the presence of git on the PATH. If git is unavailable, use an obvious null revision. --- util/Pos/Util/CompileInfo.hs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/util/Pos/Util/CompileInfo.hs b/util/Pos/Util/CompileInfo.hs index 726a78a2a3c..5a08b1bb5d8 100644 --- a/util/Pos/Util/CompileInfo.hs +++ b/util/Pos/Util/CompileInfo.hs @@ -16,6 +16,7 @@ module Pos.Util.CompileInfo import Universum +import Control.Exception.Safe (handleJust) import Data.Default (Default (def)) import Data.Reflection (Given (..), give, given) import qualified Data.Text as T @@ -26,6 +27,7 @@ import qualified Language.Haskell.TH as TH import qualified Language.Haskell.TH.Syntax as TH import System.Environment (lookupEnv) import System.Exit (ExitCode (..)) +import System.IO.Error (ioeGetErrorType, isDoesNotExistErrorType) import System.Process (readProcessWithExitCode) @@ -58,12 +60,13 @@ retrieveCompileTimeInfo = do TH.lift cti where retrieveGit :: IO String - retrieveGit = - lookupEnv "GITREV" >>= maybe retrieveFromGitExecutable pure - retrieveFromGitExecutable :: IO String - retrieveFromGitExecutable = do + retrieveGit = lookupEnv "GITREV" >>= maybe runGitRevParse pure + runGitRevParse :: IO String + runGitRevParse = handleJust missingGit (const $ pure zeroRev) $ do (exitCode, output, _) <- readProcessWithExitCode "git" ["rev-parse", "--verify", "HEAD"] "" pure $ case exitCode of ExitSuccess -> output - _ -> "Couldn't fetch git revision" + _ -> zeroRev + zeroRev = "0000000000000000000000000000000000000000" + missingGit e = if isDoesNotExistErrorType (ioeGetErrorType e) then Just () else Nothing From cf58007ded98536cf8c0dcede2bc1acd40b1c6b2 Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Mon, 18 Jun 2018 13:50:04 +0800 Subject: [PATCH 4/5] [DEVOPS-810] utils: use file-embed for git revision This will prevent needless rebuilds of cabal packages because the git revision is injected after the nix package build. --- auxx/Main.hs | 4 +- auxx/src/Command/BlockGen.hs | 2 +- auxx/src/Mode.hs | 5 +- default.nix | 30 ++++---- explorer/src/explorer/Main.hs | 4 +- generator/src/Test/Pos/Block/Logic/Mode.hs | 9 ++- .../test/Test/Pos/Binary/CommunicationSpec.hs | 3 +- node/Main.hs | 4 +- scripts/set-git-rev/default.nix | 42 +++++++++++ scripts/set-git-rev/set-git-rev.hs | 49 +++++++++++++ tools/src/launcher/Main.hs | 5 +- util/Pos/Util/CompileInfo.hs | 72 +++++++++---------- util/Pos/Util/CompileInfoGit.hs | 25 +++++++ util/cardano-sl-util.cabal | 2 + wallet-new/generate-swagger-file/Main.hs | 4 +- wallet-new/server/Main.hs | 4 +- .../Pos/Wallet/Web/Methods/PaymentSpec.hs | 3 +- 17 files changed, 184 insertions(+), 83 deletions(-) create mode 100644 scripts/set-git-rev/default.nix create mode 100644 scripts/set-git-rev/set-git-rev.hs create mode 100644 util/Pos/Util/CompileInfoGit.hs diff --git a/auxx/Main.hs b/auxx/Main.hs index 154425a71c5..f925377db16 100644 --- a/auxx/Main.hs +++ b/auxx/Main.hs @@ -26,7 +26,7 @@ import Pos.Launcher (HasConfigurations, NodeParams (..), NodeResources runRealMode, withConfigurations) import Pos.Txp (txpGlobalSettings) import Pos.Util (logException) -import Pos.Util.CompileInfo (HasCompileInfo, retrieveCompileTimeInfo, withCompileInfo) +import Pos.Util.CompileInfo (HasCompileInfo, withCompileInfo) import Pos.Util.Config (ConfigurationException (..)) import Pos.Util.UserSecret (usVss) import Pos.WorkMode (EmptyMempoolExt, RealMode) @@ -130,7 +130,7 @@ action opts@AuxxOptions {..} command = do CLI.NodeArgs {behaviorConfigPath = Nothing} main :: IO () -main = withCompileInfo $(retrieveCompileTimeInfo) $ do +main = withCompileInfo $ do opts <- getAuxxOptions let disableConsoleLog | Repl <- aoAction opts = diff --git a/auxx/src/Command/BlockGen.hs b/auxx/src/Command/BlockGen.hs index 42fea2ac8c5..4a729f9ea94 100644 --- a/auxx/src/Command/BlockGen.hs +++ b/auxx/src/Command/BlockGen.hs @@ -43,7 +43,7 @@ generateBlocks pm GenBlocksParams{..} = withStateLock HighPriority ApplyBlock $ , _bgpSkipNoKey = True , _bgpTxpGlobalSettings = txpGlobalSettings pm } - withCompileInfo def $ evalRandT (genBlocks pm bgenParams (const ())) (mkStdGen seed) + withCompileInfo $ evalRandT (genBlocks pm bgenParams (const ())) (mkStdGen seed) -- We print it twice because there can be a ton of logs and -- you don't notice the first message. logInfo $ "Generated with seed " <> show seed diff --git a/auxx/src/Mode.hs b/auxx/src/Mode.hs index 3b090cda194..ed496705a25 100644 --- a/auxx/src/Mode.hs +++ b/auxx/src/Mode.hs @@ -26,7 +26,6 @@ import Control.Lens (lens, makeLensesWith) import Control.Monad.Reader (withReaderT) import Control.Monad.Trans.Resource (transResourceT) import Data.Conduit (transPipe) -import Data.Default (def) import Mockable (Production) import System.Wlog (HasLoggerName (..)) @@ -234,8 +233,8 @@ instance ( HasConfiguration instance (HasConfigurations) => MonadTxpLocal (BlockGenMode EmptyMempoolExt AuxxMode) where - txpNormalize = withCompileInfo def $ txNormalize - txpProcessTx = withCompileInfo def $ txProcessTransactionNoLock + txpNormalize = withCompileInfo $ txNormalize + txpProcessTx = withCompileInfo $ txProcessTransactionNoLock -- | In order to create an 'Address' from a 'PublicKey' we need to -- choose suitable stake distribution. We want to pick it based on diff --git a/default.nix b/default.nix index 19d7edcdaf3..15e2c851b56 100644 --- a/default.nix +++ b/default.nix @@ -26,14 +26,10 @@ with pkgs.lib; with pkgs.haskell.lib; let - addGitRev = subject: - subject.overrideAttrs ( - drv: { - GITREV = gitrev; - librarySystemDepends = (drv.librarySystemDepends or []) ++ [ pkgs.git ]; - executableSystemDepends = (drv.executableSystemDepends or []) ++ [ pkgs.git ]; - } - ); + justStaticExecutablesGitRev = import ./scripts/set-git-rev { + inherit pkgs gitrev; + inherit (cardanoPkgs) ghc; + }; addRealTimeTestLogs = drv: overrideCabal drv (attrs: { testTarget = "--log=test.log || (sleep 10 && kill $TAILPID && false)"; preCheck = '' @@ -72,21 +68,19 @@ let }; }); - cardano-sl-wallet-static = justStaticExecutables super.cardano-sl-wallet; + cardano-sl-wallet-static = justStaticExecutablesGitRev super.cardano-sl-wallet; cardano-sl-client = addRealTimeTestLogs super.cardano-sl-client; cardano-sl-generator = addRealTimeTestLogs super.cardano-sl-generator; - # cardano-sl-auxx = addGitRev (justStaticExecutables super.cardano-sl-auxx); - cardano-sl-auxx = addGitRev (justStaticExecutables super.cardano-sl-auxx); - cardano-sl-node = addGitRev super.cardano-sl-node; - cardano-sl-wallet-new = addGitRev (justStaticExecutables super.cardano-sl-wallet-new); - cardano-sl-tools = addGitRev (justStaticExecutables (overrideCabal super.cardano-sl-tools (drv: { + cardano-sl-auxx = justStaticExecutablesGitRev super.cardano-sl-auxx; + cardano-sl-wallet-new = justStaticExecutablesGitRev super.cardano-sl-wallet-new; + cardano-sl-tools = justStaticExecutablesGitRev (overrideCabal super.cardano-sl-tools (drv: { # waiting on load-command size fix in dyld doCheck = ! pkgs.stdenv.isDarwin; - }))); + })); - cardano-sl-node-static = justStaticExecutables self.cardano-sl-node; - cardano-sl-explorer-static = addGitRev (justStaticExecutables self.cardano-sl-explorer); - cardano-report-server-static = justStaticExecutables self.cardano-report-server; + cardano-sl-node-static = justStaticExecutablesGitRev self.cardano-sl-node; + cardano-sl-explorer-static = justStaticExecutablesGitRev self.cardano-sl-explorer; + cardano-report-server-static = justStaticExecutablesGitRev self.cardano-report-server; # Undo configuration-nix.nix change to hardcode security binary on darwin # This is needed for macOS binary not to fail during update system (using http-client-tls) diff --git a/explorer/src/explorer/Main.hs b/explorer/src/explorer/Main.hs index a57d678950c..7a52ab66706 100644 --- a/explorer/src/explorer/Main.hs +++ b/explorer/src/explorer/Main.hs @@ -35,7 +35,7 @@ import Pos.Launcher (ConfigurationOptions (..), HasConfigurations, Nod import Pos.Launcher.Configuration (AssetLockPath (..)) import Pos.Update.Worker (updateTriggerWorker) import Pos.Util (logException) -import Pos.Util.CompileInfo (HasCompileInfo, retrieveCompileTimeInfo, withCompileInfo) +import Pos.Util.CompileInfo (HasCompileInfo, withCompileInfo) import Pos.Util.UserSecret (usVss) loggerName :: LoggerName @@ -56,7 +56,7 @@ main = do action :: ExplorerNodeArgs -> Production () action (ExplorerNodeArgs (cArgs@CommonNodeArgs{..}) ExplorerArgs{..}) = withConfigurations blPath conf $ \ntpConfig pm -> - withCompileInfo $(retrieveCompileTimeInfo) $ do + withCompileInfo $ do CLI.printInfoOnStart cArgs ntpConfig logInfo $ "Explorer is enabled!" currentParams <- getNodeParams loggerName cArgs nodeArgs diff --git a/generator/src/Test/Pos/Block/Logic/Mode.hs b/generator/src/Test/Pos/Block/Logic/Mode.hs index 9096cc3bcd1..fece72520da 100644 --- a/generator/src/Test/Pos/Block/Logic/Mode.hs +++ b/generator/src/Test/Pos/Block/Logic/Mode.hs @@ -45,7 +45,6 @@ module Test.Pos.Block.Logic.Mode import Universum import Control.Lens (lens, makeClassy, makeLensesWith) -import Data.Default (def) import qualified Data.Map as Map import qualified Data.Text.Buildable import Data.Time.Units (TimeUnit (..)) @@ -506,9 +505,9 @@ instance MonadBListener BlockTestMode where type instance MempoolExt BlockTestMode = EmptyMempoolExt instance HasConfigurations => MonadTxpLocal (BlockGenMode EmptyMempoolExt BlockTestMode) where - txpNormalize = withCompileInfo def $ txNormalize - txpProcessTx = withCompileInfo def $ txProcessTransactionNoLock + txpNormalize = withCompileInfo $ txNormalize + txpProcessTx = withCompileInfo $ txProcessTransactionNoLock instance HasConfigurations => MonadTxpLocal BlockTestMode where - txpNormalize = withCompileInfo def $ txNormalize - txpProcessTx = withCompileInfo def $ txProcessTransactionNoLock + txpNormalize = withCompileInfo $ txNormalize + txpProcessTx = withCompileInfo $ txProcessTransactionNoLock diff --git a/generator/test/Test/Pos/Binary/CommunicationSpec.hs b/generator/test/Test/Pos/Binary/CommunicationSpec.hs index 95501194a28..1cd13ee1b1c 100644 --- a/generator/test/Test/Pos/Binary/CommunicationSpec.hs +++ b/generator/test/Test/Pos/Binary/CommunicationSpec.hs @@ -5,7 +5,6 @@ module Test.Pos.Binary.CommunicationSpec import Universum import qualified Data.ByteString.Lazy as BSL -import Data.Default (def) import Test.Hspec (Spec, describe) import Test.Hspec.QuickCheck (prop) import Test.QuickCheck.Monadic (assert) @@ -63,7 +62,7 @@ deserializeSerilizedMsgSerializedBlockSpec = do descNoBlock = "deserialization of a serialized MsgNoSerializedBlock message should give back corresponding MsgNoBlock" spec :: Spec -spec = withStaticConfigurations $ \_ -> withCompileInfo def $ +spec = withStaticConfigurations $ \_ -> withCompileInfo $ describe "Pos.Binary.Communication" $ do describe "serializeMsgSerializedBlock" serializeMsgSerializedBlockSpec describe "decode is left inverse of serializeMsgSerializedBlock" deserializeSerilizedMsgSerializedBlockSpec diff --git a/node/Main.hs b/node/Main.hs index 5e833dd65d1..a08b2c32905 100644 --- a/node/Main.hs +++ b/node/Main.hs @@ -25,7 +25,7 @@ import Pos.Launcher.Configuration (AssetLockPath (..)) import Pos.Ssc.Types (SscParams) import Pos.Update.Worker (updateTriggerWorker) import Pos.Util (logException) -import Pos.Util.CompileInfo (HasCompileInfo, retrieveCompileTimeInfo, withCompileInfo) +import Pos.Util.CompileInfo (HasCompileInfo, withCompileInfo) import Pos.Util.UserSecret (usVss) loggerName :: LoggerName @@ -61,7 +61,7 @@ action (SimpleNodeArgs (cArgs@CommonNodeArgs {..}) (nArgs@NodeArgs {..})) ntpCon actionWithoutWallet pm sscParams currentParams main :: IO () -main = withCompileInfo $(retrieveCompileTimeInfo) $ do +main = withCompileInfo $ do args@(CLI.SimpleNodeArgs commonNodeArgs _) <- CLI.getSimpleNodeOptions let loggingParams = CLI.loggingParams loggerName commonNodeArgs let conf = CLI.configurationOptions (CLI.commonArgs commonNodeArgs) diff --git a/scripts/set-git-rev/default.nix b/scripts/set-git-rev/default.nix new file mode 100644 index 00000000000..827fd691289 --- /dev/null +++ b/scripts/set-git-rev/default.nix @@ -0,0 +1,42 @@ +# This provides a function like pkgs.haskell.lib.justStaticExecutables, +# but which also stamps the executables with the git revision. + +{ pkgs, ghc, gitrev }: + +with pkgs.lib; + +let + # Wraps a haskell package derivation so that it has all dynamic + # linking, development and doc files removed. + # + # It uses file-embed to inject the git revision into all programs of + # all derivation outputs. + # + # This is done by copying all files from the wrapped derivation + # rather than with a package override. If an override were used, the + # derivation would need to be rebuilt whenever gitrev changed. + justStaticExecutablesGitRev = drvOut: let + drvOutOutputs = drvOut.outputs or ["out"]; + in + pkgs.runCommand (staticName drvOut) { + outputs = drvOutOutputs; + passthru = drvOut.drvAttrs // { inherit gitrev; }; + } + (concatMapStrings (output: '' + cp -a "${drvOut.${output}}" "${"$"}${output}" + chmod -R +w "${"$"}${output}" + for prog in "${"$"}${output}"/bin/*; do + ${setGitRev} "${gitrev}" "$prog" || true + done + rm -rf "${"$"}${output}"/lib "${"$"}${output}"/nix-support "${"$"}${output}"/share/doc + '') drvOutOutputs); + + setGitRev = pkgs.runCommand "set-git-rev" {} '' + ${ghc.withPackages (ps: [ps.universum ps.file-embed])}/bin/ghc -o $out ${./set-git-rev.hs} + ''; + + # appends -static to the package name + staticName = drv: drv.pname + "-static-" + drv.version; + +in + justStaticExecutablesGitRev diff --git a/scripts/set-git-rev/set-git-rev.hs b/scripts/set-git-rev/set-git-rev.hs new file mode 100644 index 00000000000..0f531937adb --- /dev/null +++ b/scripts/set-git-rev/set-git-rev.hs @@ -0,0 +1,49 @@ +{-# LANGUAGE OverloadedStrings, NoImplicitPrelude, LambdaCase #-} + +module Main where + +import Universum +import Control.Exception (ErrorCall(..), handle) +import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as B8 +import Data.FileEmbed (injectWith) +import Data.List (isInfixOf) + +main :: IO () +main = do + (hash, progs) <- parseArgs + mapM_ (setGitRev hash) progs + +setGitRev :: ByteString -> FilePath -> IO () +setGitRev hash prog = do + putStr $ "Setting gitrev of " <> prog <> " ... " + bs <- B8.readFile prog + injectWith' "gitrev" hash bs >>= \case + Right bs' -> do + B8.writeFile prog bs' + B8.putStrLn "OK" + exitSuccess + Left "" -> do + B8.putStrLn $ "Failed setting gitrev to \"" <> hash <> "\"" + exitFailure + Left msg | "Size is: \"\"" `isInfixOf` msg -> do + -- Ignore programs without a gitrev injected + B8.putStrLn "File does not have dummySpace." + exitSuccess + Left msg -> do + putStrLn msg + exitFailure + +-- | Work around annoying use of error function in file-embed. +injectWith' :: ByteString -> ByteString -> ByteString -> IO (Either String ByteString) +injectWith' postfix toInj orig = handle (pure . toLeft) (toRight <$> evaluateNF inj) + where + inj = injectWith postfix toInj orig + toRight (Just a) = Right a + toRight Nothing = Left "" + toLeft (ErrorCall msg) = Left msg + +parseArgs :: IO (ByteString, [FilePath]) +parseArgs = getArgs >>= \case + (rev:prog:progs) -> pure (B8.pack rev, (prog:progs)) + _ -> die "usage: set-git-rev REV PROG [PROGS...]" >> exitFailure diff --git a/tools/src/launcher/Main.hs b/tools/src/launcher/Main.hs index 7f47d0e2118..af89d2ea377 100644 --- a/tools/src/launcher/Main.hs +++ b/tools/src/launcher/Main.hs @@ -78,8 +78,7 @@ import Pos.ReportServer.Report (ReportType (..)) import Pos.Update (installerHash) import Pos.Update.DB.Misc (affirmUpdateInstalled) import Pos.Util (HasLens (..), directory, logException, postfixLFields) -import Pos.Util.CompileInfo (HasCompileInfo, compileInfo, retrieveCompileTimeInfo, - withCompileInfo) +import Pos.Util.CompileInfo (HasCompileInfo, compileInfo, withCompileInfo) import Launcher.Environment (substituteEnvVarsValue) import Launcher.Logging (reportErrorDefault) @@ -267,7 +266,7 @@ bracketNodeDBs (NodeDbPath dbPath) = bracket (openNodeDBs False dbPath) closeNod main :: IO () main = - withCompileInfo $(retrieveCompileTimeInfo) $ + withCompileInfo $ case Sys.os of "mingw32" -> -- We don't output anything to console on Windows because on Windows the diff --git a/util/Pos/Util/CompileInfo.hs b/util/Pos/Util/CompileInfo.hs index 5a08b1bb5d8..84fc89ec67c 100644 --- a/util/Pos/Util/CompileInfo.hs +++ b/util/Pos/Util/CompileInfo.hs @@ -1,44 +1,39 @@ {-# LANGUAGE DeriveLift #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RankNTypes #-} +{-# LANGUAGE LambdaCase #-} --- | Compile time information manipulations. Was introduced as --- CSL-1563 to avoid extra library recompilations when git revision --- changes. See the issue description/comments for more details. +-- | Provides the git revision which this was compiled from. +-- +-- Stack builds will have the `git` command available to run during +-- compilation. +-- +-- Nix builds will inject the git revision into the executables after +-- compiling. If the git revision has changed but the sources have +-- not, then no haskell packages will be rebuilt, but the embedded git +-- revision will be updated. module Pos.Util.CompileInfo ( CompileTimeInfo (..) , HasCompileInfo , compileInfo , withCompileInfo - , retrieveCompileTimeInfo + , gitRev ) where import Universum -import Control.Exception.Safe (handleJust) -import Data.Default (Default (def)) +import Data.FileEmbed (dummySpaceWith) import Data.Reflection (Given (..), give, given) import qualified Data.Text as T import qualified Data.Text.Buildable import Formatting (bprint, stext, (%)) -import Instances.TH.Lift () -import qualified Language.Haskell.TH as TH -import qualified Language.Haskell.TH.Syntax as TH -import System.Environment (lookupEnv) -import System.Exit (ExitCode (..)) -import System.IO.Error (ioeGetErrorType, isDoesNotExistErrorType) -import System.Process (readProcessWithExitCode) - +import Pos.Util.CompileInfoGit -- | Data about the system that we want to retrieve in compile time. data CompileTimeInfo = CompileTimeInfo { ctiGitRevision :: Text - } deriving (Show,TH.Lift) - -instance Default CompileTimeInfo where - def = CompileTimeInfo { ctiGitRevision = "" - } + } deriving (Show) instance Buildable CompileTimeInfo where build CompileTimeInfo{..} = @@ -49,24 +44,23 @@ type HasCompileInfo = Given CompileTimeInfo compileInfo :: HasCompileInfo => CompileTimeInfo compileInfo = given -withCompileInfo :: CompileTimeInfo -> (HasCompileInfo => r) -> r -withCompileInfo = give +withCompileInfo :: (HasCompileInfo => r) -> r +withCompileInfo = give (CompileTimeInfo gitRev) + +gitRev :: Text +gitRev | gitRevEmbed /= zeroRev = gitRevEmbed + | T.null fromGit = zeroRev + | otherwise = fromGit + where + -- Git revision embedded after compilation using + -- Data.FileEmbed.injectWith. If nothing has been injected, + -- this will be filled with 0 characters. + gitRevEmbed :: Text + gitRevEmbed = decodeUtf8 $(dummySpaceWith "gitrev" 40) + + -- Git revision found during compilation by running git. If + -- git could not be run, then this will be empty. + fromGit = T.strip (fromString $(gitRevFromGit)) -retrieveCompileTimeInfo :: TH.Q TH.Exp -retrieveCompileTimeInfo = do - cti <- TH.runIO $ do - ctiGitRevision <- T.strip . fromString <$> retrieveGit - pure $ CompileTimeInfo {..} - TH.lift cti - where - retrieveGit :: IO String - retrieveGit = lookupEnv "GITREV" >>= maybe runGitRevParse pure - runGitRevParse :: IO String - runGitRevParse = handleJust missingGit (const $ pure zeroRev) $ do - (exitCode, output, _) <- - readProcessWithExitCode "git" ["rev-parse", "--verify", "HEAD"] "" - pure $ case exitCode of - ExitSuccess -> output - _ -> zeroRev - zeroRev = "0000000000000000000000000000000000000000" - missingGit e = if isDoesNotExistErrorType (ioeGetErrorType e) then Just () else Nothing +zeroRev :: Text +zeroRev = "0000000000000000000000000000000000000000" diff --git a/util/Pos/Util/CompileInfoGit.hs b/util/Pos/Util/CompileInfoGit.hs new file mode 100644 index 00000000000..efeaf1bc79f --- /dev/null +++ b/util/Pos/Util/CompileInfoGit.hs @@ -0,0 +1,25 @@ +module Pos.Util.CompileInfoGit + ( gitRevFromGit + ) where + +import Universum + +import Control.Exception.Safe (handleJust) +import qualified Language.Haskell.TH as TH +import System.IO.Error (ioeGetErrorType, isDoesNotExistErrorType) +import System.Process (readProcessWithExitCode) +import System.Exit (ExitCode (..)) + +-- | Git revision found by running git rev-parse. If git could not be +-- executed, then this will be an empty string. +gitRevFromGit :: TH.Q TH.Exp +gitRevFromGit = TH.LitE . TH.StringL <$> TH.runIO runGitRevParse + where + runGitRevParse :: IO String + runGitRevParse = handleJust missingGit (const $ pure "") $ do + (exitCode, output, _) <- + readProcessWithExitCode "git" ["rev-parse", "--verify", "HEAD"] "" + pure $ case exitCode of + ExitSuccess -> output + _ -> "" + missingGit e = if isDoesNotExistErrorType (ioeGetErrorType e) then Just () else Nothing diff --git a/util/cardano-sl-util.cabal b/util/cardano-sl-util.cabal index 57ff621e203..aecb5f3b8bc 100644 --- a/util/cardano-sl-util.cabal +++ b/util/cardano-sl-util.cabal @@ -41,6 +41,7 @@ library Pos.Util.Util other-modules: + Pos.Util.CompileInfoGit build-depends: aeson , base @@ -56,6 +57,7 @@ library , directory , ether , exceptions + , file-embed , filepath , formatting , hashable diff --git a/wallet-new/generate-swagger-file/Main.hs b/wallet-new/generate-swagger-file/Main.hs index 2c40f33eefb..2a090422c88 100644 --- a/wallet-new/generate-swagger-file/Main.hs +++ b/wallet-new/generate-swagger-file/Main.hs @@ -10,7 +10,7 @@ import Options.Applicative import Cardano.Wallet.API (devAPI, v0API, v1API) import Pos.Core (ApplicationName (..), SoftwareVersion (..)) -import Pos.Util.CompileInfo (CompileTimeInfo, retrieveCompileTimeInfo) +import Pos.Util.CompileInfo (CompileTimeInfo(CompileTimeInfo), gitRev) import qualified Cardano.Wallet.API.V1.Swagger as Swagger import qualified Data.Aeson as Aeson @@ -86,6 +86,6 @@ mkSwagger details = \case -- this if needed but for the sake of this tools, it only clutters its scope. softwareDetails :: (CompileTimeInfo, SoftwareVersion) softwareDetails = - ( $(retrieveCompileTimeInfo) + ( CompileTimeInfo gitRev , SoftwareVersion (ApplicationName "cardano-sl") 1 ) diff --git a/wallet-new/server/Main.hs b/wallet-new/server/Main.hs index 6677f636f44..ce8b3685636 100644 --- a/wallet-new/server/Main.hs +++ b/wallet-new/server/Main.hs @@ -25,7 +25,7 @@ import Pos.Launcher.Configuration (AssetLockPath (..), ConfigurationOp import Pos.Ssc.Types (SscParams) import Pos.Txp (txpGlobalSettings) import Pos.Util (logException) -import Pos.Util.CompileInfo (HasCompileInfo, retrieveCompileTimeInfo, withCompileInfo) +import Pos.Util.CompileInfo (HasCompileInfo, withCompileInfo) import Pos.Util.UserSecret (usVss) import Pos.Wallet.Web (bracketWalletWS, bracketWalletWebDB, getSKById, getWalletAddresses, runWRealMode) @@ -189,7 +189,7 @@ startEdgeNode wso = -- | The main entrypoint for the Wallet. main :: IO () -main = withCompileInfo $(retrieveCompileTimeInfo) $ do +main = withCompileInfo $ do cfg <- getWalletNodeOptions putText "Wallet is starting..." let loggingParams = CLI.loggingParams defaultLoggerName (wsoNodeArgs cfg) diff --git a/wallet/test/Test/Pos/Wallet/Web/Methods/PaymentSpec.hs b/wallet/test/Test/Pos/Wallet/Web/Methods/PaymentSpec.hs index f6c431fa483..3162f31fc70 100644 --- a/wallet/test/Test/Pos/Wallet/Web/Methods/PaymentSpec.hs +++ b/wallet/test/Test/Pos/Wallet/Web/Methods/PaymentSpec.hs @@ -14,7 +14,6 @@ module Test.Pos.Wallet.Web.Methods.PaymentSpec import Universum import Control.Exception.Safe (try) -import Data.Default (def) import Data.List ((!!), (\\)) import Data.List.NonEmpty (fromList) import Formatting (build, sformat, (%)) @@ -61,7 +60,7 @@ deriving instance Eq CTx -- TODO remove HasCompileInfo when MonadWalletWebMode will be splitted. spec :: Spec -spec = withCompileInfo def $ +spec = withCompileInfo $ withDefConfigurations $ \_ _ -> describe "Wallet.Web.Methods.Payment" $ modifyMaxSuccess (const 10) $ do describe "newPaymentBatch" $ do From b2cd80bb7e6b31490ba253cc4390ef6a499553aa Mon Sep 17 00:00:00 2001 From: Rodney Lorrimar Date: Mon, 18 Jun 2018 13:55:30 +0800 Subject: [PATCH 5/5] [DEVOPS-810] Regenerate pkgs/default.nix --- pkgs/default.nix | 86 +++++++++++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/pkgs/default.nix b/pkgs/default.nix index 2f1bfbfb25b..a1677917325 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -17163,6 +17163,44 @@ doHaddock = false; description = "Cardano SL simple node executable"; license = stdenv.lib.licenses.mit; +}) {}; +"cardano-sl-node-ipc" = callPackage +({ + mkDerivation +, aeson +, base +, binary +, bytestring +, Cabal +, cardano-sl-infra +, log-warper +, mtl +, stdenv +, universum +}: +mkDerivation { + +pname = "cardano-sl-node-ipc"; +version = "1.3.0"; +src = ./../node-ipc; +configureFlags = [ +"--ghc-option=-fwarn-redundant-constraints" +"--ghc-option=-Werror" +]; +libraryHaskellDepends = [ +aeson +base +binary +bytestring +Cabal +cardano-sl-infra +log-warper +mtl +universum +]; +doHaddock = false; +license = stdenv.lib.licenses.mit; + }) {}; "cardano-sl-ssc" = callPackage ({ @@ -17816,6 +17854,7 @@ license = stdenv.lib.licenses.mit; , directory , ether , exceptions +, file-embed , filepath , formatting , hashable @@ -17874,6 +17913,7 @@ deepseq directory ether exceptions +file-embed filepath formatting hashable @@ -18009,6 +18049,7 @@ license = stdenv.lib.licenses.mit; , cardano-sl-infra , cardano-sl-lrc , cardano-sl-networking +, cardano-sl-node-ipc , cardano-sl-ssc , cardano-sl-txp , cardano-sl-txp-test @@ -18035,7 +18076,6 @@ license = stdenv.lib.licenses.mit; , monad-control , MonadRandom , mtl -, node-ipc , QuickCheck , quickcheck-instances , random @@ -18097,6 +18137,7 @@ cardano-sl-generator cardano-sl-infra cardano-sl-lrc cardano-sl-networking +cardano-sl-node-ipc cardano-sl-ssc cardano-sl-txp cardano-sl-update @@ -18118,7 +18159,6 @@ log-warper memory monad-control mtl -node-ipc QuickCheck quickcheck-instances random @@ -18232,6 +18272,7 @@ license = stdenv.lib.licenses.mit; , cardano-sl-infra , cardano-sl-lrc , cardano-sl-networking +, cardano-sl-node-ipc , cardano-sl-ssc , cardano-sl-txp , cardano-sl-txp-test @@ -18265,7 +18306,6 @@ license = stdenv.lib.licenses.mit; , mtl , neat-interpolation , network-transport -, node-ipc , optparse-applicative , pretty-show , QuickCheck @@ -18337,6 +18377,7 @@ cardano-sl-crypto cardano-sl-db cardano-sl-infra cardano-sl-networking +cardano-sl-node-ipc cardano-sl-ssc cardano-sl-txp cardano-sl-update @@ -18362,7 +18403,6 @@ memory mtl neat-interpolation network-transport -node-ipc optparse-applicative QuickCheck reflection @@ -57687,44 +57727,6 @@ homepage = "https://github.com/peti/nix-paths"; description = "Knowledge of Nix's installation directories"; license = stdenv.lib.licenses.bsd3; -}) {}; -"node-ipc" = callPackage -({ - mkDerivation -, aeson -, base -, binary -, bytestring -, Cabal -, cardano-sl-infra -, log-warper -, mtl -, stdenv -, universum -}: -mkDerivation { - -pname = "node-ipc"; -version = "1.3.0"; -src = ./../node-ipc; -configureFlags = [ -"--ghc-option=-fwarn-redundant-constraints" -"--ghc-option=-Werror" -]; -libraryHaskellDepends = [ -aeson -base -binary -bytestring -Cabal -cardano-sl-infra -log-warper -mtl -universum -]; -doHaddock = false; -license = stdenv.lib.licenses.mit; - }) {}; "non-empty" = callPackage ({