From 28f49030e2175c35d346f7b7ad76cf4c3bb274c0 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Fri, 21 Sep 2018 13:24:49 -0300 Subject: [PATCH 1/4] [DEVOPS-1057] automated testing of yaml files --- default.nix | 1 + lib.nix | 10 ++++++++ scripts/test/yamlValidation.nix | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 scripts/test/yamlValidation.nix diff --git a/default.nix b/default.nix index b5e217f7659..ba9601a3e08 100644 --- a/default.nix +++ b/default.nix @@ -172,6 +172,7 @@ let stylishHaskell = pkgs.callPackage ./scripts/test/stylish.nix { inherit (cardanoPkgs) stylish-haskell; inherit src localLib; }; walletIntegration = pkgs.callPackage ./scripts/test/wallet/integration/build-test.nix { inherit walletIntegrationTests; }; swaggerSchemaValidation = pkgs.callPackage ./scripts/test/wallet/swaggerSchemaValidation.nix { inherit gitrev; }; + yamlValidation = pkgs.callPackage ./scripts/test/yamlValidation.nix { inherit cardanoPkgs; inherit (localLib) runHaskell; }; }; cardano-sl-explorer-frontend = (import ./explorer/frontend { inherit system config gitrev pkgs; diff --git a/lib.nix b/lib.nix index 3208e9c47ab..0ec489783be 100644 --- a/lib.nix +++ b/lib.nix @@ -82,4 +82,14 @@ in lib // (rec { forEnvironments = f: lib.mapAttrs' (name: env: lib.nameValuePair env.attr (f (env // { environment = name; }))) (lib.filterAttrs (name: env: !env.private) environments); + + runHaskell = name: hspkgs: deps: env: code: let + ghc = hspkgs.ghcWithPackages deps; + builtBinary = pkgs.runCommand "${name}-binary" { buildInputs = [ ghc ]; } '' + mkdir -p $out/bin/ + ghc ${pkgs.writeText "${name}.hs" code} -o $out/bin/${name} + ''; + in pkgs.runCommand name env '' + ${builtBinary}/bin/$name + ''; }) diff --git a/scripts/test/yamlValidation.nix b/scripts/test/yamlValidation.nix new file mode 100644 index 00000000000..6e6061a31f5 --- /dev/null +++ b/scripts/test/yamlValidation.nix @@ -0,0 +1,41 @@ +{ runHaskell, cardanoPkgs }: + +runHaskell "yamlValidation" cardanoPkgs (ps: with ps; [ + cardano-sl-util + split +]) { + logConfigs = [ + ../../log-configs/daedalus.yaml + #../../log-configs/greppable.yaml + ../../log-configs/connect-to-cluster.yaml + ../../log-configs/cluster.yaml + #../../log-configs/template-demo.yaml + ]; +} '' + import Pos.Util.Log.LoggerConfig (LoggerConfig) + import Data.List.Split + import System.Environment + import Data.Yaml + import Control.Monad.Catch + import Data.Monoid + + main :: IO () + main = do + runTest "logConfigs" checkLogConfig + outpath <- getEnv "out" + writeFile outpath "done" + + runTest :: FromJSON a => String -> (String -> IO a) -> IO () + runTest var func = do + paths <- getEnv var + let + pathList = splitOn " " paths + doTest path = do + putStrLn $ "testing: " <> path + func path + mapM_ doTest pathList + + checkLogConfig :: String -> IO LoggerConfig + checkLogConfig path = do + either throwM return =<< decodeFileEither path +'' From f1e81ac08b197fdc7f33366258c6913d2749e663 Mon Sep 17 00:00:00 2001 From: Andreas Triantafyllos Date: Fri, 21 Sep 2018 20:10:29 +0300 Subject: [PATCH 2/4] [DEVOPS-1057] Changed template-demo.yaml --- log-configs/template-demo.yaml | 4 ++-- scripts/common-functions.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/log-configs/template-demo.yaml b/log-configs/template-demo.yaml index b12bac2c694..1884068c7fd 100644 --- a/log-configs/template-demo.yaml +++ b/log-configs/template-demo.yaml @@ -5,10 +5,10 @@ rotation: keepFiles: 20 loggerTree: severity: Debug+ - file: {{file}} + file: tmpFileName handlers: - { name: "json" - , filepath: "{{file}}.json" + , filepath: "tmpFileName.json" , logsafety: PublicLogLevel , severity: Debug , backend: FileJsonBE } diff --git a/scripts/common-functions.sh b/scripts/common-functions.sh index 8edf9ab5327..46031af13da 100644 --- a/scripts/common-functions.sh +++ b/scripts/common-functions.sh @@ -65,8 +65,8 @@ function logs { local conf_file="$conf_dir/$log_file.yaml" # log files are named under the $logs_dir directory - # sed "s|{{file}}|$logs_dir/$log_file|g" "$template" > "$conf_file" - sed "s|{{file}}|$log_file|g" "$template" > "$conf_file" + # sed "s|tmpFileName|$logs_dir/$log_file|g" "$template" > "$conf_file" + sed "s|tmpFileName|$log_file|g" "$template" > "$conf_file" echo -n " --json-log=$logs_dir/logs/node$i.json " echo -n " --logs-prefix $logs_dir/logs --log-config $conf_file " } From 03e157ab4c03dcd4d68d5fb5365aa57b823ddfb2 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Tue, 25 Sep 2018 02:30:25 -0300 Subject: [PATCH 3/4] [DEVOPS-1057] also validate template-demo.yaml --- scripts/test/yamlValidation.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test/yamlValidation.nix b/scripts/test/yamlValidation.nix index 6e6061a31f5..f34cc1fb51e 100644 --- a/scripts/test/yamlValidation.nix +++ b/scripts/test/yamlValidation.nix @@ -9,7 +9,7 @@ runHaskell "yamlValidation" cardanoPkgs (ps: with ps; [ #../../log-configs/greppable.yaml ../../log-configs/connect-to-cluster.yaml ../../log-configs/cluster.yaml - #../../log-configs/template-demo.yaml + ../../log-configs/template-demo.yaml ]; } '' import Pos.Util.Log.LoggerConfig (LoggerConfig) From 8a3a5c06897facee021368dba5b8f5dd3f0855cd Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Tue, 25 Sep 2018 02:45:37 -0300 Subject: [PATCH 4/4] [DEVOPS-1057] also verify that all topology files parse --- scripts/test/yamlValidation.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scripts/test/yamlValidation.nix b/scripts/test/yamlValidation.nix index f34cc1fb51e..79bf6036e09 100644 --- a/scripts/test/yamlValidation.nix +++ b/scripts/test/yamlValidation.nix @@ -2,6 +2,7 @@ runHaskell "yamlValidation" cardanoPkgs (ps: with ps; [ cardano-sl-util + cardano-sl-infra split ]) { logConfigs = [ @@ -11,8 +12,18 @@ runHaskell "yamlValidation" cardanoPkgs (ps: with ps; [ ../../log-configs/cluster.yaml ../../log-configs/template-demo.yaml ]; + topologyConfigs = [ + ../../docs/network/example-topologies/mainnet-staging.yaml + ../../docs/network/example-topologies/behind-nat-no-dns.yaml + ../../docs/network/example-topologies/behind-nat-with-dns.yaml + ../../docs/network/example-topologies/p2p.yaml + ../../docs/network/example-topologies/static-no-dns.yaml + ../../docs/network/example-topologies/static-with-dns.yaml + ../../docs/network/example-topologies/traditional.yaml + ]; } '' import Pos.Util.Log.LoggerConfig (LoggerConfig) + import Pos.Infra.Network.Yaml (Topology) import Data.List.Split import System.Environment import Data.Yaml @@ -22,6 +33,7 @@ runHaskell "yamlValidation" cardanoPkgs (ps: with ps; [ main :: IO () main = do runTest "logConfigs" checkLogConfig + runTest "topologyConfigs" checkTopology outpath <- getEnv "out" writeFile outpath "done" @@ -38,4 +50,7 @@ runHaskell "yamlValidation" cardanoPkgs (ps: with ps; [ checkLogConfig :: String -> IO LoggerConfig checkLogConfig path = do either throwM return =<< decodeFileEither path + + checkTopology :: String -> IO Topology + checkTopology path = either throwM return =<< decodeFileEither path ''