From ba61c8adf125978eb198c37186f983c29df4172b Mon Sep 17 00:00:00 2001 From: clux Date: Tue, 12 Sep 2023 15:04:10 +0100 Subject: [PATCH 01/10] yaml merge key support Signed-off-by: clux --- Cargo.lock | 12 ++++++------ README.md | 1 + yq.rs | 6 +++++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd003f6..db1f067 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.105" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360" +checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" dependencies = [ "itoa", "ryu", @@ -371,9 +371,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.6" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", @@ -392,9 +392,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.14" +version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ "indexmap", "serde", diff --git a/README.md b/README.md index 90d744d..15be8a2 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ cargo binstall whyq - arbitrary `jq` usage on yaml/toml input with same syntax (args passed along to `jq` with json converted input) - drop-in replacement to [python-yq](https://kislyuk.github.io/yq/) (e.g. provides: yq) - handles multidoc **yaml** input (vector of documents returned when multiple docs found) +- handles duplicate yaml keys and [yaml merge keys](https://yaml.org/type/merge.html) - handles **toml** input (from [Table](https://docs.rs/toml/latest/toml/#parsing-toml)) - unpacks yaml tags (input is [singleton mapped](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map/index.html) [recursively](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map_recursive/index.html)) - allows converting `jq` output to YAML (`-y`) or TOML (`-t`) diff --git a/yq.rs b/yq.rs index daf21f4..bcd4e6a 100644 --- a/yq.rs +++ b/yq.rs @@ -72,7 +72,11 @@ impl Args { let mut docs: Vec = vec![]; for doc in yaml_de { - docs.push(singleton_map_recursive::deserialize(doc)?); + let mut yaml_doc: serde_yaml::Value = singleton_map_recursive::deserialize(doc)?; + yaml_doc.apply_merge()?; + let yaml_ser = serde_yaml::to_string(&yaml_doc)?; + let json_value: serde_json::Value = serde_json::from_str(&yaml_ser)?; + docs.push(json_value); } debug!("found {} documents", docs.len()); // if there is 1 or 0 documents, do not return as nested documents From 781ce25ff66cb9664046f615fa1cd8572b823de2 Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 13 Sep 2023 09:08:19 +0100 Subject: [PATCH 02/10] simplify release setup don't need to strip in profile if we do it in release job besides, strip fails on mac with serde now release cleanup: why are we putting readme + license in the archive? Signed-off-by: clux --- .github/workflows/release.yml | 9 --------- Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 73f543c..9bb3c72 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,17 +121,9 @@ jobs: [[ "${{ matrix.name }}" == windows-* ]] && ext=".exe" bin="target/${{ matrix.target }}/release/yq${ext}" strip "$bin" || true - - #version=$(cat VERSION) dst="yq-${{ matrix.target }}" mkdir "$dst" - - mkdir -p "target/release" - cp "$bin" "target/release/" # workaround for cargo-deb silliness with targets - cp "$bin" "$dst/" - #cp -r README.md LICENSE completions yq.1 "$dst/" - cp README.md LICENSE "$dst/" - name: Archive (tar) if: '! startsWith(matrix.name, ''windows-'')' @@ -184,7 +176,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: generate_release_notes: true - #draft: true fail_on_unmatched_files: true files: | yq-*.tar.xz diff --git a/Cargo.toml b/Cargo.toml index 901ab9c..7a5f113 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } [profile.release] lto = true panic = "abort" -strip = "symbols" +#strip = "symbols" [package.metadata.binstall] pkg-url = "{ repo }/releases/download/{ version }/yq-{ target }{ archive-suffix }" From 21316504bbfb3ebc6f3de988a6e9201b2e676e3e Mon Sep 17 00:00:00 2001 From: clux Date: Thu, 14 Sep 2023 23:46:44 +0100 Subject: [PATCH 03/10] make it work Signed-off-by: clux --- yq.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/yq.rs b/yq.rs index bcd4e6a..751cace 100644 --- a/yq.rs +++ b/yq.rs @@ -38,6 +38,10 @@ struct Args { #[arg(short = 'i', long, value_enum, default_value_t)] input: Input, + /// Apply YAML merge tags + #[arg(short = 'm', long, default_value = "false")] // TODO: yaml only? different input format? + yaml_merge: bool, + /// Arguments passed to jq /// /// These arguments must be trailing and come after the flags above. @@ -72,10 +76,14 @@ impl Args { let mut docs: Vec = vec![]; for doc in yaml_de { - let mut yaml_doc: serde_yaml::Value = singleton_map_recursive::deserialize(doc)?; - yaml_doc.apply_merge()?; - let yaml_ser = serde_yaml::to_string(&yaml_doc)?; - let json_value: serde_json::Value = serde_json::from_str(&yaml_ser)?; + let json_value: serde_json::Value = if self.yaml_merge { + let mut yaml_doc: serde_yaml::Value = singleton_map_recursive::deserialize(doc)?; + yaml_doc.apply_merge()?; + let yaml_ser = serde_yaml::to_string(&yaml_doc)?; + serde_yaml::from_str(&yaml_ser)? + } else { + singleton_map_recursive::deserialize(doc)? + }; docs.push(json_value); } debug!("found {} documents", docs.len()); @@ -99,8 +107,7 @@ impl Args { Self::try_parse_from(["cmd", "-h"])?; std::process::exit(2); } - let data = std::fs::read_to_string(f)?; - data + std::fs::read_to_string(f)? } else { Self::try_parse_from(["cmd", "-h"])?; std::process::exit(2); From 1433071f8faa7098117db009ff84290ac7f5f577 Mon Sep 17 00:00:00 2001 From: clux Date: Thu, 14 Sep 2023 23:54:48 +0100 Subject: [PATCH 04/10] tests Signed-off-by: clux --- yq.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/yq.rs b/yq.rs index 751cace..718f5d8 100644 --- a/yq.rs +++ b/yq.rs @@ -192,6 +192,7 @@ mod test { Self { yaml_output: yaml, toml_output: false, + yaml_merge: false, input: Input::Yaml, extra: args.into_iter().map(|x| x.to_string()).collect(), } From 8a003014d67a6fb74687172b6e49f41de6c28f63 Mon Sep 17 00:00:00 2001 From: clux Date: Fri, 15 Sep 2023 00:07:01 +0100 Subject: [PATCH 05/10] actual tests Signed-off-by: clux --- test/circle.yml | 58 +++++++++++++++++++++++++++++++++++++++++++++++ test/yq.test.bats | 10 +++++++- 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 test/circle.yml diff --git a/test/circle.yml b/test/circle.yml new file mode 100644 index 0000000..5f97557 --- /dev/null +++ b/test/circle.yml @@ -0,0 +1,58 @@ +version: 2.1 + +definitions: + steps: + - step: &build_binary + name: Build binary + command: chmod a+w . && cargo build --release + - step: &versions + name: Version information + command: rustc --version; cargo --version; rustup --version + + filters: + on_tag: &on_tag + tags: + only: /v[0-9]+(\.[0-9]+)*/ + branches: + ignore: /.*/ + on_every_commit: &on_every_commit + tags: + only: /.*/ + +jobs: + build: + docker: + - image: clux/muslrust:stable + resource_class: xlarge + environment: + IMAGE_NAME: whyq + steps: + - checkout + - run: *versions + - run: *build_binary + - run: echo versions + + release: + docker: + - image: clux/muslrust:stable + resource_class: xlarge + steps: + - checkout + - run: *versions + - run: *build_binary + - upload: + source: "target/x86_64-unknown-linux-musl/release/${IMAGE_NAME}" + binary_name: "${IMAGE_NAME}" + version: "${CIRCLE_TAG}" + arch: "x86_64-unknown-linux-musl" + +workflows: + version: 2 + my_flow: + jobs: + - build: + filters: + <<: *on_every_commit + - release: + filters: + <<: *on_tag diff --git a/test/yq.test.bats b/test/yq.test.bats index dc3e447..bd7dd5d 100644 --- a/test/yq.test.bats +++ b/test/yq.test.bats @@ -56,4 +56,12 @@ run yq -i=toml '.dependencies.clap.features' -c < Cargo.toml echo "$output" && echo "$output" | grep '["cargo","derive"]' -} \ No newline at end of file +} + +@test "yaml_merge" { + run yq -m '.workflows.my_flow.jobs[0].build' -c < test/circle.yml + echo "$output" && echo "$output" | grep '{"filters":{"tags":{"only":"/.*/"}}}' + + run yq -m '.jobs.build.steps[1].run.name' -r < test/circle.yml + echo "$output" && echo "$output" | grep "Version information" +} From 178d43cef16ab12ebecb410fbe7e9c21c8c5831d Mon Sep 17 00:00:00 2001 From: clux Date: Sat, 16 Sep 2023 21:21:55 +0100 Subject: [PATCH 06/10] use -x for expand instead of merge Signed-off-by: clux --- test/yq.test.bats | 4 ++-- yq.rs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/yq.test.bats b/test/yq.test.bats index bd7dd5d..87d363b 100644 --- a/test/yq.test.bats +++ b/test/yq.test.bats @@ -59,9 +59,9 @@ } @test "yaml_merge" { - run yq -m '.workflows.my_flow.jobs[0].build' -c < test/circle.yml + run yq -x '.workflows.my_flow.jobs[0].build' -c < test/circle.yml echo "$output" && echo "$output" | grep '{"filters":{"tags":{"only":"/.*/"}}}' - run yq -m '.jobs.build.steps[1].run.name' -r < test/circle.yml + run yq -x '.jobs.build.steps[1].run.name' -r < test/circle.yml echo "$output" && echo "$output" | grep "Version information" } diff --git a/yq.rs b/yq.rs index 718f5d8..64b6b69 100644 --- a/yq.rs +++ b/yq.rs @@ -38,9 +38,9 @@ struct Args { #[arg(short = 'i', long, value_enum, default_value_t)] input: Input, - /// Apply YAML merge tags - #[arg(short = 'm', long, default_value = "false")] // TODO: yaml only? different input format? - yaml_merge: bool, + /// Expand tags such as YAML <<: merge tags + #[arg(short = 'x', default_value = "false")] // Ideally should only work if input == Input::yaml + expand: bool, /// Arguments passed to jq /// @@ -76,7 +76,7 @@ impl Args { let mut docs: Vec = vec![]; for doc in yaml_de { - let json_value: serde_json::Value = if self.yaml_merge { + let json_value: serde_json::Value = if self.expand { let mut yaml_doc: serde_yaml::Value = singleton_map_recursive::deserialize(doc)?; yaml_doc.apply_merge()?; let yaml_ser = serde_yaml::to_string(&yaml_doc)?; @@ -192,7 +192,7 @@ mod test { Self { yaml_output: yaml, toml_output: false, - yaml_merge: false, + expand: false, input: Input::Yaml, extra: args.into_iter().map(|x| x.to_string()).collect(), } From 3cac338fada3ef76ebde11d3c0dfcf3c9c21df20 Mon Sep 17 00:00:00 2001 From: clux Date: Sat, 16 Sep 2023 21:54:51 +0100 Subject: [PATCH 07/10] stash this to showcase what we __could__ do, but shouldn't... Signed-off-by: clux --- Cargo.lock | 1 + Cargo.toml | 1 + test/yq.test.bats | 4 ++-- yq.rs | 33 ++++++++++++++++++++++++++------- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db1f067..57f0152 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,6 +495,7 @@ version = "0.5.0" dependencies = [ "anyhow", "clap", + "serde", "serde_json", "serde_yaml", "toml", diff --git a/Cargo.toml b/Cargo.toml index 7a5f113..e7cdd8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ toml = { version = "0.7.6", features = ["display"] } serde_yaml = "0.9.25" tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } +serde = "1.0.188" [profile.release] lto = true diff --git a/test/yq.test.bats b/test/yq.test.bats index 87d363b..66f6217 100644 --- a/test/yq.test.bats +++ b/test/yq.test.bats @@ -59,9 +59,9 @@ } @test "yaml_merge" { - run yq -x '.workflows.my_flow.jobs[0].build' -c < test/circle.yml + run yq '.workflows.my_flow.jobs[0].build' -c < test/circle.yml echo "$output" && echo "$output" | grep '{"filters":{"tags":{"only":"/.*/"}}}' - run yq -x '.jobs.build.steps[1].run.name' -r < test/circle.yml + run yq '.jobs.build.steps[1].run.name' -r < test/circle.yml echo "$output" && echo "$output" | grep "Version information" } diff --git a/yq.rs b/yq.rs index 64b6b69..d24c4f7 100644 --- a/yq.rs +++ b/yq.rs @@ -38,9 +38,21 @@ struct Args { #[arg(short = 'i', long, value_enum, default_value_t)] input: Input, - /// Expand tags such as YAML <<: merge tags - #[arg(short = 'x', default_value = "false")] // Ideally should only work if input == Input::yaml - expand: bool, + /// Do not expand tags + /// + /// This probably is not the desired behaviour for yaml given we are already + /// recursively singleton mapping the documents. + /// This is only an escape-hatch in case of early bugs, and hopefully will be removed + #[arg(long = "no-expand", default_value = "false")] + no_expand: bool, + + /// Partially expand? testing.. + #[arg( + long = "partial-expand", + default_value = "false", + conflicts_with = "no_expand" + )] + partial_expand: bool, /// Arguments passed to jq /// @@ -76,13 +88,19 @@ impl Args { let mut docs: Vec = vec![]; for doc in yaml_de { - let json_value: serde_json::Value = if self.expand { + let json_value: serde_json::Value = if self.no_expand { + use serde::Deserialize; + debug!("no expanding"); + serde_json::Value::deserialize(doc)? + } else if self.partial_expand { + debug!("some expanding"); + singleton_map_recursive::deserialize(doc)? + } else { + debug!("full expanding"); let mut yaml_doc: serde_yaml::Value = singleton_map_recursive::deserialize(doc)?; yaml_doc.apply_merge()?; let yaml_ser = serde_yaml::to_string(&yaml_doc)?; serde_yaml::from_str(&yaml_ser)? - } else { - singleton_map_recursive::deserialize(doc)? }; docs.push(json_value); } @@ -192,7 +210,8 @@ mod test { Self { yaml_output: yaml, toml_output: false, - expand: false, + no_expand: false, + partial_expand: false, // TODO: cleanup input: Input::Yaml, extra: args.into_iter().map(|x| x.to_string()).collect(), } From 9da92345a42ae706ee5d0e779526b2acec45d027 Mon Sep 17 00:00:00 2001 From: clux Date: Sat, 16 Sep 2023 21:56:53 +0100 Subject: [PATCH 08/10] fully expanding with apply_merge is the only thing that makes sense Signed-off-by: clux --- Cargo.lock | 1 - Cargo.toml | 1 - yq.rs | 28 +--------------------------- 3 files changed, 1 insertion(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 57f0152..db1f067 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -495,7 +495,6 @@ version = "0.5.0" dependencies = [ "anyhow", "clap", - "serde", "serde_json", "serde_yaml", "toml", diff --git a/Cargo.toml b/Cargo.toml index e7cdd8a..7a5f113 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ toml = { version = "0.7.6", features = ["display"] } serde_yaml = "0.9.25" tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } -serde = "1.0.188" [profile.release] lto = true diff --git a/yq.rs b/yq.rs index d24c4f7..267cb4f 100644 --- a/yq.rs +++ b/yq.rs @@ -38,22 +38,6 @@ struct Args { #[arg(short = 'i', long, value_enum, default_value_t)] input: Input, - /// Do not expand tags - /// - /// This probably is not the desired behaviour for yaml given we are already - /// recursively singleton mapping the documents. - /// This is only an escape-hatch in case of early bugs, and hopefully will be removed - #[arg(long = "no-expand", default_value = "false")] - no_expand: bool, - - /// Partially expand? testing.. - #[arg( - long = "partial-expand", - default_value = "false", - conflicts_with = "no_expand" - )] - partial_expand: bool, - /// Arguments passed to jq /// /// These arguments must be trailing and come after the flags above. @@ -88,15 +72,7 @@ impl Args { let mut docs: Vec = vec![]; for doc in yaml_de { - let json_value: serde_json::Value = if self.no_expand { - use serde::Deserialize; - debug!("no expanding"); - serde_json::Value::deserialize(doc)? - } else if self.partial_expand { - debug!("some expanding"); - singleton_map_recursive::deserialize(doc)? - } else { - debug!("full expanding"); + let json_value: serde_json::Value = { let mut yaml_doc: serde_yaml::Value = singleton_map_recursive::deserialize(doc)?; yaml_doc.apply_merge()?; let yaml_ser = serde_yaml::to_string(&yaml_doc)?; @@ -210,8 +186,6 @@ mod test { Self { yaml_output: yaml, toml_output: false, - no_expand: false, - partial_expand: false, // TODO: cleanup input: Input::Yaml, extra: args.into_iter().map(|x| x.to_string()).collect(), } From 6ce8a48465cc07aed703f17d88b78eaed7a8d2f1 Mon Sep 17 00:00:00 2001 From: clux Date: Sat, 16 Sep 2023 22:02:14 +0100 Subject: [PATCH 09/10] we don't handle duplicate keys Signed-off-by: clux --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 15be8a2..2b1b2b1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ cargo binstall whyq - arbitrary `jq` usage on yaml/toml input with same syntax (args passed along to `jq` with json converted input) - drop-in replacement to [python-yq](https://kislyuk.github.io/yq/) (e.g. provides: yq) - handles multidoc **yaml** input (vector of documents returned when multiple docs found) -- handles duplicate yaml keys and [yaml merge keys](https://yaml.org/type/merge.html) +- handles [yaml merge keys](https://yaml.org/type/merge.html) - handles **toml** input (from [Table](https://docs.rs/toml/latest/toml/#parsing-toml)) - unpacks yaml tags (input is [singleton mapped](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map/index.html) [recursively](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map_recursive/index.html)) - allows converting `jq` output to YAML (`-y`) or TOML (`-t`) From 1eee89af27436a2f95fb71771df9fc08a6f95f34 Mon Sep 17 00:00:00 2001 From: clux Date: Sat, 16 Sep 2023 22:15:01 +0100 Subject: [PATCH 10/10] clarify Signed-off-by: clux --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2b1b2b1..bc9714d 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,10 @@ cargo binstall whyq - arbitrary `jq` usage on yaml/toml input with same syntax (args passed along to `jq` with json converted input) - drop-in replacement to [python-yq](https://kislyuk.github.io/yq/) (e.g. provides: yq) - handles multidoc **yaml** input (vector of documents returned when multiple docs found) -- handles [yaml merge keys](https://yaml.org/type/merge.html) +- handles [yaml merge keys](https://yaml.org/type/merge.html) and expands yaml tags (via `serde_yaml`) - handles **toml** input (from [Table](https://docs.rs/toml/latest/toml/#parsing-toml)) -- unpacks yaml tags (input is [singleton mapped](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map/index.html) [recursively](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map_recursive/index.html)) - allows converting `jq` output to YAML (`-y`) or TOML (`-t`) -- uses <1MB in your CI image +- <1MB in binary size (for your small cloud CI images) ## YAML Input Use as [jq](https://jqlang.github.io/jq/tutorial/) either via stdin: @@ -130,4 +129,5 @@ If you pass on `-r` for raw output, then this will not be parseable as json. - Only YAML/TOML input/output is supported (no XML). - Shells out to `jq` (only supports what your jq version supports). - Does not provide rich `-h` or `--help` output (assumes you can use `jq --help` or `man jq`). -- Does not preserve [YAML tags](https://yaml.org/spec/1.2-old/spec.html#id2764295) (input is [singleton mapped](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map/index.html) [recursively](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map_recursive/index.html)). +- Does not preserve [YAML tags](https://yaml.org/spec/1.2-old/spec.html#id2764295) (input is [singleton mapped](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map/index.html) [recursively](https://docs.rs/serde_yaml/latest/serde_yaml/with/singleton_map_recursive/index.html) and then [apply_merged](https://docs.rs/serde_yaml/latest/serde_yaml/value/enum.Value.html#method.apply_merge) before `jq`) +- Does [not support duplicate keys](https://github.com/clux/whyq/issues/14) in the input document