From 20ed350192dff339949fac3871a53cbbb4282e60 Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Thu, 21 Mar 2024 10:44:45 +0100 Subject: [PATCH 1/7] fix: do not quote already quoted strings --- snakemake_interface_executor_plugins/utils.py | 12 +++++++++++- tests/tests.py | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/snakemake_interface_executor_plugins/utils.py b/snakemake_interface_executor_plugins/utils.py index 7749871..4395f60 100644 --- a/snakemake_interface_executor_plugins/utils.py +++ b/snakemake_interface_executor_plugins/utils.py @@ -6,6 +6,7 @@ import asyncio from collections import UserDict from pathlib import Path +import re import shlex import threading from typing import Any, List @@ -48,7 +49,10 @@ def format_cli_value(value: Any) -> str: elif isinstance(value, Path): return shlex.quote(str(value)) elif isinstance(value, str): - return shlex.quote(value) + if is_quoted(value): + return value + else: + return shlex.quote(value) else: return repr(value) @@ -99,3 +103,9 @@ async def async_lock(_lock: threading.Lock): yield # the lock is held finally: _lock.release() + + +_is_quoted_re = re.compile(r"^['\"].+['\"]") + +def is_quoted(value: str) -> bool: + return _is_quoted_re.match(value) is not None \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index 6cc3f02..0b76646 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -3,6 +3,7 @@ from snakemake_interface_common.plugin_registry.tests import TestRegistryBase from snakemake_interface_common.plugin_registry.plugin import PluginBase, SettingsBase from snakemake_interface_common.plugin_registry import PluginRegistryBase +from snakemake_interface_executor_plugins.utils import format_cli_arg class TestRegistry(TestRegistryBase): @@ -26,3 +27,8 @@ def validate_settings(self, settings: SettingsBase, plugin: PluginBase): def get_example_args(self) -> List[str]: return ["--cluster-generic-submit-cmd", "qsub"] + + +def test_format_cli_arg(): + fmt = format_cli_arg("--default-resources", {"slurm_extra": "'--gres=gpu:1'"}) + assert fmt == "--default-resources \"slurm_extra='--gres=gpu:1'\"" \ No newline at end of file From 2981c77d7f4a365bc0870afec7d67788c4405d2f Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Thu, 21 Mar 2024 11:34:29 +0100 Subject: [PATCH 2/7] testing --- snakemake_interface_executor_plugins/utils.py | 3 ++- tests/tests.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/snakemake_interface_executor_plugins/utils.py b/snakemake_interface_executor_plugins/utils.py index 4395f60..5c79d41 100644 --- a/snakemake_interface_executor_plugins/utils.py +++ b/snakemake_interface_executor_plugins/utils.py @@ -107,5 +107,6 @@ async def async_lock(_lock: threading.Lock): _is_quoted_re = re.compile(r"^['\"].+['\"]") + def is_quoted(value: str) -> bool: - return _is_quoted_re.match(value) is not None \ No newline at end of file + return _is_quoted_re.match(value) is not None diff --git a/tests/tests.py b/tests/tests.py index 0b76646..d0f2eb1 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -29,6 +29,11 @@ def get_example_args(self) -> List[str]: return ["--cluster-generic-submit-cmd", "qsub"] -def test_format_cli_arg(): +def test_format_cli_arg_single_quote(): fmt = format_cli_arg("--default-resources", {"slurm_extra": "'--gres=gpu:1'"}) - assert fmt == "--default-resources \"slurm_extra='--gres=gpu:1'\"" \ No newline at end of file + assert fmt == "--default-resources \"slurm_extra='--gres=gpu:1'\"" + + +def test_format_cli_arg_double_quote(): + fmt = format_cli_arg("--default-resources", {"slurm_extra": '"--gres=gpu:1"'}) + assert fmt == "--default-resources 'slurm_extra=\"--gres=gpu:1\"'" From 0094edee8662547747ef8617bd486f0f8b4fbe8f Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Thu, 21 Mar 2024 11:37:08 +0100 Subject: [PATCH 3/7] add test case for ints --- tests/tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index d0f2eb1..d1467a9 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -37,3 +37,8 @@ def test_format_cli_arg_single_quote(): def test_format_cli_arg_double_quote(): fmt = format_cli_arg("--default-resources", {"slurm_extra": '"--gres=gpu:1"'}) assert fmt == "--default-resources 'slurm_extra=\"--gres=gpu:1\"'" + + +def test_format_cli_arg_int(): + fmt = format_cli_arg("--default-resources", {"mem_mb": 200}) + assert fmt == "--default-resources 'mem_mb=200'" \ No newline at end of file From c103dad872cc3a808048b62402898a8321f19c63 Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Thu, 21 Mar 2024 11:42:13 +0100 Subject: [PATCH 4/7] fix quoting of python expressions --- snakemake_interface_executor_plugins/utils.py | 8 ++++---- tests/tests.py | 6 +++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/snakemake_interface_executor_plugins/utils.py b/snakemake_interface_executor_plugins/utils.py index 5c79d41..0956b5c 100644 --- a/snakemake_interface_executor_plugins/utils.py +++ b/snakemake_interface_executor_plugins/utils.py @@ -49,10 +49,10 @@ def format_cli_value(value: Any) -> str: elif isinstance(value, Path): return shlex.quote(str(value)) elif isinstance(value, str): - if is_quoted(value): - return value - else: - return shlex.quote(value) + # if the value is already quoted, do not quote again + # otherwise, also not quoting is necessary because it interpreted as a python + # expression + return value else: return repr(value) diff --git a/tests/tests.py b/tests/tests.py index d1467a9..565d419 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -41,4 +41,8 @@ def test_format_cli_arg_double_quote(): def test_format_cli_arg_int(): fmt = format_cli_arg("--default-resources", {"mem_mb": 200}) - assert fmt == "--default-resources 'mem_mb=200'" \ No newline at end of file + assert fmt == "--default-resources 'mem_mb=200'" + +def test_format_cli_arg_expr(): + fmt = format_cli_arg("--default-resources", {"mem_mb": 'min(2 * input.size_mb, 2000)'}) + assert fmt == "--default-resources 'mem_mb=min(2 * input.size_mb, 2000)'" \ No newline at end of file From bad9612b2bd017d1301d65017824d4e3635cc33a Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Thu, 21 Mar 2024 11:43:53 +0100 Subject: [PATCH 5/7] fmt --- snakemake_interface_executor_plugins/utils.py | 2 +- tests/tests.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/snakemake_interface_executor_plugins/utils.py b/snakemake_interface_executor_plugins/utils.py index 0956b5c..dc33886 100644 --- a/snakemake_interface_executor_plugins/utils.py +++ b/snakemake_interface_executor_plugins/utils.py @@ -50,7 +50,7 @@ def format_cli_value(value: Any) -> str: return shlex.quote(str(value)) elif isinstance(value, str): # if the value is already quoted, do not quote again - # otherwise, also not quoting is necessary because it interpreted as a python + # otherwise, also not quoting is necessary because it interpreted as a python # expression return value else: diff --git a/tests/tests.py b/tests/tests.py index 565d419..be37177 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -43,6 +43,9 @@ def test_format_cli_arg_int(): fmt = format_cli_arg("--default-resources", {"mem_mb": 200}) assert fmt == "--default-resources 'mem_mb=200'" + def test_format_cli_arg_expr(): - fmt = format_cli_arg("--default-resources", {"mem_mb": 'min(2 * input.size_mb, 2000)'}) - assert fmt == "--default-resources 'mem_mb=min(2 * input.size_mb, 2000)'" \ No newline at end of file + fmt = format_cli_arg( + "--default-resources", {"mem_mb": "min(2 * input.size_mb, 2000)"} + ) + assert fmt == "--default-resources 'mem_mb=min(2 * input.size_mb, 2000)'" From c63730a971438618c70131e4b0c12e2ed44bdd4c Mon Sep 17 00:00:00 2001 From: Johannes Koester Date: Thu, 21 Mar 2024 14:46:38 +0100 Subject: [PATCH 6/7] handle lists --- snakemake_interface_executor_plugins/utils.py | 16 ++++++++++------ tests/tests.py | 7 +++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/snakemake_interface_executor_plugins/utils.py b/snakemake_interface_executor_plugins/utils.py index dc33886..20207a6 100644 --- a/snakemake_interface_executor_plugins/utils.py +++ b/snakemake_interface_executor_plugins/utils.py @@ -40,19 +40,23 @@ def format_cli_pos_arg(value, quote=True): elif not_iterable(value): return format_cli_value(value) else: - return join_cli_args(format_cli_value(v) for v in value) + return join_cli_args(format_cli_value(v, quote_if_contains_whitespace=True) for v in value) -def format_cli_value(value: Any) -> str: +def format_cli_value(value: Any, quote_if_contains_whitespace: bool=False) -> str: if isinstance(value, SettingsEnumBase): return value.item_to_choice() elif isinstance(value, Path): return shlex.quote(str(value)) elif isinstance(value, str): - # if the value is already quoted, do not quote again - # otherwise, also not quoting is necessary because it interpreted as a python - # expression - return value + if is_quoted(value): + # the value is already quoted, do not quote again + return value + elif quote_if_contains_whitespace and " " in value: + # may be expression + return repr(value) + else: + return value else: return repr(value) diff --git a/tests/tests.py b/tests/tests.py index be37177..a70fec1 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -49,3 +49,10 @@ def test_format_cli_arg_expr(): "--default-resources", {"mem_mb": "min(2 * input.size_mb, 2000)"} ) assert fmt == "--default-resources 'mem_mb=min(2 * input.size_mb, 2000)'" + + +def test_format_cli_arg_list(): + fmt = format_cli_arg( + "--config", ["foo={'bar': 1}"] + ) + assert fmt == "--config \"foo={'bar': 1}\"" \ No newline at end of file From 4b38740fca6a5b749e32ca92536f765542ea9ed8 Mon Sep 17 00:00:00 2001 From: Christian Meesters Date: Thu, 21 Mar 2024 16:49:05 +0100 Subject: [PATCH 7/7] fix: formatting --- snakemake_interface_executor_plugins/utils.py | 6 ++++-- tests/tests.py | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/snakemake_interface_executor_plugins/utils.py b/snakemake_interface_executor_plugins/utils.py index 20207a6..b72ac7c 100644 --- a/snakemake_interface_executor_plugins/utils.py +++ b/snakemake_interface_executor_plugins/utils.py @@ -40,10 +40,12 @@ def format_cli_pos_arg(value, quote=True): elif not_iterable(value): return format_cli_value(value) else: - return join_cli_args(format_cli_value(v, quote_if_contains_whitespace=True) for v in value) + return join_cli_args( + format_cli_value(v, quote_if_contains_whitespace=True) for v in value + ) -def format_cli_value(value: Any, quote_if_contains_whitespace: bool=False) -> str: +def format_cli_value(value: Any, quote_if_contains_whitespace: bool = False) -> str: if isinstance(value, SettingsEnumBase): return value.item_to_choice() elif isinstance(value, Path): diff --git a/tests/tests.py b/tests/tests.py index a70fec1..3c3453b 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -52,7 +52,5 @@ def test_format_cli_arg_expr(): def test_format_cli_arg_list(): - fmt = format_cli_arg( - "--config", ["foo={'bar': 1}"] - ) - assert fmt == "--config \"foo={'bar': 1}\"" \ No newline at end of file + fmt = format_cli_arg("--config", ["foo={'bar': 1}"]) + assert fmt == "--config \"foo={'bar': 1}\""