From 91544b7d6a2fe8231dec9a29c626c61bef62141c Mon Sep 17 00:00:00 2001 From: John Bergvall Date: Thu, 3 Jun 2021 08:57:03 +0200 Subject: [PATCH 1/4] Limit backslash escape removal for quoted newlines Keep escape backslash for newline/tab characters. Parsing still differs from shell syntax but keeps string structure intact. Example certificate data VAR="---BEGIN---\r\n---END---" now becomes "---BEGIN---\r\n---END---" instead of "---BEGIN---rn---END---" --- environ/environ.py | 12 ++++++++++-- tests/fixtures.py | 2 ++ tests/test_env.py | 4 ++++ tests/test_env.txt | 2 ++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/environ/environ.py b/environ/environ.py index 0267add6..111e176d 100644 --- a/environ/environ.py +++ b/environ/environ.py @@ -180,7 +180,7 @@ def str(self, var, default=NOTSET, multiline=False): """ value = self.get_value(var, cast=str, default=default) if multiline: - return value.replace('\\n', '\n') + return re.sub(r'(\\r)?\\n', r'\n', value) return value def unicode(self, var, default=NOTSET): @@ -769,6 +769,13 @@ def read_env(cls, env_file=None, **overrides): logger.debug('Read environment variables from: {}'.format(env_file)) + def _keep_escaped_format_characters(match): + """Keep escaped newline/tabs in quoted strings""" + escaped_char = match.group(1) + if escaped_char in 'rnt': + return '\\' + escaped_char + return escaped_char + for line in content.splitlines(): m1 = re.match(r'\A(?:export )?([A-Za-z_0-9]+)=(.*)\Z', line) if m1: @@ -778,7 +785,8 @@ def read_env(cls, env_file=None, **overrides): val = m2.group(1) m3 = re.match(r'\A"(.*)"\Z', val) if m3: - val = re.sub(r'\\(.)', r'\1', m3.group(1)) + val = re.sub(r'\\(.)', _keep_escaped_format_characters, + m3.group(1)) cls.ENVIRON.setdefault(key, str(val)) # set defaults diff --git a/tests/fixtures.py b/tests/fixtures.py index f685f5cb..29b4dc9f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -31,6 +31,8 @@ class FakeEnv: def generate_data(cls): return dict(STR_VAR='bar', MULTILINE_STR_VAR='foo\\nbar', + MULTILINE_QUOTED_STR_VAR='---BEGIN---\\r\\n---END---', + MULTILINE_ESCAPED_STR_VAR='---BEGIN---\\\\n---END---', INT_VAR='42', FLOAT_VAR='33.3', FLOAT_COMMA_VAR='33,3', diff --git a/tests/test_env.py b/tests/test_env.py index efb09fbc..afa6470b 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -57,6 +57,10 @@ def test_contains(self): ('STR_VAR', 'bar', False), ('MULTILINE_STR_VAR', 'foo\\nbar', False), ('MULTILINE_STR_VAR', 'foo\nbar', True), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\\r\\n---END---', False), + ('MULTILINE_QUOTED_STR_VAR', '---BEGIN---\n---END---', True), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\\n---END---', False), + ('MULTILINE_ESCAPED_STR_VAR', '---BEGIN---\\\n---END---', True), ], ) def test_str(self, var, val, multiline): diff --git a/tests/test_env.txt b/tests/test_env.txt index dfbd61ef..befeed76 100644 --- a/tests/test_env.txt +++ b/tests/test_env.txt @@ -33,6 +33,8 @@ INT_VAR=42 STR_LIST_WITH_SPACES= foo, bar STR_VAR=bar MULTILINE_STR_VAR=foo\nbar +MULTILINE_QUOTED_STR_VAR="---BEGIN---\r\n---END---" +MULTILINE_ESCAPED_STR_VAR=---BEGIN---\\n---END--- INT_LIST=42,33 CYRILLIC_VAR=фуубар INT_TUPLE=(42,33) From 077f49ed9587dd1f7f9dff8951b6a94d5d828062 Mon Sep 17 00:00:00 2001 From: Serghei Iakovlev Date: Mon, 30 Aug 2021 10:34:22 +0300 Subject: [PATCH 2/4] Amend documentation --- docs/tips.rst | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/docs/tips.rst b/docs/tips.rst index a09391f7..8f6089b2 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -91,15 +91,40 @@ You can use something like this to handle similar cases. Multiline value =============== -You can set a multiline variable value: +To escape newline/tab characters pass ``multiline=False`` to ```str()```. +The following example demonstrates the above: + +**.env file**: + +.. code-block:: shell + + # .env file contents + UNQUOTED_CERT=---BEGIN---\r\n---END--- + QUOTED_CERT="---BEGIN---\r\n---END---" + +**settings.py file**: .. code-block:: python - # MULTILINE_TEXT=Hello\\nWorld - >>> print env.str('MULTILINE_TEXT', multiline=True) - Hello - World + # settings.py file contents + import environ + + + env = environ.Env() + + print(env.str('UNQUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + + print(env.str('UNQUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- + + print(env.str('QUOTED_CERT', multiline=True)) + # ---BEGIN--- + # ---END--- + print(env.str('QUOTED_CERT', multiline=False)) + # ---BEGIN---\r\n---END--- Proxy value =========== From a55864054e418e6a27ba3af03f714e0a8e444db4 Mon Sep 17 00:00:00 2001 From: John Bergvall Date: Tue, 14 Sep 2021 19:31:07 +0200 Subject: [PATCH 3/4] Update change log --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cf3b0003..c067f280 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ and this project adheres to `Semantic Versioning Date: Mon, 30 Aug 2021 11:25:12 +0300 Subject: [PATCH 4/4] Add note about escaping newlines --- docs/tips.rst | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/tips.rst b/docs/tips.rst index 8f6089b2..660f6fbc 100644 --- a/docs/tips.rst +++ b/docs/tips.rst @@ -91,7 +91,13 @@ You can use something like this to handle similar cases. Multiline value =============== -To escape newline/tab characters pass ``multiline=False`` to ```str()```. +To get multiline value pass ``multiline=True`` to ```str()```. + +.. note:: + + You shouldn't escape newline/tab characters yourself if you want to preserve + the formatting. + The following example demonstrates the above: **.env file**: @@ -101,6 +107,7 @@ The following example demonstrates the above: # .env file contents UNQUOTED_CERT=---BEGIN---\r\n---END--- QUOTED_CERT="---BEGIN---\r\n---END---" + ESCAPED_CERT=---BEGIN---\\n---END--- **settings.py file**: @@ -126,6 +133,13 @@ The following example demonstrates the above: print(env.str('QUOTED_CERT', multiline=False)) # ---BEGIN---\r\n---END--- + print(env.str('ESCAPED_CERT', multiline=True)) + # ---BEGIN---\ + # ---END--- + + print(env.str('ESCAPED_CERT', multiline=False)) + # ---BEGIN---\\n---END--- + Proxy value ===========