diff --git a/README.md b/README.md index 7be1321..a645440 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,45 @@ check = true ignore_case = true ``` +### Configuration Overrides +The `pyproject.toml` configuration file also supports configuration overrides, which are not available as command-line arguments. These overrides allow for fine-grained control of sort options for particular keys. + +Only the following options can be included in an override: + +```toml +[tool.tomlsort.overrides."path.to.key"] +table_keys = true +inline_tables = true +inline_arrays = true +``` + +In the example configuration, `path.to.key` is the key to match. Keys are matched using the [Python fnmatch function](https://docs.python.org/3/library/fnmatch.html), so glob-style wildcards are supported. + +For instance, to disable sorting the table in the following TOML file: + +```toml +[servers.beta] +ip = "10.0.0.2" +dc = "eqdc10" +country = "中国" +``` + +You can use any of the following overrides: + +```toml +# Overrides in own table +[tool.tomlsort.overrides."servers.beta"] +table_keys = false + +# Overrides in the tomlsort table +[tool.tomlsort] +overrides."servers.beta".table_keys = false + +# Override using a wildcard if config should be applied to all servers keys +[tool.tomlsort] +overrides."servers.*".table_keys = false +``` + ## Comments Due to the free form nature of comments, it is hard to include them in a sort in a generic way that will work for everyone. `toml-sort` deals with four different types of comments. They are all enabled by default, but can be disabled using CLI switches, in which case comments of that type will be removed from the output. diff --git a/tests/examples/sorted/from-toml-lang-overrides.toml b/tests/examples/sorted/from-toml-lang-overrides.toml new file mode 100644 index 0000000..c93d813 --- /dev/null +++ b/tests/examples/sorted/from-toml-lang-overrides.toml @@ -0,0 +1,44 @@ +# This is a TOML document. Boom. + +title = "TOML Example" + +[clients] +data = [["gamma", "delta"], [1, 2]] # just an update to make sure parsers support it +# Line breaks are OK when inside arrays +hosts = [ + "alpha", + "omega" +] + +[database] +connection_max = 5000 +enabled = true # Comment after a boolean +ports = [8001, 8001, 8002] +server = "192.168.1.1" + +[owner] +bio = "GitHub Cofounder & CEO\nLikes tater tots and beer." +dob = 1979-05-27T07:32:00Z # First class dates? Why not? +name = "Tom Preston-Werner" +organization = "GitHub" + +[[products]] +name = "Hammer" +sku = 738594937 + +[[products]] +color = "gray" +name = "Nail" +sku = 284758393 + +[servers] + +# You can indent as you please. Tabs or spaces. TOML don't care. +[servers.alpha] +dc = "eqdc10" +ip = "10.0.0.1" + +[servers.beta] +ip = "10.0.0.2" +dc = "eqdc10" +country = "中国" # This should be parsed as UTF-8 diff --git a/tests/test_cli.py b/tests/test_cli.py index 2cfbbf9..43a9786 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,6 +11,7 @@ import pytest from toml_sort import cli +from toml_sort.tomlsort import SortOverrideConfiguration PATH_EXAMPLES = "tests/examples" @@ -254,7 +255,8 @@ def test_multiple_files_and_errors(options): def test_load_config_file_read(): """Test no error if pyproject.toml cannot be read.""" with mock.patch("toml_sort.cli.open", side_effect=OSError): - assert not cli.load_config_file() + section = cli.load_pyproject() + assert not cli.parse_config(section) @pytest.mark.parametrize( @@ -264,6 +266,15 @@ def test_load_config_file_read(): ("[tool.other]\nfoo=2", {}), ("[tool.tomlsort]", {}), ("[tool.tomlsort]\nall=true", {"all": True}), + ( + """ + [tool.tomlsort] + all=true + [tool.tomlsort.overrides] + "a.b.c".inline_array = false + """, + {"all": True}, + ), ( "[tool.tomlsort]\nspaces_before_inline_comment=4", {"spaces_before_inline_comment": 4}, @@ -274,7 +285,8 @@ def test_load_config_file(toml, expected): """Test load_config_file.""" open_mock = mock.mock_open(read_data=toml) with mock.patch("toml_sort.cli.open", open_mock): - assert cli.load_config_file() == expected + section = cli.load_pyproject() + assert cli.parse_config(section) == expected @pytest.mark.parametrize( @@ -285,4 +297,80 @@ def test_load_config_file_invalid(toml): open_mock = mock.mock_open(read_data=toml) with mock.patch("toml_sort.cli.open", open_mock): with pytest.raises(SystemExit): - cli.load_config_file() + section = cli.load_pyproject() + cli.parse_config(section) + + +@pytest.mark.parametrize( + "toml,expected", + [ + ( + """ + [tool.tomlsort.overrides."a.b.c"] + table_keys = false + """, + {"a.b.c": SortOverrideConfiguration(table_keys=False)}, + ), + ( + """ + [tool.tomlsort.overrides."test.123"] + table_keys = false + [tool.tomlsort.overrides."test.456"] + inline_tables = false + [tool.tomlsort.overrides."test.789"] + inline_arrays = false + """, + { + "test.123": SortOverrideConfiguration(table_keys=False), + "test.456": SortOverrideConfiguration(inline_tables=False), + "test.789": SortOverrideConfiguration(inline_arrays=False), + }, + ), + ( + """ + [tool.tomlsort.overrides] + "test.123".table_keys = false + "test.456".inline_tables = false + "test.789".inline_arrays = false + """, + { + "test.123": SortOverrideConfiguration(table_keys=False), + "test.456": SortOverrideConfiguration(inline_tables=False), + "test.789": SortOverrideConfiguration(inline_arrays=False), + }, + ), + ], +) +def test_load_config_overrides(toml, expected): + """Test that we correctly turn settings in tomldocument into a + SortOverrideConfiguration dataclass.""" + open_mock = mock.mock_open(read_data=toml) + with mock.patch("toml_sort.cli.open", open_mock): + section = cli.load_pyproject() + assert expected == cli.parse_config_overrides(section) + + +@pytest.mark.parametrize( + "toml", + [ + """ + [tool.tomlsort.overrides."a.b.c"] + unknown = false + """, + """ + [tool.tomlsort.overrides."a.b.c"] + table_keys = false + inline_tables = false + inline_arrays = false + foo = "bar" + """, + ], +) +def test_load_config_overrides_fail(toml): + """Test that parse_config_overrides exits if the config contains an + unexpected key.""" + open_mock = mock.mock_open(read_data=toml) + with mock.patch("toml_sort.cli.open", open_mock): + with pytest.raises(SystemExit): + section = cli.load_pyproject() + cli.parse_config_overrides(section) diff --git a/tests/test_toml_sort.py b/tests/test_toml_sort.py index e5f7bbc..6b0b50a 100644 --- a/tests/test_toml_sort.py +++ b/tests/test_toml_sort.py @@ -11,6 +11,7 @@ CommentConfiguration, FormattingConfiguration, SortConfiguration, + SortOverrideConfiguration, ) @@ -82,6 +83,26 @@ def test_sort_toml_is_str() -> None: ), }, ), + ( + "from-toml-lang", + "from-toml-lang-overrides", + { + "sort_config": SortConfiguration( + inline_arrays=True, inline_tables=True + ), + "format_config": FormattingConfiguration( + spaces_before_inline_comment=1 + ), + "sort_config_overrides": { + "servers.beta": SortOverrideConfiguration( + table_keys=False + ), + "clients.data": SortOverrideConfiguration( + inline_arrays=False + ), + }, + }, + ), ( "pyproject-weird-order", "pyproject-weird-order", diff --git a/toml_sort/cli.py b/toml_sort/cli.py index bd4b5f5..ac057dc 100644 --- a/toml_sort/cli.py +++ b/toml_sort/cli.py @@ -1,16 +1,19 @@ """Toml Sort command line interface.""" import argparse +import dataclasses import sys from argparse import ArgumentParser from typing import Any, Dict, List, Optional, Type import tomlkit +from tomlkit import TOMLDocument from .tomlsort import ( CommentConfiguration, FormattingConfiguration, SortConfiguration, + SortOverrideConfiguration, TomlSort, ) @@ -78,18 +81,26 @@ def validate_and_copy( target[key] = data.pop(key) -def load_config_file() -> Dict[str, Any]: - """Load the configuration from pyproject.toml.""" +def load_pyproject() -> TOMLDocument: + """Load pyproject file, and return tool.tomlsort section.""" try: with open("pyproject.toml", encoding="utf-8") as file: content = file.read() except OSError: - return {} + return tomlkit.document() document = tomlkit.parse(content) tool_section = document.get("tool", tomlkit.document()) - toml_sort_section = tool_section.get("tomlsort", tomlkit.document()) - config = dict(toml_sort_section) + return tool_section.get("tomlsort", tomlkit.document()) + + +def parse_config(tomlsort_section: TOMLDocument) -> Dict[str, Any]: + """Load the toml_sort configuration from a TOMLDocument.""" + config = dict(tomlsort_section) + + # remove the overrides key, since it is parsed separately + # in parse_config_overrides. + config.pop("overrides", None) clean_config: Dict[str, Any] = {} validate_and_copy(config, clean_config, "all", bool) @@ -121,7 +132,30 @@ def load_config_file() -> Dict[str, Any]: return clean_config -def get_parser() -> ArgumentParser: +def parse_config_overrides( + tomlsort_section: TOMLDocument, +) -> Dict[str, SortOverrideConfiguration]: + """Parse the tool.tomlsort.overrides section of the config.""" + fields = dataclasses.fields(SortOverrideConfiguration) + settings_definition = {field.name: field.type for field in fields} + override_settings = dict( + tomlsort_section.get("overrides", tomlkit.document()) + ) + overrides = {} + for path, settings in override_settings.items(): + if not settings.keys() <= settings_definition.keys(): + unknown_settings = settings.keys() - settings_definition.keys() + printerr("Unexpected configuration override settings:") + for unknown_setting in unknown_settings: + printerr(f' "{path}".{unknown_setting}') + sys.exit(1) + + overrides[path] = SortOverrideConfiguration(**settings) + + return overrides + + +def get_parser(defaults: Dict[str, Any]) -> ArgumentParser: """Get the argument parser.""" parser = ArgumentParser( prog="toml-sort", @@ -286,7 +320,7 @@ def get_parser() -> ArgumentParser: type=str, nargs="*", ) - parser.set_defaults(**load_config_file()) + parser.set_defaults(**defaults) return parser @@ -294,7 +328,12 @@ def cli( # pylint: disable=too-many-branches arguments: Optional[List[str]] = None, ) -> None: """Toml sort cli implementation.""" - args = get_parser().parse_args(args=arguments) # strip command itself + settings = load_pyproject() + configuration = parse_config(settings) + configuration_overrides = parse_config_overrides(settings) + args = get_parser(configuration).parse_args( + args=arguments + ) # strip command itself if args.version: print(get_version()) sys.exit(0) @@ -353,6 +392,7 @@ def cli( # pylint: disable=too-many-branches spaces_indent_inline_array=args.spaces_indent_inline_array, trailing_comma_inline_array=args.trailing_comma_inline_array, ), + sort_config_overrides=configuration_overrides, ).sorted() if args.check: if original_toml != sorted_toml: diff --git a/toml_sort/tomlsort.py b/toml_sort/tomlsort.py index 29a52df..6a3618f 100644 --- a/toml_sort/tomlsort.py +++ b/toml_sort/tomlsort.py @@ -1,10 +1,21 @@ """Utility functions and classes to sort toml text.""" from __future__ import annotations +import fnmatch import itertools import re -from dataclasses import dataclass, field -from typing import Any, Iterable, List, Optional, Tuple, TypeVar, cast +from dataclasses import asdict, dataclass, field +from typing import ( + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + TypeVar, + Union, + cast, +) import tomlkit from tomlkit.api import item as tomlkit_item @@ -98,10 +109,10 @@ def coalesce_tables( """Merge any duplicate keys that exist in an iterable of TomlSortItem.""" coalesced = {} for table in tables: - if table.key not in coalesced: - coalesced[table.key] = table + if table.keys.base not in coalesced: + coalesced[table.keys.base] = table else: - existing = coalesced[table.key] + existing = coalesced[table.keys.base] existing.children.extend(table.children) existing.attached_comments.extend(table.attached_comments) @@ -112,12 +123,54 @@ def coalesce_tables( return coalesced.values() +class TomlSortKeys: + """Keeps track of the Keys for a particular TomlSortItem. + + We use this to keep track of the full path of an item so that we can + find the configuration overrides that apply to it. + """ + + keys: List[Key] + + def __init__(self, keys: Union[List[Key], Key]): + if isinstance(keys, Key): + self.keys = [keys] + else: + self.keys = keys + + @property + def base(self) -> Key: + """Returns the last key segment. + + For example: would return test for this.is.a.test + """ + return self.keys[-1] + + @base.setter + def base(self, value: Key) -> None: + """Setter for the base property.""" + self.keys[-1] = value + + def as_string(self) -> str: + """Returns the full set of keys as a string.""" + return ".".join(k.key for k in self.keys) + + def __add__(self, other: Union[TomlSortKeys, Key]) -> TomlSortKeys: + """Add together a TomlSortKeys object and either a Key or another + TomlSortKeys object.""" + if isinstance(other, Key): + keys = [other] + else: + keys = other.keys + return TomlSortKeys(self.keys + keys) + + @dataclass class TomlSortItem: """Dataclass used to keep track of comments attached to Toml Items while they are being sorted.""" - key: Key + keys: TomlSortKeys value: Item attached_comments: List[Comment] = field(default_factory=list) children: List[TomlSortItem] = field(default_factory=list) @@ -192,15 +245,27 @@ class FormattingConfiguration: trailing_comma_inline_array: bool = False +@dataclass +class SortOverrideConfiguration: + """Configures overrides to sort configuration for a particular key.""" + + table_keys: Optional[bool] = None + inline_tables: Optional[bool] = None + inline_arrays: Optional[bool] = None + + class TomlSort: """API to manage sorting toml files.""" - def __init__( + def __init__( # pylint: disable=too-many-arguments self, input_toml: str, comment_config: Optional[CommentConfiguration] = None, sort_config: Optional[SortConfiguration] = None, format_config: Optional[FormattingConfiguration] = None, + sort_config_overrides: Optional[ + Dict[str, SortOverrideConfiguration] + ] = None, ) -> None: """Initializer.""" self.input_toml = input_toml @@ -211,13 +276,71 @@ def __init__( if sort_config is None: sort_config = SortConfiguration() - self.sort_config = sort_config + self._sort_config = sort_config if format_config is None: format_config = FormattingConfiguration() self.format_config = format_config - def sort_array(self, array: Array, indent_depth: int = 0) -> Array: + if sort_config_overrides is None: + sort_config_overrides = {} + self.sort_config_overrides = sort_config_overrides + + def _find_config_override( + self, keys: Optional[TomlSortKeys] + ) -> Optional[SortOverrideConfiguration]: + """Returns a SortOverrideConfiguration for a particular TomlSortKeys + object, if one exists. If none exists returns None. + + Override matches are evaluated as glob patterns by the python + fnmatch function. If there are multiple matches, return the + exact match first otherwise return the first match. + """ + if keys is None: + return None + + if keys.as_string() in self.sort_config_overrides: + return self.sort_config_overrides.get(keys.as_string()) + + matches = [ + config + for pattern, config in self.sort_config_overrides.items() + if fnmatch.fnmatch(keys.as_string(), pattern) + ] + + if len(matches) > 0: + return matches[0] + + return None + + def sort_config( + self, keys: Optional[TomlSortKeys] = None + ) -> SortConfiguration: + """Returns the SortConfiguration to use for particular TomlSortKeys. + + This merges the global SortConfiguration with any matching + SortOverrideConfiguration to give the full SortConfiguration + that applies to this Key. + """ + override = self._find_config_override(keys) + if override is None: + return self._sort_config + + main_config = asdict(self._sort_config) + override_config = asdict(override) + merged_config = {} + + for key, value in main_config.items(): + if key in override_config and override_config[key] is not None: + merged_config[key] = override_config[key] + else: + merged_config[key] = value + + return SortConfiguration(**merged_config) + + def sort_array( + self, keys: TomlSortKeys, array: Array, indent_depth: int = 0 + ) -> Array: """Sort and format an inline array item while preserving comments.""" multiline = "\n" in array.as_string() indent_size = self.format_config.spaces_indent_inline_array @@ -266,13 +389,14 @@ def sort_array(self, array: Array, indent_depth: int = 0) -> Array: new_array_items.append((array_item, comments)) comments = [] array_item.value = self.sort_item( + keys, array_item.value, indent_depth=indent_depth + 1 if multiline else indent_depth, ) - if self.sort_config.inline_arrays: + if self.sort_config(keys).inline_arrays: new_array_items = sorted(new_array_items, key=self.array_sort_func) new_array_value = [] for array_item, comments in new_array_items: @@ -301,27 +425,34 @@ def sort_array(self, array: Array, indent_depth: int = 0) -> Array: ) return array - def sort_item(self, item: Item, indent_depth: int = 0) -> Item: + def sort_item( + self, keys: TomlSortKeys, item: Item, indent_depth: int = 0 + ) -> Item: """Sort an item, recursing down if the item is an inline table or array.""" if isinstance(item, Array): - return self.sort_array(item, indent_depth=indent_depth) + return self.sort_array(keys, item, indent_depth=indent_depth) if isinstance(item, InlineTable): - return self.sort_inline_table(item, indent_depth=indent_depth) + return self.sort_inline_table( + keys, item, indent_depth=indent_depth + ) return item - def sort_inline_table(self, item, indent_depth: int = 0): + def sort_inline_table( + self, keys: TomlSortKeys, item: Item, indent_depth: int = 0 + ) -> InlineTable: """Sort an inline table, recursing into its items.""" tomlsort_items = [ TomlSortItem( - key=k, value=self.sort_item(v, indent_depth=indent_depth) + keys=keys + k, + value=self.sort_item(keys + k, v, indent_depth=indent_depth), ) for k, v in item.value.body if not isinstance(v, Whitespace) and k is not None ] - if self.sort_config.inline_tables: + if self.sort_config(keys).inline_tables: tomlsort_items = sorted(tomlsort_items, key=self.key_sort_func) new_table = InlineTable( Container(parsed=True), trivia=item.trivia, new=True @@ -329,7 +460,7 @@ def sort_inline_table(self, item, indent_depth: int = 0): for tomlsort_item in tomlsort_items: normalize_trivia(tomlsort_item.value, include_comments=False) new_table.append( - self.format_key(tomlsort_item.key), tomlsort_item.value + self.format_key(tomlsort_item.keys.base), tomlsort_item.value ) new_table = normalize_trivia( new_table, @@ -357,15 +488,15 @@ def sort_items( """Sort an iterable full of TomlSortItem, making sure the key is correctly formatted and recursing into any sub-items.""" for item in items: - item.key = self.format_key(item.key) - item.value = self.sort_item(item.value) + item.keys.base = self.format_key(item.keys.base) + item.value = self.sort_item(item.keys, item.value) return items def key_sort_func(self, value: TomlSortItem) -> str: """Sort function that looks at TomlSortItems keys, respecting the configured value for ignore_case.""" - key = value.key.key - if self.sort_config.ignore_case: + key = value.keys.base.key + if self.sort_config().ignore_case: key = key.lower() return key @@ -375,12 +506,12 @@ def array_sort_func(self, value: Tuple[_ArrayItemGroup, Any]) -> str: if value[0].value is None: return "" ret = value[0].value.as_string() - if self.sort_config.ignore_case: + if self.sort_config().ignore_case: ret = ret.lower() return ret def sorted_children_table( - self, parent: List[TomlSortItem] + self, parent_keys: Optional[TomlSortKeys], parent: List[TomlSortItem] ) -> Iterable[TomlSortItem]: """Get the sorted children of a table.""" tables = coalesce_tables( @@ -395,12 +526,12 @@ def sorted_children_table( ) non_tables_final = ( sorted(non_tables, key=self.key_sort_func) - if self.sort_config.table_keys + if self.sort_config(parent_keys).table_keys else non_tables ) tables_final = ( sorted(tables, key=self.key_sort_func) - if self.sort_config.tables + if self.sort_config(parent_keys).tables else tables ) return itertools.chain(non_tables_final, tables_final) @@ -444,11 +575,14 @@ def toml_elements_sorted( if original.is_table: new_table = original.table - for item in self.sorted_children_table(original.children): + for item in self.sorted_children_table( + original.keys, original.children + ): previous_item = self.table_previous_item(new_table, parent) attach_comments(item, previous_item) new_table.add( - item.key, self.toml_elements_sorted(item, previous_item) + item.keys.base, + self.toml_elements_sorted(item, previous_item), ) return new_table @@ -489,7 +623,9 @@ def table_previous_item(parent_table, grandparent): return parent_table def body_to_tomlsortitems( - self, parent: List[Tuple[Optional[Key], Item]] + self, + parent: List[Tuple[Optional[Key], Item]], + parent_key: Optional[TomlSortKeys] = None, ) -> Tuple[List[TomlSortItem], List[Comment]]: """Iterate over Container.body, recursing down into sub-containers attaching the comments that are found to the correct TomlSortItem. We @@ -506,7 +642,7 @@ def body_to_tomlsortitems( collection, when we would like it to be attached to the [abc] collection. - So before sorting we have to iterated over the container, correctly + So before sorting we have to iterate over the container, correctly attaching the comments, then undo this process once everything is sorted. """ @@ -533,17 +669,20 @@ def body_to_tomlsortitems( self.comment_config.inline, comment_spaces=self.format_config.spaces_before_inline_comment, ) + full_key = parent_key + key if parent_key else TomlSortKeys(key) if isinstance(value, Table): comments, item = self.table_to_tomlsortitem( - comments, key, value + comments, full_key, value ) elif isinstance(value, AoT): - comments, item = self.aot_to_tomlsortitem(comments, key, value) + comments, item = self.aot_to_tomlsortitem( + comments, full_key, value + ) elif isinstance(value, Item): - item = TomlSortItem(key, value, comments) + item = TomlSortItem(full_key, value, comments) comments = [] else: @@ -556,7 +695,7 @@ def body_to_tomlsortitems( return items, comments def aot_to_tomlsortitem( - self, comments: List[Comment], key: Key, value: AoT + self, comments: List[Comment], keys: TomlSortKeys, value: AoT ) -> Tuple[List[Comment], TomlSortItem]: """Turn an AoT into a TomlSortItem, recursing down through its collections and attaching all the comments to the correct items.""" @@ -564,21 +703,21 @@ def aot_to_tomlsortitem( children = [] for table in value.body: [first_child], trailing_comments = self.body_to_tomlsortitems( - [(key, table)] + [(keys.base, table)] ) first_child.attached_comments = comments comments = trailing_comments children.append(first_child) - item = TomlSortItem(key, new_aot, children=children) + item = TomlSortItem(keys, new_aot, children=children) return comments, item def table_to_tomlsortitem( - self, comments: List[Comment], key: Key, value: Table + self, comments: List[Comment], keys: TomlSortKeys, value: Table ) -> Tuple[List[Comment], TomlSortItem]: """Turn a table into a TomlSortItem, recursing down through its collections and attaching all the comments to the correct items.""" children, trailing_comments = self.body_to_tomlsortitems( - value.value.body + value.value.body, parent_key=keys ) new_table = Table( Container(parsed=True), @@ -608,7 +747,7 @@ def table_to_tomlsortitem( child_table.attached_comments = comments comments = [] - item = TomlSortItem(key, new_table, comments, children) + item = TomlSortItem(keys, new_table, comments, children) comments = trailing_comments return comments, item @@ -624,10 +763,11 @@ def toml_doc_sorted(self, original: TOMLDocument) -> TOMLDocument: items, footer_comment = self.body_to_tomlsortitems(original_body) - for item in self.sorted_children_table(items): + for item in self.sorted_children_table(None, items): attach_comments(item, sorted_document) sorted_document.add( - item.key, self.toml_elements_sorted(item, sorted_document) + item.keys.base, + self.toml_elements_sorted(item, sorted_document), ) if self.comment_config.footer and footer_comment: