From a19b5518bb3a1be36b02c020c41043385d55e786 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Fri, 5 Nov 2021 11:27:49 +0100 Subject: [PATCH] feat: enhance extracted data for usage with API editor (#14) * feat: add new fields to model * fix: get everything running again * feat: clean docstrings * feat: extract imports and from-imports for modules * fix: use qualified name of functions as key * fix: some linter errors * fix: remaining linter errors --- .flake8 | 3 + .mega-linter.yml | 2 +- .prettierrc.json | 1 - ontology/mostly_empty_file | 1 + package-lock.json | 53 +++++ package.json | 9 + package_parser/package_parser/__main__.py | 2 +- package_parser/package_parser/cli.py | 15 +- .../commands/get_api/__init__.py | 20 +- .../commands/get_api/_ast_visitor.py | 212 ++++++++++++----- .../commands/get_api/_get_api.py | 21 +- .../package_parser/commands/get_api/_model.py | 215 ++++++++++++++---- .../commands/get_api/_package_metadata.py | 3 +- package_parser/package_parser/main.py | 4 +- .../package_parser/utils/_ASTWalker.py | 15 +- .../package_parser/utils/__init__.py | 2 +- .../package_parser/utils/_qnames.py | 3 +- 17 files changed, 443 insertions(+), 138 deletions(-) create mode 100644 .flake8 delete mode 100644 .prettierrc.json create mode 100644 ontology/mostly_empty_file create mode 100644 package-lock.json create mode 100644 package.json diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..872a7e86f --- /dev/null +++ b/.flake8 @@ -0,0 +1,3 @@ +[flake8] +extend-exclude = __init__.py +extend-ignore = E501 diff --git a/.mega-linter.yml b/.mega-linter.yml index 90f266596..366e9778f 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -20,7 +20,7 @@ TYPESCRIPT_PRETTIER_FILE_EXTENSIONS: # Commands PRE_COMMANDS: - command: npm install - cwd: root + cwd: workspace # Linters ENABLE_LINTERS: diff --git a/.prettierrc.json b/.prettierrc.json deleted file mode 100644 index 0eda2ce37..000000000 --- a/.prettierrc.json +++ /dev/null @@ -1 +0,0 @@ -"@lars-reimann/prettier-config" diff --git a/ontology/mostly_empty_file b/ontology/mostly_empty_file new file mode 100644 index 000000000..bdc3aa0b1 --- /dev/null +++ b/ontology/mostly_empty_file @@ -0,0 +1 @@ +Remove this once other content is in the ontology folder diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..d4c987963 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,53 @@ +{ + "name": "sem21", + "version": "0.0.1", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "sem21", + "version": "0.0.1", + "devDependencies": { + "@lars-reimann/prettier-config": "^4.0.1" + } + }, + "node_modules/@lars-reimann/prettier-config": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@lars-reimann/prettier-config/-/prettier-config-4.0.1.tgz", + "integrity": "sha512-mEmVgXTz6qv3NYNJUfv/P7BNiDQ7qkK/bNhdKPR6sWy1xBBvwKb6QJdML1e4e1DNjo3Es6IkX3EhAXph9pcASA==", + "dev": true, + "peerDependencies": { + "prettier": ">= 2" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "peer": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + } + }, + "dependencies": { + "@lars-reimann/prettier-config": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@lars-reimann/prettier-config/-/prettier-config-4.0.1.tgz", + "integrity": "sha512-mEmVgXTz6qv3NYNJUfv/P7BNiDQ7qkK/bNhdKPR6sWy1xBBvwKb6QJdML1e4e1DNjo3Es6IkX3EhAXph9pcASA==", + "dev": true, + "requires": {} + }, + "prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "peer": true + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 000000000..a9fe1893e --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "name": "sem21", + "version": "0.0.1", + "private": true, + "prettier": "@lars-reimann/prettier-config", + "devDependencies": { + "@lars-reimann/prettier-config": "^4.0.1" + } +} diff --git a/package_parser/package_parser/__main__.py b/package_parser/package_parser/__main__.py index 5aa48a341..e83866c46 100644 --- a/package_parser/package_parser/__main__.py +++ b/package_parser/package_parser/__main__.py @@ -1,4 +1,4 @@ from package_parser.main import main -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/package_parser/package_parser/cli.py b/package_parser/package_parser/cli.py index a28bdf125..2ef4da362 100644 --- a/package_parser/package_parser/cli.py +++ b/package_parser/package_parser/cli.py @@ -16,7 +16,9 @@ def cli() -> None: public_api = get_api(args.package) out_dir: Path = args.out - out_file = out_dir.joinpath(f"{public_api.distribution}__{public_api.package}__{public_api.version}__api.json") + out_file = out_dir.joinpath( + f"{public_api.distribution}__{public_api.package}__{public_api.version}__api.json" + ) ensure_file_exists(out_file) with out_file.open("w") as f: json.dump(public_api.to_json(), f, indent=2) @@ -33,10 +35,7 @@ def __get_args() -> argparse.Namespace: def __add_api_subparser(subparsers: _SubParsersAction) -> None: - api_parser = subparsers.add_parser( - __API_COMMAND, - help="List the API of a package." - ) + api_parser = subparsers.add_parser(__API_COMMAND, help="List the API of a package.") api_parser.add_argument( "-p", "--package", @@ -45,9 +44,5 @@ def __add_api_subparser(subparsers: _SubParsersAction) -> None: required=True, ) api_parser.add_argument( - "-o", - "--out", - help="Output directory.", - type=Path, - required=True + "-o", "--out", help="Output directory.", type=Path, required=True ) diff --git a/package_parser/package_parser/commands/get_api/__init__.py b/package_parser/package_parser/commands/get_api/__init__.py index 5e32659a3..17d7b8ccd 100644 --- a/package_parser/package_parser/commands/get_api/__init__.py +++ b/package_parser/package_parser/commands/get_api/__init__.py @@ -1,3 +1,19 @@ from ._get_api import get_api -from ._model import API, Class, Function, Parameter, ParameterAssignment, ParameterDocstring -from ._package_metadata import distribution, distribution_version, package_files, package_root +from ._model import ( + API, + Class, + FromImport, + Function, + Import, + Module, + Parameter, + ParameterAndResultDocstring, + ParameterAssignment, + Result, +) +from ._package_metadata import ( + distribution, + distribution_version, + package_files, + package_root, +) diff --git a/package_parser/package_parser/commands/get_api/_ast_visitor.py b/package_parser/package_parser/commands/get_api/_ast_visitor.py index ad792db75..29b7a496b 100644 --- a/package_parser/package_parser/commands/get_api/_ast_visitor.py +++ b/package_parser/package_parser/commands/get_api/_ast_visitor.py @@ -1,72 +1,162 @@ -from typing import Optional +import inspect +from typing import Optional, Union import astroid from numpydoc.docscrape import NumpyDocString - from package_parser.utils import parent_qname -from ._file_filters import _is_init_file -from ._model import API, Function, Parameter, Class, ParameterDocstring, ParameterAssignment - -class _CallableVisitor: +from ._file_filters import _is_init_file +from ._model import ( + API, + Class, + FromImport, + Function, + Import, + Module, + Parameter, + ParameterAndResultDocstring, + ParameterAssignment, +) + + +class _AstVisitor: def __init__(self, api: API) -> None: self.reexported: set[str] = set() self.api: API = api + self.__module_and_class_stack: list[Union[Module, Class]] = [] def enter_module(self, module_node: astroid.Module): - """ - Find re-exported declarations in __init__.py files. - """ - - if not _is_init_file(module_node.file): - return + imports: list[Import] = [] + from_imports: list[FromImport] = [] for _, global_declaration_node_list in module_node.globals.items(): global_declaration_node = global_declaration_node_list[0] + # import X as Y + if isinstance(global_declaration_node, astroid.Import): + for (name, alias) in global_declaration_node.names: + imports.append(Import(name, alias)) + + # from X import a as b if isinstance(global_declaration_node, astroid.ImportFrom): base_import_path = module_node.relative_to_absolute_name( - global_declaration_node.modname, - global_declaration_node.level + global_declaration_node.modname, global_declaration_node.level ) - for declaration, _ in global_declaration_node.names: - reexported_name = f"{base_import_path}.{declaration}" + for (name, alias) in global_declaration_node.names: + from_imports.append(FromImport(base_import_path, name, alias)) - if reexported_name.startswith(module_node.name): - self.reexported.add(reexported_name) + # Find re-exported declarations in __init__.py files + if _is_init_file(module_node.file): + for declaration, _ in global_declaration_node.names: + reexported_name = f"{base_import_path}.{declaration}" - def enter_classdef(self, node: astroid.ClassDef) -> None: - qname = node.qname() + if reexported_name.startswith(module_node.name): + self.reexported.add(reexported_name) - if qname not in self.api.classes: - self.api.add_class( - Class( - qname, - self.is_public(node.name, qname), - node.doc, - node.as_string() - ) - ) + # Remember module, so we can later add classes and global functions + module = Module( + module_node.qname(), + imports, + from_imports, + ) + self.__module_and_class_stack.append(module) - def enter_functiondef(self, node: astroid.FunctionDef) -> None: - qname = node.qname() + def leave_module(self, _: astroid.Module) -> None: + module = self.__module_and_class_stack.pop() + if not isinstance(module, Module): + raise AssertionError("Imbalanced push/pop on stack") - if qname not in self.api.functions: - is_public = self.is_public(node.name, qname) + self.api.add_module(module) - self.api.add_function( - Function( - qname, - self.__function_parameters(node, is_public), - is_public, - node.doc, - node.as_string() - ) + def enter_classdef(self, class_node: astroid.ClassDef) -> None: + qname = class_node.qname() + + decorators: Optional[astroid.Decorators] = class_node.decorators + if decorators is not None: + decorator_names = [decorator.as_string() for decorator in decorators.nodes] + else: + decorator_names = [] + + numpydoc = NumpyDocString(inspect.cleandoc(class_node.doc or "")) + + # Remember class, so we can later add methods + class_ = Class( + qname, + decorator_names, + class_node.basenames, + self.is_public(class_node.name, qname), + _AstVisitor.__description(numpydoc), + class_node.doc, + class_node.as_string(), + ) + self.__module_and_class_stack.append(class_) + + def leave_classdef(self, _: astroid.ClassDef) -> None: + class_ = self.__module_and_class_stack.pop() + if not isinstance(class_, Class): + raise AssertionError("Imbalanced push/pop on stack") + self.api.add_class(class_) + + # Add qualified name of class to containing module + if len(self.__module_and_class_stack) > 0: + parent = self.__module_and_class_stack[-1] + if isinstance(parent, Module): + parent.add_class(class_.qname) + + def enter_functiondef(self, function_node: astroid.FunctionDef) -> None: + qname = function_node.qname() + + decorators: Optional[astroid.Decorators] = function_node.decorators + if decorators is not None: + decorator_names = [decorator.as_string() for decorator in decorators.nodes] + else: + decorator_names = [] + + numpydoc = NumpyDocString(inspect.cleandoc(function_node.doc or "")) + is_public = self.is_public(function_node.name, qname) + + self.api.add_function( + Function( + qname, + decorator_names, + self.__function_parameters(function_node, is_public), + [], # TODO: results + is_public, + _AstVisitor.__description(numpydoc), + function_node.doc, + function_node.as_string(), ) + ) + + # Add qualified name of function to containing module or class + if len(self.__module_and_class_stack) > 0: + parent = self.__module_and_class_stack[-1] + if isinstance(parent, Module): + parent.add_function(qname) + elif isinstance(parent, Class): + parent.add_method(qname) + + @staticmethod + def __description(numpydoc: NumpyDocString) -> str: + has_summary = "Summary" in numpydoc and len(numpydoc["Summary"]) > 0 + has_extended_summary = ( + "Extended Summary" in numpydoc and len(numpydoc["Extended Summary"]) > 0 + ) + + result = "" + if has_summary: + result += " ".join(numpydoc["Summary"]) + if has_summary and has_extended_summary: + result += "\n\n" + if has_extended_summary: + result += " ".join(numpydoc["Extended Summary"]) + return result @staticmethod - def __function_parameters(node: astroid.FunctionDef, function_is_public: bool) -> list[Parameter]: + def __function_parameters( + node: astroid.FunctionDef, function_is_public: bool + ) -> list[Parameter]: parameters = node.args n_implicit_parameters = node.implicit_parameters() @@ -75,7 +165,7 @@ def __function_parameters(node: astroid.FunctionDef, function_is_public: bool) - docstring = node.parent.doc else: docstring = node.doc - function_numpydoc = NumpyDocString(docstring or "") + function_numpydoc = NumpyDocString(inspect.cleandoc(docstring or "")) # Arguments that can be specified positionally only ( f(1) works but not f(x=1) ) result = [ @@ -84,9 +174,8 @@ def __function_parameters(node: astroid.FunctionDef, function_is_public: bool) - default_value=None, is_public=function_is_public, assigned_by=ParameterAssignment.POSITION_ONLY, - docstring=_CallableVisitor.__parameter_docstring(function_numpydoc, it.name) + docstring=_AstVisitor.__parameter_docstring(function_numpydoc, it.name), ) - for it in parameters.posonlyargs ] @@ -94,15 +183,14 @@ def __function_parameters(node: astroid.FunctionDef, function_is_public: bool) - result += [ Parameter( it.name, - _CallableVisitor.__parameter_default( + _AstVisitor.__parameter_default( parameters.defaults, - index - len(parameters.args) + len(parameters.defaults) + index - len(parameters.args) + len(parameters.defaults), ), function_is_public, ParameterAssignment.POSITION_OR_NAME, - _CallableVisitor.__parameter_docstring(function_numpydoc, it.name) + _AstVisitor.__parameter_docstring(function_numpydoc, it.name), ) - for index, it in enumerate(parameters.args) ] @@ -110,22 +198,23 @@ def __function_parameters(node: astroid.FunctionDef, function_is_public: bool) - result += [ Parameter( it.name, - _CallableVisitor.__parameter_default( + _AstVisitor.__parameter_default( parameters.kw_defaults, - index - len(parameters.kwonlyargs) + len(parameters.kw_defaults) + index - len(parameters.kwonlyargs) + len(parameters.kw_defaults), ), function_is_public, ParameterAssignment.NAME_ONLY, - _CallableVisitor.__parameter_docstring(function_numpydoc, it.name) + _AstVisitor.__parameter_docstring(function_numpydoc, it.name), ) - for index, it in enumerate(parameters.kwonlyargs) ] return result[n_implicit_parameters:] @staticmethod - def __parameter_default(defaults: list[astroid.NodeNG], default_index: int) -> Optional[str]: + def __parameter_default( + defaults: list[astroid.NodeNG], default_index: int + ) -> Optional[str]: if 0 <= default_index < len(defaults): default = defaults[default_index] if default is None: @@ -135,18 +224,21 @@ def __parameter_default(defaults: list[astroid.NodeNG], default_index: int) -> O return None @staticmethod - def __parameter_docstring(function_numpydoc: NumpyDocString, parameter_name: str) -> ParameterDocstring: + def __parameter_docstring( + function_numpydoc: NumpyDocString, parameter_name: str + ) -> ParameterAndResultDocstring: parameters_numpydoc = function_numpydoc["Parameters"] - candidate_parameters_numpydoc = [it for it in parameters_numpydoc if it.name == parameter_name] + candidate_parameters_numpydoc = [ + it for it in parameters_numpydoc if it.name == parameter_name + ] if len(candidate_parameters_numpydoc) > 0: last_parameter_numpydoc = candidate_parameters_numpydoc[-1] - return ParameterDocstring( - last_parameter_numpydoc.type, - "\n".join(last_parameter_numpydoc.desc) + return ParameterAndResultDocstring( + last_parameter_numpydoc.type, "\n".join(last_parameter_numpydoc.desc) ) - return ParameterDocstring("", "") + return ParameterAndResultDocstring("", "") def is_public(self, name: str, qualified_name: str) -> bool: if name.startswith("_") and not name.endswith("__"): diff --git a/package_parser/package_parser/commands/get_api/_get_api.py b/package_parser/package_parser/commands/get_api/_get_api.py index 7a8f7ad06..2b5e27d78 100644 --- a/package_parser/package_parser/commands/get_api/_get_api.py +++ b/package_parser/package_parser/commands/get_api/_get_api.py @@ -1,22 +1,27 @@ from pathlib import Path import astroid - from package_parser.utils import ASTWalker -from ._ast_visitor import _CallableVisitor + +from ._ast_visitor import _AstVisitor from ._file_filters import _is_test_file from ._model import API -from ._package_metadata import distribution, distribution_version, package_files, package_root +from ._package_metadata import ( + distribution, + distribution_version, + package_files, + package_root, +) def get_api(package_name: str) -> API: root = package_root(package_name) - dist = distribution(package_name) - dist_version = distribution_version(dist) + dist = distribution(package_name) or "" + dist_version = distribution_version(dist) or "" files = package_files(package_name) api = API(dist, package_name, dist_version) - callable_visitor = _CallableVisitor(api) + callable_visitor = _AstVisitor(api) walker = ASTWalker(callable_visitor) for file in files: @@ -31,9 +36,7 @@ def get_api(package_name: str) -> API: source = f.read() walker.walk( astroid.parse( - source, - module_name=__module_name(root, Path(file)), - path=file + source, module_name=__module_name(root, Path(file)), path=file ) ) diff --git a/package_parser/package_parser/commands/get_api/_model.py b/package_parser/package_parser/commands/get_api/_model.py index d5cdab6de..39fe9c8a8 100644 --- a/package_parser/package_parser/commands/get_api/_model.py +++ b/package_parser/package_parser/commands/get_api/_model.py @@ -4,18 +4,16 @@ from enum import Enum, auto from typing import Any, Optional -from package_parser.utils import declaration_name, parent_qname +from package_parser.utils import declaration_qname_to_name, parent_qname class API: - @staticmethod def from_json(json: Any) -> API: - result = API( - json["distribution"], - json["package"], - json["version"] - ) + result = API(json["distribution"], json["package"], json["version"]) + + for module_json in json["modules"]: + result.add_module(Module.from_json(module_json)) for class_json in json["classes"]: result.add_class(Class.from_json(class_json)) @@ -29,11 +27,15 @@ def __init__(self, distribution: str, package: str, version: str) -> None: self.distribution: str = distribution self.package: str = package self.version: str = version + self.modules: dict[str, Module] = dict() self.classes: dict[str, Class] = dict() self.functions: dict[str, Function] = dict() - def add_class(self, clazz: Class) -> None: - self.classes[clazz.qname] = clazz + def add_module(self, module: Module) -> None: + self.modules[module.name] = module + + def add_class(self, class_: Class) -> None: + self.classes[class_.qname] = class_ def add_function(self, function: Function) -> None: self.functions[function.qname] = function @@ -42,7 +44,10 @@ def is_public_class(self, class_qname: str) -> bool: return class_qname in self.classes and self.classes[class_qname].is_public def is_public_function(self, function_qname: str) -> bool: - return function_qname in self.functions and self.functions[function_qname].is_public + return ( + function_qname in self.functions + and self.functions[function_qname].is_public + ) def class_count(self) -> int: return len(self.classes) @@ -74,7 +79,7 @@ def parameters(self) -> dict[str, Parameter]: def get_default_value(self, parameter_qname: str) -> Optional[str]: function_qname = parent_qname(parameter_qname) - parameter_name = declaration_name(parameter_qname) + parameter_name = declaration_qname_to_name(parameter_qname) if function_qname not in self.functions: return None @@ -90,6 +95,10 @@ def to_json(self) -> Any: "distribution": self.distribution, "package": self.package, "version": self.version, + "modules": [ + module.to_json() + for module in sorted(self.modules.values(), key=lambda it: it.name) + ], "classes": [ clazz.to_json() for clazz in sorted(self.classes.values(), key=lambda it: it.qname) @@ -97,30 +106,122 @@ def to_json(self) -> Any: "functions": [ function.to_json() for function in sorted(self.functions.values(), key=lambda it: it.qname) - ] + ], } -class Class: +class Module: + @staticmethod + def from_json(json: Any) -> Module: + result = Module( + json["name"], + [Import.from_json(import_json) for import_json in json["imports"]], + [ + FromImport.from_json(from_import_json) + for from_import_json in json["from_imports"] + ], + ) + + for class_qname in json["classes"]: + result.add_class(class_qname) + for function_qname in json["functions"]: + result.add_function(function_qname) + + return result + + def __init__( + self, name: str, imports: list[Import], from_imports: list[FromImport] + ): + self.name: str = name + self.imports: list[Import] = imports + self.from_imports: list[FromImport] = from_imports + self.classes: list[str] = [] + self.functions: list[str] = [] + + def add_class(self, class_qname: str) -> None: + self.classes.append(class_qname) + + def add_function(self, function_qname: str) -> None: + self.functions.append(function_qname) + + def to_json(self) -> Any: + return { + "name": self.name, + "imports": [import_.to_json() for import_ in self.imports], + "from_imports": [ + from_import.to_json() for from_import in self.from_imports + ], + "classes": self.classes, + "functions": self.functions, + } + + +class Import: + @staticmethod + def from_json(json: Any) -> Import: + return Import(json["module"], json["alias"]) + + def __init__(self, module_name: str, alias: Optional[str]): + self.module: str = module_name + self.alias: Optional[str] = alias + + def to_json(self) -> Any: + return {"module": self.module, "alias": self.alias} + + +class FromImport: + @staticmethod + def from_json(json: Any) -> FromImport: + return FromImport(json["module"], json["declaration"], json["alias"]) + + def __init__(self, module_name: str, declaration_name: str, alias: Optional[str]): + self.module_name: str = module_name + self.declaration_name: str = declaration_name + self.alias: Optional[str] = alias + + def to_json(self) -> Any: + return { + "module": self.module_name, + "declaration": self.declaration_name, + "alias": self.alias, + } + + +class Class: @staticmethod def from_json(json: Any) -> Class: - return Class( + result = Class( json["qname"], + json["decorators"], + json["superclasses"], json["is_public"], + json["description"], json["docstring"], - json["source_code"] + json["source_code"], ) + for method_qname in json["methods"]: + result.add_method(method_qname) + + return result + def __init__( self, qname: str, + decorators: list[str], + superclasses: list[str], is_public: bool, + description: str, docstring: str, - source_code: str + source_code: str, ) -> None: self.qname: str = qname + self.decorators: list[str] = decorators + self.superclasses: list[str] = superclasses + self.methods: list[str] = [] self.is_public: bool = is_public + self.description: str = description self.docstring: str = docstring self.source_code: str = source_code @@ -128,39 +229,57 @@ def __init__( def name(self) -> str: return self.qname.split(".")[-1] + def add_method(self, function_qname: str) -> None: + self.methods.append(function_qname) + def to_json(self) -> Any: return { "name": self.name, "qname": self.qname, + "decorators": self.decorators, + "superclasses": self.superclasses, + "methods": self.methods, "is_public": self.is_public, + "description": self.description, "docstring": self.docstring, - "source_code": self.source_code + "source_code": self.source_code, } class Function: - @staticmethod def from_json(json: Any) -> Function: return Function( json["qname"], - [Parameter.from_json(parameter_json) for parameter_json in json["parameters"]], + json["decorators"], + [ + Parameter.from_json(parameter_json) + for parameter_json in json["parameters"] + ], + [Result.from_json(result_json) for result_json in json["results"]], json["is_public"], + json["description"], json["docstring"], - json["source_code"] + json["source_code"], ) def __init__( self, qname: str, + decorators: list[str], parameters: list[Parameter], + results: list[Result], is_public: bool, + description: str, docstring: str, - source_code: str + source_code: str, ) -> None: self.qname: str = qname + self.decorators: list[str] = decorators self.parameters: list[Parameter] = parameters + self.results: list[Result] = results self.is_public: bool = is_public + self.description: str = description self.docstring: str = inspect.cleandoc(docstring or "") self.source_code: str = source_code @@ -172,18 +291,17 @@ def to_json(self) -> Any: return { "name": self.name, "qname": self.qname, - "parameters": [ - parameter.to_json() - for parameter in self.parameters - ], + "decorators": self.decorators, + "parameters": [parameter.to_json() for parameter in self.parameters], + "results": [result.to_json() for result in self.results], "is_public": self.is_public, + "description": self.description, "docstring": self.docstring, - "source_code": self.source_code + "source_code": self.source_code, } class Parameter: - @staticmethod def from_json(json: Any) -> Parameter: return Parameter( @@ -191,7 +309,7 @@ def from_json(json: Any) -> Parameter: json["default_value"], json["is_public"], ParameterAssignment[json["assigned_by"]], - ParameterDocstring.from_json(json["docstring"]) + ParameterAndResultDocstring.from_json(json["docstring"]), ) def __init__( @@ -200,7 +318,7 @@ def __init__( default_value: Optional[str], is_public: bool, assigned_by: ParameterAssignment, - docstring: ParameterDocstring + docstring: ParameterAndResultDocstring, ) -> None: self.name: str = name self.default_value: Optional[str] = default_value @@ -214,34 +332,43 @@ def to_json(self) -> Any: "default_value": self.default_value, "is_public": self.is_public, "assigned_by": self.assigned_by.name, - "docstring": self.docstring.to_json() + "docstring": self.docstring.to_json(), } class ParameterAssignment(Enum): - POSITION_ONLY = auto(), - POSITION_OR_NAME = auto(), - NAME_ONLY = auto(), + POSITION_ONLY = (auto(),) + POSITION_OR_NAME = (auto(),) + NAME_ONLY = (auto(),) -class ParameterDocstring: +class Result: @staticmethod - def from_json(json: Any) -> ParameterDocstring: - return ParameterDocstring( - json["type"], - json["description"] + def from_json(json: Any) -> Result: + return Result( + json["name"], ParameterAndResultDocstring.from_json(json["docstring"]) ) + def __init__(self, name: str, docstring: ParameterAndResultDocstring) -> None: + self.name: str = name + self.docstring = docstring + + def to_json(self) -> Any: + return {"name": self.name, "docstring": self.docstring.to_json()} + + +class ParameterAndResultDocstring: + @staticmethod + def from_json(json: Any) -> ParameterAndResultDocstring: + return ParameterAndResultDocstring(json["type"], json["description"]) + def __init__( self, - type: str, + type_: str, description: str, ) -> None: - self.type: str = type + self.type: str = type_ self.description: str = description def to_json(self) -> Any: - return { - "type": self.type, - "description": self.description - } + return {"type": self.type, "description": self.description} diff --git a/package_parser/package_parser/commands/get_api/_package_metadata.py b/package_parser/package_parser/commands/get_api/_package_metadata.py index 00bf581f1..d352d2fcd 100644 --- a/package_parser/package_parser/commands/get_api/_package_metadata.py +++ b/package_parser/package_parser/commands/get_api/_package_metadata.py @@ -2,9 +2,10 @@ from pathlib import Path from typing import Optional +# pylint: disable=no-name-in-module from importlib_metadata import packages_distributions, version - from package_parser.utils import list_files + from ._file_filters import _is_init_file diff --git a/package_parser/package_parser/main.py b/package_parser/package_parser/main.py index fa3ae3340..277ddedcc 100644 --- a/package_parser/package_parser/main.py +++ b/package_parser/package_parser/main.py @@ -8,5 +8,7 @@ def main() -> None: cli() - print("\n====================================================================================================") + print( + "\n====================================================================================================" + ) print(f"Program ran in {time.time() - start_time}s") diff --git a/package_parser/package_parser/utils/_ASTWalker.py b/package_parser/package_parser/utils/_ASTWalker.py index 8ec01094f..c07e5cb68 100644 --- a/package_parser/package_parser/utils/_ASTWalker.py +++ b/package_parser/package_parser/utils/_ASTWalker.py @@ -1,7 +1,11 @@ -from typing import Any, Callable +from typing import Any, Callable, Type import astroid +EnterAndLeaveFunctions = tuple[ + Callable[[astroid.NodeNG], None], Callable[[astroid.NodeNG], None] +] + class ASTWalker: """A walker visiting a tree in preorder, calling on the handler: @@ -15,13 +19,14 @@ class ASTWalker: def __init__(self, handler: Any) -> None: self._handler = handler - self._cache = {} + self._cache: dict[Type, EnterAndLeaveFunctions] = {} def walk(self, node: astroid.NodeNG) -> None: self.__walk(node, set()) def __walk(self, node: astroid.NodeNG, visited_nodes: set[astroid.NodeNG]) -> None: - assert node not in visited_nodes + if node in visited_nodes: + raise AssertionError("Node visited twice") visited_nodes.add(node) self.__enter(node) @@ -39,9 +44,7 @@ def __leave(self, node: astroid.NodeNG) -> None: if method is not None: method(node) - def __get_callbacks(self, node: astroid.NodeNG) -> tuple[ - Callable[[astroid.NodeNG], None], Callable[[astroid.NodeNG], None] - ]: + def __get_callbacks(self, node: astroid.NodeNG) -> EnterAndLeaveFunctions: klass = node.__class__ methods = self._cache.get(klass) diff --git a/package_parser/package_parser/utils/__init__.py b/package_parser/package_parser/utils/__init__.py index 2da7b11a9..e0be421ad 100644 --- a/package_parser/package_parser/utils/__init__.py +++ b/package_parser/package_parser/utils/__init__.py @@ -1,3 +1,3 @@ from ._ASTWalker import ASTWalker from ._files import ensure_file_exists, initialize_and_read_exclude_file, list_files -from ._qnames import declaration_name, parent_qname +from ._qnames import declaration_qname_to_name, parent_qname diff --git a/package_parser/package_parser/utils/_qnames.py b/package_parser/package_parser/utils/_qnames.py index dcf1401b8..58944e92e 100644 --- a/package_parser/package_parser/utils/_qnames.py +++ b/package_parser/package_parser/utils/_qnames.py @@ -1,5 +1,6 @@ -def declaration_name(qname: str) -> str: +def declaration_qname_to_name(qname: str) -> str: return qname.split(".")[-1] + def parent_qname(qname: str) -> str: return ".".join(qname.split(".")[:-1])