Merge pull request #64784 from yedpodtrzitko/yed/ci-add-mypy

ci: add Python static analysis check via mypy

.github/workflows/static_checks.yml

@@ -27,7 +27,7 @@ jobs:
           sudo apt-get install -qq dos2unix recode clang-format-13 libxml2-utils python3-pip moreutils
           sudo update-alternatives --remove-all clang-format || true
           sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100
-          sudo pip3 install black==22.3.0 pygments pytest
+          sudo pip3 install black==22.3.0 pygments pytest==7.1.2 mypy==0.971
 
       - name: File formatting checks (file_format.sh)
         run: |
@@ -41,6 +41,10 @@ jobs:
         run: |
           bash ./misc/scripts/black_format.sh
 
+      - name: Python scripts static analysis (mypy_check.sh)
+        run: |
+          bash ./misc/scripts/mypy_check.sh
+
       - name: Python builders checks via pytest (pytest_builders.sh)
         run: |
           bash ./misc/scripts/pytest_builders.sh

core/core_builders.py

@@ -2,6 +2,7 @@
 
 All such functions are invoked in a subprocess on Windows to prevent build flakiness.
 """
+import zlib
 
 from platform_methods import subprocess_main
 
@@ -33,7 +34,6 @@ def make_certs_header(target, source, env):
     g = open(dst, "w", encoding="utf-8")
     buf = f.read()
     decomp_size = len(buf)
-    import zlib
 
     # Use maximum zlib compression level to further reduce file size
     # (at the cost of initial build times).
@@ -208,7 +208,7 @@ def next_tag(self):
 
     from collections import OrderedDict
 
-    projects = OrderedDict()
+    projects: dict = OrderedDict()
     license_list = []
 
     with open(src_copyright, "r", encoding="utf-8") as copyright_file:
@@ -230,7 +230,7 @@ def next_tag(self):
                 part = {}
                 reader.next_line()
 
-    data_list = []
+    data_list: list = []
     for project in iter(projects.values()):
         for part in project:
             part["file_index"] = len(data_list)

core/input/input_builders.py

@@ -16,7 +16,7 @@ def make_default_controller_mappings(target, source, env):
     g.write('#include "core/input/default_controller_mappings.h"\n')
 
     # ensure mappings have a consistent order
-    platform_mappings = OrderedDict()
+    platform_mappings: dict = OrderedDict()
     for src_path in source:
         with open(src_path, "r") as f:
             # read mapping file and skip header

doc/tools/make_rst.py

@@ -526,7 +526,7 @@ def main() -> None:
         )
         if os.path.exists(lang_file):
             try:
-                import polib
+                import polib  # type: ignore
             except ImportError:
                 print("Base template strings localization requires `polib`.")
                 exit(1)
@@ -739,9 +739,10 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir:
             f.write(f"- {make_link(url, title)}\n\n")
 
     # Properties overview
+    ml: List[Tuple[Optional[str], ...]] = []
     if len(class_def.properties) > 0:
         f.write(make_heading("Properties", "-"))
-        ml: List[Tuple[Optional[str], ...]] = []
+        ml = []
         for property_def in class_def.properties.values():
             type_rst = property_def.type_name.to_rst(state)
             default = property_def.default_value
@@ -757,23 +758,23 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir:
     # Constructors, Methods, Operators overview
     if len(class_def.constructors) > 0:
         f.write(make_heading("Constructors", "-"))
-        ml: List[Tuple[Optional[str], ...]] = []
+        ml = []
         for method_list in class_def.constructors.values():
             for m in method_list:
                 ml.append(make_method_signature(class_def, m, "constructor", state))
         format_table(f, ml)
 
     if len(class_def.methods) > 0:
         f.write(make_heading("Methods", "-"))
-        ml: List[Tuple[Optional[str], ...]] = []
+        ml = []
         for method_list in class_def.methods.values():
             for m in method_list:
                 ml.append(make_method_signature(class_def, m, "method", state))
         format_table(f, ml)
 
     if len(class_def.operators) > 0:
         f.write(make_heading("Operators", "-"))
-        ml: List[Tuple[Optional[str], ...]] = []
+        ml = []
         for method_list in class_def.operators.values():
             for m in method_list:
                 ml.append(make_method_signature(class_def, m, "operator", state))
@@ -858,7 +859,7 @@ def make_rst_class(class_def: ClassDef, state: State, dry_run: bool, output_dir:
         f.write(make_heading("Annotations", "-"))
         index = 0
 
-        for method_list in class_def.annotations.values():
+        for method_list in class_def.annotations.values():  # type: ignore
             for i, m in enumerate(method_list):
                 if index != 0:
                     f.write("----\n\n")
@@ -1039,17 +1040,15 @@ def make_method_signature(
 ) -> Tuple[str, str]:
     ret_type = ""
 
-    is_method_def = isinstance(definition, MethodDef)
-    if is_method_def:
+    if isinstance(definition, MethodDef):
         ret_type = definition.return_type.to_rst(state)
 
     qualifiers = None
-    if is_method_def or isinstance(definition, AnnotationDef):
+    if isinstance(definition, (MethodDef, AnnotationDef)):
         qualifiers = definition.qualifiers
 
     out = ""
-
-    if is_method_def and ref_type != "":
+    if isinstance(definition, MethodDef) and ref_type != "":
         if ref_type == "operator":
             op_name = definition.name.replace("<", "\\<")  # So operator "<" gets correctly displayed.
             out += f":ref:`{op_name}<class_{class_def.name}_{ref_type}_{sanitize_operator_name(definition.name, state)}_{definition.return_type.type_name}>` "
@@ -1456,18 +1455,14 @@ def format_text_block(
                         escape_post = True
 
                     elif cmd.startswith("param"):
-                        valid_context = (
-                            isinstance(context, MethodDef)
-                            or isinstance(context, SignalDef)
-                            or isinstance(context, AnnotationDef)
-                        )
+                        valid_context = isinstance(context, (MethodDef, SignalDef, AnnotationDef))
                         if not valid_context:
                             print_error(
                                 f'{state.current_class}.xml: Argument reference "{link_target}" used outside of method, signal, or annotation context in {context_name}.',
                                 state,
                             )
                         else:
-                            context_params: List[ParameterDef] = context.parameters
+                            context_params: List[ParameterDef] = context.parameters  # type: ignore
                             found = False
                             for param_def in context_params:
                                 if param_def.name == link_target:

doc/translations/extract.py

@@ -60,7 +60,7 @@
 ## <xml-line-number-hack from="https://stackoverflow.com/a/36430270/10846399">
 import sys
 
-sys.modules["_elementtree"] = None
+sys.modules["_elementtree"] = None  # type: ignore
 import xml.etree.ElementTree as ET
 
 ## override the parser to get the line number

editor/editor_builders.py

@@ -9,6 +9,7 @@
 import subprocess
 import tempfile
 import uuid
+import zlib
 from platform_methods import subprocess_main
 
 
@@ -28,7 +29,6 @@ def make_doc_header(target, source, env):
 
     buf = (docbegin + buf + docend).encode("utf-8")
     decomp_size = len(buf)
-    import zlib
 
     # Use maximum zlib compression level to further reduce file size
     # (at the cost of initial build times).
@@ -88,9 +88,6 @@ def make_translations_header(target, source, env, category):
     g.write("#ifndef _{}_TRANSLATIONS_H\n".format(category.upper()))
     g.write("#define _{}_TRANSLATIONS_H\n".format(category.upper()))
 
-    import zlib
-    import os.path
-
     sorted_paths = sorted(source, key=lambda path: os.path.splitext(os.path.basename(path))[0])
 
     msgfmt_available = shutil.which("msgfmt") is not None

editor/translations/extract.py

@@ -8,6 +8,7 @@
 import shutil
 import subprocess
 import sys
+from typing import Dict, Tuple
 
 
 class Message:
@@ -42,7 +43,7 @@ def format(self):
         return "\n".join(lines)
 
 
-messages_map = {}  # (id, context) -> Message.
+messages_map: Dict[Tuple[str, str], Message] = {}  # (id, context) -> Message.
 
 line_nb = False
 
@@ -51,11 +52,11 @@ def format(self):
         print("Enabling line numbers in the context locations.")
         line_nb = True
     else:
-        os.sys.exit("Non supported argument '" + arg + "'. Aborting.")
+        sys.exit("Non supported argument '" + arg + "'. Aborting.")
 
 
 if not os.path.exists("editor"):
-    os.sys.exit("ERROR: This script should be started from the root of the git repo.")
+    sys.exit("ERROR: This script should be started from the root of the git repo.")
 
 
 matches = []

gles3_builders.py

@@ -3,6 +3,10 @@
 All such functions are invoked in a subprocess on Windows to prevent build flakiness.
 
 """
+import os.path
+
+from typing import Optional
+
 from platform_methods import subprocess_main
 
 
@@ -30,7 +34,7 @@ def __init__(self):
         self.specialization_values = []
 
 
-def include_file_in_gles3_header(filename, header_data, depth):
+def include_file_in_gles3_header(filename: str, header_data: GLES3HeaderStruct, depth: int):
     fs = open(filename, "r")
     line = fs.readline()
 
@@ -91,8 +95,6 @@ def include_file_in_gles3_header(filename, header_data, depth):
         while line.find("#include ") != -1:
             includeline = line.replace("#include ", "").strip()[1:-1]
 
-            import os.path
-
             included_file = os.path.relpath(os.path.dirname(filename) + "/" + includeline)
             if not included_file in header_data.vertex_included_files and header_data.reading == "vertex":
                 header_data.vertex_included_files += [included_file]
@@ -182,7 +184,7 @@ def include_file_in_gles3_header(filename, header_data, depth):
     return header_data
 
 
-def build_gles3_header(filename, include, class_suffix, header_data=None):
+def build_gles3_header(filename: str, include: str, class_suffix: str, header_data: Optional[GLES3HeaderStruct] = None):
     header_data = header_data or GLES3HeaderStruct()
     include_file_in_gles3_header(filename, header_data, 0)
 

glsl_builders.py

@@ -4,13 +4,15 @@
 
 """
 import os.path
+from typing import Optional, Iterable
+
 from platform_methods import subprocess_main
 
 
-def generate_inline_code(input_lines, insert_newline=True):
+def generate_inline_code(input_lines: Iterable[str], insert_newline: bool = True):
     """Take header data and generate inline code
 
-    :param: list input_lines: values for shared inline code
+    :param: input_lines: values for shared inline code
     :return: str - generated inline value
     """
     output = []
@@ -40,7 +42,7 @@ def __init__(self):
         self.compute_offset = 0
 
 
-def include_file_in_rd_header(filename, header_data, depth):
+def include_file_in_rd_header(filename: str, header_data: RDHeaderStruct, depth: int) -> RDHeaderStruct:
     fs = open(filename, "r")
     line = fs.readline()
 
@@ -112,7 +114,7 @@ def include_file_in_rd_header(filename, header_data, depth):
     return header_data
 
 
-def build_rd_header(filename, header_data=None):
+def build_rd_header(filename: str, header_data: Optional[RDHeaderStruct] = None) -> None:
     header_data = header_data or RDHeaderStruct()
     include_file_in_rd_header(filename, header_data, 0)
 
@@ -171,7 +173,7 @@ def __init__(self):
         self.code = ""
 
 
-def include_file_in_raw_header(filename, header_data, depth):
+def include_file_in_raw_header(filename: str, header_data: RAWHeaderStruct, depth: int) -> None:
     fs = open(filename, "r")
     line = fs.readline()
 
@@ -191,7 +193,7 @@ def include_file_in_raw_header(filename, header_data, depth):
     fs.close()
 
 
-def build_raw_header(filename, header_data=None):
+def build_raw_header(filename: str, header_data: Optional[RAWHeaderStruct] = None):
     header_data = header_data or RAWHeaderStruct()
     include_file_in_raw_header(filename, header_data, 0)
 

methods.py

@@ -1,6 +1,5 @@
 import os
 import re
-import sys
 import glob
 import subprocess
 from collections import OrderedDict
@@ -663,7 +662,6 @@ def detect_visual_c_compiler_version(tools_env):
         if vc_x86_amd64_compiler_detection_index > -1 and (
             vc_chosen_compiler_index == -1 or vc_chosen_compiler_index > vc_x86_amd64_compiler_detection_index
         ):
-            vc_chosen_compiler_index = vc_x86_amd64_compiler_detection_index
             vc_chosen_compiler_str = "x86_amd64"
 
     return vc_chosen_compiler_str

misc/scripts/mypy.ini

@@ -0,0 +1,11 @@
+[mypy]
+ignore_missing_imports = true
+disallow_any_generics = True
+pretty = True
+show_column_numbers = True
+warn_redundant_casts = True
+warn_return_any = True
+warn_unreachable = True
+
+namespace_packages = True
+explicit_package_bases = True

misc/scripts/mypy_check.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+echo -e "Python: mypy static analysis..."
+mypy --config-file=./misc/scripts/mypy.ini .

modules/mono/build_scripts/build_assemblies.py

@@ -5,6 +5,7 @@
 import shlex
 import subprocess
 from dataclasses import dataclass
+from typing import Optional, List
 
 
 def find_dotnet_cli():
@@ -150,10 +151,7 @@ def find_any_msbuild_tool(mono_prefix):
     return None
 
 
-def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: [str] = None):
-    if msbuild_args is None:
-        msbuild_args = []
-
+def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: Optional[List[str]] = None):
     using_msbuild_mono = False
 
     # Preference order: dotnet CLI > Standalone MSBuild > Mono's MSBuild
@@ -169,7 +167,7 @@ def run_msbuild(tools: ToolsLocation, sln: str, msbuild_args: [str] = None):
 
     args += [sln]
 
-    if len(msbuild_args) > 0:
+    if msbuild_args:
         args += msbuild_args
 
     print("Running MSBuild: ", " ".join(shlex.quote(arg) for arg in args), flush=True)