Adding support for other language plugins

.github/actions/build-package/action.yml

@@ -40,7 +40,7 @@ runs:
           with:
               name: pypi-packages
               path: |
-                  dist/*.gz
-                  dist/*.whl
+                  packages/python/dist/*.gz
+                  packages/python/dist/*.whl
               if-no-files-found: error
               retention-days: 7

.github/workflows/version_check.py

@@ -2,7 +2,12 @@
 import pathlib
 import sys
 
-pyproject = pathlib.Path(__file__).parent.parent.parent / "pyproject.toml"
+pyproject = (
+    pathlib.Path(__file__).parent.parent.parent
+    / "packages"
+    / "python"
+    / "pyproject.toml"
+)
 
 content = pyproject.read_text(encoding="utf-8")
 

.vscode/launch.json

@@ -10,21 +10,20 @@
             "request": "launch",
             "module": "generator",
             "console": "integratedTerminal",
-            "justMyCode": true,
-            "args": [
-                "--output",
-                "lsprotocol"
-            ]
+            "justMyCode": true
         },
         {
             "name": "DON'T SELECT (test debug config)",
             "type": "python",
             "request": "launch",
             "console": "integratedTerminal",
             "justMyCode": false,
-            "purpose": [
-                "debug-test"
-            ]
+            "purpose": ["debug-test"],
+            "presentation": {
+                "hidden": true,
+                "group": "",
+                "order": 4
+            }
         }
     ]
-}
+}

.vscode/settings.json

@@ -7,5 +7,6 @@
     "isort.args": ["--profile", "black"],
     "python.testing.pytestArgs": ["tests"],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.testing.pytestEnabled": true,
+    "python.analysis.extraPaths": ["packages/python", "tests/python/common"]
 }

README.md

@@ -1,40 +1,66 @@
-# Language Server Protocol implementation for Python
+# Language Server Protocol types code generator
 
-`lsprotocol` is a python implementation of object types used in the Language Server Protocol (LSP). This repository contains the code generator and the generated types for LSP.
+This repository contains code to generate Language Server Protocol types and classes for various languages.
 
-## Overview
+# Code Generator usage
 
-LSP is used by editors to communicate with various tools to enables services like code completion, documentation on hover, formatting, code analysis, etc. The intent of this library is to allow you to build on top of the types used by LSP. This repository will be kept up to date with the latest version of LSP as it is updated.
+## Usage
 
-## Installation
+### Command line
 
-`python -m pip install lsprotocol`
+Clone this repository and run `generator` like a module.
 
-## Usage
+```console
+>python -m generator --help
+usage: __main__.py [-h] [--schema SCHEMA] [--model MODEL]
 
-### Using LSP types
+Generate types from LSP JSON model.
 
-```python
-from lsprotocol import types
+optional arguments:
+  -h, --help            show this help message and exit
+  --schema SCHEMA, -s SCHEMA
+                        Path to a model schema file. By default uses packaged
+                        schema.
+  --model MODEL, -m MODEL
+                        Path to a model JSON file. By default uses packaged
+                        model file.
+```
+
+### using `nox`
 
-position = types.Position(line=10, character=3)
+This project uses `nox` as a task runner to run the code generator. You can install `nox` and run `build_lsp` session to generate code from spec available in the repo.
+
+```console
+> python -m pip install nox
+> nox --session build_lsp
 ```
 
-### Using built-in type converters
+# Contributing plugins
 
-```python
-# test.py
-import json
-from lsprotocol import converters, types
+You can contribute plugins by adding your code generator under `generator-plugins` directory. The `generator` module will load the plugin (`myplugin`), and call `myplugin.generate()` on it. See, the `python` plugin for layout.
+
+This is the expected signature of generate:
 
-position = types.Position(line=10, character=3)
-converter = converters.get_converter()
-print(json.dumps(converter.unstructure(position, unstructure_as=types.Position)))
+```python
+def (spec: model.LSPModel, output_dir: str) -> None: ...
 ```
 
-Output:
+Expected directory structure:
 
-```console
-> python test.py
-{"line": 10, "character": 3}
 ```
+generator-plugins
+├───myplugin
+│      __init__.py (required)
+│      <your code files>
+│
+└───python
+       utils.py
+       __init__.py
+
+```
+
+# Supported plugins
+
+| Language | Plugin                  | Package                                                            | Notes  |
+| -------- | ----------------------- | ------------------------------------------------------------------ | ------ |
+| Python   | generator-plugin.python | ![PyPI](https://img.shields.io/pypi/v/lsprotocol?label=lsprotocol) | Active |

conftest.py

@@ -1,12 +1,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+import os
 import pathlib
 import sys
 
 import pytest
 
-
-@pytest.fixture
-def customize_sys_path():
-    sys.path.append(str(pathlib.Path(__file__).parent))
+sys.path.append(os.fspath(pathlib.Path(__file__).parent / "packages" / "python"))
+sys.path.append(
+    os.fspath(pathlib.Path(__file__).parent / "tests" / "python" / "common")
+)

generator-plugins/python/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from .utils import generate_from_spec as generate

generator/utils.py → generator-plugins/python/utils.py

@@ -5,13 +5,23 @@
 import copy
 import itertools
 import keyword
+import pathlib
 import re
 from typing import Dict, List, Optional, OrderedDict, Sequence, Tuple, Union
 
-from . import model
+import generator.model as model
 
 METHOD_NAME_RE_1 = re.compile(r"(.)([A-Z][a-z]+)")
 METHOD_NAME_RE_2 = re.compile(r"([a-z0-9])([A-Z])")
+PACKAGE_NAME = "lsprotocol"
+
+
+def generate_from_spec(spec: model.LSPModel, output_dir: str) -> None:
+    code = TypesCodeGenerator(spec).get_code()
+    for file_name in code:
+        pathlib.Path(output_dir, PACKAGE_NAME, file_name).write_text(
+            code[file_name], encoding="utf-8"
+        )
 
 
 def _generate_field_validator(

generator-plugins/rust/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from .rust_utils import generate_from_spec as generate

generator-plugins/rust/rust_utils.py

@@ -0,0 +1,8 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import generator.model as model
+
+
+def generate_from_spec(spec: model.LSPModel, output_dir: str) -> None:
+    pass

generator/__main__.py

@@ -3,15 +3,30 @@
 
 
 import argparse
+import importlib
 import json
+import logging
+import os
 import pathlib
 import sys
 from typing import Sequence
 
 import importlib_resources as ir
 import jsonschema
 
-from . import model, utils
+from . import model
+
+PACKAGES_ROOT = pathlib.Path(__file__).parent.parent / "packages"
+LOGGER = logging.getLogger("generator")
+
+
+def setup_logging() -> None:
+    logging.basicConfig(
+        stream=sys.stdout,
+        level=logging.DEBUG,
+        format="[%(levelname)s][%(asctime)s]  %(name)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
 
 
 def get_parser() -> argparse.ArgumentParser:
@@ -28,13 +43,6 @@ def get_parser() -> argparse.ArgumentParser:
         help="Path to a model JSON file. By default uses packaged model file.",
         type=str,
     )
-    parser.add_argument(
-        "--output",
-        "-o",
-        help="Path to a where the types should be written. By default uses stdout.",
-        type=str,
-        default="-",
-    )
     return parser
 
 
@@ -43,31 +51,54 @@ def main(argv: Sequence[str]) -> None:
     args = parser.parse_args(argv)
 
     # Validate against LSP model JSON schema.
+
     if args.schema:
-        schema = json.load(pathlib.Path(args.schema).open("rb"))
+        schema_file = pathlib.Path(args.schema)
     else:
         schema_file = ir.files("generator") / "lsp.schema.json"
-        schema = json.load(schema_file.open("rb"))
+
+    LOGGER.info("Using schema file %s", os.fspath(schema_file))
+    schema = json.load(schema_file.open("rb"))
 
     if args.model:
-        json_model = json.load(pathlib.Path(args.model).open("rb"))
+        model_file = pathlib.Path(args.model)
     else:
         model_file = ir.files("generator") / "lsp.json"
-        json_model = json.load(model_file.open("rb"))
 
+    LOGGER.info("Using model file %s", os.fspath(model_file))
+    json_model = json.load(model_file.open("rb"))
+
+    LOGGER.info("Validating model.")
     jsonschema.validate(json_model, schema)
 
-    # load model and generate types.
-    spec = model.create_lsp_model(json_model)
-    code = utils.TypesCodeGenerator(spec).get_code()
-    if args.output:
-        for file_name in code:
-            pathlib.Path(args.output, file_name).write_text(
-                code[file_name], encoding="utf-8"
-            )
-    else:
-        print(code)
+    LOGGER.info("Finding plugins.")
+    plugin_root = pathlib.Path(__file__).parent.parent / "generator-plugins"
+    plugins = []
+    for item in plugin_root.iterdir():
+        if (
+            item.is_dir()
+            and (item / "__init__.py").exists()
+            and not item.name.startswith("_")
+        ):
+            plugins.append(item.name)
+    LOGGER.info(f"Found plugins: {plugins}")
+    LOGGER.info("Starting code generation.")
+
+    for plugin in plugins:
+        LOGGER.info(f"Running plugin {plugin}.")
+
+        # load model and generate types for each plugin to avoid
+        # any conflicts between plugins.
+        spec: model.LSPModel = model.create_lsp_model(json_model)
+
+        try:
+            plugin_module = importlib.import_module(f"generator-plugins.{plugin}")
+            plugin_module.generate(spec, os.fspath(PACKAGES_ROOT / plugin))
+            LOGGER.info(f"Plugin {plugin} completed.")
+        except Exception as e:
+            LOGGER.error(f"Error running plugin {plugin}:", exc_info=e)
 
 
 if __name__ == "__main__":
+    setup_logging()
     main(sys.argv[1:])

generator/requirements.txt

@@ -55,9 +55,9 @@ pyrsistent==0.19.3 \
     --hash=sha256:f0774bf48631f3a20471dd7c5989657b639fd2d285b861237ea9e82c36a415a9 \
     --hash=sha256:f0e7c4b2f77593871e918be000b96c8107da48444d57005b6a6bc61fb4331b2c
     # via jsonschema
-typing-extensions==4.4.0 \
-    --hash=sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa \
-    --hash=sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e
+typing-extensions==4.5.0 \
+    --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \
+    --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4
     # via
     #   importlib-metadata
     #   jsonschema