Add extended example for metadata and fix some docstrings

Instagram · Aug 15, 2019 · 796f864 · 796f864
1 parent b248b1a
commit 796f864
Show file tree

Hide file tree

Showing 6 changed files with 123 additions and 135 deletions.
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -17,7 +17,8 @@ LibCST
 
    why_libcst
    motivation
-   usage
+   tutorial
+   Metadata Tutorial <metadata_tutorial>
    parser
    nodes
    visitors

diff --git a/docs/source/metadata.rst b/docs/source/metadata.rst
@@ -6,14 +6,29 @@ Metadata
 LibCST ships with a metadata interface that defines a standardized way to
 associate nodes in a CST with arbitrary metadata while maintaining the immutability
 of the tree. The metadata interface is designed to be declarative and type safe.
+Here's a quick example of using the metadata interface to get line and column
+numbers of nodes:
+
+.. code-block:: python
+
+    class NamePrinter(cst.CSTVisitor):
+        METADATA_DEPENDENCIES = (cst.SyntacticPositionProvider,)
+
+        def visit_Name(self, node: cst.Name) -> None:
+            pos = self.get_metadata(cst.SyntacticPositionProvider, node).start
+            print(f"{node.value} found at line {pos.line}, column {pos.column}")
+
+    wrapper = cst.MetadataWrapper(cst.parse_module("x = 1"))
+    result = wrapper.visit(NamePrinter())  # should print "x found at line 1, column 0"
 
 Accessing Metadata
 ------------------
 
 Metadata providers are declared as dependencies by a :class:`~libcst.MetadataDependent`
-or any of its subclasses and will be computed automatically whenever one of the
-resolve methods is used. Alternatively, you can use one of the visit methods in
-the wrapper when working with visitors.
+or any of its subclasses and will be computed automatically whenever visited
+by a :class:`~libcst.MetadataWrapper`. Alternatively, you can use 
+:func:`~libcst.MetadataWrapper.resolve` or :func:`~libcst.MetadataWrapper.resolve_many`
+in the wrapper to generate and access metadata directly.
 
 .. autoclass:: libcst.MetadataDependent
 .. autoclass:: libcst.MetadataWrapper
@@ -34,14 +49,12 @@ Position Metadata
 
 Position (line and column numbers) metadata are accessible through the metadata
 interface by declaring the one of the following providers as a dependency. For
-most cases, :class:`~libcst.SytacticPositionProvider` is what you probably want.
+most cases, :class:`~libcst.SyntacticPositionProvider` is what you probably want.
 Accessing position metadata through the :class:`~libcst.MetadataDepedent`
 interface will return a :class:`~libcst.CodeRange` object.
 
 .. autoclass:: libcst.BasicPositionProvider
 .. autoclass:: libcst.SyntacticPositionProvider
 
 .. autoclass:: libcst.CodeRange
-    :members:
 .. autoclass:: libcst.CodePosition
-    :members:
diff --git a/docs/source/metadata_tutorial.ipynb b/docs/source/metadata_tutorial.ipynb
@@ -0,0 +1,79 @@
+{
+ "cells": [
+  {
+   "cell_type": "raw",
+   "metadata": {
+    "raw_mimetype": "text/restructuredtext"
+   },
+   "source": [
+    "=====================\n",
+    "Working with Metadata\n",
+    "=====================\n",
+    "LibCST handles node metadata in a somewhat unusal manner in order to maintain the immutability of the tree. See :doc:`Metadata <metadata>` for the complete documentation. Here's an example of a provider that lablels marks nodes as function parameters that is used by a visitor that prints all names that are function parameters."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import libcst as cst\n",
+    "\n",
+    "\n",
+    "class IsParamProvider(cst.BatchableMetadataProvider[bool]):\n",
+    "    \"\"\"\n",
+    "    Marks Name nodes found as a parameter to a function.\n",
+    "    \"\"\"\n",
+    "    def __init__(self):\n",
+    "        super().__init__()\n",
+    "        self.is_param = False\n",
+    "    \n",
+    "    def visit_Param(self, node: cst.Param) -> None:\n",
+    "        # Mark the child Name node as a parameter \n",
+    "        self.set_metadata(node.name, True)\n",
+    "        \n",
+    "    def visit_Name(self, node: cst.Name) -> None:\n",
+    "        # Mark all other Name nodes as not parameters\n",
+    "        if not self.get_metadata(type(self), node, False):\n",
+    "            self.set_metadata(node, False)\n",
+    "\n",
+    "\n",
+    "class ParamPrinter(cst.CSTVisitor):\n",
+    "    METADATA_DEPENDENCIES = (IsParamProvider, cst.SyntacticPositionProvider,)\n",
+    "\n",
+    "    def visit_Name(self, node: cst.Name) -> None:\n",
+    "        # Only print out names that are parameters\n",
+    "        if self.get_metadata(IsParamProvider, node):\n",
+    "            pos = self.get_metadata(cst.SyntacticPositionProvider, node).start\n",
+    "            print(f\"{node.value} found at line {pos.line}, column {pos.column}\")\n",
+    "\n",
+    "\n",
+    "wrapper = cst.MetadataWrapper(cst.parse_module(\"def foo(x):\\n    y = 1\\n    return x + y\"))\n",
+    "result = wrapper.visit(ParamPrinter())"
+   ]
+  }
+ ],
+ "metadata": {
+  "celltoolbar": "Raw Cell Format",
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.2+"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docs/source/usage.ipynb → docs/source/tutorial.ipynb b/docs/source/usage.ipynb → docs/source/tutorial.ipynb
@@ -6,16 +6,15 @@
     "raw_mimetype": "text/restructuredtext"
    },
    "source": [
-    "===== \n",
-    "Usage\n",
-    "=====\n",
+    "========\n",
+    "Tutorial\n",
+    "========\n",
     "\n",
     "LibCST provides helpers to parse source code string as concrete syntax tree. In order to perform static analysis to identify patterns in the tree or modify the tree programmatically, we can use visitor pattern to traverse the tree. In this tutorial, we demonstrate a common three-step-workflow to build an automated refactoring (codemod) application:\n",
     "\n",
     "1. `Parse Source Code <#Parse-Source-Code>`_\n",
     "2. `Build Visitor or Transformer <#Build-Visitor-or-Transformer>`_\n",
     "3. `Generate Source Code <#Generate-Source-Code>`_\n",
-    "4. `Work with Metadata <#Working with Metadata>`_\n",
     "\n",
     "Parse Source Code\n",
     "=================\n",
@@ -24,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
     "nbsphinx": "hidden"
    },
@@ -36,41 +35,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "BinaryOperation(\n",
-       "    left=Integer(\n",
-       "        value='1',\n",
-       "        lpar=[],\n",
-       "        rpar=[],\n",
-       "    ),\n",
-       "    operator=Add(\n",
-       "        whitespace_before=SimpleWhitespace(\n",
-       "            value=' ',\n",
-       "        ),\n",
-       "        whitespace_after=SimpleWhitespace(\n",
-       "            value=' ',\n",
-       "        ),\n",
-       "    ),\n",
-       "    right=Integer(\n",
-       "        value='2',\n",
-       "        lpar=[],\n",
-       "        rpar=[],\n",
-       "    ),\n",
-       "    lpar=[],\n",
-       "    rpar=[],\n",
-       ")"
-      ]
-     },
-     "execution_count": 2,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
+   "outputs": [],
    "source": [
     "import libcst as cst\n",
     "\n",
@@ -90,7 +57,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -133,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -224,61 +191,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "\n",
-      "class PythonToken(Token):\n",
-      "    def __repr__(self) -> str:\n",
-      "        return ('TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r)' %\n",
-      "                self._replace(type=self.type.name))\n",
-      "\n",
-      "def tokenize(code: str, version_info: PythonVersionInfo, start_pos: Tuple[int, int] = (1, 0)\n",
-      ") -> Generator[PythonToken, None, None]:\n",
-      "    \"\"\"Generate tokens from a the source code (string).\"\"\"\n",
-      "    lines = split_lines(code, keepends=True)\n",
-      "    return tokenize_lines(lines, version_info, start_pos=start_pos)\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "print(modified_tree.code)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "--- \n",
-      "+++ \n",
-      "@@ -1,10 +1,11 @@\n",
-      " \n",
-      " class PythonToken(Token):\n",
-      "-    def __repr__(self):\n",
-      "+    def __repr__(self) -> str:\n",
-      "         return ('TokenInfo(type=%s, string=%r, start_pos=%r, prefix=%r)' %\n",
-      "                 self._replace(type=self.type.name))\n",
-      " \n",
-      "-def tokenize(code, version_info, start_pos=(1, 0)):\n",
-      "+def tokenize(code: str, version_info: PythonVersionInfo, start_pos: Tuple[int, int] = (1, 0)\n",
-      "+) -> Generator[PythonToken, None, None]:\n",
-      "     \"\"\"Generate tokens from a the source code (string).\"\"\"\n",
-      "     lines = split_lines(code, keepends=True)\n",
-      "     return tokenize_lines(lines, version_info, start_pos=start_pos)\n",
-      "\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# Use difflib to show the changes to verify type annotations were added as expected.\n",
     "import difflib\n",
@@ -301,51 +225,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
     "if not modified_tree.deep_equals(source_tree):\n",
     "    ...  # write to file"
    ]
-  },
-  {
-   "cell_type": "raw",
-   "metadata": {
-    "raw_mimetype": "text/restructuredtext"
-   },
-   "source": [
-    "Work with Metadata\n",
-    "==================\n",
-    "LibCST handles node metadata in a somewhat unusal manner in order to maintain the immutability of the tree. See :doc:`Metadata <metadata>` for the complete documentation. Here's an example of a visitor that prints out the starting position of every :class:`~libcst.Name` node."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 13,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "x found at line 1, column 0\n"
-     ]
-    }
-   ],
-   "source": [
-    "import libcst as cst\n",
-    "\n",
-    "class NamePrinter(cst.CSTVisitor):\n",
-    "    METADATA_DEPENDENCIES = (cst.SyntacticPositionProvider,)\n",
-    "\n",
-    "    def visit_Name(self, node: cst.Name) -> None:\n",
-    "        pos = self.get_metadata(cst.SyntacticPositionProvider, node).start\n",
-    "        print(f\"{node.value} found at line {pos.line}, column {pos.column}\")\n",
-    "\n",
-    "wrapper = cst.MetadataWrapper(cst.parse_module(\"x = 1\"))\n",
-    "result = wrapper.visit(NamePrinter())  # should print \"x found at line 1, column 0\""
-   ]
   }
  ],
  "metadata": {

diff --git a/docs/source/visitors.rst b/docs/source/visitors.rst
@@ -48,7 +48,7 @@ An example Python REPL using the above visitor is as follows::
 Batched Visitors
 ----------------
 
-A batchable visitor classs is provided to facilitate performing operations that
+A batchable visitor class is provided to facilitate performing operations that
 can be performed in parallel in a single traversal over a CST. An example of this
 is :ref:`metadata computation<libcst-metadata>`.
 

diff --git a/libcst/metadata/base_provider.py b/libcst/metadata/base_provider.py
@@ -41,6 +41,9 @@ class BaseMetadataProvider(MetadataDependent, Generic[_T]):
     """
     The low-level base class for all metadata providers. This class should be
     extended for metadata providers that are not visitor-based.
+
+    This class is generic. A subclass of ``BaseMetadataProvider[T]`` will
+    provider metadata of type ``T``.
     """
 
     #: Cache of metadata computed by this provider
@@ -102,6 +105,9 @@ class VisitorMetadataProvider(CSTVisitor, BaseMetadataProvider[_T]):
     """
     The low-level base class for all non-batchable visitor-based metadata
     providers. Inherits from :class:`~libcst.CSTVisitor`.
+
+    This class is generic. A subclass of ``VisitorMetadataProvider[T]`` will
+    provider metadata of type ``T``.
     """
 
     def _gen_impl(self, module: "_ModuleT") -> None:
@@ -112,6 +118,9 @@ class BatchableMetadataProvider(BatchableCSTVisitor, BaseMetadataProvider[_T]):
     """
     The low-level base class for all batchable visitor-based metadata providers.
     Inherits from :class:`~libcst.BatchableCSTVisitor`.
+
+    This class is generic. A subclass of ``BatchableMetadataProvider[T]`` will
+    provider metadata of type ``T``.
     """
 
     def _gen_impl(self, module: "Module") -> None: