Sdk markdown generation (#3407)

* fix(docs): added margin between sections * Fixes * Update stocks_helper.py * update git-actions set-output to new format * Update stocks_helper.py * Update terminal_helper.py * removed LineAnnotateDrawer from qa_view * lint * few changes * updates * sdk auto gen modules done * Update stocks_helper.py * updates to changed imports, and remove first sdk_modules * Update generate_sdk.py * Update generate_sdk.py * pylint * revert stocks_helper * Update generate_sdk.py * Update sdk.py * Update generate_sdk.py * full auto generation, added sdk.py/controllers creation * missed enable forecasting * added running black in subprocess after sdk files generation completes * removed deleted sdk_arg_logger * comment out tests * property doc fix * clean up * Update generate_sdk.py * make trailmap classes useable for doc generation * Update generate_sdk.py * added lineon to trailmap class for linking to func in markdown * changed lineon to dict * added full_path to trailmap for linking in docs * updated portfolio * feat: initial files * feat: added meta head * feat: added funcdef * added func_def to trailmap attributes for markdown in docs, added missing type hints to covid functions * feat: added view and merged with jaun * Update generate_sdk.py * Update generate_sdk.py * Update generate_sdk.py * Update generate_sdk.py * init * fix returns * fix: random stuff * fix: random * fixed encoding issue on windows * fix: generate tabs * Added changes * Added fixes * Updated dependencies * Added Jose fixes, pls work * FIxed small issue * Automated Change * Added forecast, will prolly also need to pip install * Added fix * Automated Change * init * Update generate_terminal_markdown.py * Update generate_terminal_markdown.py * moved folders out of common into root * remove cat from trail if sub in [ ba, ta, qa ] * Update generate_terminal_markdown.py * un-indented example images * Update generate_terminal_markdown.py * Update generate_terminal_markdown.py * Update generate_terminal_markdown.py * replace ✨ with 🦋 * Update regression_model.py * updates * up * Automated Change * removed Juan sdk * update * updates * fix missing type hints, added defaults to sdk generation * typing * typing * datetime defaults, fix dict_keys([]) in choices * fix regression docstring, added divider on markdowns * fix get_hcp portfolio trailmap * Update generate_sdk_markdown.py * Update generate_sdk_markdown.py * Update generate_sdk_markdown.py * Update generate_sdk_markdown.py * fix: some more stuff * fix: bump ga to 3.10 * fix: examples * fix: removed functions from root and moved ga * fix: run pages only when main is merged * fix: new poetry.lock * Update dependencies Co-authored-by: teh_coderer <me@tehcoderer.com> Co-authored-by: Colin Delahunty <72827203+colin99d@users.noreply.github.com> Co-authored-by: James Maslek <jmaslek11@gmail.com> Co-authored-by: colin99d <colin99delahunty@gmail.com> Co-authored-by: colin99d <colin99d@users.noreply.github.com> Co-authored-by: tehcoderer <tehcoderer@users.noreply.github.com> Co-authored-by: Chavithra PARANA <chavithra@gmail.com>
OpenBB-finance · Nov 22, 2022 · eb0c305 · eb0c305
1 parent 60f8c0f
commit eb0c305
Show file tree

Hide file tree

Showing 8 changed files with 4,235 additions and 1,074 deletions.
diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml
@@ -1,16 +1,19 @@
 name: github pages
-on: push
+on:
+  push:
+    branches:
+      - main
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
       - name: Git checkout
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b  # actions/checkout v3.0.2
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # actions/checkout v3.0.2
 
       - name: Setup hugo
         uses: peaceiris/actions-hugo@v2
         with:
-          hugo-version: '0.87.0'
+          hugo-version: "0.87.0"
           extended: true
 
       - name: Build

diff --git a/.gitignore b/.gitignore
@@ -30,6 +30,8 @@ openbb_terminal/portfolio/portfolios/*
 .coverage.*
 !openbb_terminal/reports/templates/*.ipynb
 *_tests.csv
+*_sdk_audit.csv
+website/functions
 website/terminaltest
 
 # pyinstaller artifacts

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -106,6 +106,7 @@ papermill = "^2.4.0"
 stocksera = "^0.1.21"
 ipython = "8.5.0"
 protobuf = "3.20.1"
+docstring-parser = {version = "^0.15", optional = true}
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.2"
@@ -137,6 +138,7 @@ nbmake = ">0.5.0"
 prediction = ["torch"]
 jupyterlab = ["jupyterlab-code-formatter", "jupyterlab-lsp", "jedi-language-server"]
 installer = ["pyinstaller"]
+doc = ["docstring-parser"]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

diff --git a/requirements-full.txt b/requirements-full.txt
diff --git a/requirements.txt b/requirements.txt
diff --git a/website/gen-docs.yml b/website/gen-docs.yml
@@ -0,0 +1,46 @@
+name: gen docs
+on: push
+jobs:
+  deploy:
+    name: Creating Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # actions/checkout v3.0.2
+
+      - name: Setup Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          architecture: x64
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.1.13
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-v1-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Install Package
+        run: poetry install --no-interaction -E prediction
+
+      - run: |
+          source $VENV
+          pip install "u8darts[torch]"
+          python website/generate_sdk_markdown.py
+
+      - name: Commit Documentation
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: Automated Change
diff --git a/website/generate_sdk_markdown.py b/website/generate_sdk_markdown.py
@@ -102,7 +102,6 @@ def get_docstrings(self) -> None:
                 self.short_doc[key] = clean_attr_desc(func_attr)
 
                 self.params[key] = {}
-
                 for k, p in get_signature_parameters(
                     func_attr, func_attr.__globals__
                 ).items():
@@ -316,11 +315,30 @@ def generate_markdown_section(meta):
     return markdown
 
 
+def add_todict(d: dict, location_path: list, tmap: Trailmap) -> dict:
+    """Adds the trailmap to the dictionary. A trailmap is a path to a function
+    in the sdk. This function creates the dictionary paths to the function."""
+
+    if location_path[0] not in d:
+        d[location_path[0]] = {}
+
+    if len(location_path) > 1:
+        add_todict(d[location_path[0]], location_path[1:], tmap)
+    else:
+        d[location_path[0]][tmap.class_attr] = (
+            "/sdk/functions/" + "/".join(tmap.location_path) + "/" + tmap.class_attr
+        )  # noqa: E501
+
+    return d
+
+
 def main():
     print("Loading trailmaps...")
     trailmaps = get_trailmaps()
     print("Generating markdown files...")
+    functions_dict = {}
     for trailmap in trailmaps:
+        functions_dict = add_todict(functions_dict, trailmap.location_path, trailmap)
         model_meta = get_function_meta(trailmap, "model") if trailmap.model else None
         view_meta = get_function_meta(trailmap, "view") if trailmap.view else None
         markdown = generate_markdown(model_meta, view_meta)
@@ -338,8 +356,22 @@ def main():
         os.makedirs(os.path.dirname(filepath), exist_ok=True)
         with open(filepath, "w", encoding="utf-8") as f:
             f.write(markdown)
+    index_markdown = "# OpenBB SDK Reference\n\n"
+    index_markdown += generate_index_markdown("", functions_dict, 2)
+    with open("functions/index.md", "w", encoding="utf-8") as f:
+        f.write(index_markdown)
     print("Markdown files generated, check the functions folder")
 
 
+def generate_index_markdown(markdown, d, level):
+    for key in d:
+        if isinstance(d[key], dict):
+            markdown += f"{'#' * level} {key}\n"
+            markdown = generate_index_markdown(markdown, d[key], level + 1)
+        else:
+            markdown += f"- [{key}]({d[key]})\n"
+    return markdown
+
+
 if __name__ == "__main__":
     main()