Added flyteidl documentation (#167)

* Added flyteidl documentation Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> * Generated doc requirements and updated CI to not generate docs for now Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> * Moved to rst file for events Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> * Moved to rst file for all index files Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> (cherry picked from commit 6dc3c4785dc95b05412a9bec2377016cf16b5ea8) * Added tmp doc generation step for getting doc dependencies Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com>
flyteorg · May 21, 2021 · b056e1c · b056e1c
1 parent 5def3d5
commit b056e1c
Show file tree

Hide file tree

Showing 111 changed files with 6,751 additions and 28,939 deletions.
diff --git a/flyteidl/.github/workflows/verification.yml b/flyteidl/.github/workflows/verification.yml
@@ -101,20 +101,6 @@ jobs:
       - name: Proto-Datacatalog-Java
         run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l java
 
-      # Docs
-      - name: Docs-Service
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/service -l protodoc
-      - name: Docs-Admin
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/admin -l protodoc
-      - name: Docs-Core
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/core -l protodoc
-      - name: Docs-Event
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/event -l protodoc
-      - name: Docs-Plugins
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/plugins -l protodoc
-      - name: Docs-Datacatalog
-        run: python3 /usr/local/bin/entrypoint.py -i ./protos -d protos/flyteidl/datacatalog -l protodoc
-
       - uses: actions/upload-artifact@master
         with:
           name: generated

diff --git a/flyteidl/.gitignore b/flyteidl/.gitignore
@@ -12,4 +12,5 @@ gen/pb_python/flyteidl.egg-info/
 .virtualgo
 docs/build/
 
-.vscode/
+.vscode/
+tmp/
diff --git a/flyteidl/Makefile b/flyteidl/Makefile
@@ -12,7 +12,7 @@ update_boilerplate:
 	@boilerplate/update.sh
 
 .PHONY: generate
-generate: install # install tools, generate protos, mocks and pflags
+generate: install doc_gen_deps # install tools, generate protos, doc dependecies, mocks and pflags
 	./generate_protos.sh
 	./generate_mocks.sh
 	go generate ./...
@@ -38,6 +38,23 @@ build_python:
 install-piptools:
 	pip install -U pip-tools
 
+.PHONY: doc_gen_deps # these dependencies are required by protoc gen doc for the protos which have external library dependencies. 
+# which includes grpc-gateway, googleapis, k8.io/api and apimachinery, protocolbuffers 
+doc_gen_deps:
+	rm -rf tmp/doc_gen_deps tmp/protocolbuffers tmp/grpc-gateway tmp/k8s.io
+	git clone --depth 1 https://github.com/googleapis/googleapis tmp/doc_gen_deps
+	rm -rf tmp/doc_gen_deps/.git
+	git clone --depth 1 https://github.com/protocolbuffers/protobuf tmp/protocolbuffers
+	cp -r tmp/protocolbuffers/src/* tmp/doc_gen_deps/
+	rm -rf tmp/protocolbuffers
+	git -c advice.detachedHead=false clone --depth 1 --branch v1.15.2 https://github.com/grpc-ecosystem/grpc-gateway tmp/grpc-gateway #v1.15.2 is used to keep the grpc-gateway version in sync with generated protos which is using the LYFT image
+	cp -r tmp/grpc-gateway/protoc-gen-swagger  tmp/doc_gen_deps/
+	rm -rf tmp/grpc-gateway
+	git clone --depth 1 https://github.com/kubernetes/api tmp/k8s.io/api
+	git clone --depth 1 https://github.com/kubernetes/apimachinery tmp/k8s.io/apimachinery
+	cp -r tmp/k8s.io tmp/doc_gen_deps/
+	rm -rf tmp/k8s.io
+
 .PHONY: doc-requirements.txt
 doc-requirements.txt: doc-requirements.in install-piptools
 	$(call PIP_COMPILE,doc-requirements.in)
@@ -53,4 +70,4 @@ update_pyversion:
 .PHONY: update_npmversion
 update_npmversion:
 	grep "$(PLACEHOLDER_NPM)" "package.json"
-	sed -i "s/$(PLACEHOLDER_NPM)/\"version\":  \"${VERSION}\"/g" "package.json"
+	sed -i "s/$(PLACEHOLDER_NPM)/\"version\":  \"${VERSION}\"/g" "package.json"
diff --git a/flyteidl/conf.py b/flyteidl/conf.py
@@ -17,6 +17,8 @@
 import sys
 
 sys.path.insert(0, os.path.abspath("gen/pb-protodoc/"))
+import recommonmark
+from recommonmark.transform import AutoStructify
 
 
 # -- Project information -----------------------------------------------------
@@ -48,6 +50,8 @@
     "sphinx-prompt",
     "sphinx_copybutton",
     "sphinx_search.extension",
+    "recommonmark",
+    "sphinx_markdown_tables",
 ]
 
 # build the templated autosummary files
@@ -64,8 +68,7 @@
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
-# source_suffix = ['.rst', '.md']
-source_suffix = ".rst"
+source_suffix = ['.rst', '.md']
 
 # The master toctree document.
 master_doc = "index"
@@ -80,7 +83,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path .
-exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store', 'tmp/doc_gen_deps', 'gen/*/*/*/*/*', 'boilerplate', 'pull_request_template.md', 'README.rst', 'reference/*', 'CODE_OF_CONDUCT.md']
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "tango"
@@ -189,3 +192,12 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/{.major}".format(sys.version_info), None),
 }
+
+def setup(app):
+    app.add_config_value('recommonmark_config', {
+        'auto_toc_tree_section': 'Contents',
+        'enable_math': False,
+        'enable_inline_math': False,
+        'enable_eval_rst': True,
+    }, True)
+    app.add_transform(AutoStructify)
diff --git a/flyteidl/doc-requirements.in b/flyteidl/doc-requirements.in
@@ -1,7 +1,9 @@
 git+git://github.com/flyteorg/furo@main
 readthedocs-sphinx-search
+recommonmark
 sphinx
 sphinx-prompt
 sphinx-material
 sphinx-code-include
 sphinx-copybutton
+sphinx_markdown_tables
diff --git a/flyteidl/doc-requirements.txt b/flyteidl/doc-requirements.txt
@@ -17,10 +17,14 @@ certifi==2020.12.5
     # via requests
 chardet==4.0.0
     # via requests
+commonmark==0.9.1
+    # via recommonmark
 css-html-js-minify==2.5.5
     # via sphinx-material
 docutils==0.16
-    # via sphinx
+    # via
+    #   recommonmark
+    #   sphinx
 git+git://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
 idna==2.10
@@ -31,6 +35,8 @@ jinja2==2.11.3
     # via sphinx
 lxml==4.6.3
     # via sphinx-material
+markdown==3.3.4
+    # via sphinx-markdown-tables
 markupsafe==1.1.1
     # via jinja2
 packaging==20.9
@@ -47,6 +53,8 @@ pytz==2021.1
     # via babel
 readthedocs-sphinx-search==0.1.0
     # via -r doc-requirements.in
+recommonmark==0.7.1
+    # via -r doc-requirements.in
 requests==2.25.1
     # via sphinx
 six==1.15.0
@@ -59,6 +67,8 @@ sphinx-code-include==1.1.1
     # via -r doc-requirements.in
 sphinx-copybutton==0.3.1
     # via -r doc-requirements.in
+sphinx-markdown-tables==0.0.15
+    # via -r doc-requirements.in
 sphinx-material==0.0.32
     # via -r doc-requirements.in
 sphinx-prompt==1.4.0
@@ -67,6 +77,7 @@ sphinx==3.5.4
     # via
     #   -r doc-requirements.in
     #   furo
+    #   recommonmark
     #   sphinx-code-include
     #   sphinx-copybutton
     #   sphinx-material