Bumped up idl to use insecureVerify flag for remote and fixed docs (#125

) * Bumped up idl to use insecureVerify flag for remote Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> * Fixed the doc links Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com> * Added sphinx-tabs Signed-off-by: Prafulla Mahindrakar <prafulla.mahindrakar@gmail.com>
flyteorg · Jul 1, 2021 · bf60ec1 · bf60ec1
1 parent 67c6ff9
commit bf60ec1
Show file tree

Hide file tree

Showing 14 changed files with 184 additions and 160 deletions.
diff --git a/cmd/sandbox/start.go b/cmd/sandbox/start.go
@@ -27,6 +27,7 @@ Mount your flytesnacks repository code inside sandbox
 ::
 
  bin/flytectl sandbox start --sourcesPath=$HOME/flyteorg/flytesnacks 
+
 Usage
 	`
 )

diff --git a/doc-requirements.in b/doc-requirements.in
@@ -6,3 +6,4 @@ sphinx-material
 sphinx-code-include
 sphinx-copybutton
 sphinx_fontawesome
+sphinx_tabs
diff --git a/doc-requirements.txt b/doc-requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile doc-requirements.in
@@ -20,7 +20,9 @@ chardet==4.0.0
 css-html-js-minify==2.5.5
     # via sphinx-material
 docutils==0.16
-    # via sphinx
+    # via
+    #   sphinx
+    #   sphinx-tabs
 git+git://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
 idna==2.10
@@ -39,6 +41,7 @@ pygments==2.9.0
     # via
     #   sphinx
     #   sphinx-prompt
+    #   sphinx-tabs
 pyparsing==2.4.7
     # via packaging
 python-slugify[unidecode]==5.0.2
@@ -55,6 +58,16 @@ snowballstemmer==2.1.0
     # via sphinx
 soupsieve==2.2.1
     # via beautifulsoup4
+sphinx==3.5.4
+    # via
+    #   -r doc-requirements.in
+    #   furo
+    #   sphinx-code-include
+    #   sphinx-copybutton
+    #   sphinx-fontawesome
+    #   sphinx-material
+    #   sphinx-prompt
+    #   sphinx-tabs
 sphinx-code-include==1.1.1
     # via -r doc-requirements.in
 sphinx-copybutton==0.3.1
@@ -65,15 +78,8 @@ sphinx-material==0.0.32
     # via -r doc-requirements.in
 sphinx-prompt==1.4.0
     # via -r doc-requirements.in
-sphinx==3.5.4
-    # via
-    #   -r doc-requirements.in
-    #   furo
-    #   sphinx-code-include
-    #   sphinx-copybutton
-    #   sphinx-fontawesome
-    #   sphinx-material
-    #   sphinx-prompt
+sphinx-tabs==3.1.0
+    # via -r doc-requirements.in
 sphinxcontrib-applehelp==1.0.2
     # via sphinx
 sphinxcontrib-devhelp==1.0.2

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -43,6 +43,7 @@
     "sphinx_copybutton",
     "sphinx_search.extension",
     "sphinx_fontawesome",
+    "sphinx_tabs.tabs",
 ]
 
 # build the templated autosummary files

diff --git a/docs/source/gen/flytectl_register_examples.rst b/docs/source/gen/flytectl_register_examples.rst
@@ -34,6 +34,7 @@ Options
   -h, --help                          help for examples
   -k, --k8ServiceAccount string        custom kubernetes service account auth role to register launch plans with.
   -l, --outputLocationPrefix string    custom output location prefix for offloaded types (files/schemas).
+      --sourceUploadPath string        Location for source code in storage.
   -v, --version string                version of the entity to be registered with flyte. (default "v1")
 
 Options inherited from parent commands

diff --git a/docs/source/gen/flytectl_register_files.rst b/docs/source/gen/flytectl_register_files.rst
@@ -15,7 +15,18 @@ If there are already registered entities with v1 version then the command will f
 ::
 
  bin/flytectl register file  _pb_output/* -d development  -p flytesnacks
+	
+There is no difference between registration and fast registration, In fast registration, the input provided by the user is fast serialized proto that is generated by pyflyte. If Flytectl finds any source code in users's input then it will consider registration as fast registration. Flytectl finds input file by searching an archive file whose name starts with fast and has .tar.gz extension When the user runs pyflyte with --fast flag then pyflyte creates serialize proto and it also archive create source code archive file in the same directory. 
+SourceUploadPath is an optional flag. By default, flytectl will create SourceUploadPath from your storage config. In case of s3 flytectl will upload code base in s3://{{DEFINE_BUCKET_IN_STORAGE_CONFIG}}/fast/{{VERSION}}-fast{{MD5_CREATED_BY_PYFLYTE}.tar.gz}. 
+::
+
+ bin/flytectl register file  _pb_output/* -d development  -p flytesnacks  -v v2 
+	
+In case of fast registration, If the SourceUploadPath flag is defined then In this case flytectl will not use the default directory for uploading the source code, it will override the destination path on the registration  
+::
 
+ bin/flytectl register file  _pb_output/* -d development  -p flytesnacks  -v v2 --SourceUploadPath="s3://dummy/fast" 
+	
 Using archive file.Currently supported are .tgz and .tar extension files and can be local or remote file served through http/https.
 Use --archive flag.
 
@@ -29,8 +40,7 @@ Using  local tgz file.
 
  bin/flytectl register files  _pb_output.tgz -d development  -p flytesnacks --archive
 
-If you want to continue executing registration on other files ignoring the errors including version conflicts then pass in
-the continueOnError flag.
+If you want to continue executing registration on other files ignoring the errors including version conflicts then pass in the continueOnError flag.
 
 ::
 
@@ -69,7 +79,7 @@ Override Output location prefix during registration.
 ::
 
  bin/flytectl register file  _pb_output/* -d development  -p flytesnacks --continueOnError -v v2 -l "s3://dummy/prefix"
-
+	
 Usage
 
 
@@ -88,6 +98,7 @@ Options
   -h, --help                          help for files
   -k, --k8ServiceAccount string        custom kubernetes service account auth role to register launch plans with.
   -l, --outputLocationPrefix string    custom output location prefix for offloaded types (files/schemas).
+      --sourceUploadPath string        Location for source code in storage.
   -v, --version string                version of the entity to be registered with flyte. (default "v1")
 
 Options inherited from parent commands

diff --git a/docs/source/gen/flytectl_sandbox.rst b/docs/source/gen/flytectl_sandbox.rst
@@ -43,6 +43,7 @@ SEE ALSO
 ~~~~~~~~
 
 * :doc:`flytectl` 	 - flyetcl CLI tool
+* :doc:`flytectl_sandbox_exec` 	 - Execute any command in sandbox
 * :doc:`flytectl_sandbox_start` 	 - Start the flyte sandbox
 * :doc:`flytectl_sandbox_status` 	 - Get the status of the sandbox environment.
 * :doc:`flytectl_sandbox_teardown` 	 - Teardown will cleanup the sandbox environment

diff --git a/docs/source/gen/flytectl_sandbox_exec.rst b/docs/source/gen/flytectl_sandbox_exec.rst
@@ -0,0 +1,46 @@
+.. _flytectl_sandbox_exec:
+
+flytectl sandbox exec
+---------------------
+
+Execute any command in sandbox
+
+Synopsis
+~~~~~~~~
+
+
+
+Execute command will Will run non-interactive commands and return immediately with the output.
+
+::
+ bin/flytectl sandbox exec -- ls -al 
+
+Usage
+
+::
+
+  flytectl sandbox exec [flags]
+
+Options
+~~~~~~~
+
+::
+
+  -h, --help            help for exec
+      --source string    Path of your source code
+
+Options inherited from parent commands
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+  -c, --config string    config file (default is $HOME/.flyte/config.yaml)
+  -d, --domain string    Specifies the Flyte project's domain.
+  -o, --output string    Specifies the output type - supported formats [TABLE JSON YAML DOT DOTURL]. NOTE: dot, doturl are only supported for Workflow (default "TABLE")
+  -p, --project string   Specifies the Flyte project.
+
+SEE ALSO
+~~~~~~~~
+
+* :doc:`flytectl_sandbox` 	 - Used for testing flyte sandbox.
+
diff --git a/docs/source/gen/flytectl_sandbox_start.rst b/docs/source/gen/flytectl_sandbox_start.rst
@@ -19,6 +19,7 @@ Mount your flytesnacks repository code inside sandbox
 ::
 
  bin/flytectl sandbox start --sourcesPath=$HOME/flyteorg/flytesnacks 
+
 Usage
 
 
@@ -31,8 +32,8 @@ Options
 
 ::
 
-      --flytesnacks string    Path of your flytesnacks repository
-  -h, --help                 help for start
+  -h, --help            help for start
+      --source string    Path of your source code
 
 Options inherited from parent commands
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -10,19 +10,32 @@ Install
 Flytectl is a Golang binary and can be installed on any platform supported by
 golang
 
-Install flytectl with homebrew tap
 
-.. prompt:: bash
+.. tabs::
 
-   brew install flyteorg/homebrew-tap/flytectl
-   # Upgrade flytectl
-   brew upgrade flytectl
+  .. tab:: OSX
 
-Install flytectl with shell script
+    .. prompt:: bash $
 
-.. prompt:: bash
+       brew install flyteorg/homebrew-tap/flytectl
 
-   curl -s https://raw.githubusercontent.com/lyft/flytectl/master/install.sh | bash
+    *Upgrade* existing installation using the following command:
+
+    .. prompt:: bash $
+
+       brew upgrade flytectl
+
+  .. tab:: Other Operating systems
+
+    .. prompt:: bash $
+
+        curl -s https://raw.githubusercontent.com/lyft/flytectl/master/install.sh | bash
+
+**Test** if Flytectl is installed correctly (your Flytectl version should be > 0.2.0) using the following command:
+
+.. prompt:: bash $
+
+  flytectl version
 
 Configure
 =========
@@ -32,25 +45,73 @@ on command-line. The following configuration is useful to setup.
 Basic Configuration
 --------------------
 
-.. code-block:: yaml
-
-  admin:
-    # For GRPC endpoints you might want to use dns:///flyte.myexample.com
-    endpoint: dns:///flyte.lyft.net
-    authType: Pkce
-  logger:
-    # Logger settings to control logger output. Useful to debug logger:
-    show-source: true
-    level: 1
-
-
-Place this in $HOME/.flyte directory with name config.yaml.
-This file is searched in
-
-- $HOME/.flyte
-- currDir from where you run flytectl
-- /etc/flyte/config
-- You can pass it commandline using --config <config-file-path>
+.. tabs:: Flytectl configuration
+
+   .. tab:: Local Flyte Sandbox
+
+       Automatically configured for you by ``flytectl sandbox`` command.
+
+       .. code-block:: yaml
+
+           admin:
+             # For GRPC endpoints you might want to use dns:///flyte.myexample.com
+             endpoint: dns:///localhost:30081
+             insecure: true
+             authType: Pkce # if using authentication or just drop this. If insecure set insecure: True
+           storage:
+             connection:
+               access-key: minio
+               auth-type: accesskey
+               disable-ssl: true
+               endpoint: http://localhost:30084
+               region: my-region-here
+               secret-key: miniostorage
+             container: my-s3-bucket
+             type: minio
+
+   .. tab:: AWS Configuration
+
+       .. code-block:: yaml
+
+           admin:
+             # For GRPC endpoints you might want to use dns:///flyte.myexample.com
+             endpoint: dns:///<replace-me>
+             authType: Pkce # if using authentication or just drop this. If insecure set insecure: True
+           storage:
+             kind: s3
+             config:
+               auth_type: iam
+               region: <replace> # Example: us-east-2
+             container: <replace> # Example my-bucket. Flyte k8s cluster / service account for execution should have read access to this bucket
+
+   .. tab:: GCS Configuration
+
+       .. code-block:: yaml
+
+           admin:
+             # For GRPC endpoints you might want to use dns:///flyte.myexample.com
+             endpoint: dns:///<replace-me>
+             authType: Pkce # if using authentication or just drop this. If insecure set insecure: True
+           storage:
+             kind: google
+             config:
+               json: ""
+               project_id: <replace-me> # TODO: replace <project-id> with the GCP project ID
+               scopes: https://www.googleapis.com/auth/devstorage.read_write
+             container: <replace> # Example my-bucket. Flyte k8s cluster / service account for execution should have access to this bucket
+
+   .. tab:: Others
+
+           For other supported storage backends like Oracle, Azure, etc., refer to the configuration structure `here <https://pkg.go.dev/github.com/flyteorg/flytestdlib/storage#Config>`__.
+
+
+    Place this in $HOME/.flyte directory with name config.yaml.
+    This file is searched in
+
+    * $HOME/.flyte
+    * currDir from where you run flytectl
+    * /etc/flyte/config
+        You can pass it commandline using --config <config-file-path> aswell
 
 
 .. toctree::

diff --git a/docs/source/nouns.rst b/docs/source/nouns.rst
@@ -28,12 +28,16 @@ Flytectl noun specify the resource on which the action needs to be performed eg:
     gen/flytectl_update_execution-queue-attribute
     gen/flytectl_update_plugin-override
     gen/flytectl_register_files
+    gen/flytectl_register_examples
     gen/flytectl_delete_execution
     gen/flytectl_delete_task-resource-attribute
     gen/flytectl_delete_cluster-resource-attribute
     gen/flytectl_delete_execution-cluster-label
     gen/flytectl_delete_execution-queue-attribute
     gen/flytectl_delete_plugin-override
-    gen/flytectl_version
     gen/flytectl_config_validate
     gen/flytectl_config_discover
+    gen/flytectl_sandbox_start
+    gen/flytectl_sandbox_status
+    gen/flytectl_sandbox_teardown
+    gen/flytectl_sandbox_exec
diff --git a/docs/source/verbs.rst b/docs/source/verbs.rst
@@ -7,8 +7,11 @@ Flytectl verbs specify the actions to be performed on the resources like create/
     :caption: Verbs
 
     gen/flytectl_create
+    gen/flytectl_completion
     gen/flytectl_get
     gen/flytectl_update
     gen/flytectl_delete
     gen/flytectl_register
     gen/flytectl_config
+    gen/flytectl_sandbox
+    gen/flytectl_version
diff --git a/go.mod b/go.mod
@@ -9,8 +9,7 @@ require (
 	github.com/docker/docker v20.10.7+incompatible
 	github.com/docker/go-connections v0.4.0
 	github.com/enescakir/emoji v1.0.0
-	github.com/flyteorg/datacatalog v0.3.6
-	github.com/flyteorg/flyteidl v0.19.3
+	github.com/flyteorg/flyteidl v0.19.8
 	github.com/flyteorg/flytestdlib v0.3.24
 	github.com/ghodss/yaml v1.0.0
 	github.com/golang/protobuf v1.4.3
@@ -30,7 +29,6 @@ require (
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/opencontainers/image-spec v1.0.1
 	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sirupsen/logrus v1.8.0
 	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,6 +27,7 @@ Mount your flytesnacks repository code inside sandbox @@
     ::
      bin/flytectl sandbox start --sourcesPath=$HOME/flyteorg/flytesnacks
     Usage
     	`
     )
@@ Expand Down @@