diff --git a/.github/workflows/github_pages.yml b/.github/workflows/github_pages.yml index 7c24937a4d7..6c3d58da39a 100644 --- a/.github/workflows/github_pages.yml +++ b/.github/workflows/github_pages.yml @@ -32,7 +32,7 @@ jobs: - name: Build docs run: | - pip install gitpython packaging toml + pip install -r site/requirements.txt python site/build_docs.py - name: Deploy diff --git a/CHANGELOG.md b/CHANGELOG.md index d266e99f327..2305f17cfe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## \[2.3.0] - Unreleased ### Added -- TDB +- SDK section in docs () +- An env variable to enable or disable host certificate checking in CLI () ### Changed -- TDB +- `api/docs`, `api/swagger`, `api/schema` endpoints now allow unauthorized access () ### Deprecated - TDB diff --git a/README.md b/README.md index 4695d747a33..87c6d9982ad 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,10 @@ Start using CVAT online for free: [cvat.ai](https://cvat.ai). Or set it up as a - [Installation guide](https://opencv.github.io/cvat/docs/administration/basics/installation/) - [Manual](https://opencv.github.io/cvat/docs/manual/) - [Contributing](https://opencv.github.io/cvat/docs/contributing/) -- [Django REST API documentation](https://opencv.github.io/cvat/docs/administration/basics/rest_api_guide/) - [Datumaro dataset framework](https://github.com/cvat-ai/datumaro/blob/develop/README.md) -- [Command line interface](https://opencv.github.io/cvat/docs/manual/advanced/cli/) +- [Server API](#api) +- [Python SDK](#sdk) +- [Command line tool](#cli) - [XML annotation format](https://opencv.github.io/cvat/docs/manual/advanced/xml_format/) - [AWS Deployment Guide](https://opencv.github.io/cvat/docs/administration/basics/aws-deployment-guide/) - [Frequently asked questions](https://opencv.github.io/cvat/docs/faq/) @@ -85,11 +86,6 @@ Prebuilt docker images are the easiest way to start using CVAT locally. They are The images have been downloaded more than 1M times so far. -## REST API - -CVAT has a REST API: [documentation](https://opencv.github.io/cvat/docs/administration/basics/rest_api_guide/). -Its current version is `2.0-alpha`. We focus on its improvement, and the API may be changed in the next releases. - ## Screencasts 🎦 Here are some screencasts showing how to use CVAT. @@ -104,6 +100,22 @@ Here are some screencasts showing how to use CVAT. - [Tutorial for polygons](https://youtu.be/C7-r9lZbjBw) - [Semi-automatic segmentation](https://youtu.be/9HszWP_qsRQ) +## API + +- [Documentation](https://opencv.github.io/cvat/docs/api_sdk/api/) + +## SDK + +- Install with `pip install cvat-sdk` +- [PyPI package homepage](https://pypi.org/project/cvat-sdk/) +- [Documentation](https://opencv.github.io/cvat/docs/api_sdk/sdk/) + +## CLI + +- Install with `pip install cvat-cli` +- [PyPI package homepage](https://pypi.org/project/cvat-cli/) +- [Documentation](https://opencv.github.io/cvat/docs/api_sdk/cli/) + ## Supported annotation formats CVAT supports multiple annotation formats. You can select the format after clicking the "Upload annotation" and "Dump diff --git a/cvat-cli/src/cvat_cli/__main__.py b/cvat-cli/src/cvat_cli/__main__.py index 0ac29d5e697..63cb31a0b3a 100755 --- a/cvat-cli/src/cvat_cli/__main__.py +++ b/cvat-cli/src/cvat_cli/__main__.py @@ -6,9 +6,11 @@ import logging import sys from http.client import HTTPConnection +from types import SimpleNamespace from typing import List -from cvat_sdk import exceptions, make_client +from cvat_sdk import exceptions +from cvat_sdk.core.client import Client, Config from cvat_cli.cli import CLI from cvat_cli.parser import get_action_args, make_cmdline_parser @@ -28,6 +30,16 @@ def configure_logger(level): HTTPConnection.debuglevel = 1 +def build_client(parsed_args: SimpleNamespace, logger: logging.Logger) -> Client: + config = Config(verify_ssl=not parsed_args.insecure) + + return Client( + url="{host}:{port}".format(host=parsed_args.server_host, port=parsed_args.server_port), + logger=logger, + config=config, + ) + + def main(args: List[str] = None): actions = { "create": CLI.tasks_create, @@ -43,9 +55,7 @@ def main(args: List[str] = None): parsed_args = parser.parse_args(args) configure_logger(parsed_args.loglevel) - with make_client(parsed_args.server_host, port=parsed_args.server_port) as client: - client.logger = logger - + with build_client(parsed_args, logger=logger) as client: action_args = get_action_args(parser, parsed_args) try: cli = CLI(client=client, credentials=parsed_args.auth) diff --git a/cvat-cli/src/cvat_cli/parser.py b/cvat-cli/src/cvat_cli/parser.py index 43bca8ebc67..24d4709dca6 100644 --- a/cvat-cli/src/cvat_cli/parser.py +++ b/cvat-cli/src/cvat_cli/parser.py @@ -47,6 +47,11 @@ def make_cmdline_parser() -> argparse.ArgumentParser: description="Perform common operations related to CVAT tasks.\n\n" ) parser.add_argument("--version", action="version", version=VERSION) + parser.add_argument( + "--insecure", + action="store_true", + help="Allows to disable SSL certificate check", + ) task_subparser = parser.add_subparsers(dest="action") diff --git a/cvat-sdk/cvat_sdk/core/client.py b/cvat-sdk/cvat_sdk/core/client.py index cb8c42051c8..99033f37e9e 100644 --- a/cvat-sdk/cvat_sdk/core/client.py +++ b/cvat-sdk/cvat_sdk/core/client.py @@ -31,6 +31,11 @@ class Config: status_check_period: float = 5 """In seconds""" + verify_ssl: Optional[bool] = None + """ + Whether to verify host SSL certificate or not. + """ + class Client: """ @@ -41,10 +46,12 @@ def __init__( self, url: str, *, logger: Optional[logging.Logger] = None, config: Optional[Config] = None ): url = self._validate_and_prepare_url(url) - self.api_map = CVAT_API_V2(url) - self.api_client = ApiClient(Configuration(host=self.api_map.host)) self.logger = logger or logging.getLogger(__name__) self.config = config or Config() + self.api_map = CVAT_API_V2(url) + self.api_client = ApiClient( + Configuration(host=self.api_map.host, verify_ssl=self.config.verify_ssl) + ) self._repos: Dict[str, Repo] = {} @@ -76,7 +83,7 @@ def _detect_schema(cls, base_url: str) -> str: for schema in cls.ALLOWED_SCHEMAS: with ApiClient(Configuration(host=f"{schema}://{base_url}")) as api_client: with suppress(urllib3.exceptions.RequestError): - (_, response) = api_client.schema_api.retrieve( + (_, response) = api_client.server_api.retrieve_about( _request_timeout=5, _parse_response=False, _check_status=False ) diff --git a/cvat-sdk/developer_guide.md b/cvat-sdk/developer_guide.md deleted file mode 100644 index 9cbee779074..00000000000 --- a/cvat-sdk/developer_guide.md +++ /dev/null @@ -1,48 +0,0 @@ -# Developer guide - -## General info - -Most of the files in this package are generated. The `gen/` directory -contains generator config and templates. - -## How to generate API - -1. Obtain the REST API schema: -```bash -python manage.py spectacular --file schema.yml && mkdir -p cvat-sdk/schema/ && mv schema.yml cvat-sdk/schema/ -``` - -2. Generate package code (call from the package root directory): -```bash -# pip install -r gen/requirements.txt - -./gen/generate.sh -``` - -## How to edit templates - -If you want to edit templates, obtain them from the generator first: - -```bash -docker run --rm -v $PWD:/local \ - openapitools/openapi-generator-cli author template \ - -o /local/generator_templates -g python -``` - -Then, you can copy the modified version of the template you need into -the `gen/templates/openapi-generator/` directory. - -Relevant links: -- [Generator implementation, available variables in templates](https://github.com/OpenAPITools/openapi-generator/tree/master/modules/openapi-generator/src/main/java/org/openapitools/codegen) -- [Mustache syntax in the generator](https://github.com/OpenAPITools/openapi-generator/wiki/Mustache-Template-Variables) - -## How to test - -API client tests are integrated into REST API tests (`/tests/python/rest_api`) -and SDK tests are placed next to them (`/tests/python/sdk`). -To execute, run: -```bash -pytest tests/python/rest_api tests/python/sdk -``` - -To allow editing of the package, install it with `pip install -e cvat-sdk/`. diff --git a/cvat-sdk/gen/templates/openapi-generator/README.mustache b/cvat-sdk/gen/templates/openapi-generator/README.mustache index 6e37baf11df..d48a0ac0c57 100644 --- a/cvat-sdk/gen/templates/openapi-generator/README.mustache +++ b/cvat-sdk/gen/templates/openapi-generator/README.mustache @@ -16,36 +16,21 @@ For more information, please visit [{{{infoUrl}}}]({{{infoUrl}}}) {{/infoUrl}} ## Installation & Usage -### pip install -If the python package is hosted on a repository, you can install directly using: +To install a prebuilt package, run the following command in the terminal: ```sh -pip install git+https://{{gitHost}}/{{{gitUserId}}}/{{{gitRepoId}}}.git +pip install cvat-sdk ``` -(you may need to run `pip` with root permission: `sudo pip install git+https://{{gitHost}}/{{{gitUserId}}}/{{{gitRepoId}}}.git`) -Then import the package: -```python -import {{{packageName}}} -``` +To install from the local directory, follow [the developer guide](https://opencv.github.io/cvat/docs/integration/sdk/developer_guide). -### Setuptools +After installation you can import the package: -Install via [Setuptools](http://pypi.python.org/pypi/setuptools). - -```sh -python setup.py install --user -``` -(or `sudo python setup.py install` to install the package for all users) - -Then import the package: ```python -import {{{packageName}}} +import cvat_sdk ``` ## Getting Started -Please follow the [installation procedure](#installation--usage) and then run the following: - {{> README_common }} diff --git a/cvat-sdk/gen/templates/openapi-generator/configuration.mustache b/cvat-sdk/gen/templates/openapi-generator/configuration.mustache index 878e0920265..cb64afb6126 100644 --- a/cvat-sdk/gen/templates/openapi-generator/configuration.mustache +++ b/cvat-sdk/gen/templates/openapi-generator/configuration.mustache @@ -6,6 +6,7 @@ import logging import multiprocessing {{/asyncio}} import sys +import typing import urllib3 from http import client as http_client @@ -18,7 +19,7 @@ JSON_SCHEMA_VALIDATION_KEYWORDS = { 'minLength', 'pattern', 'maxItems', 'minItems' } -class Configuration(object): +class Configuration: """ NOTE: This class is auto generated by OpenAPI Generator @@ -30,9 +31,13 @@ class Configuration(object): Each entry in the dict specifies an API key. The dict key is the name of the security scheme in the OAS specification. The dict value is the API key secret. + Supported key names:{{#authMethods}}{{#isApiKey}} + '{{name}}'{{/isApiKey}}{{/authMethods}} :param api_key_prefix: Dict to store API prefix (e.g. Bearer) The dict key is the name of the security scheme in the OAS specification. The dict value is an API key prefix when generating the auth data. + {{#authMethods}}{{#-first}}Default prefixes for API keys:{{/-first}}{{#isApiKey}}{{#vendorExtensions.x-token-prefix}} + {{name}}: '{{.}}'{{/vendorExtensions.x-token-prefix}}{{/isApiKey}}{{/authMethods}} :param username: Username for HTTP basic authentication :param password: Password for HTTP basic authentication :param discard_unknown_keys: Boolean value indicating whether to discard @@ -72,31 +77,30 @@ class Configuration(object): :param server_operation_variables: Mapping from operation ID to a mapping with string values to replace variables in templated server configuration. The validation of enums is performed for variables with defined enum values before. - :param ssl_ca_cert: str - the path to a file of concatenated CA certificates - in PEM format + :param ssl_ca_cert: the path to a file of concatenated CA certificates in PEM format + :param verify_ssl: whether to verify server SSL certificates or not. {{#hasAuthMethods}} :Example: {{#hasApiKeyMethods}} API Key Authentication Example. - Given the following security scheme in the OpenAPI specification: - components: - securitySchemes: - cookieAuth: # name for the security scheme - type: apiKey - in: cookie - name: JSESSIONID # cookie name - You can programmatically set the cookie: + You can authorize with API token after doing the basic auth the following way: conf = {{{packageName}}}.Configuration( - api_key={'cookieAuth': 'abc123'} - api_key_prefix={'cookieAuth': 'JSESSIONID'} + ... + api_key={ + "sessionAuth": , + "csrfAuth": , + "tokenAuth": , + } ) - The following cookie will be added to the HTTP request: - Cookie: JSESSIONID abc123 + You need to specify all the 3 keys for this kind of auth. + + If your custom server uses another token prefix, use the 'api_key_prefix' parameter. + {{/hasApiKeyMethods}} {{#hasHttpBasicMethods}} @@ -111,6 +115,7 @@ class Configuration(object): Configure API client with HTTP basic authentication: conf = {{{packageName}}}.Configuration( + ..., username='the-user', password='the-password', ) @@ -162,60 +167,73 @@ class Configuration(object): _default = None - def __init__(self, host=None, - api_key=None, api_key_prefix=None, - access_token=None, - username=None, password=None, - discard_unknown_keys=False, - disabled_client_side_validations="", + def __init__(self, + host: typing.Optional[str] = None, + api_key: typing.Optional[typing.Dict[str, str]] = None, + api_key_prefix: typing.Optional[typing.Dict[str, str]] = None, + username: typing.Optional[str] = None, + password: typing.Optional[str]=None, + discard_unknown_keys: bool = False, + disabled_client_side_validations: str = "", {{#hasHttpSignatureMethods}} - signing_info=None, + signing_info=None, {{/hasHttpSignatureMethods}} - server_index=None, server_variables=None, - server_operation_index=None, server_operation_variables=None, - ssl_ca_cert=None, - ): - """Constructor - """ + server_index: typing.Optional[int] = None, + server_variables: typing.Optional[typing.Dict[str, str]] = None, + server_operation_index: typing.Optional[int] = None, + server_operation_variables: typing.Optional[typing.Dict[str, str]] = None, + ssl_ca_cert: typing.Optional[str] = None, + verify_ssl: typing.Optional[bool] = None, + ) -> None: self._base_path = self._fix_host_url("{{{basePath}}}" if host is None else host) - """Default Base url - """ + """Default Base url""" + self.server_index = 0 if server_index is None and host is None else server_index + """Default server index""" + self.server_operation_index = server_operation_index or {} - """Default server index - """ + """Default server operation index""" + self.server_variables = server_variables or {} + """Default server variables""" + self.server_operation_variables = server_operation_variables or {} - """Default server variables - """ + """Default server variables""" + self.temp_folder_path = None - """Temp file folder for downloading files - """ + """Temp file folder for downloading files""" + # Authentication Settings - self.access_token = access_token + self.access_token = None + """Bearer API token""" + self.api_key = {} - """dict to store API key(s) - """ + """dict to store API key(s)""" if api_key: self.api_key = api_key - self.api_key_prefix = {} - """dict to store API prefix (e.g. Bearer) - """ + self.api_key_prefix = { {{#authMethods}}{{#isApiKey}}{{#vendorExtensions.x-token-prefix}} + '{{name}}': '{{.}}'{{/vendorExtensions.x-token-prefix}}{{/isApiKey}}{{/authMethods}} + } + """dict to store API prefix (e.g. Bearer)""" if api_key_prefix: - self.api_key_prefix = api_key_prefix + self.api_key_prefix.update(api_key_prefix) self.refresh_api_key_hook = None - """function hook to refresh API key if expired - """ + """function hook to refresh API key if expired""" + self.username = username - """Username for HTTP basic authentication - """ + """Username for HTTP basic authentication""" + self.password = password - """Password for HTTP basic authentication - """ + """Password for HTTP basic authentication""" + self.discard_unknown_keys = discard_unknown_keys + """A flag to control unknown key deserialization behaviour""" + self.disabled_client_side_validations = disabled_client_side_validations + """A flag to enable or disable specific model field validation in the client""" + {{#hasHttpSignatureMethods}} if signing_info is not None: signing_info.host = host @@ -224,43 +242,44 @@ class Configuration(object): """ {{/hasHttpSignatureMethods}} self.logger = {} - """Logging Settings - """ + """Logging Settings""" + self.logger["package_logger"] = logging.getLogger("{{packageName}}") self.logger["urllib3_logger"] = logging.getLogger("urllib3") + self.logger_format = '%(asctime)s %(levelname)s %(message)s' - """Log format - """ + """Log format""" + self.logger_stream_handler = None - """Log stream handler - """ + """Log stream handler""" + self.logger_file_handler = None - """Log file handler - """ + """Log file handler""" + self.logger_file = None - """Debug file location - """ + """Debug file location""" + self.debug = False - """Debug switch - """ + """Debug switch""" - self.verify_ssl = True - """SSL/TLS verification - Set this to false to skip verifying SSL certificate when calling API - from https server. + self.verify_ssl = verify_ssl if verify_ssl is not None else True """ - self.ssl_ca_cert = ssl_ca_cert - """Set this to customize the certificate file to verify the peer. + SSL/TLS verification + Set this to false to skip verifying SSL certificate when calling API + from https server. """ + + self.ssl_ca_cert = ssl_ca_cert + """Set this to customize the certificate file to verify the peer.""" + self.cert_file = None - """client certificate file - """ + """client certificate file""" + self.key_file = None - """client key file - """ + """client key file""" + self.assert_hostname = None - """Set this to True/False to enable/disable SSL hostname verification. - """ + """Set this to True/False to enable/disable SSL hostname verification.""" {{#asyncio}} self.connection_pool_maxsize = 100 @@ -279,20 +298,20 @@ class Configuration(object): {{/asyncio}} self.proxy = None - """Proxy URL - """ + """Proxy URL""" + self.no_proxy = None - """bypass proxy for host in the no_proxy list. - """ + """bypass proxy for host in the no_proxy list.""" + self.proxy_headers = None - """Proxy headers - """ + """Proxy headers""" + self.safe_chars_for_path_param = '' - """Safe chars for path_param - """ + """Safe chars for path_param""" + self.retries = None - """Adding retries to override urllib3 default value 3 - """ + """Adding retries to override urllib3 default value 3""" + # Enable client side validation self.client_side_validation = True @@ -439,7 +458,9 @@ class Configuration(object): self.__logger_format = value self.logger_formatter = logging.Formatter(self.__logger_format) - def get_api_key_with_prefix(self, identifier, alias=None): + def get_api_key_with_prefix(self, identifier: str, *, + alias: typing.Optional[str] = None + ) -> typing.Optional[str]: """Gets API key (with prefix if set). :param identifier: The identifier of apiKey. diff --git a/cvat/apps/engine/urls.py b/cvat/apps/engine/urls.py index 7edbd07b7ca..aa2537a04c7 100644 --- a/cvat/apps/engine/urls.py +++ b/cvat/apps/engine/urls.py @@ -30,9 +30,17 @@ query_string=True)), # documentation for API - path('api/schema/', SpectacularAPIView.as_view(), name='schema'), - path('api/swagger/', SpectacularSwaggerView.as_view(url_name='schema'), name='swagger'), - path('api/docs/', SpectacularRedocView.as_view(url_name='schema'), name='redoc'), + path('api/schema/', SpectacularAPIView.as_view( + permission_classes=[] # This endpoint is available for everyone + ), name='schema'), + path('api/swagger/', SpectacularSwaggerView.as_view( + url_name='schema', + permission_classes=[] # This endpoint is available for everyone + ), name='swagger'), + path('api/docs/', SpectacularRedocView.as_view( + url_name='schema', + permission_classes=[] # This endpoint is available for everyone + ), name='redoc'), # entry point for API path('api/', include('cvat.apps.iam.urls')), diff --git a/cvat/apps/iam/schema.py b/cvat/apps/iam/schema.py index e9a510044a5..0fb74b63f40 100644 --- a/cvat/apps/iam/schema.py +++ b/cvat/apps/iam/schema.py @@ -4,14 +4,18 @@ # SPDX-License-Identifier: MIT import re +import textwrap from drf_spectacular.openapi import AutoSchema from drf_spectacular.extensions import OpenApiFilterExtension, OpenApiAuthenticationExtension +from drf_spectacular.authentication import TokenScheme, SessionScheme from drf_spectacular.plumbing import build_parameter_type from drf_spectacular.utils import OpenApiParameter # https://drf-spectacular.readthedocs.io/en/latest/customization.html?highlight=OpenApiFilterExtension#step-5-extensions class OrganizationFilterExtension(OpenApiFilterExtension): - """Describe OrganizationFilterBackend filter""" + """ + Describe OrganizationFilterBackend filter + """ target_class = 'cvat.apps.iam.filters.OrganizationFilterBackend' priority = 1 @@ -36,19 +40,77 @@ def get_schema_operation_parameters(self, auto_schema, *args, **kwargs): ] class SignatureAuthenticationScheme(OpenApiAuthenticationExtension): + """ + Adds the signature auth method to schema + """ + target_class = 'cvat.apps.iam.authentication.SignatureAuthentication' - name = 'SignatureAuthentication' # name used in the schema + name = 'signatureAuth' # name used in the schema def get_security_definition(self, auto_schema): return { 'type': 'apiKey', 'in': 'query', 'name': 'sign', + 'description': 'Can be used to share URLs to private links', + } + +class TokenAuthenticationScheme(TokenScheme): + """ + Adds the token auth method to schema. The description includes extra info + comparing to what is generated by default. + """ + + name = 'tokenAuth' + priority = 0 + match_subclasses = True + + def get_security_requirement(self, auto_schema): + # These schemes must be used together + return {'sessionAuth': [], 'csrfAuth': [], self.name: []} + + def get_security_definition(self, auto_schema): + schema = super().get_security_definition(auto_schema) + schema['x-token-prefix'] = self.target.keyword + schema['description'] = textwrap.dedent(f""" + To authenticate using a token (or API key), you need to have 3 components in a request: + - the 'sessionid' cookie + - the 'csrftoken' cookie or 'X-CSRFTOKEN' header + - the 'Authentication' header with the '{self.target.keyword} ' prefix + + You can obtain an API key (the token) from the server response on + the basic auth request. + """) + return schema + +class CookieAuthenticationScheme(SessionScheme): + """ + This class adds csrftoken cookie into security sections. It must be used together with + the 'sessionid' cookie. + """ + + name = ['sessionAuth', 'csrfAuth'] + priority = 0 + + def get_security_requirement(self, auto_schema): + # These schemes cannot be used separately + return None + + def get_security_definition(self, auto_schema): + sessionid_schema = super().get_security_definition(auto_schema) + csrftoken_schema = { + 'type': 'apiKey', + 'in': 'cookie', + 'name': 'csrftoken', + 'description': 'Can be sent as a cookie or as the X-CSRFTOKEN header' } + return [sessionid_schema, csrftoken_schema] class CustomAutoSchema(AutoSchema): - # https://github.com/tfranzel/drf-spectacular/issues/111 - # Adds organization context parameters to all endpoints + """ + https://github.com/tfranzel/drf-spectacular/issues/111 + Adds organization context parameters to all endpoints + """ def get_override_parameters(self): return [ diff --git a/site/build_docs.py b/site/build_docs.py old mode 100644 new mode 100755 index c2378b8f133..7a41699126d --- a/site/build_docs.py +++ b/site/build_docs.py @@ -1,87 +1,111 @@ +#!/usr/bin/env python3 + # Copyright (C) 2021-2022 Intel Corporation +# Copyright (C) 2022 CVAT.ai Corporation # # SPDX-License-Identifier: MIT -import os import shutil import subprocess +import tarfile +import tempfile +from pathlib import Path -from packaging import version import git import toml +from packaging import version + +# the initial version for the documentation site +MINIMUM_VERSION = version.Version("1.5.0") -MINIMUM_VERSION='1.5.0' def prepare_tags(repo): tags = {} for tag in repo.tags: tag_version = version.parse(tag.name) - if tag_version >= version.Version(MINIMUM_VERSION) and not tag_version.is_prerelease: + if tag_version >= MINIMUM_VERSION and not tag_version.is_prerelease: release_version = (tag_version.major, tag_version.minor) - if not release_version in tags or tag_version > version.parse(tags[release_version].name): + if release_version not in tags or tag_version > version.parse( + tags[release_version].name + ): tags[release_version] = tag return tags.values() -def generate_versioning_config(filename, versions, url_prefix=''): + +def generate_versioning_config(filename, versions, url_prefix=""): def write_version_item(file_object, version, url): - file_object.write('[[params.versions]]\n') + file_object.write("[[params.versions]]\n") file_object.write('version = "{}"\n'.format(version)) file_object.write('url = "{}"\n\n'.format(url)) - with open(filename, 'w') as f: - write_version_item(f, 'Latest version', '{}/'.format(url_prefix)) + with open(filename, "w") as f: + write_version_item(f, "Latest version", "{}/".format(url_prefix)) for v in versions: - write_version_item(f, v, '{}/{}'.format(url_prefix, v)) + write_version_item(f, v, "{}/{}".format(url_prefix, v)) + + +def git_checkout(tagname, repo, temp_dir): + subdirs = ["site/content/en/docs", "site/content/en/images"] + + for subdir in subdirs: + shutil.rmtree(temp_dir / subdir) + + with tempfile.TemporaryFile() as archive: + # `git checkout` doesn't work for this, as it modifies the index. + # `git restore` would work, but it's only available since Git 2.23. + repo.git.archive(tagname, "--", subdir, output_stream=archive) + archive.seek(0) + with tarfile.open(fileobj=archive) as tar: + tar.extractall(temp_dir) -def git_checkout(tagname, cwd): - docs_dir = os.path.join(cwd, 'site', 'content', 'en', 'docs') - shutil.rmtree(docs_dir) - repo.git.checkout(tagname, '--', 'site/content/en/docs') - images_dir = os.path.join(cwd, 'site', 'content', 'en', 'images') - shutil.rmtree(images_dir) - repo.git.checkout(tagname, '--', 'site/content/en/images') def change_version_menu_toml(filename, version): data = toml.load(filename) - data['params']['version_menu'] = version + data["params"]["version_menu"] = version - with open(filename,'w') as f: + with open(filename, "w") as f: toml.dump(data, f) + def generate_docs(repo, output_dir, tags): - def run_hugo(content_loc, destination_dir): - subprocess.run([ # nosec - 'hugo', - '--destination', - destination_dir, - '--config', - 'config.toml,versioning.toml', - ], - cwd=content_loc, - ) - - cwd = repo.working_tree_dir - content_loc = os.path.join(cwd, 'site') - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - generate_versioning_config(os.path.join(cwd, 'site', 'versioning.toml'), (t.name for t in tags)) - change_version_menu_toml(os.path.join(cwd, 'site', 'versioning.toml'), 'Latest version') - run_hugo(content_loc, output_dir) - - generate_versioning_config(os.path.join(cwd, 'site', 'versioning.toml'), (t.name for t in tags), '/..') - for tag in tags: - git_checkout(tag.name, cwd) - destination_dir = os.path.join(output_dir, tag.name) - change_version_menu_toml(os.path.join(cwd, 'site', 'versioning.toml'), tag.name) - os.makedirs(destination_dir) - run_hugo(content_loc, destination_dir) + repo_root = Path(repo.working_tree_dir) + + with tempfile.TemporaryDirectory() as temp_dir: + content_loc = Path(temp_dir, "site") + shutil.copytree(repo_root / "site", content_loc, symlinks=True) + + def run_hugo(destination_dir): + subprocess.run( # nosec + [ + "hugo", + "--destination", + str(destination_dir), + "--config", + "config.toml,versioning.toml", + ], + cwd=content_loc, + check=True, + ) + + versioning_toml_path = content_loc / "versioning.toml" + + # Handle the develop version + generate_versioning_config(versioning_toml_path, (t.name for t in tags)) + change_version_menu_toml(versioning_toml_path, "develop") + run_hugo(output_dir) + + generate_versioning_config(versioning_toml_path, (t.name for t in tags), "/..") + for tag in tags: + git_checkout(tag.name, repo, Path(temp_dir)) + change_version_menu_toml(versioning_toml_path, tag.name) + run_hugo(output_dir / tag.name) + if __name__ == "__main__": - repo_root = os.getcwd() - repo = git.Repo(repo_root) - output_dir = os.path.join(repo_root, 'public') + repo_root = Path(__file__).resolve().parents[1] + output_dir = repo_root / "public" - tags = prepare_tags(repo) - generate_docs(repo, output_dir, tags) + with git.Repo(repo_root) as repo: + tags = prepare_tags(repo) + generate_docs(repo, output_dir, tags) diff --git a/site/config.toml b/site/config.toml index 7d2f6a6563a..a343c86bdfa 100644 --- a/site/config.toml +++ b/site/config.toml @@ -8,7 +8,7 @@ enableRobotsTXT = true theme = ["docsy"] # Will give values to .Lastmod etc. -enableGitInfo = true +enableGitInfo = false # Language settings contentDir = "content/en" diff --git a/site/content/en/docs/api_sdk/_index.md b/site/content/en/docs/api_sdk/_index.md new file mode 100644 index 00000000000..374f971024e --- /dev/null +++ b/site/content/en/docs/api_sdk/_index.md @@ -0,0 +1,26 @@ +--- +title: "API & SDK" +linkTitle: "API & SDK" +weight: 4 +description: 'How to interact with CVAT' +--- + +## Overview + +In the modern world, it is often necessary to integrate different tools to work together. +CVAT provides the following integration layers: + +- Server REST API + Swagger schema +- Python client library (SDK) + - REST API client + - High-level wrappers +- Command-line tool (CLI) + +In this section, you can find documentation about each separate layer. + +## Component compatibility + +Currently, the only supported configuration is when the server API major and minor versions +are the same as SDK and CLI major and minor versions, e.g. server v2.1.* is supported by +SDK and CLI v2.1.*. Different versions may have incompatibilities, which lead to some functions +in SDK or CLI may not work properly. diff --git a/site/content/en/docs/manual/advanced/cli.md b/site/content/en/docs/api_sdk/cli/_index.md similarity index 92% rename from site/content/en/docs/manual/advanced/cli.md rename to site/content/en/docs/api_sdk/cli/_index.md index 047f7f61ad6..03da4a9e767 100644 --- a/site/content/en/docs/manual/advanced/cli.md +++ b/site/content/en/docs/api_sdk/cli/_index.md @@ -1,11 +1,11 @@ --- title: 'Command line interface (CLI)' linkTitle: 'CLI' -weight: 29 -description: 'Guide to working with CVAT tasks in the command line interface. This section on [GitHub](https://github.com/cvat-ai/cvat/tree/develop/cvat-cli).' +weight: 4 +description: '' --- -## Description +## Overview A simple command line interface for working with CVAT tasks. At the moment it implements a basic feature set but may serve as the starting point for a more @@ -22,21 +22,24 @@ Overview of functionality: - Export and download a whole task - Import a task -## Usage +## Installation -To access the CLI, you need to have python in environment, -as well as a clone of the CVAT repository and the necessary modules: +To install an [official release of CVAT CLI](https://pypi.org/project/cvat-cli/), use this command: ```bash pip install cvat-cli ``` +We support Python versions 3.7 - 3.9. + +## Usage + You can get help with `cvat-cli --help`. ``` -usage: cvat-cli [-h] [--auth USER:[PASS]] [--server-host SERVER_HOST] - [--server-port SERVER_PORT] [--debug] - {create,delete,ls,frames,dump,upload,export,import} ... +usage: cvat-cli [-h] [--auth USER:[PASS]] + [--server-host SERVER_HOST] [--server-port SERVER_PORT] [--debug] + {create,delete,ls,frames,dump,upload,export,import} ... Perform common operations related to CVAT tasks. diff --git a/site/content/en/docs/api_sdk/faq.md b/site/content/en/docs/api_sdk/faq.md new file mode 100644 index 00000000000..5cdd096c1d6 --- /dev/null +++ b/site/content/en/docs/api_sdk/faq.md @@ -0,0 +1,11 @@ +--- +title: 'Frequently asked questions' +linkTitle: 'FAQ' +weight: 100 +description: '' +--- + +### My server uses a custom SSL certificate and I don't want to check it. + +You can call control SSL certificate check with the `--insecure` CLI argument. +For SDK, you can specify `ssl_verify = True/False` in the `cvat_sdk.core.client.Config` object. diff --git a/site/content/en/docs/api_sdk/sdk/_index.md b/site/content/en/docs/api_sdk/sdk/_index.md new file mode 100644 index 00000000000..51a30d5b18d --- /dev/null +++ b/site/content/en/docs/api_sdk/sdk/_index.md @@ -0,0 +1,47 @@ +--- +title: 'Python SDK' +linkTitle: 'SDK' +weight: 3 +description: '' +--- + +## Overview + +SDK is a Python library. It provides you access to Python function and objects, which +simplify server interaction and provide additional functionality like data validation. + +SDK API includes 2 layers: +- Low-level API with REST API wrappers. Located in at `cvat_sdk.api_client`. [Read more](/api_sdk/sdk/lowlevel-api) +- High-level API. Located at `cvat_sdk.core`. [Read more](/api_sdk/sdk/highlevel-api) + +Roughly, low-level API provides single-request operations, while the high-level one allows you +to use composite, multi-request operations and have local counterparts for server objects. +For most uses, the high-level API should be good enough and sufficient, and it should be +right point to start your integration with CVAT. + +## Installation + +To install an [official release of CVAT SDK](https://pypi.org/project/cvat-sdk/) use this command: +```bash +pip install cvat-sdk +``` + +We support Python versions 3.7 - 3.9. + +## Usage + +To import the package components, use the following code: + +For high-level API: + +```python +import cvat_sdk +# or +import cvat_sdk.core +``` + +For low-level API: + +```python +import cvat_sdk.api_client +``` diff --git a/site/content/en/docs/api_sdk/sdk/developer-guide.md b/site/content/en/docs/api_sdk/sdk/developer-guide.md new file mode 100644 index 00000000000..f9287cf3244 --- /dev/null +++ b/site/content/en/docs/api_sdk/sdk/developer-guide.md @@ -0,0 +1,80 @@ +--- +title: 'Developer guide' +linkTitle: 'Developer guide' +weight: 10 +description: '' +--- + +## Overview + +This package contains manually written and generated files. We store only sources in +the repository. To get the full package, one need to generate missing package files. + +## Package file layout + +- `gen/` - generator files +- `cvat_sdk/` - Python package root +- `cvat_sdk/api_client` - autogenerated low-level package code +- `cvat_sdk/core` - high-level package code + +## How to generate package code + +1. Obtain the server API schema + +If you have a local custom version of the server, run the following command in the terminal. +You need to be able to execute django server. Server installation instructions are available +[here](/contributing/development-environment). +```bash +mkdir -p cvat-sdk/schema/ && python manage.py spectacular --file cvat-sdk/schema/schema.yml +``` + +If you want to use docker instead: +```bash +docker-compose -f docker-compose.yml -f docker-compose.dev.yml run \ + --no-deps --entrypoint '/usr/bin/env python' --rm -u "$(id -u)":"$(id -g)" -v "$PWD":"/local" \ + cvat_server \ + manage.py spectacular --file /local/cvat-sdk/schema/schema.yml +``` + +If you don't have access to the server sources, but have a working instance, +you can also get schema from `/api/docs`: + +![Download server schema button image](/images/download_server_schema.png) + +2. Install generator dependencies: +```bash +pip install -r gen/requirements.txt +``` + +3. Generate package code (call from the package root directory!): +```bash +./gen/generate.sh +``` + +4. To allow editing of the package, install it with `pip install -e cvat-sdk/`. + +## How to edit templates + +If you want to edit templates, obtain them from the generator first: + +```bash +docker run --rm -v $PWD:/local \ + openapitools/openapi-generator-cli author template \ + -o /local/generator_templates -g python +``` + +Then, you can copy the modified version of the template you need into +the `gen/templates/openapi-generator/` directory. + +Relevant links: +- [Generator implementation, available variables in templates](https://github.com/OpenAPITools/openapi-generator/tree/master/modules/openapi-generator/src/main/java/org/openapitools/codegen) +- [Mustache syntax in the generator](https://github.com/OpenAPITools/openapi-generator/wiki/Mustache-Template-Variables) + +## How to test + +API client tests are integrated into REST API tests in `/tests/python/rest_api` +and SDK tests are placed next to them in `/tests/python/sdk`. +To execute, run: +```bash +pytest tests/python/rest_api tests/python/sdk +``` diff --git a/site/content/en/docs/api_sdk/sdk/highlevel-api.md b/site/content/en/docs/api_sdk/sdk/highlevel-api.md new file mode 100644 index 00000000000..a86e4a081ec --- /dev/null +++ b/site/content/en/docs/api_sdk/sdk/highlevel-api.md @@ -0,0 +1,63 @@ +--- +title: 'High-level API' +linkTitle: 'High-level API' +weight: 4 +description: '' +--- + +## Overview + +This layer provides high-level APIs, allowing easier access to server operations. +API includes _Repositories_ and _Entities_. Repositories provide management +operations for Entitites. Entitites represent separate objects on the server +(e.g. tasks, jobs etc). + +Code of this component is located in `cvat_sdk.core`. + +## Example + +```python +from cvat_sdk import make_client, models +from cvat_sdk.core.proxies.tasks import ResourceType, Task + +with make_client(host="http://localhost") as client: + # Authorize using the basic auth + client.login(('YOUR_USERNAME', 'YOUR_PASSWORD')) + + # Models are used the same way as in the layer 1 + task_spec = { + "name": "example task 2", + "labels": [ + { + "name": "car", + "color": "#ff00ff", + "attributes": [ + { + "name": "a", + "mutable": True, + "input_type": "number", + "default_value": "5", + "values": ["4", "5", "6"], + } + ], + } + ], + } + + # Different repositories can be accessed as the Client class members. + # They may provide both simple and complex operations, + # such as entity creation, retrieval and removal. + task = client.tasks.create_from_data( + spec=task_spec, + resource_type=ResourceType.LOCAL, + resources=['image1.jpg', 'image2.png'], + ) + + # Task object is already up-to-date with its server counterpart + assert task.size == 2 + + # An entity needs to be fetch()-ed to reflect the latest changes. + # It can be update()-d and remove()-d depending on the entity type. + task.update({'name': 'mytask'}) + task.remove() +``` diff --git a/site/content/en/docs/api_sdk/sdk/lowlevel-api.md b/site/content/en/docs/api_sdk/sdk/lowlevel-api.md new file mode 100644 index 00000000000..20ed4c52e86 --- /dev/null +++ b/site/content/en/docs/api_sdk/sdk/lowlevel-api.md @@ -0,0 +1,386 @@ +--- +title: 'Low-level API' +linkTitle: 'Low-level API' +weight: 3 +description: '' +--- + +## Overview + +Low-level API is useful if you need to work directly with REST API, but want +to have data validation and syntax assistance from your code editor. The code +on this layer is autogenerated. + +Code of this component is located in `cvat_sdk.api_client`. + +## Example + +Let's see how a task with local files can be created. We will use the basic auth +to make things simpler. + +```python +from time import sleep +from cvat_sdk.api_client import Configuration, ApiClient, models, apis, exceptions + +configuration = Configuration( + host="http://localhost", + username='YOUR_USERNAME', + password='YOUR_PASSWORD', +) + +# Enter a context with an instance of the API client +with ApiClient(configuration) as api_client: + # Parameters can be passed as a plain dict with JSON-serialized data + # or as model objects (from cvat_sdk.api_client.models), including + # mixed variants. + # + # In case of dicts, keys must be the same as members of models.I + # interfaces and values must be convertible to the corresponding member + # value types (e.g. a date or string enum value can be parsed from a string). + # + # In case of model objects, data must be of the corresponding + # models. types. + # + # Let's use a dict here. It should look like models.ITaskWriteRequest + task_spec = { + 'name': 'example task', + "labels": [{ + "name": "car", + "color": "#ff00ff", + "attributes": [ + { + "name": "a", + "mutable": True, + "input_type": "number", + "default_value": "5", + "values": ["4", "5", "6"] + } + ] + }], + } + + try: + # Apis can be accessed as ApiClient class members + # We use different models for input and output data. For input data, + # models are typically called like "*Request". Output data models have + # no suffix. + (task, response) = api_client.tasks_api.create(task_spec) + except exceptions.ApiException as e: + # We can catch the basic exception type, or a derived type + print("Exception when trying to create a task: %s\n" % e) + + # Here we will use models instead of a dict + task_data = models.DataRequest( + image_quality=75, + client_files=[ + open('image1.jpg', 'rb'), + open('image2.jpg', 'rb'), + ], + ) + + # If we pass binary file objects, we need to specify content type. + # For this endpoint, we don't have response data + (_, response) = api_client.tasks_api.create_data(task.id, + data_request=task_data, + _content_type="multipart/form-data", + + # we can choose to check the response status manually + # and disable the response data parsing + _check_status=False, _parse_response=False + ) + assert response.status == 202, response.msg + + # Wait till task data is processed + for _ in range(100): + (status, _) = api_client.tasks_api.retrieve_status(task.id) + if status.state.value in ['Finished', 'Failed']: + break + sleep(0.1) + assert status.state.value == 'Finished', status.message + + # Update the task object and check the task size + (task, _) = api_client.tasks_api.retrieve(task.id) + assert task.size == 4 +``` + +## Available API Endpoints + +All URIs are relative to _`http://localhost`_ + +APIs can be instanted directly like this: + +```python +from cvat_sdk.api_client import ApiClient, apis + +api_client = ApiClient(...) +auth_api = apis.AuthApi(api_client) +auth_api.(...) +``` + +Or they can be accessed as `ApiClient` object members: +``` +from cvat_sdk.api_client import ApiClient + +api_client = ApiClient(...) +api_client.auth_api.(...) +``` + + + +Class | Method | HTTP request | Description +------------ | ------------- | ------------- | ------------- +_AuthApi_ | **auth_create_login** | **POST** /api/auth/login | +_AuthApi_ | **auth_create_logout** | **POST** /api/auth/logout | +_AuthApi_ | **auth_create_password_change** | **POST** /api/auth/password/change | +_AuthApi_ | **auth_create_password_reset** | **POST** /api/auth/password/reset | +_AuthApi_ | **auth_create_password_reset_confirm** | **POST** /api/auth/password/reset/confirm | +_AuthApi_ | **auth_create_register** | **POST** /api/auth/register | +_AuthApi_ | **auth_create_signing** | **POST** /api/auth/signing | This method signs URL for access to the server +_CloudstoragesApi_ | **cloudstorages_create** | **POST** /api/cloudstorages | Method creates a cloud storage with a specified characteristics +_CloudstoragesApi_ | **cloudstorages_destroy** | **DELETE** /api/cloudstorages/{id} | Method deletes a specific cloud storage +_CloudstoragesApi_ | **cloudstorages_list** | **GET** /api/cloudstorages | Returns a paginated list of storages according to query parameters +_CloudstoragesApi_ | **cloudstorages_partial_update** | **PATCH** /api/cloudstorages/{id} | Methods does a partial update of chosen fields in a cloud storage instance +_CloudstoragesApi_ | **cloudstorages_retrieve** | **GET** /api/cloudstorages/{id} | Method returns details of a specific cloud storage +_CloudstoragesApi_ | **cloudstorages_retrieve_actions** | **GET** /api/cloudstorages/{id}/actions | Method returns allowed actions for the cloud storage +_CloudstoragesApi_ | **cloudstorages_retrieve_content** | **GET** /api/cloudstorages/{id}/content | Method returns a manifest content +_CloudstoragesApi_ | **cloudstorages_retrieve_preview** | **GET** /api/cloudstorages/{id}/preview | Method returns a preview image from a cloud storage +_CloudstoragesApi_ | **cloudstorages_retrieve_status** | **GET** /api/cloudstorages/{id}/status | Method returns a cloud storage status +_CommentsApi_ | **comments_create** | **POST** /api/comments | Method creates a comment +_CommentsApi_ | **comments_destroy** | **DELETE** /api/comments/{id} | Method deletes a comment +_CommentsApi_ | **comments_list** | **GET** /api/comments | Method returns a paginated list of comments according to query parameters +_CommentsApi_ | **comments_partial_update** | **PATCH** /api/comments/{id} | Methods does a partial update of chosen fields in a comment +_CommentsApi_ | **comments_retrieve** | **GET** /api/comments/{id} | Method returns details of a comment +_InvitationsApi_ | **invitations_create** | **POST** /api/invitations | Method creates an invitation +_InvitationsApi_ | **invitations_destroy** | **DELETE** /api/invitations/{key} | Method deletes an invitation +_InvitationsApi_ | **invitations_list** | **GET** /api/invitations | Method returns a paginated list of invitations according to query parameters +_InvitationsApi_ | **invitations_partial_update** | **PATCH** /api/invitations/{key} | Methods does a partial update of chosen fields in an invitation +_InvitationsApi_ | **invitations_retrieve** | **GET** /api/invitations/{key} | Method returns details of an invitation +_IssuesApi_ | **issues_create** | **POST** /api/issues | Method creates an issue +_IssuesApi_ | **issues_destroy** | **DELETE** /api/issues/{id} | Method deletes an issue +_IssuesApi_ | **issues_list** | **GET** /api/issues | Method returns a paginated list of issues according to query parameters +_IssuesApi_ | **issues_list_comments** | **GET** /api/issues/{id}/comments | The action returns all comments of a specific issue +_IssuesApi_ | **issues_partial_update** | **PATCH** /api/issues/{id} | Methods does a partial update of chosen fields in an issue +_IssuesApi_ | **issues_retrieve** | **GET** /api/issues/{id} | Method returns details of an issue +_JobsApi_ | **jobs_create_annotations** | **POST** /api/jobs/{id}/annotations/ | Method allows to upload job annotations +_JobsApi_ | **jobs_destroy_annotations** | **DELETE** /api/jobs/{id}/annotations/ | Method deletes all annotations for a specific job +_JobsApi_ | **jobs_list** | **GET** /api/jobs | Method returns a paginated list of jobs according to query parameters +_JobsApi_ | **jobs_list_commits** | **GET** /api/jobs/{id}/commits | The action returns the list of tracked changes for the job +_JobsApi_ | **jobs_list_issues** | **GET** /api/jobs/{id}/issues | Method returns list of issues for the job +_JobsApi_ | **jobs_partial_update** | **PATCH** /api/jobs/{id} | Methods does a partial update of chosen fields in a job +_JobsApi_ | **jobs_partial_update_annotations** | **PATCH** /api/jobs/{id}/annotations/ | Method performs a partial update of annotations in a specific job +_JobsApi_ | **jobs_partial_update_annotations_file** | **PATCH** /api/jobs/{id}/annotations/{file_id} | Allows to upload an annotation file chunk. Implements TUS file uploading protocol. +_JobsApi_ | **jobs_retrieve** | **GET** /api/jobs/{id} | Method returns details of a job +_JobsApi_ | **jobs_retrieve_annotations** | **GET** /api/jobs/{id}/annotations/ | Method returns annotations for a specific job as a JSON document. If format is specified a zip archive is returned. +_JobsApi_ | **jobs_retrieve_data** | **GET** /api/jobs/{id}/data | Method returns data for a specific job +_JobsApi_ | **jobs_retrieve_data_meta** | **GET** /api/jobs/{id}/data/meta | Method provides a meta information about media files which are related with the job +_JobsApi_ | **jobs_retrieve_dataset** | **GET** /api/jobs/{id}/dataset | Export job as a dataset in a specific format +_JobsApi_ | **jobs_update_annotations** | **PUT** /api/jobs/{id}/annotations/ | Method performs an update of all annotations in a specific job +_LambdaApi_ | **lambda_create_functions** | **POST** /api/lambda/functions/{func_id} | +_LambdaApi_ | **lambda_create_requests** | **POST** /api/lambda/requests | Method calls the function +_LambdaApi_ | **lambda_list_functions** | **GET** /api/lambda/functions | Method returns a list of functions +_LambdaApi_ | **lambda_list_requests** | **GET** /api/lambda/requests | Method returns a list of requests +_LambdaApi_ | **lambda_retrieve_functions** | **GET** /api/lambda/functions/{func_id} | Method returns the information about the function +_LambdaApi_ | **lambda_retrieve_requests** | **GET** /api/lambda/requests/{id} | Method returns the status of the request +_MembershipsApi_ | **memberships_destroy** | **DELETE** /api/memberships/{id} | Method deletes a membership +_MembershipsApi_ | **memberships_list** | **GET** /api/memberships | Method returns a paginated list of memberships according to query parameters +_MembershipsApi_ | **memberships_partial_update** | **PATCH** /api/memberships/{id} | Methods does a partial update of chosen fields in a membership +_MembershipsApi_ | **memberships_retrieve** | **GET** /api/memberships/{id} | Method returns details of a membership +_OrganizationsApi_ | **organizations_create** | **POST** /api/organizations | Method creates an organization +_OrganizationsApi_ | **organizations_destroy** | **DELETE** /api/organizations/{id} | Method deletes an organization +_OrganizationsApi_ | **organizations_list** | **GET** /api/organizations | Method returns a paginated list of organizatins according to query parameters +_OrganizationsApi_ | **organizations_partial_update** | **PATCH** /api/organizations/{id} | Methods does a partial update of chosen fields in an organization +_OrganizationsApi_ | **organizations_retrieve** | **GET** /api/organizations/{id} | Method returns details of an organization +_ProjectsApi_ | **projects_create** | **POST** /api/projects | Method creates a new project +_ProjectsApi_ | **projects_create_backup** | **POST** /api/projects/backup/ | Methods create a project from a backup +_ProjectsApi_ | **projects_create_dataset** | **POST** /api/projects/{id}/dataset/ | Import dataset in specific format as a project +_ProjectsApi_ | **projects_destroy** | **DELETE** /api/projects/{id} | Method deletes a specific project +_ProjectsApi_ | **projects_list** | **GET** /api/projects | Returns a paginated list of projects according to query parameters (12 projects per page) +_ProjectsApi_ | **projects_list_tasks** | **GET** /api/projects/{id}/tasks | Method returns information of the tasks of the project with the selected id +_ProjectsApi_ | **projects_partial_update** | **PATCH** /api/projects/{id} | Methods does a partial update of chosen fields in a project +_ProjectsApi_ | **projects_partial_update_backup_file** | **PATCH** /api/projects/backup/{file_id} | Allows to upload a file chunk. Implements TUS file uploading protocol +_ProjectsApi_ | **projects_partial_update_dataset_file** | **PATCH** /api/projects/{id}/dataset/{file_id} | Allows to upload a file chunk. Implements TUS file uploading protocol. +_ProjectsApi_ | **projects_retrieve** | **GET** /api/projects/{id} | Method returns details of a specific project +_ProjectsApi_ | **projects_retrieve_annotations** | **GET** /api/projects/{id}/annotations | Method allows to download project annotations +_ProjectsApi_ | **projects_retrieve_backup** | **GET** /api/projects/{id}/backup | Methods creates a backup copy of a project +_ProjectsApi_ | **projects_retrieve_dataset** | **GET** /api/projects/{id}/dataset/ | Export project as a dataset in a specific format +_RestrictionsApi_ | **restrictions_retrieve_terms_of_use** | **GET** /api/restrictions/terms-of-use | Method provides CVAT terms of use +_RestrictionsApi_ | **restrictions_retrieve_user_agreements** | **GET** /api/restrictions/user-agreements | Method provides user agreements that the user must accept to register +_SchemaApi_ | **schema_retrieve** | **GET** /api/schema/ | +_ServerApi_ | **server_create_exception** | **POST** /api/server/exception | Method saves an exception from a client on the server +_ServerApi_ | **server_create_logs** | **POST** /api/server/logs | Method saves logs from a client on the server +_ServerApi_ | **server_list_share** | **GET** /api/server/share | Returns all files and folders that are on the server along specified path +_ServerApi_ | **server_retrieve_about** | **GET** /api/server/about | Method provides basic CVAT information +_ServerApi_ | **server_retrieve_annotation_formats** | **GET** /api/server/annotation/formats | Method provides the list of supported annotations formats +_ServerApi_ | **server_retrieve_plugins** | **GET** /api/server/plugins | Method provides allowed plugins +_TasksApi_ | **jobs_partial_update_data_meta** | **PATCH** /api/jobs/{id}/data/meta | Method provides a meta information about media files which are related with the job +_TasksApi_ | **tasks_create** | **POST** /api/tasks | Method creates a new task in a database without any attached images and videos +_TasksApi_ | **tasks_create_annotations** | **POST** /api/tasks/{id}/annotations/ | Method allows to upload task annotations from a local file or a cloud storage +_TasksApi_ | **tasks_create_backup** | **POST** /api/tasks/backup/ | Method recreates a task from an attached task backup file +_TasksApi_ | **tasks_create_data** | **POST** /api/tasks/{id}/data/ | Method permanently attaches images or video to a task. Supports tus uploads, see more +_TasksApi_ | **tasks_destroy** | **DELETE** /api/tasks/{id} | Method deletes a specific task, all attached jobs, annotations, and data +_TasksApi_ | **tasks_destroy_annotations** | **DELETE** /api/tasks/{id}/annotations/ | Method deletes all annotations for a specific task +_TasksApi_ | **tasks_list** | **GET** /api/tasks | Returns a paginated list of tasks according to query parameters (10 tasks per page) +_TasksApi_ | **tasks_list_jobs** | **GET** /api/tasks/{id}/jobs | Method returns a list of jobs for a specific task +_TasksApi_ | **tasks_partial_update** | **PATCH** /api/tasks/{id} | Methods does a partial update of chosen fields in a task +_TasksApi_ | **tasks_partial_update_annotations** | **PATCH** /api/tasks/{id}/annotations/ | Method performs a partial update of annotations in a specific task +_TasksApi_ | **tasks_partial_update_annotations_file** | **PATCH** /api/tasks/{id}/annotations/{file_id} | Allows to upload an annotation file chunk. Implements TUS file uploading protocol. +_TasksApi_ | **tasks_partial_update_backup_file** | **PATCH** /api/tasks/backup/{file_id} | Allows to upload a file chunk. Implements TUS file uploading protocol. +_TasksApi_ | **tasks_partial_update_data_file** | **PATCH** /api/tasks/{id}/data/{file_id} | Allows to upload a file chunk. Implements TUS file uploading protocol. +_TasksApi_ | **tasks_partial_update_data_meta** | **PATCH** /api/tasks/{id}/data/meta | Method provides a meta information about media files which are related with _he_task +_TasksApi_ | **tasks_retrieve** | **GET** /api/tasks/{id} | Method returns details of a specific task +_TasksApi_ | **tasks_retrieve_annotations** | **GET** /api/tasks/{id}/annotations/ | Method allows to download task annotations +_TasksApi_ | **tasks_retrieve_backup** | **GET** /api/tasks/{id}/backup | Method backup a specified task +_TasksApi_ | **tasks_retrieve_data** | **GET** /api/tasks/{id}/data/ | Method returns data for a specific task +_TasksApi_ | **tasks_retrieve_data_meta** | **GET** /api/tasks/{id}/data/meta | Method provides a meta information about media files which are related with the task +_TasksApi_ | **tasks_retrieve_dataset** | **GET** /api/tasks/{id}/dataset | Export task as a dataset in a specific format +_TasksApi_ | **tasks_retrieve_status** | **GET** /api/tasks/{id}/status | When task is being created the method returns information about a status of the creation process +_TasksApi_ | **tasks_update_annotations** | **PUT** /api/tasks/{id}/annotations/ | Method allows to upload task annotations +_UsersApi_ | **users_destroy** | **DELETE** /api/users/{id} | Method deletes a specific user from the server +_UsersApi_ | **users_list** | **GET** /api/users | Method provides a paginated list of users registered on the server +_UsersApi_ | **users_partial_update** | **PATCH** /api/users/{id} | Method updates chosen fields of a user +_UsersApi_ | **users_retrieve** | **GET** /api/users/{id} | Method provides information of a specific user +_UsersApi_ | **users_retrieve_self** | **GET** /api/users/self | Method returns an instance of a user who is currently authorized + + +## Available Models + +Models can be instantiated like this: + +```python +from cvat_sdk.api_client import models + +user_model = models.User(...) +``` + +- About +- AnnotationFileRequest +- AnnotationsRead +- Attribute +- AttributeRequest +- AttributeVal +- AttributeValRequest +- BackupWriteRequest +- BasicUser +- BasicUserRequest +- ChunkType +- CloudStorageRead +- CloudStorageWriteRequest +- CommentRead +- CommentReadOwner +- CommentWriteRequest +- CredentialsTypeEnum +- DataMetaRead +- DataRequest +- DatasetFileRequest +- DatasetFormat +- DatasetFormats +- DatasetWriteRequest +- Exception +- ExceptionRequest +- FileInfo +- FileInfoTypeEnum +- FrameMeta +- InputTypeEnum +- InvitationRead +- InvitationWrite +- InvitationWriteRequest +- IssueRead +- IssueWriteRequest +- JobAnnotationsUpdateRequest +- JobCommit +- JobRead +- JobStage +- JobStatus +- Label +- LabeledData +- LabeledDataRequest +- LabeledImage +- LabeledImageRequest +- LabeledShape +- LabeledShapeRequest +- LabeledTrack +- LabeledTrackRequest +- LocationEnum +- LogEvent +- LogEventRequest +- LoginRequest +- Manifest +- ManifestRequest +- MembershipRead +- MembershipWrite +- MetaUser +- OperationStatus +- OrganizationRead +- OrganizationWrite +- OrganizationWriteRequest +- PaginatedCloudStorageReadList +- PaginatedCommentReadList +- PaginatedInvitationReadList +- PaginatedIssueReadList +- PaginatedJobCommitList +- PaginatedJobReadList +- PaginatedMembershipReadList +- PaginatedMetaUserList +- PaginatedPolymorphicProjectList +- PaginatedTaskReadList +- PasswordChangeRequest +- PasswordResetConfirmRequest +- PasswordResetSerializerExRequest +- PatchedCloudStorageWriteRequest +- PatchedCommentWriteRequest +- PatchedDataMetaWriteRequest +- PatchedInvitationWriteRequest +- PatchedIssueWriteRequest +- PatchedJobWriteRequest +- PatchedLabelRequest +- PatchedLabeledDataRequest +- PatchedMembershipWriteRequest +- PatchedOrganizationWriteRequest +- PatchedProjectWriteRequest +- PatchedProjectWriteRequestTargetStorage +- PatchedTaskWriteRequest +- PatchedTaskWriteRequestTargetStorage +- PatchedUserRequest +- Plugins +- PolymorphicProject +- ProjectFileRequest +- ProjectRead +- ProjectReadAssignee +- ProjectReadOwner +- ProjectReadTargetStorage +- ProjectSearch +- ProjectWriteRequest +- ProviderTypeEnum +- RestAuthDetail +- RestrictedRegister +- RestrictedRegisterRequest +- RoleEnum +- RqStatus +- RqStatusStateEnum +- Segment +- ShapeType +- SigningRequest +- SimpleJob +- SortingMethod +- Storage +- StorageMethod +- StorageRequest +- StorageType +- SubLabeledShape +- SubLabeledShapeRequest +- SubLabeledTrack +- SubLabeledTrackRequest +- Sublabel +- SublabelRequest +- TaskAnnotationsUpdateRequest +- TaskAnnotationsWriteRequest +- TaskFileRequest +- TaskRead +- TaskReadTargetStorage +- TaskWriteRequest +- Token +- TrackedShape +- TrackedShapeRequest +- User +- UserAgreement +- UserAgreementRequest diff --git a/site/content/en/docs/contributing/rest-api-design.md b/site/content/en/docs/api_sdk/server-api/_index.md similarity index 69% rename from site/content/en/docs/contributing/rest-api-design.md rename to site/content/en/docs/api_sdk/server-api/_index.md index 4a1eb81b950..1f2b2fb6d04 100644 --- a/site/content/en/docs/contributing/rest-api-design.md +++ b/site/content/en/docs/api_sdk/server-api/_index.md @@ -1,13 +1,37 @@ --- -title: 'REST API design principles' -linkTitle: 'REST API design principles' -weight: 100 -description: 'Information on using the REST API scheme and principles of its design.' +title: 'Server API' +linkTitle: 'API' +weight: 2 +description: '' --- -## REST API scheme +## Overview -Common scheme for our REST API is ` [namespace] `. +CVAT server provides HTTP REST API for interaction. Each client application - be it +a command line tool, browser or a script - all interact with CVAT via HTTP requests and +responses: + +![CVAT server interaction image](/images/server_api_interaction.svg) + +## API schema + +You can obtain schema for your server at `/api/docs`. For example, +the official CVAT.ai application has API documentation [here](https://app.cvat.ai/api/docs/). + +## Examples + +Here you can see how a task is created in CVAT: + +![Task creation example](/images/server_api_create_task_example.png) + +1. At first, we have to login +1. Then we create a task from its configuration +1. Then we send task data (images, videos etc.) +1. We wait for data processing and finish + +## Design principles + +Common pattern for our REST API is ` [namespace] `. - `VERB` can be `POST`, `GET`, `PATCH`, `PUT`, `DELETE`. - `namespace` should scope some specific functionality like `auth`, `lambda`. @@ -18,7 +42,7 @@ Common scheme for our REST API is ` [namespace] `. without `objects` endpoint like `annotations`, `data`, `data/meta`. Note: action should not duplicate other endpoints without a reason. -## Design principles +When you're developing new endpoints, follow these guidelines: - Use nouns instead of verbs in endpoint paths. For example, `POST /api/tasks` instead of `POST /api/tasks/create`. diff --git a/site/content/en/docs/contributing/_index.md b/site/content/en/docs/contributing/_index.md index bf75235b252..9f986f39dba 100644 --- a/site/content/en/docs/contributing/_index.md +++ b/site/content/en/docs/contributing/_index.md @@ -1,7 +1,7 @@ --- title: 'Contributing to this project' linkTitle: 'Contributing' -weight: 4 +weight: 5 description: 'This section contains documents for CVAT developers.' --- diff --git a/site/content/en/docs/manual/advanced/backup.md b/site/content/en/docs/manual/advanced/backup.md index 6ee41be52ac..cd25d4f4186 100644 --- a/site/content/en/docs/manual/advanced/backup.md +++ b/site/content/en/docs/manual/advanced/backup.md @@ -4,16 +4,45 @@ linkTitle: 'Backup' weight: 17 --- +## Overview + In CVAT you can backup tasks and projects. This can be used to backup a task or project on your PC or to transfer to another server. -## Backup +## Create backup To backup a task or project, open the action menu and select `Backup Task` or `Backup Project`. ![](/images/image219.jpg) -### Backup structure +## Create backup APIs + +- endpoints: + - `/tasks/{id}/backup` + - `/projects/{id}/backup` +- method: `GET` +- responses: 202, 201 with zip archive payload + +### Upload backup APIs + +- endpoints: + - `/api/tasks/backup` + - `/api/projects/backup` +- method: `POST` +- Content-Type: `multipart/form-data` +- responses: 202, 201 with json payload + +## Create from backup + +To create a task or project from a backup, go to the tasks or projects page, +click the `Create from backup` button and select the archive you need. + +![](/images/image220.jpg) + +As a result, you'll get a task containing data, parameters, and annotations of +the previously exported task. + +## Backup file structure As a result, you'll get a zip archive containing data, task or project and task specification and annotations with the following structure: @@ -36,25 +65,3 @@ task or project and task specification and annotations with the following struct └── project.json {{< /tab >}} {{< /tabpane >}} - -### Backup API - -- endpoint: `/tasks/{id}/backup` or `/projects/{id}/backup` -- method: `GET` -- responses: 202, 201 with zip archive payload - -## Create from backup - -To create a task or project from a backup, go to the tasks or projects page, -click the `Create from backup` button and select the archive you need. - -![](/images/image220.jpg) - -As a result, you'll get a task containing data, parameters, and annotations of the previously exported task. - -### Create from backup API - -- endpoint: `/api/tasks/backup` or `/api/projects/backup` -- method: `POST` -- Content-Type: `multipart/form-data` -- responses: 202, 201 with json payload diff --git a/site/content/en/images/download_server_schema.png b/site/content/en/images/download_server_schema.png new file mode 100644 index 00000000000..4678f743ebd Binary files /dev/null and b/site/content/en/images/download_server_schema.png differ diff --git a/site/content/en/images/server_api_create_task_example.png b/site/content/en/images/server_api_create_task_example.png new file mode 100644 index 00000000000..7c846f6591a Binary files /dev/null and b/site/content/en/images/server_api_create_task_example.png differ diff --git a/site/content/en/images/server_api_interaction.svg b/site/content/en/images/server_api_interaction.svg new file mode 100644 index 00000000000..950b723f7fe --- /dev/null +++ b/site/content/en/images/server_api_interaction.svg @@ -0,0 +1,478 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/site/requirements.txt b/site/requirements.txt new file mode 100644 index 00000000000..d0817559014 --- /dev/null +++ b/site/requirements.txt @@ -0,0 +1,3 @@ +gitpython +packaging +toml \ No newline at end of file diff --git a/tests/python/cli/test_cli.py b/tests/python/cli/test_cli.py index a2d9db5074a..f437f9ba815 100644 --- a/tests/python/cli/test_cli.py +++ b/tests/python/cli/test_cli.py @@ -8,6 +8,7 @@ from pathlib import Path import pytest +from cvat_cli.cli import CLI from cvat_sdk import make_client from cvat_sdk.api_client import exceptions from cvat_sdk.core.proxies.tasks import ResourceType, Task @@ -188,3 +189,22 @@ def test_can_create_from_backup(self, fxt_new_task: Task, fxt_backup_file: Path) assert task_id assert task_id != fxt_new_task.id assert self.client.tasks.retrieve(task_id).size == fxt_new_task.size + + @pytest.mark.parametrize("verify", [True, False]) + def test_can_control_ssl_verification_with_arg(self, monkeypatch, verify: bool): + # TODO: Very hacky implementation, improve it, if possible + class MyException(Exception): + pass + + normal_init = CLI.__init__ + + def my_init(self, *args, **kwargs): + normal_init(self, *args, **kwargs) + raise MyException(self.client.api_client.configuration.verify_ssl) + + monkeypatch.setattr(CLI, "__init__", my_init) + + with pytest.raises(MyException) as capture: + self.run_cli(*(["--insecure"] if not verify else []), "ls") + + assert capture.value.args[0] == verify diff --git a/tests/python/rest_api/test_auth.py b/tests/python/rest_api/test_auth.py index 12d21062374..c9c632a0cfa 100644 --- a/tests/python/rest_api/test_auth.py +++ b/tests/python/rest_api/test_auth.py @@ -7,6 +7,7 @@ import pytest from cvat_sdk.api_client import ApiClient, Configuration, models + from shared.utils.config import BASE_URL, USER_PASS, make_api_client @@ -39,23 +40,48 @@ def make_client(cls, username: str) -> ApiClient: def test_can_do_token_auth_and_manage_cookies(self, admin_user: str): username = admin_user - with ApiClient(Configuration(host=BASE_URL)) as client: - auth = self.login(client, username=username) - assert "sessionid" in client.cookies - assert "csrftoken" in client.cookies + with ApiClient(Configuration(host=BASE_URL)) as api_client: + auth = self.login(api_client, username=username) + assert "sessionid" in api_client.cookies + assert "csrftoken" in api_client.cookies assert auth.key - (user, response) = client.users_api.retrieve_self() + (user, response) = api_client.users_api.retrieve_self() + assert response.status == HTTPStatus.OK + assert user.username == username + + def test_can_do_token_auth_from_config(self, admin_user: str): + username = admin_user + + with make_api_client(username) as api_client: + auth = self.login(api_client, username=username) + + config = Configuration( + host=BASE_URL, + api_key={ + "sessionAuth": api_client.cookies["sessionid"].value, + "csrfAuth": api_client.cookies["csrftoken"].value, + "tokenAuth": auth.key, + }, + ) + + with ApiClient(config) as api_client: + auth = self.login(api_client, username=username) + assert "sessionid" in api_client.cookies + assert "csrftoken" in api_client.cookies + assert auth.key + + (user, response) = api_client.users_api.retrieve_self() assert response.status == HTTPStatus.OK assert user.username == username def test_can_do_logout(self, admin_user: str): username = admin_user - with self.make_client(username) as client: - (_, response) = client.auth_api.create_logout() + with self.make_client(username) as api_client: + (_, response) = api_client.auth_api.create_logout() assert response.status == HTTPStatus.OK - (_, response) = client.users_api.retrieve_self( + (_, response) = api_client.users_api.retrieve_self( _parse_response=False, _check_status=False ) assert response.status == HTTPStatus.UNAUTHORIZED @@ -66,8 +92,8 @@ class TestCredentialsManagement: def test_can_register(self): username = "newuser" email = "123@456.com" - with ApiClient(Configuration(host=BASE_URL)) as client: - (user, response) = client.auth_api.create_register( + with ApiClient(Configuration(host=BASE_URL)) as api_client: + (user, response) = api_client.auth_api.create_register( models.RestrictedRegisterRequest( username=username, password1=USER_PASS, password2=USER_PASS, email=email ) @@ -75,8 +101,8 @@ def test_can_register(self): assert response.status == HTTPStatus.CREATED assert user.username == username - with make_api_client(username) as client: - (user, response) = client.users_api.retrieve_self() + with make_api_client(username) as api_client: + (user, response) = api_client.users_api.retrieve_self() assert response.status == HTTPStatus.OK assert user.username == username assert user.email == email @@ -84,8 +110,8 @@ def test_can_register(self): def test_can_change_password(self, admin_user: str): username = admin_user new_pass = "5w4knrqaW#$@gewa" - with make_api_client(username) as client: - (info, response) = client.auth_api.create_password_change( + with make_api_client(username) as api_client: + (info, response) = api_client.auth_api.create_password_change( models.PasswordChangeRequest( old_password=USER_PASS, new_password1=new_pass, new_password2=new_pass ) @@ -93,21 +119,21 @@ def test_can_change_password(self, admin_user: str): assert response.status == HTTPStatus.OK assert info.detail == "New password has been saved." - (_, response) = client.users_api.retrieve_self( + (_, response) = api_client.users_api.retrieve_self( _parse_response=False, _check_status=False ) assert response.status == HTTPStatus.UNAUTHORIZED - client.configuration.password = new_pass - (user, response) = client.users_api.retrieve_self() + api_client.configuration.password = new_pass + (user, response) = api_client.users_api.retrieve_self() assert response.status == HTTPStatus.OK assert user.username == username def test_can_report_weak_password(self, admin_user: str): username = admin_user new_pass = "pass" - with make_api_client(username) as client: - (_, response) = client.auth_api.create_password_change( + with make_api_client(username) as api_client: + (_, response) = api_client.auth_api.create_password_change( models.PasswordChangeRequest( old_password=USER_PASS, new_password1=new_pass, new_password2=new_pass ), @@ -124,8 +150,8 @@ def test_can_report_weak_password(self, admin_user: str): def test_can_report_mismatching_passwords(self, admin_user: str): username = admin_user - with make_api_client(username) as client: - (_, response) = client.auth_api.create_password_change( + with make_api_client(username) as api_client: + (_, response) = api_client.auth_api.create_password_change( models.PasswordChangeRequest( old_password=USER_PASS, new_password1="3j4tb13/T$#", new_password2="q#@$n34g5" ), diff --git a/tests/python/sdk/test_client.py b/tests/python/sdk/test_client.py index d36bf52bbb9..e240e3d7625 100644 --- a/tests/python/sdk/test_client.py +++ b/tests/python/sdk/test_client.py @@ -8,7 +8,7 @@ import pytest from cvat_sdk import Client -from cvat_sdk.core.client import make_client +from cvat_sdk.core.client import Config, make_client from cvat_sdk.core.exceptions import InvalidHostException from cvat_sdk.exceptions import ApiException @@ -69,3 +69,12 @@ def test_can_reject_invalid_server_schema(): make_client(host="ftp://" + host, port=int(port) + 1) assert capture.match(r"Invalid url schema 'ftp'") + + +@pytest.mark.parametrize("verify", [True, False]) +def test_can_control_ssl_verification_with_config(verify: bool): + config = Config(verify_ssl=verify) + + client = Client(BASE_URL, config=config) + + assert client.api_client.configuration.verify_ssl == verify