From e72bc3bc11dd4575e970ea2d29ebbf5e1ac83e8d Mon Sep 17 00:00:00 2001 From: JosepSampe Date: Thu, 23 May 2024 19:49:57 +0200 Subject: [PATCH] Update docs --- .gitignore | 1 + CHANGELOG.md | 2 ++ README.md | 2 +- config/README.md | 23 ++++++++++----- docs/source/compute_config/aws_batch.md | 4 +-- docs/source/compute_config/aws_ec2.md | 6 ++-- docs/source/compute_config/aws_lambda.md | 4 +-- .../source/compute_config/azure_containers.md | 2 +- docs/source/compute_config/azure_functions.md | 2 +- docs/source/compute_config/azure_vms.md | 8 +++--- docs/source/compute_config/localhost.md | 12 ++++---- docs/source/compute_config/vm.md | 6 ++-- docs/source/storage_config/aliyun_oss.md | 2 +- docs/source/storage_config/azure_blob.md | 2 +- docs/source/storage_config/ceph.md | 7 ++--- docs/source/storage_config/gcp_storage.md | 28 +++++++++---------- docs/source/storage_config/infinispan.md | 6 ++-- docs/source/storage_config/minio.md | 4 +-- docs/source/storage_config/redis.md | 6 ++-- docs/source/storage_config/swift.md | 2 +- lithops/storage/backends/localhost/config.py | 4 ++- 21 files changed, 71 insertions(+), 62 deletions(-) diff --git a/.gitignore b/.gitignore index 0c3d70627..1817e5ee3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ lithops_*.zip *.log *.txt *.csv +*.coverage* # Virtual environments .env diff --git a/CHANGELOG.md b/CHANGELOG.md index a17ce17d7..bc5f3c484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added - [CLI] Allow to pass a name in the "lithops runtime list" command +- [Ceph] Added extra region parameter to Ceph backend ### Changed - [Setup] Removed unused 'lxml', 'docker' and 'python-dateutil' packages from the setup.py @@ -16,6 +17,7 @@ - [AWS Lambda] Fixed wrong AWS Lambda delete runtime_name match semantics - [Worker] Fixed potential issue that can appear during 'func_obj' loading from cache - [Monitor] Fixed potential 'keyerror' exceptions +- [Swift] Fixed OpenStack Swift parameters and authentication by adding domain information ## [v3.3.0] diff --git a/README.md b/README.md index 4f5be6d2a..91680394d 100644 --- a/README.md +++ b/README.md @@ -195,4 +195,4 @@ If you are interested in contributing, see [CONTRIBUTING.md](./CONTRIBUTING.md). # Acknowledgements -This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 825184. +This project has received funding from the European Union's Horizon 2020 research and innovation programme under grant agreement No 825184 (CloudButton). diff --git a/config/README.md b/config/README.md index 638670e94..8ae04961e 100644 --- a/config/README.md +++ b/config/README.md @@ -117,11 +117,20 @@ Example of providing configuration keys for IBM Code Engine and IBM Cloud Object ```python import lithops -config = {'lithops': {'backend': 'code_engine', 'storage': 'ibm_cos'}, - 'ibm': {'region': 'REGION', - 'iam_api_key': 'IAM_API_KEY', - 'resource_group_id': 'RESOURCE_GROUP_ID'}, - 'ibm_cos': {'storage_bucket': 'STORAGE_BUCKET'}} +config = { + 'lithops': { + 'backend': 'code_engine', + 'storage': 'ibm_cos' + }, + 'ibm': { + 'region': 'REGION', + 'iam_api_key': 'IAM_API_KEY', + 'resource_group_id': 'RESOURCE_GROUP_ID' + }, + 'ibm_cos': { + 'storage_bucket': 'STORAGE_BUCKET' + } +} def hello_world(name): return f'Hello {name}!' @@ -136,8 +145,8 @@ if __name__ == '__main__': |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| -|lithops | backend | aws_lambda | no | Compute backend implementation. IBM Cloud Functions is the default | -|lithops | storage | aws_s3 | no | Storage backend implementation. IBM Cloud Object Storage is the default | +|lithops | backend | aws_lambda | no | Compute backend implementation. `localhost` is the default if no config or config file is provided| +|lithops | storage | aws_s3 | no | Storage backend implementation. `localhost` is the default if no config or config file is provided| |lithops | data_cleaner | True | no |If set to True, then the cleaner will automatically delete all the temporary data that was written into `storage_bucket/lithops.jobs`| |lithops | monitoring | storage | no | Monitoring system implementation. One of: **storage** or **rabbitmq** | |lithops | monitoring_interval | 2 | no | Monitoring check interval in seconds in case of **storage** monitoring | diff --git a/docs/source/compute_config/aws_batch.md b/docs/source/compute_config/aws_batch.md index c8aa2f67e..f4db379e9 100644 --- a/docs/source/compute_config/aws_batch.md +++ b/docs/source/compute_config/aws_batch.md @@ -14,13 +14,13 @@ python3 -m pip install lithops[aws] 1. [Login](https://console.aws.amazon.com/?nc2=h_m_mc) to Amazon Web Services Console (or signup if you don't have an account) -2. Navigate to **IAM > Roles** to create the ECS Task Execution Role. AWS provides a defualt role named `ecsTaskExecutionRole`, which can be used instead. If you want to create another role or it is missing, create a new role attached to `Elastic Container Service Task`, and add the following policies: +2. Navigate to **IAM > Roles** to create the ECS Task Execution Role. AWS provides a default role named `ecsTaskExecutionRole`, which can be used instead. If you want to create another role or it is missing, create a new role attached to `Elastic Container Service Task`, and add the following policies: - `SecretsManagerReadWrite` - `AmazonEC2ContainerRegistryFullAccess` - `CloudWatchFullAccess` - `AmazonECSTaskExecutionRolePolicy` -3. Navigate to **IAM > Roles** to create the ECS Instance Role. AWS provides a defualt role named `ecsInstanceRole`, which can be used instead. If you want to create another role or it is missing, create a new role attached to `EC2`, and add the following policy: +3. Navigate to **IAM > Roles** to create the ECS Instance Role. AWS provides a default role named `ecsInstanceRole`, which can be used instead. If you want to create another role or it is missing, create a new role attached to `EC2`, and add the following policy: - `AmazonEC2ContainerServiceforEC2Role` ## AWS Credential setup diff --git a/docs/source/compute_config/aws_ec2.md b/docs/source/compute_config/aws_ec2.md index a96420f2a..074620ab8 100644 --- a/docs/source/compute_config/aws_ec2.md +++ b/docs/source/compute_config/aws_ec2.md @@ -77,7 +77,7 @@ In summary, you can use one of the following settings: |aws_ec2 | region | |yes | Region name of the VPC. For example `us-east-1`. Lithops will use the region set under the `aws` section if it is not set here | |aws_ec2 | ssh_username | ubuntu |no | Username to access the VM | |aws_ec2 | ssh_key_filename | ~/.ssh/id_rsa | no | Path to the ssh key file provided to create the VM. It will use the default path if not provided | -|aws_ec2 | worker_processes | AUTO | no | Number of Lithops processes within a given worker. This is used to parallelize function activations within the worker. By default it detects the amount of CPUs in the VM| +|aws_ec2 | worker_processes | AUTO | no | Number of parallel Lithops processes in a worker. This is used to parallelize function activations within the worker. By default it detects the amount of CPUs in the VM| |aws_ec2 | runtime | python3 | no | Runtime name to run the functions. Can be a container image name. If not set Lithops will use the defeuv python3 interpreter of the VM | |aws_ec2 | auto_dismantle | True |no | If False then the VM is not stopped automatically.| |aws_ec2 | soft_dismantle_timeout | 300 |no| Time in seconds to stop the VM instance after a job **completed** its execution | @@ -151,7 +151,7 @@ In summary, you can use one of the following settings: |aws_ec2 | worker_instance_type | t2.medium | no | Profile name for the worker VMs | |aws_ec2 | delete_on_dismantle | True | no | Delete the worker VMs when they are stopped. Master VM is never deleted when stopped | |aws_ec2 | max_workers | 100 | no | Max number of workers per `FunctionExecutor()`| -|aws_ec2 | worker_processes | 2 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker. It is recommendable to set this value to the same number of CPUs of a worker VM. | +|aws_ec2 | worker_processes | AUTO | no | Number of parallel Lithops processes in a worker. This is used to parallelize function activations within the worker. By default it detects the amount of CPUs in the `worker_instance_type` VM| |aws_ec2 | runtime | python3 | no | Runtime name to run the functions. Can be a container image name. If not set Lithops will use the default python3 interpreter of the VM | |aws_ec2 | auto_dismantle | True |no | If False then the VM is not stopped automatically.| |aws_ec2 | soft_dismantle_timeout | 300 |no| Time in seconds to stop the VM instance after a job **completed** its execution | @@ -176,7 +176,7 @@ lithops logs poll ## VM Management -Lithops for AWS EC2 follows a Mater-Worker architecrue (1:N). +Lithops for AWS EC2 follows a Mater-Worker architecture (1:N). All the VMs, including the master VM, are automatically stopped after a configurable timeout (see hard/soft dismantle timeouts). diff --git a/docs/source/compute_config/aws_lambda.md b/docs/source/compute_config/aws_lambda.md index 7e4842914..514c4ee96 100644 --- a/docs/source/compute_config/aws_lambda.md +++ b/docs/source/compute_config/aws_lambda.md @@ -39,7 +39,7 @@ python3 -m pip install lithops[aws] } ``` -4. Click **Next: Tags** and **Next: Review**. Fill the policy name field (you can name it `lithops-policy` or simmilar) and create the policy. +4. Click **Next: Tags** and **Next: Review**. Fill the policy name field (you can name it `lithops-policy` or similar) and create the policy. 5. Go back to **IAM** and navigate to **Roles** tab. Click **Create role**. @@ -105,7 +105,7 @@ In summary, you can use one of the following settings: | aws_lambda | architecture | x86_64 | no | Runtime architecture. One of **x86_64** or **arm64** | | aws_lambda | ephemeral_storage | 512 | no | Ephemeral storage (`/tmp`) size in MB (must be between 512 MB and 10240 MB) | | aws_lambda | env_vars | {} | no | List of {name: ..., value: ...} pairs for Lambda instance environment variables | -| aws_lambda | namespace | | no | Virtual namespace. This can be usefull to virtually group Lithops function workers. The functions deployed by lithops will be prefixed by this namespace. For example you can set it to differentiate between `prod`, `dev` and `stage` environments. | +| aws_lambda | namespace | | no | Virtual namespace. This can be useful to virtually group Lithops function workers. The functions deployed by lithops will be prefixed by this namespace. For example you can set it to differentiate between `prod`, `dev` and `stage` environments. | | aws_lambda | runtime_include_function | False | no | If set to true, Lithops will automatically build a new runtime, including the function's code, instead of transferring it through the storage backend at invocation time. This is useful when the function's code size is large (in the order of 10s of MB) and the code does not change frequently | diff --git a/docs/source/compute_config/azure_containers.md b/docs/source/compute_config/azure_containers.md index 122dedf31..637042d35 100644 --- a/docs/source/compute_config/azure_containers.md +++ b/docs/source/compute_config/azure_containers.md @@ -95,7 +95,7 @@ az containerapp env create --name lithops --resource-group LithopsResourceGroup |---|---|---|---|---| |azure_containers| resource_group | |no | Name of a resource group, for example: `LithopsResourceGroup`. Lithops will use the `resource_group` set under the `azure` section if it is not set here | |azure_containers| region | |no | The location where you created the `lithops` Container APP environment. For example: `westeurope`, `westus2`, etc. Lithops will use the `region` set under the `azure` section if it is not set here| -|azure_containers| environment | lithops |no | The environemnt name you created in the step 5 of the installation | +|azure_containers| environment | lithops |no | The environment name you created in the step 5 of the installation | |azure_containers | docker_server | index.docker.io |no | Container registry URL | |azure_containers | docker_user | |no | Container registry user name | |azure_containers | docker_password | |no | Container registry password/token. In case of Docker hub, login to your docker hub account and generate a new access token [here](https://hub.docker.com/settings/security)| diff --git a/docs/source/compute_config/azure_functions.md b/docs/source/compute_config/azure_functions.md index d4e3d4e19..d48d83160 100644 --- a/docs/source/compute_config/azure_functions.md +++ b/docs/source/compute_config/azure_functions.md @@ -89,7 +89,7 @@ az login |---|---|---|---|---| |azure_functions| resource_group | |no | Name of a resource group, for example: `LithopsResourceGroup`. Lithops will use the `resource_group` set under the `azure` section if it is not set here | |azure_functions| region | |no | The location of the consumption plan for the runtime. Use `az functionapp list-consumption-locations` to view the available locations. For example: `westeurope`, `westus2`, etc. Lithops will use the `region` set under the `azure` section if it is not set here| -|azure_functions | max_workers | 1000 | no | Max number of parallel workers. Altough Azure limits the number of workrs to 200, it is convenient to keep this value high| +|azure_functions | max_workers | 1000 | no | Max number of parallel workers. Although Azure limits the number of workers to 200, it is convenient to keep this value high| |azure_functions | worker_processes | 1 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker | |azure_functions| runtime | |no | Runtime name already deployed in the service| |azure_functions | runtime_timeout | 300 |no | Runtime timeout in seconds. Default 5 minutes | diff --git a/docs/source/compute_config/azure_vms.md b/docs/source/compute_config/azure_vms.md index 242a188f4..6cef81e14 100644 --- a/docs/source/compute_config/azure_vms.md +++ b/docs/source/compute_config/azure_vms.md @@ -89,7 +89,7 @@ Edit your lithops config and add the relevant keys: |azure_vms | ssh_username | ubuntu | yes | Username to access the VM. It will use `ubuntu` if not provided | |azure_vms | ssh_key_filename | ~/.ssh/id_rsa | yes | Path to the ssh key file provided to create the VM. It will use the default path if not provided | |azure_vms | region | |no | Location of the resource group, for example: `westeurope`, `westus2`, etc. Lithops will use the region set under the `azure` section if it is not set here | -|azure_vms | worker_processes | AUTO | no | Number of Lithops processes within a given worker. This is used to parallelize function activations within the worker. By default it detects the amount of CPUs in the VM| +|azure_vms | worker_processes | AUTO | no | Number of parallel Lithops processes in a worker. This is used to parallelize function activations within the worker. By default it detects the amount of CPUs in the VM| |azure_vms | runtime | python3 | no | Runtime name to run the functions. Can be a container image name. If not set Lithops will use the defeuv python3 interpreter of the VM | |azure_vms | auto_dismantle | True |no | If False then the VM is not stopped automatically.| |azure_vms | soft_dismantle_timeout | 300 |no| Time in seconds to stop the VM instance after a job **completed** its execution | @@ -139,12 +139,12 @@ Edit your lithops config and add the relevant keys: |azure_vms | worker_instance_type | Standard_B2s | no | Profile name for the worker VMs | |azure_vms | delete_on_dismantle | False | no | Delete the worker VMs when they are stopped. Master VM is never deleted when stopped. `True` is NOT YET SUPPORTED | |azure_vms | max_workers | 100 | no | Max number of workers per `FunctionExecutor()`| -|azure_vms | worker_processes | 2 | no | Number of Lithops processes within a given worker. This can be used to parallelize function activations within a worker. It is recommendable to set this value to the same number of CPUs of a worker VM. | +|azure_vms | worker_processes | AUTO | no | Number of parallel Lithops processes in a worker. This is used to parallelize function activations within the worker. By default it detects the amount of CPUs in the `worker_instance_type` VM| |azure_vms | runtime | python3 | no | Runtime name to run the functions. Can be a container image name. If not set Lithops will use the default python3 interpreter of the VM | |azure_vms | auto_dismantle | True |no | If False then the VM is not stopped automatically.| |azure_vms | soft_dismantle_timeout | 300 |no| Time in seconds to stop the VM instance after a job **completed** its execution | |azure_vms | hard_dismantle_timeout | 3600 | no | Time in seconds to stop the VM instance after a job **started** its execution | -|azure_vms | exec_mode | reuse | no | One of: **consume**, **create** or **reuse**. If set to **create**, Lithops will automatically create new VMs for each map() call based on the number of elements in iterdata. If set to **reuse** will try to reuse running workers if exist | +|azure_vms | exec_mode | reuse | no | One of: **consume**, **create** or **reuse**. If set to **create**, Lithops will automatically create new VMs for each map() call based on the number of elements in `iterdata`. If set to **reuse** will try to reuse running workers if exist | ## Test Lithops @@ -164,7 +164,7 @@ lithops logs poll ## VM Management -Lithops for Azure VMs follows a Mater-Worker architecrue (1:N). +Lithops for Azure VMs follows a Mater-Worker architecture (1:N). All the VMs, including the master VM, are automatically stopped after a configurable timeout (see hard/soft dismantle timeouts). diff --git a/docs/source/compute_config/localhost.md b/docs/source/compute_config/localhost.md index 2afaf99ea..9a47f1498 100644 --- a/docs/source/compute_config/localhost.md +++ b/docs/source/compute_config/localhost.md @@ -14,7 +14,7 @@ lithops: ## Execution Environments -The localhost backend can run functions both using the local ``python3`` interpreter, or using a ``docker container`` image. The environment is automatically chosen depending on whether or not you provided a Docker image as a runtime. +The localhost backend can run functions both using the local ``python3`` interpreter, or using a ``container`` image. The environment is automatically chosen depending on whether or not you provided a Docker image as a runtime. In both cases, you can view the executions logs in your local machine using the *lithops client*: @@ -24,7 +24,7 @@ lithops logs poll ### Default Environment -By default Lithops uses the local python interpreter to run the functions. That is, if for example you executed the main script with ``python3.8``, your functions will run with ``python3.8``. in this case, you must ensure that all the dependencies of your script are installed in your machine. +By default Lithops uses the local python interpreter to run the functions. That is, if for example you executed the main script with ``python3.12``, your functions will run with ``python3.12``. in this case, you must ensure that all the dependencies of your script are installed in your machine. ```python # As we use the default FunctionExecutor(), backend must be set to localhost in config @@ -44,19 +44,19 @@ The Docker environment runs the functions within a ``docker container``. In this ```yaml localhost: - runtime: lithopscloud/ibmcf-python-v310 + runtime: lithopscloud/ibmcf-python-v312 ``` or by using the ``runtime`` param in a function executor: ```python # As we use the default FunctionExecutor(), the "backend" config parameter must be set to localhost in config -fexec = lithops.FunctionExecutor(runtime='lithopscloud/ibmcf-python-v310') +fexec = lithops.FunctionExecutor(runtime='lithopscloud/ibmcf-python-v312') ``` ```python # As we use/force the LocalhostExecutor(), the "backend" config parameter does not need to be set to localhost in config -fexec = lithops.LocalhostExecutor(runtime='lithopscloud/ibmcf-python-v310') +fexec = lithops.LocalhostExecutor(runtime='lithopscloud/ibmcf-python-v312') ``` In this mode of execution, you can use any docker image that contains all the required dependencies. For example, the IBM Cloud Functions and Knative runtimes are compatible with it. @@ -66,7 +66,7 @@ In this mode of execution, you can use any docker image that contains all the re |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |localhost | runtime | python3 | no | By default it uses the `python3` interpreter. It can be a container image name | -|localhost | version | 1 | no | There are 2 diferent localhost implementations. Use '2' for using the alterantive version (beta) | +|localhost | version | 1 | no | There are 2 different localhost implementations. Use '2' for using the alternative version (beta) | |localhost | worker_processes | CPU_COUNT | no | Number of Lithops processes. This is used to parallelize function activations. By default it is set to the number of CPUs of your machine | ## Test Lithops diff --git a/docs/source/compute_config/vm.md b/docs/source/compute_config/vm.md index 4e8d77da4..c828c0c1d 100644 --- a/docs/source/compute_config/vm.md +++ b/docs/source/compute_config/vm.md @@ -1,6 +1,6 @@ # Virtual Machine -Lithops can run functions using a remote host or a virtual machine (VM). In this backend, Lithops uses all the available VM CPUs to parallelize the tasks of a job. For testing purposes, it is preferable to have an Ubuntu > 20.04 host. +Lithops can run functions using a remote host or a virtual machine (VM). In this backend, Lithops uses all the available VM CPUs to parallelize the tasks of a job. For testing purposes, it is preferable to have an Ubuntu > 22.04 host. ## Configuration @@ -33,13 +33,13 @@ The Docker environment runs the functions within a ``docker container``. In this ```yaml vm: - runtime: lithopscloud/ibmcf-python-v38 + runtime: lithopscloud/ibmcf-python-v312 ``` or by using the ``runtime`` param in a function executor: ```python -fexec = lithops.FunctionExecutor(runtime='lithopscloud/ibmcf-python-v38') +fexec = lithops.FunctionExecutor(runtime='lithopscloud/ibmcf-python-v312') ``` In this backend, you can use any docker image that contains all the required dependencies. For example, the IBM Cloud Functions and Knative runtimes are compatible with it. diff --git a/docs/source/storage_config/aliyun_oss.md b/docs/source/storage_config/aliyun_oss.md index 3b282ecd5..e818ddec8 100644 --- a/docs/source/storage_config/aliyun_oss.md +++ b/docs/source/storage_config/aliyun_oss.md @@ -41,5 +41,5 @@ aliyun: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| -|aliyun_oss | region | | no | Region Name from [here](https://www.alibabacloud.com/help/en/object-storage-service/latest/regions-and-endpoints). Ommit the `oss-` prefix. For example: `eu-west-1`. Lithops will use the region set under the `aliyun` section if it is not set here | +|aliyun_oss | region | | no | Region Name from [here](https://www.alibabacloud.com/help/en/object-storage-service/latest/regions-and-endpoints). Omit the `oss-` prefix. For example: `eu-west-1`. Lithops will use the region set under the `aliyun` section if it is not set here | |aliyun_oss | storage_bucket | | no | The name of a bucket that exists in you account. This will be used by Lithops for intermediate data. Lithops will automatically create a new one if it is not provided| \ No newline at end of file diff --git a/docs/source/storage_config/azure_blob.md b/docs/source/storage_config/azure_blob.md index 255312791..4233a38e5 100644 --- a/docs/source/storage_config/azure_blob.md +++ b/docs/source/storage_config/azure_blob.md @@ -11,7 +11,7 @@ Lithops with Azure Blob Storage as storage backend. $ python3 -m pip install lithops[azure] ``` -5. Create a Resource Group and a Storage Account: +2. Create a Resource Group and a Storage Account: Option 1: diff --git a/docs/source/storage_config/ceph.md b/docs/source/storage_config/ceph.md index 9c37504c1..c53729e67 100644 --- a/docs/source/storage_config/ceph.md +++ b/docs/source/storage_config/ceph.md @@ -15,11 +15,10 @@ python3 -m pip install lithops[ceph] 3. Create a new user. -4. Create a new bucket (e.g. `lithops-data`). Remember to update the corresponding Lithops config field with this bucket name. ## Configuration -3. Edit your lithops config file and add the following keys: +1. Edit your lithops config file and add the following keys: ```yaml lithops: @@ -36,9 +35,9 @@ python3 -m pip install lithops[ceph] |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| -|ceph | endpoint | |yes | The host ip adress where you installed the Ceph server. Must start with http:// or https:// | +|ceph | endpoint | |yes | The host ip address where you installed the Ceph server. Must start with http:// or https:// | +|ceph | region | |no | Region name. For example 'eu-west-1' | |ceph | access_key_id | |yes | Account user access key | |ceph | secret_access_key | |yes | Account user secret access key | |ceph | session_token | |no | Session token for temporary AWS credentials | |ceph | storage_bucket | | no | The name of a bucket that exists in you account. This will be used by Lithops for intermediate data. Lithops will automatically create a new one if it is not provided | -|ceph | region | |no | Region name. For example 'eu-west-1' | diff --git a/docs/source/storage_config/gcp_storage.md b/docs/source/storage_config/gcp_storage.md index 1bb17c2f1..177a93511 100644 --- a/docs/source/storage_config/gcp_storage.md +++ b/docs/source/storage_config/gcp_storage.md @@ -10,27 +10,25 @@ Lithops with GCP Storage as storage backend. python3 -m pip install lithops[gcp] ``` - 2. [Login](https://console.cloud.google.com) to Google Cloud Console (or signup if you don't have an account). +## Configuration + +1. [Login](https://console.cloud.google.com) to Google Cloud Console (or signup if you don't have an account). - 3. Create a new project. Name it `lithops` or similar. +2. Create a new project. Name it `lithops` or similar. - 4. Navigate to *IAM & Admin* > *Service Accounts*. +3. Navigate to *IAM & Admin* > *Service Accounts*. - 5. Click on *Create Service Account*. Name the service account `lithops-executor` or similar. Then click on *Create*. +4. Click on *Create Service Account*. Name the service account `lithops-executor` or similar. Then click on *Create*. - 6. Add the following roles to the service account: - - Service Account User - - Cloud Functions Admin - - Pub/Sub Admin - - Storage Admin - - 7. Click on *Continue*. Then, click on *Create key*. Select *JSON* and then *Create*. Download the JSON file to a secure location in you computer. Click *Done*. +5. Add the following roles to the service account: + - Service Account User + - Cloud Functions Admin + - Pub/Sub Admin + - Storage Admin - 8. Navigate to *Storage* on the menu. Create a bucket and name it `lithops-data` or similar. Remember to update the corresponding Lithops config field with this bucket name. - -## Configuration +6. Click on *Continue*. Then, click on *Create key*. Select *JSON* and then *Create*. Download the JSON file to a secure location in you computer. Click *Done*. -9. Edit your lithops config file and add the following keys: +7. Edit your lithops config file and add the following keys: ```yaml lithops: diff --git a/docs/source/storage_config/infinispan.md b/docs/source/storage_config/infinispan.md index fbe7c3a1d..af117f476 100644 --- a/docs/source/storage_config/infinispan.md +++ b/docs/source/storage_config/infinispan.md @@ -11,7 +11,7 @@ infinispan_hotrod (native binary) endpoint. ## Configuration -2. Edit your lithops config file and add the following keys: +Edit your lithops config file and add the following keys: ### Rest endpoint ```yaml @@ -30,7 +30,7 @@ infinispan_hotrod (native binary) endpoint. - ... ``` -### Summary of configuration keys for Infinispan: +#### Summary of configuration keys for Infinispan: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| @@ -61,7 +61,7 @@ To run this endpoint you need to compile and install the Infinispan python clien - ... ``` -### Summary of configuration keys for Infinispan_hotrod: +#### Summary of configuration keys for Infinispan_hotrod: |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| diff --git a/docs/source/storage_config/minio.md b/docs/source/storage_config/minio.md index aa0a5bc5b..1a91947e8 100644 --- a/docs/source/storage_config/minio.md +++ b/docs/source/storage_config/minio.md @@ -15,11 +15,9 @@ python3 -m pip install lithops[minio] 3. Create a new user. -4. Create a new bucket (e.g. `lithops-data`). Remember to update the corresponding Lithops config field with this bucket name. - ## Configuration -3. Edit your lithops config file and add the following keys: +Edit your lithops config file and add the following keys: ```yaml lithops: diff --git a/docs/source/storage_config/redis.md b/docs/source/storage_config/redis.md index f5cf30b97..859b186c9 100644 --- a/docs/source/storage_config/redis.md +++ b/docs/source/storage_config/redis.md @@ -18,7 +18,7 @@ python3 -m pip install lithops[redis] ## Configuration -1. Edit your lithops config file and add the following keys: +Edit your lithops config file and add the following keys: ```yaml lithops: @@ -35,10 +35,10 @@ python3 -m pip install lithops[redis] |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| -|redis | host | localhost |no | The host ip adress where you installed the Redis server. | +|redis | host | localhost |no | The host ip address where you installed the Redis server. | |redis | port | 6379 |no | The port where the redis server is listening | |redis | username | None |no | The username (if any)| |redis | password | None |no | The password you set in the Redis configuration file (if any) | |redis | db | 0 |no | Number of database to use | -|redis | ssl | False |no | Activate ssl conection | +|redis | ssl | False |no | Activate ssl connection | |redis | ... | |no | All the parameters set in this lithops `redis` config section are directly passed to a [`reds.Redis()`](https://redis-py.readthedocs.io/en/stable/index.html#redis.Redis) instance, so you can set all the same parameters if necessary. | diff --git a/docs/source/storage_config/swift.md b/docs/source/storage_config/swift.md index 3183db02b..d9c8e793d 100644 --- a/docs/source/storage_config/swift.md +++ b/docs/source/storage_config/swift.md @@ -36,7 +36,7 @@ Lithops with OpenStack Swift as storage backend. |Group|Key|Default|Mandatory|Additional info| |---|---|---|---|---| |swift | storage_bucket | | yes | The name of a container that exists in you account. This will be used by Lithops for intermediate data. If set, this will overwrite the `storage_bucket` set in `lithops` section | -|swift | auth_url | |yes | The keystone endpoint for authenthication | +|swift | auth_url | |yes | The keystone endpoint for authentication | |swift | region | |yes | The region of your container | |swift | project_id | |yes | The Project ID | |swift | user_id | |yes | The user ID | diff --git a/lithops/storage/backends/localhost/config.py b/lithops/storage/backends/localhost/config.py index e8c447530..068c3c34c 100644 --- a/lithops/storage/backends/localhost/config.py +++ b/lithops/storage/backends/localhost/config.py @@ -19,5 +19,7 @@ def load_config(config_data): if 'localhost' not in config_data or config_data['localhost'] is None: config_data['localhost'] = {} - config_data['lithops']['monitoring_interval'] = 0.1 + if 'monitoring_interval' not in config_data['lithops']: + config_data['lithops']['monitoring_interval'] = 0.1 + config_data['localhost']['storage_bucket'] = 'storage'