diff --git a/.gitignore b/.gitignore
index f6b4b7d..10a7f3b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -138,3 +138,6 @@ dmypy.json
# Terraform
.terraform/
*.tfstate
+
+# serverless
+.serverless/
diff --git a/README.md b/README.md
index 80a1842..91efec4 100644
--- a/README.md
+++ b/README.md
@@ -1,65 +1,309 @@
Lambda Cache
Simple Caching for AWS Lambda
-   
-
- [](https://coveralls.io/github/keithrozario/lambda-cache?branch=release) [](https://lambda-cache.readthedocs.io/en/latest/?badge=latest) [](https://github.com/psf/black)
-
+    [](https://lambda-cache.readthedocs.io/en/latest/?badge=latest) [](https://lambda-cache.readthedocs.io/en/latest/?badge=latest)
+ [](https://coveralls.io/github/keithrozario/lambda-cache?branch=release) [](https://github.com/psf/black)
# Introduction
-
+
-_lambda-cache_ helps you cache data in your Lambda function **across** invocations. The package utilizes the internal memory of the lambda function's [execution context](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-context.html) to assist in caching, which in turn:
+_lambda-cache_ helps you cache data in your Lambda function **across** invocations. The package utilizes the internal memory of the lambda function's [execution context](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-context.html) to store data across multiple invocations. Doing this:
* Reduces load on back-end systems
* Reduces the execution time of the lambda
-* Guarantees functions will reference latest data after caches have expired
-* Reduces expensive network calls to APIs with throttling limits (or high charges)
+* Guarantees functions will reference latest data after cache expiry
+* Reduces calls to APIs throttling limits or high charges
+
+_lambda-cache_ prioritizes simplicity over performance and flexibility.
+
-_lambda-cache_ prioritizes simplicity over performance or flexibility. The goal is to provide the **simplest** way for developers to cache data across lambda invocations.
+The package is purpose-built for AWS Lambda functions, and currently supports SSM Parameters, Secrets from Secrets Manager and S3 Objects.
-The package is purpose-built for running in AWS Lambda functions, and currently supports SSM Parameters, Secrets Manager and S3 Objects.
# Installation
- $ pip install lambda-cache
+Include the package in your functions code zip-file using the following:
-Refer to [docs](https://lambda-cache.readthedocs.io/en/latest/) for how to include into your lambda function.
+ $ pip install lambda-cache -t /path/of/function
+
+Refer to [docs](https://lambda-cache.readthedocs.io/en/latest/install/) for how to include into your lambda function.
# Usage
-To begin caching parameters, secrets or S3 objects, decorate your function's handler with the right decorator:
+Below we describe further features of the package. For more info refer to the [user guide](https://lambda-cache.readthedocs.io/en/latest/user_guide/).
+
+
+* [SSM - Parameter Store](#SSM-ParameterStore)
+ * [Cache single parameter](#Cachesingleparameter)
+ * [Change cache expiry](#Changecacheexpiry)
+ * [Change cache entry settings](#Changecacheentrysettings)
+ * [Cache multiple parameters](#Cachemultipleparameters)
+ * [Decorator stacking](#Decoratorstacking)
+ * [Cache invalidation](#Cacheinvalidation)
+ * [Return Values](#ReturnValues)
+* [Secrets Manager](#SecretsManager)
+ * [Cache single secret](#Cachesinglesecret)
+ * [Change Cache expiry](#ChangeCacheexpiry)
+ * [Change Cache entry settings](#ChangeCacheentrysettings)
+ * [Decorator stacking](#Decoratorstacking-1)
+ * [Cache Invalidation](#CacheInvalidation)
+ * [Return Values](#ReturnValues-1)
+* [S3](#S3)
+ * [Cache a single file](#Cacheasinglefile)
+ * [Change Cache expiry](#ChangeCacheexpiry-1)
+ * [Check file before download](#Checkfilebeforedownload)
+
+## SSM - Parameter Store
+
+### Cache single parameter
+To cache a parameter from ssm, decorate your handler function like so:
```python
-from lambda_cache import ssm, secrets_manager, s3
+from lambda_cache import ssm
-# Decorators injects cache entry into context object
-@ssm.cache(parameter='/prod/app/var')
-@secrets_manager.cache(name='/prod/db/conn_string')
-@s3.cache(s3Uri='s3://bucket_name/path/to/object.json')
+@ssm.cache(parameter='/production/app/var')
+def handler(event, context):
+ var = getattr(context,'var')
+ response = do_something(var)
+ return response
+```
+All invocations of this function over in the next minute will reference the parameter from the function's internal cache, rather than making a network call to ssm. After one minute has lapsed, the the next invocation will invoke `get_parameter` to refresh the cache. The parameter value will be injected into the `context` object of your lambda handler.
+
+### Change cache expiry
+
+The default `max_age_in_seconds` settings is 60 seconds (1 minute), it defines the maximum age of a parameter that is acceptable to the handler function. Cache entries older than this, will be refreshed. To set a longer cache duration (e.g 5 minutes), change the setting like so:
+
+```python
+from lambda_cache import ssm
+
+@ssm.cache(parameter='/production/app/var', max_age_in_seconds=300)
+def handler(event, context):
+ var = getattr(context,'var')
+ response = do_something(var)
+ return response
+```
+
+_Note: The caching logic runs only at invocation, regardless of how long the function runs. A 15 minute lambda function will not refresh the parameter, unless explicitly refreshed using `get_entry` method. The library is primary interested in caching 'across' invocation rather than 'within' an invocation_
+
+### Change cache entry settings
+
+The default name of the parameter is the string after the last slash('/') character of its name. This means `/production/app/var` and `test/app/var` both resolve to just `var`. To over-ride this default, use `entry_name` setting like so:
+
+```python
+from lambda_cache import ssm
+
+@ssm.cache(parameter='/production/app/var', entry_name='new_var')
+def handler(event, context):
+ var = getattr(context,'new_var')
+ response = do_something(var)
+ return response
+```
+
+### Cache multiple parameters
+
+To cache multiple entries at once, pass a list of parameters to the parameter argument. This method groups all the parameter value under one python dictionary, stored in the Lambda Context under the `entry_name`.
+
+_Note: When using this method, `entry_name` is a required parameter, if not present `NoEntryNameError` exception is thrown._
+
+```python
+from lambda_cache import ssm
+
+@ssm.cache(parameter=['/app/var1', '/app/var2'], entry_name='parameters')
+def handler(event, context):
+ var1 = getattr(context,'parameters').get('var1')
+ var2 = getattr(context,'parameters').get('var2')
+ response = do_something(var)
+ return response
+```
+
+Under the hood, we use the `get_parameters` API call for boto3, which translate to a single network call for multiple parameters. You can group all parameters types in a single call, including `String`, `StringList` and `SecureString`. `StringList` will return as a list, while all other types will return as plain-text strings. The library does not support returning `SecureString` parameters in encrypted form, and will only return plain-text strings regardless of String type.
+
+_Note: for this method to work, ensure you have both `ssm:GetParameter` **and** `ssm:GetParameters` (with the 's' at the end) in your function's permission policy_
+
+### Decorator stacking
+
+If you wish to cache multiple parameters with different expiry times, stack the decorators. In this example, `var1` will be refreshed every 30 seconds, `var2` will be refreshed after 60.
+
+```python
+@ssm.cache(parameter='/production/app/var1', max_age_in_seconds=30)
+@ssm.cache(parameter='/production/app/var2', max_age_in_seconds=60)
+def handler(event, context):
+ var1 = getattr(context,'var1')
+ var2 = getattr(context,'var2')
+ response = do_something(var)
+ return response
+```
+_Note: Decorator stacking performs one API call per decorator, which might result is slower performance_
+
+### Cache invalidation
+
+If you require a fresh value at some point of the code, you can force a refresh using the `ssm.get_entry` function, and setting the `max_age_in_seconds` argument to 0.
+
+```python
+from lambda_cache import ssm
+
+@ssm.cache(parameter='/prod/var')
def handler(event, context):
- # Parameter from SSM
- var = getattr(context, 'var')
+ if event.get('refresh'):
+ # refresh parameter
+ var = ssm.get_entry(parameter='/prod/var', max_age_in_seconds=0)
+ else:
+ var = getattr(context,'var')
+
+ response = do_something(var)
+ return response
+```
+
+You may also use `ssm.get_entry` to get a parameter entry from anywhere in your functions code.
+
+To only get parameter once in the lifetime of the function, set `max_age_in_seconds` to some arbitary large number ~36000 (10 hours).
- # Secret from Secrets Manager
- secret = getattr(context, 'conn_string')
+### Return Values
+
+Caching supports `String`, `SecureString` and `StringList` parameters with no change required (ensure you have `kms:Decrypt` permission for `SecureString`). For simplicity, `StringList` parameters are automatically converted into list (delimited by comma), while `String` and `SecureString` both return the single string value of the parameter.
+
+## Secrets Manager
+
+### Cache single secret
+
+Secret support is similar, but uses the `secret.cache` decorator.
+
+```python
+from lambda_cache import secret
+
+@secret.cache(name='/prod/db/conn_string')
+def handler(event, context):
+ conn_string = getattr(context,'conn_string')
+ return context
+```
+
+
+### Change Cache expiry
+
+The default `max_age_in_seconds` settings is 60 seconds (1 minute), it defines how long a parameter should be kept in cache before it is refreshed from ssm. To configure longer or shorter times, modify this argument like so:
+
+```python
+from lambda_cache import secret
+
+@secret.cache(name='/prod/db/conn_string', max_age_in_seconds=300)
+def handler(event, context):
+ var = getattr(context,'conn_string')
+ response = do_something(var)
+ return response
+```
+
+_Note: The caching logic runs only at invocation, regardless of how long the function runs. A 15 minute lambda function will not refresh the parameter, unless explicitly refreshed using get_cache_ssm. The library is primary interested in caching 'across' invocation rather than 'within' an invocation_
+
+### Change Cache entry settings
+
+The name of the secret is simply shortened to the string after the last slash('/') character of the secret's name. This means `/prod/db/conn_string` and `/test/db/conn_string` resolve to just `conn_string`. To over-ride this default, use `entry_name`:
+
+```python
+from lambda_cache import secret
+
+@secret.cache(name='/prod/db/conn_string', entry_name='new_var')
+def handler(event, context):
+ var = getattr(context,'new_var')
+ response = do_something(var)
+ return response
+```
+
+### Decorator stacking
+
+If you wish to cache multiple secrets, you can use decorator stacking.
+
+```python
+@secret.cache(name='/prod/db/conn_string', max_age_in_seconds=30)
+@secret.cache(name='/prod/app/elk_username_password', max_age_in_seconds=60)
+def handler(event, context):
+ var1 = getattr(context,'conn_string')
+ var2 = getattr(context,'elk_username_password')
+ response = do_something(var)
+ return response
+```
+
+_Note: Decorator stacking performs one API call per decorator, which might result is slower performance._
+
+### Cache Invalidation
+
+To invalidate a secret, use the `get_entry`, setting the `max_age_in_seconds=0`.
+```python
+from lambda_cache import secret
+
+@secret.cache(name='/prod/db/conn_string')
+def handler(event, context):
+
+ if event.get('refresh'):
+ var = secret.get_entry(name='/prod/db/conn_string', max_age_in_seconds=0)
+ else:
+ var = getattr(context,'conn_string')
+ response = do_something(var)
+ return response
+```
+
+### Return Values
+
+Secrets Manager supports both string and binary secrets. For simplicity we will cache the secret in the format it is stored. It is up to the calling application to process the return as Binary or Strings.
+
+## S3
+
+S3 support is considered _experimental_ for now, but withing the python community we see a lot of folks pull down files from S3 for use in AI/ML models.
+
+Files downloaded from s3 are automatically stored in the `/tmp` directory of the lambda function. This is the only writable directory within lambda, and has a 512MB of storage space.
+
+### Cache a single file
+To download a file from S3 use the the same decorator pattern:
+
+```python
+from lambda_cache import s3
+
+@s3.cache(s3Uri='s3://bucket_name/path/to/object.json')
+def s3_download_entry_name(event, context):
# Object from S3 automatically saved to /tmp directory
with open("/tmp/object.json") as file_data:
status = json.loads(file_data.read())['status']
- response = do_something(var,secret,status)
+ return status
+```
+
+### Change Cache expiry
- return response
+The default `max_age_in_seconds` settings is 60 seconds (1 minute), it defines how long a file should be kept in `/tmp` before it is refreshed from S3. To configure longer or shorter times, modify this argument like so:
+
+```python
+from lambda_cache import s3
+
+@s3.cache(s3Uri='s3://bucket_name/path/to/object.json', max_age_in_seconds=300)
+def s3_download_entry_name(event, context):
+ with open("/tmp/object.json") as file_data:
+ status = json.loads(file_data.read())['status']
+
+ return status
```
-The first invocation of the function will populate the cache, after which all invocations over the next 60 seconds, will reference the parameter from the function's internal cache, without making a network calls to ssm, secrets manager or S3. After 60 seconds, the next invocation will refresh the cache from the respective back-ends.
+_Note: The caching logic runs only at invocation, regardless of how long the function runs. A 15 minute lambda function will not refresh the object, unless explicitly refreshed using `s3.get_entry`. The library is primary interested in caching 'across' invocation rather than 'within' an invocation_
+
+### Check file before download
+
+By default, _lambda_cache_ will download the file once at cache has expired, however, to save on network bandwidth (and possibly time), we can set the `check_before_download` parameter to True. This will check the age of the object in S3 and download **only** if the object has changed since the last download.
+
+```python
+from lambda_cache import s3
+
+@s3.cache(s3Uri='s3://bucket_name/path/to/object.json', max_age_in_seconds=300, check_before_download=True)
+def s3_download_entry_name(event, context):
+ with open("/tmp/object.json") as file_data:
+ status = json.loads(file_data.read())['status']
+
+ return status
+```
-Refer to [docs](https://lambda-cache.readthedocs.io/en/latest/user_guide/) for how to change cache timings, change the cache entry names,and invalidate caches.
+_Note: we use the GetHead object call to verify the objects `last_modified_date`. This simplifies the IAM policy of the function, as it still only requires the `s3:GetObject` permission. However, this is still a GET requests, and will be charged as such, for smaller objects it might be cheaper to just download the object_
# Credit
diff --git a/docs/images/installed_package.png b/docs/images/installed_package.png
new file mode 100644
index 0000000..2cae9de
Binary files /dev/null and b/docs/images/installed_package.png differ
diff --git a/docs/install.md b/docs/install.md
index 2cbba04..d26c6e8 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -4,11 +4,15 @@ Unlike other packages, _lambda_cache_ was designed to operate specifically withi
There are two general options to using it.
-## Using the publicly available layer from Klayers
+## Manual Installation
-[Klayers](https://github.com/keithrozario/Klayers) is a project that publishes AWS Lambda Layers for public consumption. A Lambda layer is way to pre-package code for easy deployments into any Lambda function.
+Because _lambda-cache_ is a pure python package, you can manually include it in your lambda function, like so:
-You can 'install' _lambda_cache_ by simply including the latest layer arn in your lambda function.
+ $ pip install lambda-cache -t /path/to/function
+
+Once installed you will see the following directory structure in your lambda function via the console:
+
+
## Using Serverless Framework
@@ -18,10 +22,12 @@ simply ensure that _simple_lambda_cache_ is part of your `requirements.txt` file
$ pip install lambda-cache
+## Using the publicly available layer from Klayers
+
+[Klayers](https://github.com/keithrozario/Klayers) is a project that publishes AWS Lambda Layers for public consumption. A Lambda layer is way to pre-package code for easy deployments into any Lambda function.
+
+You can 'install' _lambda_cache_ by simply including the latest layer arn in your lambda function.
-## Manual Installation
-Because _lambda_cache_ is a pure python package, you can also manually include it in your lambda function, like so:
- $ pip install lambda-cache -t /path/to/function
diff --git a/lambda_cache/__init__.py b/lambda_cache/__init__.py
index 1fae966..15001a2 100644
--- a/lambda_cache/__init__.py
+++ b/lambda_cache/__init__.py
@@ -1,5 +1,16 @@
-__all__ = ["cache", "get_entry"]
-__version__ = "0.8.0"
+# -*- coding: utf-8 -*-
+
+"""
+lambda-cache
+~~~~~~~~~~~~
+
+A python package for caching within AWS Lambda Functions
+
+Full Documentation is at .
+:license: MIT, see LICENSE for more details.
+"""
+
+__version__ = "0.8.1"
from .ssm import cache, get_entry
from .secrets_manager import cache, get_entry
diff --git a/lambda_cache/caching_logic.py b/lambda_cache/caching_logic.py
index a8c4912..adf4f3a 100644
--- a/lambda_cache/caching_logic.py
+++ b/lambda_cache/caching_logic.py
@@ -5,6 +5,7 @@
def get_decorator(**kwargs):
+
"""
Args:
argument (string, list, dict) : argument to be passed to the missed function
@@ -34,6 +35,7 @@ def inner_function(event, context):
def get_value(**kwargs):
+
"""
returns value of check_cache.
"""
@@ -50,6 +52,7 @@ def check_cache(
send_details=False,
**kwargs
):
+
"""
Executes the caching logic, checks cache for entry
If entry doesn't exist, returns entry_value by calling the miss function with entry_name and var_name
@@ -97,6 +100,7 @@ def check_cache(
def get_entry_name(argument, entry_name):
+
"""
argument is either SSM Parameter, Secret in Secrets Manager or Key in S3 bucket:
SSM Parameter names can include only the following symbols and letters: a-zA-Z0-9_.-/
@@ -136,6 +140,7 @@ def get_entry_name(argument, entry_name):
def get_entry_age(entry_name):
+
"""
Args:
entry_name(string): Name of entry to get age for
@@ -180,6 +185,7 @@ def update_cache(entry_name, entry_value):
def get_entry_from_cache(entry_name):
+
"""
Gets entry value from the cache
diff --git a/lambda_cache/exceptions.py b/lambda_cache/exceptions.py
index 9874911..16545a6 100644
--- a/lambda_cache/exceptions.py
+++ b/lambda_cache/exceptions.py
@@ -1,11 +1,17 @@
class LambdaCacheError(Exception):
- """Base class for exceptions in this module."""
+
+ """
+ Base class for exceptions in this module.
+ """
pass
class ArgumentTypeNotSupportedError(LambdaCacheError):
- """Raised when Argument is not supported by the function."""
+
+ """
+ Raised when Argument is not supported by the function.
+ """
def __init__(self, message):
self.message = message
@@ -13,7 +19,10 @@ def __init__(self, message):
class NoEntryNameError(LambdaCacheError):
- """Raised when No entry_name is provided."""
+
+ """
+ Raised when No entry_name is provided.
+ """
def __init__(self, message=False):
self.message = "No entry_name provided"
@@ -21,6 +30,11 @@ def __init__(self, message=False):
class InvalidS3UriError(LambdaCacheError):
+
+ """
+ s3Uri provided in invalid format
+ """
+
def __init__(self, invalid_uri):
self.message = f"Expected Valid s3uri of the form 's3://bucket-name/path/to/file', given: {invalid_uri}"
self.Code = "InvalidS3UriError"
diff --git a/lambda_cache/s3.py b/lambda_cache/s3.py
index 05a7a3d..35e57a5 100644
--- a/lambda_cache/s3.py
+++ b/lambda_cache/s3.py
@@ -1,8 +1,8 @@
import boto3
from datetime import datetime, timezone
-from .caching_logic import get_decorator, get_value, get_entry_name
-from .exceptions import ArgumentTypeNotSupportedError, InvalidS3UriError
+from .caching_logic import get_decorator, get_value
+from .exceptions import InvalidS3UriError
def cache(
@@ -62,6 +62,7 @@ def get_entry(
def get_object_from_s3(**kwargs):
+
"""
Gets parameter value from the System manager Parameter store
diff --git a/pyproject.toml b/pyproject.toml
index 35a0a6e..60c676b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,11 +1,11 @@
[tool.poetry]
name = "lambda-cache"
-version = "0.8.0"
+version = "0.8.1"
description = "Python utility for simple caching in AWS Lambda functions"
authors = ["keithrozario "]
-documentation = "https://simple-lambda-cache.readthedocs.io/en/latest/"
-repository = "https://github.com/keithrozario/simple_lambda_cache"
-homepage = "https://github.com/keithrozario/simple_lambda_cache"
+documentation = "https://lambda-cache.readthedocs.io/en/latest/"
+repository = "https://github.com/keithrozario/lambda-cache"
+homepage = "https://github.com/keithrozario/lambda-cache"
classifiers=[
"Development Status :: 4 - Beta",
"Intended Audience :: Developers",
diff --git a/tests/acceptance_tests/_test_ssm.py b/tests/acceptance_tests/_test_ssm.py
index a0a8b0f..c891429 100644
--- a/tests/acceptance_tests/_test_ssm.py
+++ b/tests/acceptance_tests/_test_ssm.py
@@ -2,6 +2,7 @@
import boto3
from lambda_cache import ssm
+from datetime import datetime
# this file is packaged in the lambda using serverless.yml
from variables_data import *
@@ -30,19 +31,8 @@ def multi_parameter_2(event, context):
client = boto3.client('ssm')
response = client.put_parameter(
Name=ssm_parameter,
- Value='string',
- Type='String'|'StringList'|'SecureString',
- KeyId='string',
- Overwrite=True|False,
- AllowedPattern='string',
- Tags=[
- {
- 'Key': 'string',
- 'Value': 'string'
- },
- ],
- Tier='Standard'|'Advanced'|'Intelligent-Tiering',
- Policies='string'
-)
+ Value=datetime.now().isoformat(),
+ Type='String',
+ Overwrite=True)
return generic_return(message)
\ No newline at end of file
diff --git a/tests/acceptance_tests/serverless.yml b/tests/acceptance_tests/serverless.yml
index c9c7484..b30f959 100644
--- a/tests/acceptance_tests/serverless.yml
+++ b/tests/acceptance_tests/serverless.yml
@@ -19,6 +19,7 @@ provider:
Action:
- ssm:GetParameter
- ssm:GetParameters
+ - ssm:PutParameter
Resource:
- Fn::Join:
- ":"
@@ -89,7 +90,7 @@ package:
- tests/**
functions:
- single_handler:
+ acceptance_test:
handler: _test_ssm.multi_parameter_2
layers:
- arn:aws:lambda:ap-southeast-1:908645878701:layer:pylayers-python38-defaultlambda-cache:1