Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: parse JUJU_* env in one place #1313

Merged
merged 26 commits into from
Aug 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
8b7eb0e
chore: poc parsing juju env in one place
IronCore864 Aug 9, 2024
384fa9c
feat: parse all juju env vars in one place
IronCore864 Aug 15, 2024
4f146be
chore: add some more env vars that are used in models
IronCore864 Aug 15, 2024
2f4694a
chore: update juju version and model
IronCore864 Aug 15, 2024
a21a204
chore: update juju version to use juju context and update model
IronCore864 Aug 15, 2024
239a595
test: fix ut
IronCore864 Aug 15, 2024
a296905
chore: refactor after self review
IronCore864 Aug 15, 2024
c7f7104
chore: add ut and lint
IronCore864 Aug 15, 2024
d9981e5
chore: add ut and lint
IronCore864 Aug 15, 2024
aad4788
chore: add jujuversion from environ back
IronCore864 Aug 15, 2024
982a2f7
chore: remove JujuVersion.from_context
IronCore864 Aug 15, 2024
f477b51
chore: refactor according to discussion and code review
IronCore864 Aug 16, 2024
aced15d
chore: some final refactor after self review
IronCore864 Aug 19, 2024
b77f23e
Merge branch 'main' into parse-juju-env-vars
IronCore864 Aug 20, 2024
e8b44d2
chore: refactor according to code review comments
IronCore864 Aug 20, 2024
0d8fffc
chore: some refactor after self review and testing
IronCore864 Aug 20, 2024
fef2b9d
chore: moving JujuVersion back to ops/jujuversion.py
IronCore864 Aug 21, 2024
4020b6f
chore: add juju_debug_at to _JujuContext and some refactor
IronCore864 Aug 21, 2024
1d3d07d
chore: move tests for _JujuContext to a new file
IronCore864 Aug 21, 2024
a440143
chore: fix tests after refactoring
IronCore864 Aug 21, 2024
b9cfbdc
chore: fix tests and backward compatibility
IronCore864 Aug 21, 2024
922dffc
chore: fix tests and backward compatibility
IronCore864 Aug 21, 2024
9f702d1
Merge branch 'parse-juju-env-vars' of github.com:IronCore864/operator…
IronCore864 Aug 21, 2024
deef8c8
chore: fix tests and backward compatibility
IronCore864 Aug 21, 2024
4733687
Merge branch 'parse-juju-env-vars' of github.com:IronCore864/operator…
IronCore864 Aug 21, 2024
ccd3acb
chore: minor refactor
IronCore864 Aug 22, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 2 additions & 9 deletions ops/framework.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import keyword
import logging
import marshal
import os
import pathlib
import pdb
import re
Expand Down Expand Up @@ -604,6 +603,7 @@ def __init__(
meta: 'charm.CharmMeta',
model: 'Model',
event_name: Optional[str] = None,
juju_debug_at: Optional[Set[str]] = None,
):
super().__init__(self, None)

Expand All @@ -618,7 +618,6 @@ def __init__(
if event_name:
event_name = event_name.replace('-', '_')
self._event_name = event_name

self.meta = meta
self.model = model
# [(observer_path, method_name, parent_path, event_key)]
Expand Down Expand Up @@ -650,13 +649,7 @@ def __init__(

# Flag to indicate that we already presented the welcome message in a debugger breakpoint
self._breakpoint_welcomed: bool = False

# Parse the env var once, which may be used multiple times later
debug_at = os.environ.get('JUJU_DEBUG_AT')
if debug_at:
self._juju_debug_at = {x.strip() for x in debug_at.split(',')}
else:
self._juju_debug_at: Set[str] = set()
self._juju_debug_at = juju_debug_at or set()

def set_breakpointhook(self) -> Optional[Any]:
"""Hook into ``sys.breakpointhook`` so the builtin ``breakpoint()`` works as expected.
Expand Down
229 changes: 229 additions & 0 deletions ops/jujucontext.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
# Copyright 2024 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""A helper to work with the Juju context and version."""

import dataclasses
from pathlib import Path
from typing import Any, Mapping, Optional, Set

from ops.jujuversion import JujuVersion


@dataclasses.dataclass(frozen=True)
class _JujuContext:
"""_JujuContext collects information from environment variables named 'JUJU_*'.

Source: https://juju.is/docs/juju/charm-environment-variables.
The HookVars function: https://github.com/juju/juju/blob/3.6/worker/uniter/runner/context/context.go#L1398.
Only a subset of the above source, because these are what are used in ops.
"""

action_name: Optional[str] = None
"""The action's name.
IronCore864 marked this conversation as resolved.
Show resolved Hide resolved

For example 'backup' (from JUJU_ACTION_NAME).
"""

action_uuid: Optional[str] = None
"""The action's uuid.

For example '1' (from JUJU_ACTION_UUID).
"""

charm_dir: Path = dataclasses.field(
default_factory=lambda: Path(f'{__file__}/../../..').resolve()
)
"""The root directory of the charm where it is running.

For example '/var/lib/juju/agents/unit-bare-0/charm' (from JUJU_CHARM_DIR).

If JUJU_CHARM_DIR is None or set to an empty string, use Path(f'{__file__}/../../..') as
default (assuming the '$JUJU_CHARM_DIR/lib/op/main.py' structure).
"""

debug: bool = False
"""Debug mode.

If true, write logs to stderr as well as to juju-log (from JUJU_DEBUG).
"""

debug_at: Set[str] = dataclasses.field(default_factory=set)
"""Where you want to stop when debugging.

For example 'all' (from JUJU_DEBUG_AT).
"""

dispatch_path: str = ''
"""The dispatch path in the format of 'actions/do-something'.

For example 'hooks/workload-pebble-ready' (from JUJU_DISPATCH_PATH).
"""

model_name: str = ''
"""The name of the model.

For example 'foo' (from JUJU_MODEL_NAME).
"""

model_uuid: str = ''
"""The uuid of the model.

For example 'cdac5656-2423-4388-8f30-41854b4cca7d' (from JUJU_MODEL_UUID).
"""

notice_id: Optional[str] = None
"""The ID of the notice.

For example '1', (from JUJU_NOTICE_ID).
"""

notice_key: Optional[str] = None
"""The key of the notice.

For example 'example.com/a', (from JUJU_NOTICE_KEY).
"""

notice_type: Optional[str] = None
"""The type of the notice.

For example 'custom' (from JUJU_NOTICE_TYPE).
"""

pebble_check_name: Optional[str] = None
"""The name of the pebble check.

For example 'http-check' (from JUJU_PEBBLE_CHECK_NAME).
"""

relation_departing_unit_name: Optional[str] = None
"""The unit that is departing a relation.

For example 'remote/42' (from JUJU_DEPARTING_UNIT).
"""

relation_name: Optional[str] = None
"""The name of the relation.

For example 'database' (from JUJU_RELATION).
"""

relation_id: Optional[int] = None
"""The id of the relation.

For example 1 (integer) if the original environment variable's value is 'database:1'
(from JUJU_RELATION_ID).
"""

remote_app_name: Optional[str] = None
"""The name of the remote app.

For example 'remoteapp1' (from JUJU_REMOTE_APP).
"""

remote_unit_name: Optional[str] = None
"""The name of the remote unit.

For example 'remoteapp1/0' (from JUJU_REMOTE_UNIT).
"""

secret_id: Optional[str] = None
"""The ID of the secret.

For example 'secret:dcc7aa9c-7202-4da6-8d5f-0fbbaa4e1a41' (from JUJU_SECRET_ID).
"""

secret_label: Optional[str] = None
"""The label of the secret.

For example 'db-password' (from JUJU_SECRET_LABEL).
"""

secret_revision: Optional[int] = None
"""The revision of the secret.

For example 42 (integer) (from JUJU_SECRET_REVISION).
"""

storage_name: Optional[str] = None
"""The storage name.

For example 'my-storage' if the original environment variable's value is 'my-storage/1'
(from JUJU_STORAGE_ID).
"""

unit_name: str = ''
"""The name of the unit.

For example 'myapp/0' (from JUJU_UNIT_NAME).
"""

version: JujuVersion = dataclasses.field(default_factory=lambda: JujuVersion('0.0.0'))
"""The version of Juju.

For example '3.4.0' (from JUJU_VERSION).
"""

workload_name: Optional[str] = None
"""The name of the workload.

For example 'workload' (from JUJU_WORKLOAD_NAME).
"""

@classmethod
def from_dict(cls, env: Mapping[str, Any]) -> '_JujuContext':
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Design question:

The half-baked idea I imagined was that from_dict returns a Union of types, and the caller always uses a match expression on the result. It would imply py3.10 (ubuntu 22.04) which isn't great, but I wonder (a) if it's a feasible design goal in the first place, and (b) if we should plan the design today with match in mind.

I.e. it would be great if this new class brought significant new added value over poking into os.environ directly.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If I understand correctly, there is already some similar logic in main.py, specifically, in _get_event_args, where it tries to match the type of the event then pull related environment variables.

I think it would be great if we could somehow integrate it with JujuContext, maybe in the future we can build on top of this PR and implement that.

Regarding the scope of this PR, we had a short discussion today in the daily and we think this PR serves two purposes: 1, parse all ENV vars from a single place (previously it was scattered across main.py and framework.py); and 2, provide some unified object so that others can build on top of it if they want to implement some experiments.

So, for now, I'm sorry that there is no implementation of the event type matching thingy and we are going to merge this _JujuContext. Thanks!

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The half-baked idea I imagined was that from_dict returns a Union of types, and the caller always uses a match expression on the result.

If the intention is to find out which event Juju has emitted, then it's much simpler to just look at that directly rather than trying to match against which context variables have been set. If the intention is to get back a collection of types (e.g. which fields are set are the same with different events - start and stop would be setting the same) then I'm not sure I really see the use-case.

I don't think it's good to have a hierarchy of context objects when there's already a 1:1 mapping of what that would be in the hierarchy of event classes. A SecretChangedJujuContext would only be useful for SecretChangedEvent, for example.

As @IronCore864 mentioned, _get_event_args provides the "given a context, what are the arguments needed to create an event object". It would be simple enough for someone to subclass _JujuContext to add a to_event method (or whatever), if they wanted to build an "alt ops" package.

What would be most useful in that sort of situation, I believe, is being able to create SecretChangedEvent (and on on) objects without opting in to the whole framework/handle system (and registering, in particular). My feeling is that's what we could provide (not this cycle) to continue making it easier for people to explore alternative ways to do event handling without re-implementing everything.

I.e. it would be great if this new class brought significant new added value over poking into os.environ directly.

"significant" is subjective, of course, but I think there is value in not just collecting all the uses together as @IronCore864 mentioned, but also doing all the conversion so that you have a bunch of Python objects rather than a bunch of strings. With this, plus the similar hook tool wrapping in the model/model backend (the border of those could be a bit cleaner) that's the main interface with Juju (other than Pebble).

return _JujuContext(
action_name=env.get('JUJU_ACTION_NAME') or None,
action_uuid=env.get('JUJU_ACTION_UUID') or None,
charm_dir=(
Path(env['JUJU_CHARM_DIR']).resolve()
if env.get('JUJU_CHARM_DIR')
else Path(f'{__file__}/../../..').resolve()
),
debug='JUJU_DEBUG' in env,
debug_at=(
{x.strip() for x in env['JUJU_DEBUG_AT'].split(',')}
if env.get('JUJU_DEBUG_AT')
else set()
),
dispatch_path=env.get('JUJU_DISPATCH_PATH', ''),
model_name=env.get('JUJU_MODEL_NAME', ''),
model_uuid=env.get('JUJU_MODEL_UUID', ''),
notice_id=env.get('JUJU_NOTICE_ID') or None,
notice_key=env.get('JUJU_NOTICE_KEY') or None,
notice_type=env.get('JUJU_NOTICE_TYPE') or None,
pebble_check_name=env.get('JUJU_PEBBLE_CHECK_NAME') or None,
relation_departing_unit_name=env.get('JUJU_DEPARTING_UNIT') or None,
relation_name=env.get('JUJU_RELATION') or None,
relation_id=(
int(env['JUJU_RELATION_ID'].split(':')[-1])
if env.get('JUJU_RELATION_ID')
else None
),
remote_app_name=env.get('JUJU_REMOTE_APP') or None,
remote_unit_name=env.get('JUJU_REMOTE_UNIT') or None,
secret_id=env.get('JUJU_SECRET_ID') or None,
secret_label=env.get('JUJU_SECRET_LABEL') or None,
secret_revision=(
int(env['JUJU_SECRET_REVISION']) if env.get('JUJU_SECRET_REVISION') else None
),
storage_name=(
env.get('JUJU_STORAGE_ID', '').split('/')[0]
if env.get('JUJU_STORAGE_ID')
else None
),
unit_name=env.get('JUJU_UNIT_NAME', ''),
version=JujuVersion(env['JUJU_VERSION']),
workload_name=env.get('JUJU_WORKLOAD_NAME') or None,
)
Loading
Loading