Skip to content

Commit

Permalink
Deps lock by justbldwn (#8408)
Browse files Browse the repository at this point in the history
* ✨ adding installed_packages.json functionality

* ✅ update test_simple_dependency_deps test

* 📝 adding changelog for deps feature via changie

* ✨ restructure deps command, include lock/add

* ✅ add new deps event types to sample_values

* ✅ fix test_simple_dependency_deps test

* 🐛 attempting to fix cli commands

* 🐛 convert dbt deps to dbt deps install

also leave dbt deps as just a new click group

* ✅ update test_command_mutually_exclusive_option

change deps command to deps install

* ✅ update functional tests from deps > deps install

* ✅ change missing deps to deps install

* ✅ convert adapter tests to deps install from deps

* move back to deps and merge more with main

* fix-unittest

* add hash

* foramt yml and update command structure

* nits

* add new param

* nits

* nits

* nits

* fix_tests

* pr_feedback

* nits

* nits

* move_check

* Update Features-20230125-165933.yaml

---------

Co-authored-by: Justin Baldwin <91483530+justbldwn@users.noreply.github.com>
  • Loading branch information
ChenyuLInx and justbldwn authored Oct 10, 2023
1 parent 70b2e15 commit 549dbf3
Show file tree
Hide file tree
Showing 14 changed files with 1,390 additions and 903 deletions.
6 changes: 6 additions & 0 deletions .changes/unreleased/Features-20230125-165933.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Features
body: add log file of installed packages via dbt deps
time: 2023-01-25T16:59:33.786304-05:00
custom:
Author: jusbaldw ChenyuLInx
Issue: "6643"
32 changes: 29 additions & 3 deletions core/dbt/cli/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,13 +454,40 @@ def debug(ctx, **kwargs):
@p.project_dir
@p.target
@p.vars
@p.source
@p.dry_run
@p.lock
@p.upgrade
@p.add_package
@requires.postflight
@requires.preflight
@requires.unset_profile
@requires.project
def deps(ctx, **kwargs):
"""Pull the most recent version of the dependencies listed in packages.yml"""
task = DepsTask(ctx.obj["flags"], ctx.obj["project"])
"""Install dbt packages specified.
In the following case, a new `package-lock.yml` will be generated and the packages are installed:
- user updated the packages.yml
- user specify the flag --update, which means for packages that are specified as a
range, dbt-core will try to install the newer version
Otherwise, deps will use `package-lock.yml` as source of truth to install packages.
There is a way to add new packages by providing an `--add-package` flag to deps command
which will allow user to specify a package they want to add in the format of packagename@version.
"""
flags = ctx.obj["flags"]
if flags.ADD_PACKAGE:
if not flags.ADD_PACKAGE["version"] and flags.SOURCE != "local":
raise BadOptionUsage(
message=f"Version is required in --add-package when a package when source is {flags.SOURCE}",
option_name="--add-package",
)
else:
if flags.DRY_RUN:
raise BadOptionUsage(
message="Invalid flag `--dry-run` when not using `--add-package`.",
option_name="--dry-run",
)
task = DepsTask(flags, ctx.obj["project"])
results = task.run()
success = task.interpret_results(results)
return results, success
Expand Down Expand Up @@ -555,7 +582,6 @@ def list(ctx, **kwargs):
def parse(ctx, **kwargs):
"""Parses the project and provides information on performance"""
# manifest generation and writing happens in @requires.manifest

return ctx.obj["manifest"], True


Expand Down
20 changes: 20 additions & 0 deletions core/dbt/cli/option_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,26 @@ def convert(self, value, param, ctx):
self.fail(f"String '{value}' is not valid YAML", param, ctx)


class Package(ParamType):
"""The Click STRING type. Converts string into dict with package name and version.
Example package:
package-name@1.0.0
package-name
"""

name = "NewPackage"

def convert(self, value, param, ctx):
# assume non-string values are a problem
if not isinstance(value, str):
self.fail(f"Cannot load Package from type {type(value)}", param, ctx)
try:
package_name, package_version = value.split("@")
return {"name": package_name, "version": package_version}
except ValueError:
return {"name": value, "version": None}


class WarnErrorOptionsType(YAML):
"""The Click WarnErrorOptions type. Converts YAML strings into objects."""

Expand Down
38 changes: 37 additions & 1 deletion core/dbt/cli/params.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,16 @@

import click
from dbt.cli.options import MultiOption
from dbt.cli.option_types import YAML, ChoiceTuple, WarnErrorOptionsType
from dbt.cli.option_types import YAML, ChoiceTuple, WarnErrorOptionsType, Package
from dbt.cli.resolvers import default_project_dir, default_profiles_dir
from dbt.version import get_version_information

add_package = click.option(
"--add-package",
help="Add a package to current package spec, specify it as package-name@version. Change the source with --source flag.",
envvar=None,
type=Package(),
)
args = click.option(
"--args",
envvar=None,
Expand Down Expand Up @@ -77,6 +83,14 @@
hidden=True,
)

dry_run = click.option(
"--dry-run",
envvar=None,
help="Option to run `dbt deps --add-package` without updating package-lock.yml file.",
is_flag=True,
)


enable_legacy_logger = click.option(
"--enable-legacy-logger/--no-enable-legacy-logger",
envvar="DBT_ENABLE_LEGACY_LOGGER",
Expand Down Expand Up @@ -127,6 +141,13 @@
default="eager",
)

lock = click.option(
"--lock",
envvar=None,
help="Generate the package-lock.yml file without install the packages.",
is_flag=True,
)

log_cache_events = click.option(
"--log-cache-events/--no-log-cache-events",
help="Enable verbose logging for relational cache events to help when debugging.",
Expand Down Expand Up @@ -465,6 +486,14 @@
is_flag=True,
)

source = click.option(
"--source",
envvar=None,
help="Source to download page from, must be one of hub, git, or local. Defaults to hub.",
type=click.Choice(["hub", "git", "local"], case_sensitive=True),
default="hub",
)

static = click.option(
"--static",
help="Generate an additional static_index.html with manifest and catalog built-in.",
Expand Down Expand Up @@ -540,6 +569,13 @@
type=click.Path(),
)

upgrade = click.option(
"--upgrade",
envvar=None,
help="Upgrade packages to the latest version.",
is_flag=True,
)

debug_connection = click.option(
"--connection",
envvar=None,
Expand Down
28 changes: 18 additions & 10 deletions core/dbt/config/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,12 @@

from dbt.flags import get_flags
from dbt import deprecations
from dbt.constants import DEPENDENCIES_FILE_NAME, PACKAGES_FILE_NAME
from dbt.clients.system import path_exists, resolve_path_from_base, load_file_contents
from dbt.constants import (
DEPENDENCIES_FILE_NAME,
PACKAGES_FILE_NAME,
PACKAGE_LOCK_HASH_KEY,
)
from dbt.clients.system import path_exists, load_file_contents
from dbt.clients.yaml_helper import load_yaml_text
from dbt.contracts.connection import QueryComment
from dbt.exceptions import (
Expand Down Expand Up @@ -94,16 +98,17 @@ def _load_yaml(path):
return load_yaml_text(contents)


def load_yml_dict(file_path):
ret = {}
if path_exists(file_path):
ret = _load_yaml(file_path) or {}
return ret


def package_and_project_data_from_root(project_root):
package_filepath = resolve_path_from_base(PACKAGES_FILE_NAME, project_root)
dependencies_filepath = resolve_path_from_base(DEPENDENCIES_FILE_NAME, project_root)

packages_yml_dict = {}
dependencies_yml_dict = {}
if path_exists(package_filepath):
packages_yml_dict = _load_yaml(package_filepath) or {}
if path_exists(dependencies_filepath):
dependencies_yml_dict = _load_yaml(dependencies_filepath) or {}
packages_yml_dict = load_yml_dict(f"{project_root}/{PACKAGES_FILE_NAME}")
dependencies_yml_dict = load_yml_dict(f"{project_root}/{DEPENDENCIES_FILE_NAME}")

if "packages" in packages_yml_dict and "packages" in dependencies_yml_dict:
msg = "The 'packages' key cannot be specified in both packages.yml and dependencies.yml"
Expand All @@ -127,6 +132,8 @@ def package_config_from_data(packages_data: Dict[str, Any]) -> PackageConfig:
if not packages_data:
packages_data = {"packages": []}

if PACKAGE_LOCK_HASH_KEY in packages_data:
packages_data.pop(PACKAGE_LOCK_HASH_KEY)
try:
PackageConfig.validate(packages_data)
packages = PackageConfig.from_dict(packages_data)
Expand Down Expand Up @@ -548,6 +555,7 @@ def from_project_root(
packages_specified_path,
) = package_and_project_data_from_root(project_root)
selectors_dict = selector_data_from_root(project_root)

return cls.from_dicts(
project_root=project_root,
project_dict=project_dict,
Expand Down
2 changes: 2 additions & 0 deletions core/dbt/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
DBT_PROJECT_FILE_NAME = "dbt_project.yml"
PACKAGES_FILE_NAME = "packages.yml"
DEPENDENCIES_FILE_NAME = "dependencies.yml"
PACKAGE_LOCK_FILE_NAME = "package-lock.yml"
MANIFEST_FILE_NAME = "manifest.json"
SEMANTIC_MANIFEST_FILE_NAME = "semantic_manifest.json"
PARTIAL_PARSE_FILE_NAME = "partial_parse.msgpack"
PACKAGE_LOCK_HASH_KEY = "sha1_hash"
12 changes: 12 additions & 0 deletions core/dbt/deps/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,15 @@ def resolve_packages(
resolved = final.resolved()
_check_for_duplicate_project_names(resolved, project, renderer)
return resolved


def resolve_lock_packages(packages: List[PackageContract]) -> List[PinnedPackage]:
lock_packages = PackageListing.from_contracts(packages)
final = PackageListing()

for package in lock_packages:
final.incorporate(package)

resolved = final.resolved()

return resolved
42 changes: 42 additions & 0 deletions core/dbt/events/types.proto
Original file line number Diff line number Diff line change
Expand Up @@ -1586,6 +1586,48 @@ message NoNodesForSelectionCriteriaMsg {
NoNodesForSelectionCriteria data = 2;
}

// M031
message DepsLockUpdating{
string lock_filepath = 1;
}

message DepsLockUpdatingMsg{
EventInfo info = 1;
DepsLockUpdating data = 2;
}

// M032
message DepsAddPackage{
string package_name = 1;
string version = 2;
string packages_filepath = 3;
}

message DepsAddPackageMsg{
EventInfo info = 1;
DepsAddPackage data = 2;
}

//M033
message DepsFoundDuplicatePackage{
map<string, string> removed_package = 1;
}

message DepsFoundDuplicatePackageMsg{
EventInfo info = 1;
DepsFoundDuplicatePackage data = 2;
}

//M034
message DepsVersionMissing{
string source = 1;
}

message DepsVersionMissingMsg{
EventInfo info = 1;
DepsVersionMissing data = 2;
}

// Q - Node execution

// Q001
Expand Down
24 changes: 24 additions & 0 deletions core/dbt/events/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -1504,6 +1504,30 @@ def message(self) -> str:
return f"The selection criterion '{self.spec_raw}' does not match any nodes"


class DepsLockUpdating(InfoLevel):
def code(self):
return "M031"

def message(self) -> str:
return f"Updating lock file in file path: {self.lock_filepath}"


class DepsAddPackage(InfoLevel):
def code(self):
return "M032"

def message(self) -> str:
return f"Added new package {self.package_name}@{self.version} to {self.packages_filepath}"


class DepsFoundDuplicatePackage(InfoLevel):
def code(self):
return "M033"

def message(self) -> str:
return f"Found duplicate package in packages.yml, removing: {self.removed_package}"


# =======================================================
# Q - Node execution
# =======================================================
Expand Down
1,781 changes: 901 additions & 880 deletions core/dbt/events/types_pb2.py

Large diffs are not rendered by default.

Loading

0 comments on commit 549dbf3

Please sign in to comment.