diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c2894a98b..c5b8b543fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -238,7 +238,12 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon + + - name: Install crun + run: | + sudo curl -L -o /usr/local/bin/crun https://github.com/containers/crun/releases/download/1.6/crun-1.6-linux-amd64 + sudo chmod +x /usr/local/bin/crun - name: Build and install Apptainer run: | @@ -268,7 +273,12 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session conmon + + - name: Install crun + run: | + sudo curl -L -o /usr/local/bin/crun https://github.com/containers/crun/releases/download/1.6/crun-1.6-linux-amd64 + sudo chmod +x /usr/local/bin/crun - name: Build and install Apptainer run: | @@ -314,7 +324,12 @@ jobs: - name: Fetch deps if: env.run_tests - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon + + - name: Install crun + run: | + sudo curl -L -o /usr/local/bin/crun https://github.com/containers/crun/releases/download/1.6/crun-1.6-linux-amd64 + sudo chmod +x /usr/local/bin/crun - name: Fetch gocryptfs run: wget -O gocryptfs.tar.gz https://github.com/rfjakob/gocryptfs/releases/download/v2.3/gocryptfs_v2.3_linux-static_amd64.tar.gz && sudo tar xzvf gocryptfs.tar.gz -C /usr/local/bin gocryptfs @@ -362,6 +377,7 @@ jobs: retention-days: 7 check_pkg_no_buildcfg: + if: ${{ github.base_ref != 'oci-action' }} name: check_pkg_no_buildcfg runs-on: ubuntu-22.04 steps: diff --git a/.gitignore b/.gitignore index e8e1026cca..ebaba871bc 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,12 @@ pkg/library/client/test[0-9]* /debian LICENSE_DEPENDENCIES.csv + +# VSCode debugging build targets +__debug_bin +*/__debug_bin +*/*/__debug_bin +*/*/*/__debug_bin +*/*/*/*/__debug_bin +*/*/*/*/*/__debug_bin +*/*/*/*/*/*/__debug_bin diff --git a/CHANGELOG.md b/CHANGELOG.md index f72ae6ca1f..7d118b3e6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ For older changes see the [archived Singularity change log](https://github.com/a working directory, though `--pwd` is still supported for compatibility. - When building RPM, we will now use `/var/lib/apptainer` (rather than `/var/apptainer`) to store local state files. +- The `apptainer oci` command group now uses `crun`, when available, or otherwise + `runc` to manage containers. +- The `apptainer oci` flags `--sync-socket`, `--empty-process`, and + `--timeout` have been removed. +- `sessiondir maxsize` in `apptainer.conf` now defaults to 64 MiB for new + installations. This is an increase from 16 MiB in prior versions. ### New Features & Functionality @@ -20,6 +26,64 @@ For older changes see the [archived Singularity change log](https://github.com/a of the logged-in user, if available. - New option `--warn-unused-build-args` is provided to output warnings rather than fatal errors for any unused variables given in --build-arg or --build-arg-file. +- A new `--oci` flag for `run/exec/shell` enables the experimental OCI runtime + mode. This mode: + - Runs OCI container images from an OCI bundle, using `runc` or `crun`. + - Supports `docker://`, `docker-archive:`, `docker-daemon:`, `oci:`, + `oci-archive:` image sources. + - Does not support running Apptainer SIF, SquashFS, or EXT3 images. + - Provides an environment similar to Apptainer's native runtime, running + with `--compat`. + - Supports the following options / flags. Other options are not yet supported: + - `--fakeroot` for effective root in the container. Requires subuid/subgid + mappings. + - Bind mounts via `--bind` or `--mount`. No image mounts. + - Additional namespaces requests with `--net`, `--uts`, `--user`. + - Container environment variables via `--env`, `--env-file`, and + `APPTAINERENV_` host env vars. + - `--rocm` to bind ROCm GPU libraries and devices into the container. + - `--nv` to bind Nvidia driver / basic CUDA libraries and devices into + the container. + - `--apply-cgroups`, and the `--cpu*`, `--blkio*`, `--memory*`, + `--pids-limit` flags to apply resource limits. +- Added `--device` flag to "action" commands (`run`/`exec`/`shell`) when run in + OCI mode (`--oci`). Currently supports passing one or more (comma-separated) + fully-qualified CDI device names, and those devices will then be made + available inside the container. +- Added `--cdi-dirs` flag to override the default search locations for CDI + json files, allowing, for example, users who don't have root access on their + host machine to nevertheless create CDI mappings (into containers run with + `--fakeroot`, for example). +- OCI mode now supports `--hostname` (requires UTS namespace, therefore this + flag will infer `--uts`). +- OCI mode now supports `--scratch` (shorthand: `-S`) to mount a tmpfs scratch + directory in the container. +- Support `--pwd` in OCI mode. +- OCI mode now supports `--home`. Supplying a single location (e.g. + `--home /myhomedir`) will result in a new tmpfs directory being created at the + specified location inside the container, and that dir being set as the + in-container user's home dir. Supplying two locations separated by a colon + (e.g. `--home /home/user:/myhomedir`) will result in the first location on the + host being bind-mounted as the second location in-container, and set as + the in-container user's home dir. +- OCI mode now handles `--dns` and `resolv.conf` on par with native mode: the + `--dns` flag can be used to pass a comma-separated list of DNS servers that + will be used in the container; if this flag is not used, the container will + use the same `resolv.conf` settings as the host. +- OCI-mode now supports the `--overlay ` flag. `` can be the path to a + writable directory or writable extfs image, in which case changes to the + filesystem will persist across runs of the OCI container. Alternatively, + `--overlay :ro` can be used, where `` is the path to a directory, to + a squashfs image, or to an extfs image, to be mounted as a read-only overlay. + Multiple overlays can be specified, but all but one must be read-only. +- OCI-mode now supports the `--workdir ` option. If this option is + specified, `/tmp` and `/var/tmp` will be mapped, respectively, to + `/tmp` and `/var_tmp` on the host, rather than to tmpfs + storage. If `--scratch ` is used in conjunction with `--workdir`, + scratch directories will be mapped to subdirectories nested under + `/scratch` on the host, rather than to tmpfs storage. +- If kernel does not support unprivileged overlays, OCI-mode will attempt to use + `fuse-overlayfs` and `fusermount` for overlay mounting and unmounting. ### New Features & Functionality diff --git a/INSTALL.md b/INSTALL.md index eb5e687f89..9abd960b1a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -28,9 +28,16 @@ sudo apt-get install -y \ fuse-overlayfs \ fakeroot \ cryptsetup \ - curl wget git + curl wget git \ + conmon crun ``` +_Note_: on Ubuntu 18.04 or Debian 10 leave out `conmon`, `crun`, and +`fuse-overlayfs` because they are not available, or install them from another +source. Leaving out the first two will prevent the `--oci` option from working +and leaving out the third will prevent `--overlay` and `--writable-tmpfs` +options from working without suid mode. + On CentOS/RHEL: ```sh @@ -47,9 +54,12 @@ sudo yum install -y \ fakeroot \ /usr/*bin/fuse2fs \ cryptsetup \ - wget git + wget git \ + conmon crun ``` +_Note - use `runc` instead of `crun` on CentOS/RHEL 7._ + On SLE/openSUSE ```sh @@ -59,9 +69,13 @@ sudo zypper install -y \ libuuid-devel \ openssl-devel \ cryptsetup sysuser-tools \ - gcc go + gcc go \ + conmon crun ``` +_Note - `crun` / `runc` can be omitted if you will not use the `apptainer oci` +commands, or the `--oci` execution mode._ + ## Install Go Apptainer is written in Go, and may require a newer version of Go than is diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md index e698e11f14..7ed457ca8a 100644 --- a/LICENSE_DEPENDENCIES.md +++ b/LICENSE_DEPENDENCIES.md @@ -17,6 +17,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/container-orchestrated-devices/container-device-interface + +**License:** Apache-2.0 + +**License URL:** + ## github.com/containerd/containerd **License:** Apache-2.0 @@ -35,6 +41,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/containers/common + +**License:** Apache-2.0 + +**License URL:** + ## github.com/containers/image/v5 **License:** Apache-2.0 @@ -53,11 +65,11 @@ The dependencies and their licenses are as follows: **License URL:** -## github.com/containers/storage/pkg +## github.com/containers/storage **License:** Apache-2.0 -**License URL:** +**License URL:** ## github.com/coreos/go-iptables/iptables @@ -287,6 +299,18 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/opencontainers/runtime-tools + +**License:** Apache-2.0 + +**License URL:** + +## github.com/opencontainers/selinux + +**License:** Apache-2.0 + +**License URL:** + ## github.com/opencontainers/umoci **License:** Apache-2.0 @@ -521,6 +545,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/fsnotify/fsnotify + +**License:** BSD-3-Clause + +**License URL:** + ## github.com/gogo/protobuf/proto **License:** BSD-3-Clause @@ -623,6 +653,12 @@ The dependencies and their licenses are as follows: **Project URL:** +## golang.org/x/mod/semver + +**License:** BSD-3-Clause + +**Project URL:** + ## golang.org/x/net **License:** BSD-3-Clause @@ -869,6 +905,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/samber/lo + +**License:** MIT + +**License URL:** + ## github.com/secure-systems-lab/go-securesystemslib/dsse **License:** MIT @@ -917,6 +959,12 @@ The dependencies and their licenses are as follows: **Project URL:** +## sigs.k8s.io/yaml + +**License:** MIT + +**Project URL:** + ## github.com/gosimple/slug **License:** MPL-2.0 diff --git a/LICENSE_THIRD_PARTY.md b/LICENSE_THIRD_PARTY.md index a2374b0469..3f827622e9 100644 --- a/LICENSE_THIRD_PARTY.md +++ b/LICENSE_THIRD_PARTY.md @@ -238,6 +238,7 @@ The source files: * `pkg/sypgp/testdata_test.go` * `internal/pkg/util/user/cgo_lookup_unix.go` +* `internal/pkg/util/passwdfile/passwdfile_unix.go` Contain code from the Go project. @@ -288,3 +289,217 @@ The source files: * `internal/app/apptainer/instance_linux.go` Contain code from the docker cli project, under the Apache License, Version 2.0. + +## github.com/containers/podman + +The source files: + +* `internal/app/apptainer/oci_linux.go` +* `internal/app/apptainer/oci_attach_linux.go` +* `internal/app/apptainer/oci_create_linux.go` + +Contain code from the podman project, under the Apache License, Version 2.0. + +```text + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/cmd/internal/cli/action_flags.go b/cmd/internal/cli/action_flags.go index 6947c35b66..ac7c3453dd 100644 --- a/cmd/internal/cli/action_flags.go +++ b/cmd/internal/cli/action_flags.go @@ -42,6 +42,8 @@ var ( noMount []string dmtcpLaunch string dmtcpRestart string + device []string + cdiDirs []string isBoot bool isFakeroot bool @@ -95,6 +97,8 @@ var ( ignoreUserns bool underlay bool // whether using underlay instead of overlay + + ociRuntime bool ) // --app @@ -114,7 +118,7 @@ var actionBindFlag = cmdline.Flag{ DefaultValue: cmdline.StringArray{}, // to allow commas in bind path Name: "bind", ShortHand: "B", - Usage: "a user-bind path specification. spec has the format src[:dest[:opts]], where src and dest are outside and inside paths. If dest is not given, it is set equal to src. Mount options ('opts') may be specified as 'ro' (read-only) or 'rw' (read/write, which is the default). Multiple bind paths can be given by a comma separated list.", + Usage: "a user-bind path specification. spec has the format src[:dest[:opts]], where src and dest are outside and inside paths. If dest is not given, it is set equal to src. Mount options ('opts') may be specified as 'ro' (read-only) or 'rw' (read/write, which is the default). Multiple bind paths can be given by a comma separated list.", EnvKeys: []string{"BIND", "BINDPATH"}, Tag: "", EnvHandler: cmdline.EnvAppendValue, @@ -139,7 +143,7 @@ var actionHomeFlag = cmdline.Flag{ DefaultValue: CurrentUser.HomeDir, Name: "home", ShortHand: "H", - Usage: "a home directory specification. spec can either be a src path or src:dest pair. src is the source path of the home directory outside the container and dest overrides the home directory within the container.", + Usage: "a home directory specification. spec can either be a src path or src:dest pair. src is the source path of the home directory outside the container and dest overrides the home directory within the container.", EnvKeys: []string{"HOME"}, Tag: "", } @@ -175,7 +179,7 @@ var actionWorkdirFlag = cmdline.Flag{ DefaultValue: "", Name: "workdir", ShortHand: "W", - Usage: "working directory to be used for /tmp, /var/tmp and $HOME (if -c/--contain was also used)", + Usage: "working directory to be used for /tmp and /var/tmp (if -c/--contain was also used)", EnvKeys: []string{"WORKDIR"}, Tag: "", } @@ -231,7 +235,7 @@ var actionHostnameFlag = cmdline.Flag{ Value: &hostname, DefaultValue: "", Name: "hostname", - Usage: "set container hostname", + Usage: "set container hostname. Infers --uts.", EnvKeys: []string{"HOSTNAME"}, Tag: "", } @@ -875,6 +879,34 @@ var actionUnderlayFlag = cmdline.Flag{ Hidden: false, } +// --oci +var actionOCIFlag = cmdline.Flag{ + ID: "actionOCI", + Value: &ociRuntime, + DefaultValue: false, + Name: "oci", + Usage: "Launch container with OCI runtime (experimental)", + EnvKeys: []string{"OCI"}, +} + +// --device +var actionDevice = cmdline.Flag{ + ID: "actionDevice", + Value: &device, + DefaultValue: []string{}, + Name: "device", + Usage: "fully-qualified CDI device name(s). A fully-qualified CDI device name consists of a VENDOR, CLASS, and NAME, which are combined as follows: /= (e.g. vendor.com/device=mydevice). Multiple fully-qualified CDI device names can be given as a comma separated list.", +} + +// --cdi-dirs +var actionCdiDirs = cmdline.Flag{ + ID: "actionCdiDirs", + Value: &cdiDirs, + DefaultValue: []string{}, + Name: "cdi-dirs", + Usage: "comma-separated list of directories in which CDI should look for device definition JSON files. If omitted, default will be: /etc/cdi,/var/run/cdi", +} + func init() { addCmdInit(func(cmdManager *cmdline.CommandManager) { cmdManager.RegisterCmd(ExecCmd) @@ -972,5 +1004,8 @@ func init() { cmdManager.RegisterFlagForCmd(&actionIgnoreFakerootCommand, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionIgnoreUsernsFlag, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionUnderlayFlag, actionsInstanceCmd...) + cmdManager.RegisterFlagForCmd(&actionOCIFlag, actionsCmd...) + cmdManager.RegisterFlagForCmd(&actionDevice, actionsCmd...) + cmdManager.RegisterFlagForCmd(&actionCdiDirs, actionsCmd...) }) } diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index 16763e9219..40a1d9eac9 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -24,10 +24,15 @@ import ( "github.com/apptainer/apptainer/internal/pkg/client/oci" "github.com/apptainer/apptainer/internal/pkg/client/oras" "github.com/apptainer/apptainer/internal/pkg/client/shub" - "github.com/apptainer/apptainer/internal/pkg/runtime/launch" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/native" + ocilauncher "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/uri" + "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" + useragent "github.com/apptainer/apptainer/pkg/util/user-agent" + "github.com/containers/image/v5/types" "github.com/spf13/cobra" ) @@ -48,7 +53,12 @@ func getCacheHandle(cfg cache.Config) *cache.Handle { return h } -// actionPreRun will run replaceURIWithImage and will also do the proper path unsetting +// actionPreRun will: +// - run replaceURIWithImage; +// - do the proper path unsetting; +// - and implement flag inferences for: +// --compat +// --hostname func actionPreRun(cmd *cobra.Command, args []string) { // For compatibility - we still set USER_PATH so it will be visible in the // container, and can be used there if needed. USER_PATH is not used by @@ -70,6 +80,11 @@ func actionPreRun(cmd *cobra.Command, args []string) { noUmask = true noEval = true } + + // --hostname requires UTS namespace + if len(hostname) > 0 { + utsNamespace = true + } } func handleOCI(ctx context.Context, imgCache *cache.Handle, cmd *cobra.Command, pullFrom string) (string, error) { @@ -143,6 +158,14 @@ func replaceURIWithImage(ctx context.Context, cmd *cobra.Command, args []string) sylog.Fatalf("failed to create a new image cache handle") } + // The OCI runtime launcher will handle OCI image sources directly. + if ociRuntime { + if oci.IsSupported(t) != t { + sylog.Fatalf("OCI runtime only supports OCI image sources. %s is not supported.", t) + } + return + } + switch t { case uri.Library: image, err = handleLibrary(ctx, imgCache, args[0]) @@ -193,13 +216,21 @@ var ExecCmd = &cobra.Command{ Args: cobra.MinimumNArgs(2), PreRun: actionPreRun, Run: func(cmd *cobra.Command, args []string) { - a := append([]string{"/.singularity.d/actions/exec"}, args[1:]...) + // apptainer exec [args...] + image := args[0] + containerCmd := "/.singularity.d/actions/exec" + containerArgs := args[1:] + // OCI runtime does not use an action script + if ociRuntime { + containerCmd = args[1] + containerArgs = args[2:] + } setVM(cmd) if vm { - execVM(cmd, args[0], a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -221,13 +252,30 @@ var ShellCmd = &cobra.Command{ sylog.Warningf("Parameters to shell command are ignored") } - a := []string{"/.singularity.d/actions/shell"} + // apptainer shell + image := args[0] + containerCmd := "/.singularity.d/actions/shell" + containerArgs := []string{} + // OCI runtime does not use an action script, but must match behavior. + // See - internal/pkg/util/fs/files/action_scripts.go (case shell). + if ociRuntime { + // APPTAINER_SHELL or --shell has priority + if shellPath != "" { + containerCmd = shellPath + // Clear the shellPath - not handled internally by the OCI runtime, as we exec directly without an action script. + shellPath = "" + } else { + // Otherwise try to exec /bin/bash --norc, falling back to /bin/sh + containerCmd = "/bin/sh" + containerArgs = []string{"-c", "test -x /bin/bash && PS1='Apptainer> ' exec /bin/bash --norc || PS1='Apptainer> ' exec /bin/sh"} + } + } setVM(cmd) if vm { - execVM(cmd, args[0], a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -245,13 +293,20 @@ var RunCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), PreRun: actionPreRun, Run: func(cmd *cobra.Command, args []string) { - a := append([]string{"/.singularity.d/actions/run"}, args[1:]...) + // apptainer run [args...] + image := args[0] + containerCmd := "/.singularity.d/actions/run" + containerArgs := args[1:] + // OCI runtime does not use an action script + if ociRuntime { + containerCmd = "" + } setVM(cmd) if vm { - execVM(cmd, args[0], a) + execVM(cmd, args[0], containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -269,13 +324,15 @@ var TestCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), PreRun: actionPreRun, Run: func(cmd *cobra.Command, args []string) { - a := append([]string{"/.singularity.d/actions/test"}, args[1:]...) - setVM(cmd) + // apptainer test [args...] + image := args[0] + containerCmd := "/.singularity.d/actions/test" + containerArgs := args[1:] if vm { - execVM(cmd, args[0], a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -286,8 +343,8 @@ var TestCmd = &cobra.Command{ Example: docs.RunTestExample, } -func launchContainer(cmd *cobra.Command, image string, args []string, instanceName string) error { - ns := launch.Namespaces{ +func launchContainer(cmd *cobra.Command, image string, containerCmd string, containerArgs []string, instanceName string) error { + ns := launcher.Namespaces{ User: userNamespace, UTS: utsNamespace, PID: pidNamespace, @@ -309,63 +366,90 @@ func launchContainer(cmd *cobra.Command, image string, args []string, instanceNa return err } - opts := []launch.Option{ - launch.OptWritable(isWritable), - launch.OptWritableTmpfs(isWritableTmpfs), - launch.OptOverlayPaths(overlayPath), - launch.OptScratchDirs(scratchPath), - launch.OptWorkDir(workdirPath), - launch.OptHome( + opts := []launcher.Option{ + launcher.OptWritable(isWritable), + launcher.OptWritableTmpfs(isWritableTmpfs), + launcher.OptOverlayPaths(overlayPath), + launcher.OptScratchDirs(scratchPath), + launcher.OptWorkDir(workdirPath), + launcher.OptHome( homePath, cmd.Flag(actionHomeFlag.Name).Changed, noHome, ), - launch.OptMounts(bindPaths, mounts, fuseMount), - launch.OptNoMount(noMount), - launch.OptNvidia(nvidia, nvCCLI), - launch.OptNoNvidia(noNvidia), - launch.OptRocm(rocm), - launch.OptNoRocm(noRocm), - launch.OptContainLibs(containLibsPath), - launch.OptEnv(apptainerEnv, apptainerEnvFile, isCleanEnv), - launch.OptNoEval(noEval), - launch.OptNamespaces(ns), - launch.OptNetwork(network, networkArgs), - launch.OptHostname(hostname), - launch.OptDNS(dns), - launch.OptCaps(addCaps, dropCaps), - launch.OptAllowSUID(allowSUID), - launch.OptKeepPrivs(keepPrivs), - launch.OptNoPrivs(noPrivs), - launch.OptSecurity(security), - launch.OptNoUmask(noUmask), - launch.OptCgroupsJSON(cgJSON), - launch.OptConfigFile(configurationFile), - launch.OptShellPath(shellPath), - launch.OptCwdPath(cwdPath), - launch.OptFakeroot(isFakeroot), - launch.OptBoot(isBoot), - launch.OptNoInit(noInit), - launch.OptContain(isContained), - launch.OptContainAll(isContainAll), - launch.OptAppName(appName), - launch.OptKeyInfo(ki), - launch.OptCacheDisabled(disableCache), - launch.OptDMTCPLaunch(dmtcpLaunch), - launch.OptDMTCPRestart(dmtcpRestart), - launch.OptUnsquash(unsquash), - launch.OptIgnoreSubuid(ignoreSubuid), - launch.OptIgnoreFakerootCmd(ignoreFakerootCmd), - launch.OptIgnoreUserns(ignoreUserns), - launch.OptUseBuildConfig(useBuildConfig), - launch.OptTmpDir(tmpDir), - launch.OptUnderlay(underlay), + launcher.OptMounts(bindPaths, mounts, fuseMount), + launcher.OptNoMount(noMount), + launcher.OptNvidia(nvidia, nvCCLI), + launcher.OptNoNvidia(noNvidia), + launcher.OptRocm(rocm), + launcher.OptNoRocm(noRocm), + launcher.OptContainLibs(containLibsPath), + launcher.OptEnv(apptainerEnv, apptainerEnvFile, isCleanEnv), + launcher.OptNoEval(noEval), + launcher.OptNamespaces(ns), + launcher.OptNetwork(network, networkArgs), + launcher.OptHostname(hostname), + launcher.OptDNS(dns), + launcher.OptCaps(addCaps, dropCaps), + launcher.OptAllowSUID(allowSUID), + launcher.OptKeepPrivs(keepPrivs), + launcher.OptNoPrivs(noPrivs), + launcher.OptSecurity(security), + launcher.OptNoUmask(noUmask), + launcher.OptCgroupsJSON(cgJSON), + launcher.OptConfigFile(configurationFile), + launcher.OptShellPath(shellPath), + launcher.OptCwdPath(cwdPath), + launcher.OptFakeroot(isFakeroot), + launcher.OptBoot(isBoot), + launcher.OptNoInit(noInit), + launcher.OptContain(isContained), + launcher.OptContainAll(isContainAll), + launcher.OptAppName(appName), + launcher.OptKeyInfo(ki), + launcher.OptCacheDisabled(disableCache), + launcher.OptDMTCPLaunch(dmtcpLaunch), + launcher.OptDMTCPRestart(dmtcpRestart), + launcher.OptUnsquash(unsquash), + launcher.OptIgnoreSubuid(ignoreSubuid), + launcher.OptIgnoreFakerootCmd(ignoreFakerootCmd), + launcher.OptIgnoreUserns(ignoreUserns), + launcher.OptUseBuildConfig(useBuildConfig), + launcher.OptTmpDir(tmpDir), + launcher.OptUnderlay(underlay), + launcher.OptDevice(device), + launcher.OptCdiDirs(cdiDirs), } - l, err := launch.NewLauncher(opts...) - if err != nil { - return fmt.Errorf("while configuring container: %s", err) + var l launcher.Launcher + + if ociRuntime { + sylog.Debugf("Using OCI runtime launcher.") + + sysCtx := &types.SystemContext{ + OCIInsecureSkipTLSVerify: noHTTPS, + DockerAuthConfig: &dockerAuthConfig, + DockerDaemonHost: dockerHost, + OSChoice: "linux", + AuthFilePath: syfs.DockerConf(), + DockerRegistryUserAgent: useragent.Value(), + } + if noHTTPS { + sysCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) + } + opts = append(opts, launcher.OptSysContext(sysCtx)) + + l, err = ocilauncher.NewLauncher(opts...) + if err != nil { + return fmt.Errorf("while configuring container: %s", err) + } + } else { + sylog.Debugf("Using native runtime launcher.") + l, err = native.NewLauncher(opts...) + if err != nil { + return fmt.Errorf("while configuring container: %s", err) + } } - return l.Exec(cmd.Context(), image, args, instanceName) + return l.Exec(cmd.Context(), image, containerCmd, containerArgs, instanceName) } diff --git a/cmd/internal/cli/build_linux.go b/cmd/internal/cli/build_linux.go index 099164213d..de6c8ac9fa 100644 --- a/cmd/internal/cli/build_linux.go +++ b/cmd/internal/cli/build_linux.go @@ -27,6 +27,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/build" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/remote/endpoint" fakerootConfig "github.com/apptainer/apptainer/internal/pkg/runtime/engine/fakeroot/config" @@ -95,7 +96,7 @@ func fakerootExec(isDeffile, unprivEncrypt bool) { if buildArgs.ignoreUserns { err = errors.New("could not start root-mapped namespace because --ignore-userns is set") } else { - err = fakeroot.UnshareRootMapped(args) + err = fakefake.UnshareRootMapped(args) } if err == nil { // All the work has been done by the child process @@ -173,7 +174,7 @@ func runBuild(cmd *cobra.Command, args []string) { if buildArgs.ignoreFakerootCmd { err = errors.New("fakeroot command is ignored because of --ignore-fakeroot-command") } else { - fakerootPath, err = fakeroot.FindFake() + fakerootPath, err = fakefake.FindFake() } if err != nil { sylog.Infof("fakeroot command not found") diff --git a/cmd/internal/cli/checkpoint.go b/cmd/internal/cli/checkpoint.go index 04f17f0a25..f37a150805 100644 --- a/cmd/internal/cli/checkpoint.go +++ b/cmd/internal/cli/checkpoint.go @@ -166,8 +166,9 @@ var CheckpointInstanceCmd = &cobra.Command{ sylog.Infof("Using checkpoint %q", e.Name()) - a := append([]string{"/.singularity.d/actions/exec"}, dmtcp.CheckpointArgs(port)...) - if err := launchContainer(cmd, "instance://"+args[0], a, ""); err != nil { + containerCmd := "/.singularity.d/actions/exec" + containerArgs := dmtcp.CheckpointArgs(port) + if err := launchContainer(cmd, "instance://"+args[0], containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, diff --git a/cmd/internal/cli/instance_actions_linux.go b/cmd/internal/cli/instance_actions_linux.go index da351533fe..e3a19eecba 100644 --- a/cmd/internal/cli/instance_actions_linux.go +++ b/cmd/internal/cli/instance_actions_linux.go @@ -49,13 +49,14 @@ func instanceAction(cmd *cobra.Command, args []string) { script = "run" killCont = "kill -CONT 1; " } - a := append([]string{killCont + "/.singularity.d/actions/" + script}, args[2:]...) + containerCmd := killCont + "/.singularity.d/actions/" + script + containerArgs := args[2:] setVM(cmd) if vm { - execVM(cmd, image, a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, image, a, name); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, name); err != nil { sylog.Fatalf("%s", err) } diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index 4113e85feb..3d8afe5971 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -10,8 +10,13 @@ package cli import ( + "errors" + "os" + "os/exec" + "github.com/apptainer/apptainer/docs" "github.com/apptainer/apptainer/internal/app/apptainer" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" "github.com/apptainer/apptainer/pkg/cmdline" "github.com/apptainer/apptainer/pkg/sylog" "github.com/spf13/cobra" @@ -32,26 +37,15 @@ var ociBundleFlag = cmdline.Flag{ EnvKeys: []string{"BUNDLE"}, } -// -s|--sync-socket -var ociSyncSocketFlag = cmdline.Flag{ - ID: "ociSyncSocketFlag", - Value: &ociArgs.SyncSocketPath, - DefaultValue: "", - Name: "sync-socket", - ShortHand: "s", - Usage: "specify the path to unix socket for state synchronization", +// -o|--overlay +var ociOverlayFlag = cmdline.Flag{ + ID: "ociOverlayFlag", + Value: &ociArgs.OverlayPaths, + DefaultValue: []string{}, + Name: "overlay", + ShortHand: "o", + Usage: "specify an overlay dir to use in lieu of a writable tmpfs", Tag: "", - EnvKeys: []string{"SYNC_SOCKET"}, -} - -// --empty-process -var ociCreateEmptyProcessFlag = cmdline.Flag{ - ID: "ociCreateEmptyProcessFlag", - Value: &ociArgs.EmptyProcess, - DefaultValue: false, - Name: "empty-process", - Usage: "run container without executing container process (eg: for POD container)", - EnvKeys: []string{"EMPTY_PROCESS"}, } // -l|--log-path @@ -111,16 +105,6 @@ var ociKillForceFlag = cmdline.Flag{ EnvKeys: []string{"FORCE"}, } -// -t|--timeout -var ociKillTimeoutFlag = cmdline.Flag{ - ID: "ociKillTimeoutFlag", - Value: &ociArgs.KillTimeout, - DefaultValue: uint32(0), - Name: "timeout", - ShortHand: "t", - Usage: "timeout in second before killing container", -} - // -f|--from-file var ociUpdateFromFileFlag = cmdline.Flag{ ID: "ociUpdateFromFileFlag", @@ -138,6 +122,7 @@ func init() { cmdManager.RegisterSubCmd(OciCmd, OciStartCmd) cmdManager.RegisterSubCmd(OciCmd, OciCreateCmd) cmdManager.RegisterSubCmd(OciCmd, OciRunCmd) + cmdManager.RegisterSubCmd(OciCmd, OciRunWrappedCmd) cmdManager.RegisterSubCmd(OciCmd, OciDeleteCmd) cmdManager.RegisterSubCmd(OciCmd, OciKillCmd) cmdManager.RegisterSubCmd(OciCmd, OciStateCmd) @@ -149,20 +134,17 @@ func init() { cmdManager.RegisterSubCmd(OciCmd, OciMountCmd) cmdManager.RegisterSubCmd(OciCmd, OciUmountCmd) - cmdManager.SetCmdGroup("create_run", OciCreateCmd, OciRunCmd) + cmdManager.SetCmdGroup("create_run", OciCreateCmd, OciRunCmd, OciRunWrappedCmd) createRunCmd := cmdManager.GetCmdGroup("create_run") cmdManager.RegisterFlagForCmd(&ociBundleFlag, createRunCmd...) - cmdManager.RegisterFlagForCmd(&ociSyncSocketFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociLogPathFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociLogFormatFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociPidFileFlag, createRunCmd...) - cmdManager.RegisterFlagForCmd(&ociCreateEmptyProcessFlag, OciCreateCmd) + cmdManager.RegisterFlagForCmd(&ociOverlayFlag, OciRunWrappedCmd) cmdManager.RegisterFlagForCmd(&ociKillForceFlag, OciKillCmd) cmdManager.RegisterFlagForCmd(&ociKillSignalFlag, OciKillCmd) - cmdManager.RegisterFlagForCmd(&ociKillTimeoutFlag, OciKillCmd) cmdManager.RegisterFlagForCmd(&ociUpdateFromFileFlag, OciUpdateCmd) - cmdManager.RegisterFlagForCmd(&ociSyncSocketFlag, OciStateCmd) }) } @@ -189,6 +171,10 @@ var OciRunCmd = &cobra.Command{ PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { if err := apptainer.OciRun(cmd.Context(), args[0], &ociArgs); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } sylog.Fatalf("%s", err) } }, @@ -198,6 +184,25 @@ var OciRunCmd = &cobra.Command{ Example: docs.OciRunExample, } +// OciRunWrappedCmd is for internal OCI launcher use. +// Executes an oci run, wrapped with preparation / cleanup code. +var OciRunWrappedCmd = &cobra.Command{ + Args: cobra.ExactArgs(1), + DisableFlagsInUseLine: true, + PreRun: CheckRoot, + Run: func(cmd *cobra.Command, args []string) { + if err := apptainer.OciRunWrapped(cmd.Context(), args[0], &ociArgs); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + sylog.Fatalf("%s", err) + } + }, + Use: docs.OciRunWrappedUse, + Hidden: true, +} + // OciStartCmd represents oci start command. var OciStartCmd = &cobra.Command{ Args: cobra.ExactArgs(1), @@ -236,7 +241,6 @@ var OciKillCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - timeout := int(ociArgs.KillTimeout) killSignal := "" if len(args) > 1 && args[1] != "" { killSignal = args[1] @@ -246,7 +250,7 @@ var OciKillCmd = &cobra.Command{ if ociArgs.ForceKill { killSignal = "SIGKILL" } - if err := apptainer.OciKill(args[0], killSignal, timeout); err != nil { + if err := apptainer.OciKill(args[0], killSignal); err != nil { sylog.Fatalf("%s", err) } }, @@ -278,7 +282,7 @@ var OciAttachCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciAttach(cmd.Context(), args[0]); err != nil { + if err := oci.Attach(cmd.Context(), args[0]); err != nil { sylog.Fatalf("%s", err) } }, @@ -326,7 +330,7 @@ var OciPauseCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciPauseResume(args[0], true); err != nil { + if err := apptainer.OciPause(args[0]); err != nil { sylog.Fatalf("%s", err) } }, @@ -342,7 +346,7 @@ var OciResumeCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciPauseResume(args[0], false); err != nil { + if err := apptainer.OciResume(args[0]); err != nil { sylog.Fatalf("%s", err) } }, @@ -358,7 +362,7 @@ var OciMountCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciMount(args[0], args[1]); err != nil { + if err := apptainer.OciMount(cmd.Context(), args[0], args[1]); err != nil { sylog.Fatalf("%s", err) } }, diff --git a/cmd/internal/cli/startvm.go b/cmd/internal/cli/startvm.go index 53d050d99f..b1917bfbff 100644 --- a/cmd/internal/cli/startvm.go +++ b/cmd/internal/cli/startvm.go @@ -32,7 +32,7 @@ func getHypervisorArgs(sifImage, bzImage, initramfs, singAction, cliExtra string return args } -func execVM(cmd *cobra.Command, image string, args []string) { +func execVM(cmd *cobra.Command, image string, containerCmd string, containerArgs []string) { // SIF image we are running sifImage := image @@ -46,8 +46,8 @@ func execVM(cmd *cobra.Command, image string, args []string) { isInternal = true } else { // Get our "action" (run, exec, shell) based on the action script being called - singAction = filepath.Base(args[0]) - cliExtra = strings.Join(args[1:], " ") + singAction = filepath.Base(containerCmd) + cliExtra = strings.Join(containerArgs, " ") } if err := startVM(sifImage, singAction, cliExtra, isInternal); err != nil { diff --git a/cmd/starter/engines/oci_linux.go b/cmd/starter/engines/oci_linux.go deleted file mode 100644 index d2df39e32d..0000000000 --- a/cmd/starter/engines/oci_linux.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -//go:build oci_engine - -package engines - -import ( - // register the oci runtime engine - _ "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" -) diff --git a/dist/debian/control b/dist/debian/control index 4867a0c046..3943922b2b 100644 --- a/dist/debian/control +++ b/dist/debian/control @@ -36,7 +36,9 @@ Depends: squashfuse, fuse2fs, fuse-overlayfs, - fakeroot + fakeroot, + conmon, + crun | runc Conflicts: singularity-container Description: container platform focused on supporting "Mobility of Compute" formerly known as Singularity Mobility of Compute encapsulates the development to compute model diff --git a/dist/rpm/apptainer.rpmlintrc b/dist/rpm/apptainer.rpmlintrc new file mode 100644 index 0000000000..bac4de24ba --- /dev/null +++ b/dist/rpm/apptainer.rpmlintrc @@ -0,0 +1,7 @@ +addFilter(r'setuid-binary /usr/libexec/apptainer/bin/starter-suid') +addFilter(r'non-standard-executable-perm /usr/libexec/apptainer/bin/starter-suid') +addFilter(r'zero-length /etc/apptainer/capability.json') +addFilter(r'zero-length /etc/apptainer/global-pgp-public') +addFilter(r'readelf-failed /usr/bin/apptainer 'utf-8' codec can't decode byte 0xc2') +addFilter(r'readelf-failed /usr/libexec/apptainer/bin/starter 'utf-8' codec can't decode byte 0xc2') +addFilter(r'readelf-failed /usr/libexec/apptainer/bin/starter-suid 'utf-8' codec can't decode byte 0xc2') diff --git a/dist/rpm/apptainer.spec.in b/dist/rpm/apptainer.spec.in index 0ea8f3a128..5742b620da 100644 --- a/dist/rpm/apptainer.spec.in +++ b/dist/rpm/apptainer.spec.in @@ -41,15 +41,17 @@ # The last singularity version number in EPEL/Fedora %global last_singularity_version 3.8.7-3 -Summary: Application and environment virtualization formerly known as Singularity Name: apptainer Version: @PACKAGE_RPM_VERSION@ Release: @PACKAGE_RELEASE@%{?dist} +Summary: Application and environment virtualization formerly known as Singularity + # See LICENSE.md for first party code (BSD-3-Clause and LBNL BSD) # See LICENSE_THIRD_PARTY.md for incorporated code (ASL 2.0) # See LICENSE_DEPENDENCIES.md for dependencies # License identifiers taken from: https://fedoraproject.org/wiki/Licensing License: BSD and LBNL BSD and ASL 2.0 + URL: https://apptainer.org Source: https://github.com/%{name}/%{name}/releases/download/v%{package_version}/%{name}-%{package_version}.tar.gz @PACKAGE_GOLANG_SOURCE@ @@ -91,11 +93,20 @@ Conflicts: sif-runtime BuildRequires: binutils-gold %endif BuildRequires: golang -BuildRequires: git BuildRequires: gcc BuildRequires: make -BuildRequires: libseccomp-devel +# Paths to runtime dependencies detected by mconfig, so must be present at build time. BuildRequires: cryptsetup +# Required for building bundled conmon +BuildRequires: libseccomp-devel +Requires: conmon +# crun requirement not satisfied on EL7 or SLES default repos - use runc there. +%if "%{_target_vendor}" == "suse" || 0%{?rhel} < 8 +Requires: runc +%else +Requires: crun +%endif +Requires: cryptsetup %if "%{?squashfuse_version}" != "" BuildRequires: autoconf BuildRequires: automake @@ -107,6 +118,7 @@ BuildRequires: zlib-devel %if "%{_target_vendor}" == "suse" Requires: squashfs %else +Requires: shadow-utils Requires: squashfs-tools %endif Requires: squashfuse @@ -294,6 +306,7 @@ fi %if "%{?gocryptfs_version}" != "" %{_libexecdir}/%{name}/bin/gocryptfs %endif +%dir %{_libexecdir}/%{name}/cni %if "%{?squashfuse_version}" != "" %{_libexecdir}/%{name}/bin/squashfuse_ll %endif @@ -301,6 +314,9 @@ fi %{_libexecdir}/%{name}/lib %dir %{_sysconfdir}/%{name} %config(noreplace) %{_sysconfdir}/%{name}/* +%dir %{_sysconfdir}/%{name}/cgroups +%dir %{_sysconfdir}/%{name}/network +%dir %{_sysconfdir}/%{name}/seccomp-profiles %{_datadir}/bash-completion/completions/* %dir %{_sharedstatedir}/%{name} %dir %{_sharedstatedir}/%{name}/mnt @@ -312,9 +328,9 @@ fi %license LICENSE_DEPENDENCIES.md %doc README.md %doc CHANGELOG.md +%doc CONTRIBUTING.md %files suid %attr(4755, root, root) %{_libexecdir}/%{name}/bin/starter-suid %changelog - diff --git a/docs/content.go b/docs/content.go index 8155d97579..1354270a23 100644 --- a/docs/content.go +++ b/docs/content.go @@ -1035,6 +1035,9 @@ Enterprise Performance Computing (EPC)` $ apptainer oci attach mycontainer $ apptainer oci delete mycontainer` + // Internal oci launcher use only - no user-facing docs + OciRunWrappedUse string = `run-wrapped -b [-o ] [run options...] ` + OciUpdateUse string = `update [update options...] ` OciUpdateShort string = `Update container cgroups resources (root user only)` OciUpdateLong string = ` diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 5a87a6b7c6..30bf7f7c45 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -104,7 +104,11 @@ func (c actionTests) actionExec(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(testdata) + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(testdata) + } + }) testdataTmp := filepath.Join(testdata, "tmp") if err := os.Mkdir(testdataTmp, 0o755); err != nil { @@ -124,9 +128,10 @@ func (c actionTests) actionExec(t *testing.T) { homePath := filepath.Join("/home", basename) tests := []struct { - name string - argv []string - exit int + name string + argv []string + exit int + wantOutputs []e2e.ApptainerCmdResultOp }{ { name: "NoCommand", @@ -237,7 +242,10 @@ func (c actionTests) actionExec(t *testing.T) { }, { name: "Home", - argv: []string{"--home", testdata, c.env.ImagePath, "test", "-f", tmpfile.Name()}, + argv: []string{"--home", "/myhomeloc", c.env.ImagePath, "env"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/myhomeloc\b`), + }, exit: 0, }, { @@ -270,6 +278,30 @@ func (c actionTests) actionExec(t *testing.T) { argv: []string{"--no-home", c.env.ImagePath, "ls", "-ld", user.Dir}, exit: 1, }, + { + name: "Hostname", + argv: []string{"--hostname", "whats-in-a-native-name", c.env.ImagePath, "hostname"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "whats-in-a-native-name"), + }, + }, + { + name: "ResolvConfGoogle", + argv: []string{"--dns", "8.8.8.8,8.8.4.4", c.env.ImagePath, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(8\.8\.8\.8|8\.8\.4\.4)(\s*)\n`), + }, + }, + { + name: "ResolvConfCloudflare", + argv: []string{"--dns", "1.1.1.1", c.env.ImagePath, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(1\.1\.1\.1)(\s*)\n`), + }, + }, } for _, tt := range tests { @@ -280,7 +312,7 @@ func (c actionTests) actionExec(t *testing.T) { e2e.WithCommand("exec"), e2e.WithDir("/tmp"), e2e.WithArgs(tt.argv...), - e2e.ExpectExit(tt.exit), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), ) } } @@ -929,7 +961,7 @@ func (c actionTests) actionBasicProfiles(t *testing.T) { }, } - for _, profile := range e2e.Profiles { + for _, profile := range e2e.NativeProfiles { profile := profile t.Run(profile.String(), func(t *testing.T) { @@ -1100,39 +1132,7 @@ func (c actionTests) actionBinds(t *testing.T) { hostWorkDir := filepath.Join(workspace, "workdir") createWorkspaceDirs := func(t *testing.T) { - e2e.Privileged(func(t *testing.T) { - if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete canary_dir: %s", err) - } - if err := os.RemoveAll(hostHomeDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete workspace home: %s", err) - } - if err := os.RemoveAll(hostWorkDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete workspace work: %s", err) - } - })(t) - - if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { - t.Fatalf("failed to create canary_dir: %s", err) - } - if err := fs.Touch(hostCanaryFile); err != nil { - t.Fatalf("failed to create canary_file: %s", err) - } - if err := fs.Touch(hostCanaryFileWithComma); err != nil { - t.Fatalf("failed to create canary_file_comma: %s", err) - } - if err := fs.Touch(hostCanaryFileWithColon); err != nil { - t.Fatalf("failed to create canary_file_colon: %s", err) - } - if err := os.Chmod(hostCanaryFile, 0o777); err != nil { - t.Fatalf("failed to apply permissions on canary_file: %s", err) - } - if err := fs.Mkdir(hostHomeDir, 0o777); err != nil { - t.Fatalf("failed to create workspace home directory: %s", err) - } - if err := fs.Mkdir(hostWorkDir, 0o777); err != nil { - t.Fatalf("failed to create workspace work directory: %s", err) - } + mkWorkspaceDirs(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) } // convert test image to sandbox @@ -1165,10 +1165,11 @@ func (c actionTests) actionBinds(t *testing.T) { } tests := []struct { - name string - args []string - postRun func(*testing.T) - exit int + name string + args []string + wantOutputs []e2e.ApptainerCmdResultOp + postRun func(*testing.T) + exit int }{ { name: "NonExistentSource", @@ -1731,6 +1732,28 @@ func (c actionTests) actionBinds(t *testing.T) { postRun: checkHostDir(filepath.Join(hostWorkDir, "var_tmp", "canary/dir")), exit: 0, }, + { + name: "IsScratchTmpfs", + args: []string{ + "--scratch", "/name-of-a-scratch", + sandbox, + "mount", + }, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /name-of-a-scratch\b`), + }, + exit: 0, + }, + { + name: "BindOverScratch", + args: []string{ + "--scratch", "/name-of-a-scratch", + "--bind", hostCanaryDir + ":/name-of-a-scratch", + sandbox, + "test", "-f", "/name-of-a-scratch/file", + }, + exit: 0, + }, { name: "ScratchTmpfsBind", args: []string{ @@ -1850,7 +1873,7 @@ func (c actionTests) actionBinds(t *testing.T) { }, } - for _, profile := range e2e.Profiles { + for _, profile := range e2e.NativeProfiles { profile := profile createWorkspaceDirs(t) @@ -1863,7 +1886,7 @@ func (c actionTests) actionBinds(t *testing.T) { e2e.WithCommand("exec"), e2e.WithArgs(tt.args...), e2e.PostRun(tt.postRun), - e2e.ExpectExit(tt.exit), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), ) } }) @@ -2870,44 +2893,60 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { np := testhelper.NoParallel return testhelper.Tests{ - "action URI": c.RunFromURI, // action_URI - "singularity link": c.singularityLink, // singularity symlink - "exec": c.actionExec, // apptainer exec - "persistent overlay": c.PersistentOverlay, // Persistent Overlay - "persistent overlay unpriv": c.PersistentOverlayUnpriv, // Persistent Overlay Unprivileged - "run": c.actionRun, // apptainer run - "shell": c.actionShell, // shell interaction - "STDPIPE": c.STDPipe, // stdin/stdout pipe - "action basic profiles": c.actionBasicProfiles, // run basic action under different profiles - "issue 4488": c.issue4488, // https://github.com/apptainer/singularity/issues/4488 - "issue 4587": c.issue4587, // https://github.com/apptainer/singularity/issues/4587 - "issue 4755": c.issue4755, // https://github.com/apptainer/singularity/issues/4755 - "issue 4768": c.issue4768, // https://github.com/apptainer/singularity/issues/4768 - "issue 4797": c.issue4797, // https://github.com/apptainer/singularity/issues/4797 - "issue 4823": c.issue4823, // https://github.com/apptainer/singularity/issues/4823 - "issue 4836": c.issue4836, // https://github.com/apptainer/singularity/issues/4836 - "issue 5211": c.issue5211, // https://github.com/apptainer/singularity/issues/5211 - "issue 5228": c.issue5228, // https://github.com/apptainer/singularity/issues/5228 - "issue 5271": c.issue5271, // https://github.com/apptainer/singularity/issues/5271 - "issue 5399": c.issue5399, // https://github.com/apptainer/singularity/issues/5399 - "issue 5455": c.issue5455, // https://github.com/apptainer/singularity/issues/5455 - "issue 5465": c.issue5465, // https://github.com/apptainer/singularity/issues/5465 - "issue 5599": c.issue5599, // https://github.com/apptainer/singularity/issues/5599 - "issue 5631": c.issue5631, // https://github.com/apptainer/singularity/issues/5631 - "issue 5690": c.issue5690, // https://github.com/apptainer/singularity/issues/5690 - "issue 6165": c.issue6165, // https://github.com/apptainer/singularity/issues/6165 - "issue 619": c.issue619, // https://github.com/apptainer/apptainer/issues/619 - "network": c.actionNetwork, // test basic networking - "binds": c.actionBinds, // test various binds with --bind and --mount - "exit and signals": c.exitSignals, // test exit and signals propagation - "fuse mount": c.fuseMount, // test fusemount option - "bind image": c.bindImage, // test bind image with --bind and --mount - "unsquash": c.actionUnsquash, // test --unsquash - "no-mount": c.actionNoMount, // test --no-mount - "compat": np(c.actionCompat), // test --compat - "umask": np(c.actionUmask), // test umask propagation - "invalidRemote": np(c.invalidRemote), // GHSA-5mv9-q7fq-9394 - "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot - "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch + "action URI": c.RunFromURI, // action_URI + "singularity link": c.singularityLink, // singularity symlink + "exec": c.actionExec, // apptainer exec + "persistent overlay": c.PersistentOverlay, // Persistent Overlay + "persistent overlay unpriv": c.PersistentOverlayUnpriv, // Persistent Overlay Unprivileged + "run": c.actionRun, // apptainer run + "shell": c.actionShell, // shell interaction + "STDPIPE": c.STDPipe, // stdin/stdout pipe + "action basic profiles": c.actionBasicProfiles, // run basic action under different profiles + "issue 4488": c.issue4488, // https://github.com/apptainer/singularity/issues/4488 + "issue 4587": c.issue4587, // https://github.com/apptainer/singularity/issues/4587 + "issue 4755": c.issue4755, // https://github.com/apptainer/singularity/issues/4755 + "issue 4768": c.issue4768, // https://github.com/apptainer/singularity/issues/4768 + "issue 4797": c.issue4797, // https://github.com/apptainer/singularity/issues/4797 + "issue 4823": c.issue4823, // https://github.com/apptainer/singularity/issues/4823 + "issue 4836": c.issue4836, // https://github.com/apptainer/singularity/issues/4836 + "issue 5211": c.issue5211, // https://github.com/apptainer/singularity/issues/5211 + "issue 5228": c.issue5228, // https://github.com/apptainer/singularity/issues/5228 + "issue 5271": c.issue5271, // https://github.com/apptainer/singularity/issues/5271 + "issue 5399": c.issue5399, // https://github.com/apptainer/singularity/issues/5399 + "issue 5455": c.issue5455, // https://github.com/apptainer/singularity/issues/5455 + "issue 5465": c.issue5465, // https://github.com/apptainer/singularity/issues/5465 + "issue 5599": c.issue5599, // https://github.com/apptainer/singularity/issues/5599 + "issue 5631": c.issue5631, // https://github.com/apptainer/singularity/issues/5631 + "issue 5690": c.issue5690, // https://github.com/apptainer/singularity/issues/5690 + "issue 6165": c.issue6165, // https://github.com/apptainer/singularity/issues/6165 + "issue 619": c.issue619, // https://github.com/apptainer/apptainer/issues/619 + "network": c.actionNetwork, // test basic networking + "binds": c.actionBinds, // test various binds with --bind and --mount + "exit and signals": c.exitSignals, // test exit and signals propagation + "fuse mount": c.fuseMount, // test fusemount option + "bind image": c.bindImage, // test bind image with --bind and --mount + "unsquash": c.actionUnsquash, // test --unsquash + "no-mount": c.actionNoMount, // test --no-mount + "compat": np(c.actionCompat), // test --compat + "umask": np(c.actionUmask), // test umask propagation + "invalidRemote": np(c.invalidRemote), // GHSA-5mv9-q7fq-9394 + "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot + "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch + "ociRelWorkdirScratch": np(c.ociRelWorkdirScratch), // test relative --workdir with --scratch in OCI mode + // + // OCI Runtime Mode + // + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci + "ociShell": c.actionOciShell, // apptainer shell --oci + "ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci + "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net + "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount + "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi + "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot + "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat + "ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode + "ociOverlayExtfsPerms": (c.actionOciOverlayExtfsPerms), // permissions in writable extfs overlays mounted with FUSE in OCI mode + "ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode } } diff --git a/e2e/actions/common.go b/e2e/actions/common.go new file mode 100644 index 0000000000..05fddff251 --- /dev/null +++ b/e2e/actions/common.go @@ -0,0 +1,54 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package actions + +import ( + "os" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" + "github.com/apptainer/apptainer/internal/pkg/util/fs" +) + +func mkWorkspaceDirs(t *testing.T, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon string) { + e2e.Privileged(func(t *testing.T) { + if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete canary_dir: %s", err) + } + if err := os.RemoveAll(hostHomeDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete workspace home: %s", err) + } + if err := os.RemoveAll(hostWorkDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete workspace home: %s", err) + } + })(t) + + if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { + t.Fatalf("failed to create canary_dir: %s", err) + } + if err := fs.Touch(hostCanaryFile); err != nil { + t.Fatalf("failed to create canary_file: %s", err) + } + if err := fs.Touch(hostCanaryFileWithComma); err != nil { + t.Fatalf("failed to create canary_file_comma: %s", err) + } + if err := fs.Touch(hostCanaryFileWithColon); err != nil { + t.Fatalf("failed to create canary_file_colon: %s", err) + } + if err := os.Chmod(hostCanaryFile, 0o777); err != nil { + t.Fatalf("failed to apply permissions on canary_file: %s", err) + } + if err := fs.Mkdir(hostHomeDir, 0o777); err != nil { + t.Fatalf("failed to create workspace home directory: %s", err) + } + if err := fs.Mkdir(hostWorkDir, 0o777); err != nil { + t.Fatalf("failed to create workspace home directory: %s", err) + } +} diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go new file mode 100644 index 0000000000..f638883f4e --- /dev/null +++ b/e2e/actions/oci.go @@ -0,0 +1,1625 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package actions + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "text/template" + + "github.com/apptainer/apptainer/e2e/internal/e2e" + "github.com/apptainer/apptainer/internal/pkg/test/tool/dirs" + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/apptainer/apptainer/internal/pkg/util/fs" + cdispecs "github.com/container-orchestrated-devices/container-device-interface/specs-go" + "gotest.tools/v3/assert" +) + +const ( + imgTestFilePath string = "file-for-testing" + squashfsTestString string = "squashfs-test-string" + extfsTestString string = "extfs-test-string" +) + +var ( + imgsPath = filepath.Join("..", "test", "images") + squashfsImgPath = filepath.Join(imgsPath, "squashfs-for-overlay.img") + extfsImgPath = filepath.Join(imgsPath, "extfs-for-overlay.img") +) + +func (c actionTests) actionOciRun(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + e2e.EnsureDockerArchive(t, c.env) + + // Prepare oci source (oci directory layout) + ociLayout := t.TempDir() + cmd := exec.Command("tar", "-C", ociLayout, "-xf", c.env.OCIArchivePath) + err := cmd.Run() + if err != nil { + t.Fatalf("Error extracting oci archive to layout: %v", err) + } + + tests := []struct { + name string + imageRef string + argv []string + exit int + }{ + { + name: "docker-archive", + imageRef: "docker-archive:" + c.env.DockerArchivePath, + exit: 0, + }, + { + name: "oci-archive", + imageRef: "oci-archive:" + c.env.OCIArchivePath, + exit: 0, + }, + { + name: "oci", + imageRef: "oci:" + ociLayout, + exit: 0, + }, + { + name: "true", + imageRef: "oci:" + ociLayout, + argv: []string{"true"}, + exit: 0, + }, + { + name: "false", + imageRef: "oci:" + ociLayout, + argv: []string{"false"}, + exit: 1, + }, + } + + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + cmdArgs := []string{tt.imageRef} + cmdArgs = append(cmdArgs, tt.argv...) + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("run"), + // While we don't support args we are entering a /bin/sh interactively. + e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), + e2e.WithArgs(cmdArgs...), + e2e.ExpectExit(tt.exit), + ) + } + }) + } +} + +// exec tests min fuctionality for apptainer exec +func (c actionTests) actionOciExec(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + + imageRef := "oci-archive:" + c.env.OCIArchivePath + + // Create a temp testfile + testdata, err := fs.MakeTmpDir(c.env.TestDir, "testdata", 0o755) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(testdata) + } + }) + + testdataTmp := filepath.Join(testdata, "tmp") + if err := os.Mkdir(testdataTmp, 0o755); err != nil { + t.Fatal(err) + } + + // Create a temp testfile + tmpfile, err := fs.MakeTmpFile(testdataTmp, "testApptainerExec.", 0o644) + if err != nil { + t.Fatal(err) + } + tmpfile.Close() + + basename := filepath.Base(tmpfile.Name()) + tmpfilePath := filepath.Join("/tmp", basename) + homePath := filepath.Join("/home", basename) + + tests := []struct { + name string + argv []string + exit int + wantOutputs []e2e.ApptainerCmdResultOp + skipProfiles map[string]bool + }{ + { + name: "NoCommand", + argv: []string{imageRef}, + exit: 1, + }, + { + name: "True", + argv: []string{imageRef, "true"}, + exit: 0, + }, + { + name: "TrueAbsPAth", + argv: []string{imageRef, "/bin/true"}, + exit: 0, + }, + { + name: "False", + argv: []string{imageRef, "false"}, + exit: 1, + }, + { + name: "FalseAbsPath", + argv: []string{imageRef, "/bin/false"}, + exit: 1, + }, + { + name: "TouchTmp", + argv: []string{imageRef, "/bin/touch", "/tmp/test"}, + exit: 0, + }, + { + name: "TouchVarTmp", + argv: []string{imageRef, "/bin/touch", "/var/tmp/test"}, + exit: 0, + }, + { + name: "TouchHome", + argv: []string{imageRef, "/bin/sh", "-c", "touch $HOME"}, + exit: 0, + }, + { + name: "Home", + argv: []string{"--home", "/myhomeloc", imageRef, "sh", "-c", "env; mount"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/myhomeloc\b`), + e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /myhomeloc\b`), + }, + exit: 0, + }, + { + name: "HomePath", + argv: []string{"--home", testdataTmp + ":/home", imageRef, "test", "-f", homePath}, + exit: 0, + }, + { + name: "HomeTmp", + argv: []string{"--home", "/tmp", imageRef, "true"}, + exit: 0, + }, + { + name: "HomeTmpExplicit", + argv: []string{"--home", "/tmp:/home", imageRef, "true"}, + exit: 0, + }, + { + name: "UTSNamespace", + argv: []string{"--uts", imageRef, "true"}, + exit: 0, + }, + { + name: "Hostname", + argv: []string{"--hostname", "whats-in-an-oci-name", imageRef, "hostname"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "whats-in-an-oci-name"), + }, + }, + { + name: "Workdir", + argv: []string{"--workdir", testdata, imageRef, "test", "-f", tmpfilePath}, + exit: 0, + }, + { + name: "Pwd", + argv: []string{"--pwd", "/etc", imageRef, "pwd"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "/etc"), + }, + }, + { + name: "Cwd", + argv: []string{"--cwd", "/etc", imageRef, "pwd"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "/etc"), + }, + }, + { + name: "ResolvConfGoogle", + argv: []string{"--dns", "8.8.8.8,8.8.4.4", imageRef, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(8\.8\.8\.8|8\.8\.4\.4)(\s*)\n`), + }, + }, + { + name: "ResolvConfCloudflare", + argv: []string{"--dns", "1.1.1.1", imageRef, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(1\.1\.1\.1)(\s*)\n`), + }, + }, + } + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + skip, ok := tt.skipProfiles[profile.String()] + if ok && skip { + continue + } + + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithDir("/tmp"), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), + ) + } + }) + } +} + +// Shell interaction tests +func (c actionTests) actionOciShell(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + + tests := []struct { + name string + argv []string + consoleOps []e2e.ApptainerConsoleOp + exit int + }{ + { + name: "ShellExit", + argv: []string{"oci-archive:" + c.env.OCIArchivePath}, + consoleOps: []e2e.ApptainerConsoleOp{ + // "cd /" to work around issue where a long + // working directory name causes the test + // to fail because the "Apptainer" that + // we are looking for is chopped from the + // front. + // TODO(mem): This test was added back in 491a71716013654acb2276e4b37c2e015d2dfe09 + e2e.ConsoleSendLine("cd /"), + e2e.ConsoleExpect("Apptainer"), + e2e.ConsoleSendLine("exit"), + }, + exit: 0, + }, + { + name: "ShellBadCommand", + argv: []string{"oci-archive:" + c.env.OCIArchivePath}, + consoleOps: []e2e.ApptainerConsoleOp{ + e2e.ConsoleSendLine("_a_fake_command"), + e2e.ConsoleSendLine("exit"), + }, + exit: 127, + }, + } + + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("shell"), + e2e.WithArgs(tt.argv...), + e2e.ConsoleRun(tt.consoleOps...), + e2e.ExpectExit(tt.exit), + ) + } + }) + } +} + +func (c actionTests) actionOciNetwork(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + tests := []struct { + name string + profile e2e.Profile + netType string + expectExit int + }{ + { + name: "InvalidNetworkRoot", + profile: e2e.OCIRootProfile, + netType: "bridge", + expectExit: 255, + }, + { + name: "InvalidNetworkUser", + profile: e2e.OCIUserProfile, + netType: "bridge", + expectExit: 255, + }, + { + name: "InvalidNetworkFakeroot", + profile: e2e.OCIFakerootProfile, + netType: "bridge", + expectExit: 255, + }, + { + name: "NoneNetworkRoot", + profile: e2e.OCIRootProfile, + netType: "none", + expectExit: 0, + }, + { + name: "NoneNetworkUser", + profile: e2e.OCIUserProfile, + netType: "none", + expectExit: 0, + }, + { + name: "NoneNetworkFakeRoot", + profile: e2e.OCIFakerootProfile, + netType: "none", + expectExit: 0, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(tt.profile), + e2e.WithCommand("exec"), + e2e.WithArgs("--net", "--network", tt.netType, imageRef, "id"), + e2e.ExpectExit(tt.expectExit), + ) + } +} + +//nolint:maintidx +func (c actionTests) actionOciBinds(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + workspace, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "bind-workspace-", "") + t.Cleanup(func() { + if !t.Failed() { + e2e.Privileged(cleanup) + } + }) + + contCanaryDir := "/canary" + hostCanaryDir := filepath.Join(workspace, "canary") + + contCanaryFile := "/canary/file" + hostCanaryFile := filepath.Join(hostCanaryDir, "file") + hostCanaryFileWithComma := filepath.Join(hostCanaryDir, "file,comma") + hostCanaryFileWithColon := filepath.Join(hostCanaryDir, "file:colon") + + canaryFileBind := hostCanaryFile + ":" + contCanaryFile + canaryFileMount := "type=bind,source=" + hostCanaryFile + ",destination=" + contCanaryFile + canaryDirBind := hostCanaryDir + ":" + contCanaryDir + canaryDirMount := "type=bind,source=" + hostCanaryDir + ",destination=" + contCanaryDir + + hostHomeDir := filepath.Join(workspace, "home") + hostWorkDir := filepath.Join(workspace, "workdir") + + createWorkspaceDirs := func(t *testing.T) { + mkWorkspaceDirs(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) + } + + checkHostFn := func(path string, fn func(string) bool) func(*testing.T) { + return func(t *testing.T) { + if t.Failed() { + return + } + if !fn(path) { + t.Errorf("%s not found on host", path) + } + // When a nested bind is performed under workdir, the bind + // destination will be created (if necessary) by runc/crun inside + // workdir on the host. The bind destination will be created with + // subuid:subgid ownership. This requires privilege, or a userns + + // id mapping, to remove. (Relevant to tests like WorkdirTmpBind, + // below.) + e2e.Privileged(func(t *testing.T) { + if err := os.RemoveAll(path); err != nil { + t.Errorf("failed to delete %s: %s", path, err) + } + })(t) + } + } + checkHostFile := func(path string) func(*testing.T) { + return checkHostFn(path, fs.IsFile) + } + checkHostDir := func(path string) func(*testing.T) { + return checkHostFn(path, fs.IsDir) + } + + tests := []struct { + name string + args []string + wantOutputs []e2e.ApptainerCmdResultOp + postRun func(*testing.T) + exit int + }{ + { + name: "NonExistentSource", + args: []string{ + "--bind", "/non/existent/source/path", + imageRef, + "true", + }, + exit: 255, + }, + { + name: "RelativeBindDestination", + args: []string{ + "--bind", hostCanaryFile + ":relative", + imageRef, + "true", + }, + exit: 255, + }, + { + name: "SimpleFile", + args: []string{ + "--bind", canaryFileBind, + imageRef, + "test", "-f", contCanaryFile, + }, + exit: 0, + }, + { + name: "SimpleDir", + args: []string{ + "--bind", canaryDirBind, + imageRef, + "test", "-f", contCanaryFile, + }, + exit: 0, + }, + { + name: "HomeOverride", + args: []string{ + "--bind", hostCanaryDir + ":/home", + imageRef, + "test", "-f", "/home/file", + }, + exit: 0, + }, + { + name: "TmpOverride", + args: []string{ + "--bind", hostCanaryDir + ":/tmp", + imageRef, + "test", "-f", "/tmp/file", + }, + exit: 0, + }, + { + name: "VarTmpOverride", + args: []string{ + "--bind", hostCanaryDir + ":/var/tmp", + imageRef, + "test", "-f", "/var/tmp/file", + }, + exit: 0, + }, + { + name: "NestedBindFile", + args: []string{ + "--bind", canaryDirBind, + "--bind", hostCanaryFile + ":" + filepath.Join(contCanaryDir, "file2"), + imageRef, + "test", "-f", "/canary/file2", + }, + postRun: checkHostFile(filepath.Join(hostCanaryDir, "file2")), + exit: 0, + }, + { + name: "NestedBindDir", + args: []string{ + "--bind", canaryDirBind, + "--bind", hostCanaryDir + ":" + filepath.Join(contCanaryDir, "dir2"), + imageRef, + "test", "-d", "/canary/dir2", + }, + postRun: checkHostDir(filepath.Join(hostCanaryDir, "dir2")), + exit: 0, + }, + { + name: "MultipleNestedBindDir", + args: []string{ + "--bind", canaryDirBind, + "--bind", hostCanaryDir + ":" + filepath.Join(contCanaryDir, "dir2"), + "--bind", hostCanaryFile + ":" + filepath.Join(filepath.Join(contCanaryDir, "dir2"), "nested"), + imageRef, + "test", "-f", "/canary/dir2/nested", + }, + postRun: checkHostFile(filepath.Join(hostCanaryDir, "nested")), + exit: 0, + }, + { + name: "WorkdirTmpBind", + args: []string{ + "--workdir", hostWorkDir, + "--bind", hostCanaryDir + ":/tmp/canary/dir", + imageRef, + "test", "-f", "/tmp/canary/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "tmp", "canary/dir")), + exit: 0, + }, + { + name: "WorkdirVarTmpBind", + args: []string{ + "--workdir", hostWorkDir, + "--bind", hostCanaryDir + ":/var/tmp/canary/dir", + imageRef, + "test", "-f", "/var/tmp/canary/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "var_tmp", "canary/dir")), + exit: 0, + }, + { + name: "WorkdirVarTmpBindWritable", + args: []string{ + "--workdir", hostWorkDir, + "--bind", hostCanaryDir + ":/var/tmp/canary/dir", + imageRef, + "test", "-f", "/var/tmp/canary/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "var_tmp", "canary/dir")), + exit: 0, + }, + { + name: "IsScratchTmpfs", + args: []string{ + "--scratch", "/name-of-a-scratch", + imageRef, + "mount", + }, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /name-of-a-scratch\b`), + }, + exit: 0, + }, + { + name: "BindOverScratch", + args: []string{ + "--scratch", "/name-of-a-scratch", + "--bind", hostCanaryDir + ":/name-of-a-scratch", + imageRef, + "test", "-f", "/name-of-a-scratch/file", + }, + exit: 0, + }, + { + name: "ScratchTmpfsBind", + args: []string{ + "--scratch", "/scratch", + "--bind", hostCanaryDir + ":/scratch/dir", + imageRef, + "test", "-f", "/scratch/dir/file", + }, + exit: 0, + }, + { + name: "ScratchWorkdirBind", + args: []string{ + "--workdir", hostWorkDir, + "--scratch", "/scratch", + "--bind", hostCanaryDir + ":/scratch/dir", + imageRef, + "test", "-f", "/scratch/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "scratch/scratch", "dir")), + exit: 0, + }, + { + name: "CustomHomeOneToOne", + args: []string{ + "--home", hostHomeDir + ":" + hostHomeDir, + "--bind", hostCanaryDir + ":" + filepath.Join(hostHomeDir, "canary121RO"), + imageRef, + "test", "-f", filepath.Join(hostHomeDir, "canary121RO/file"), + }, + postRun: checkHostDir(filepath.Join(hostHomeDir, "canary121RO")), + exit: 0, + }, + { + name: "CustomHomeBind", + args: []string{ + "--home", hostHomeDir + ":/home/e2e", + "--bind", hostCanaryDir + ":/home/e2e/canaryRO", + imageRef, + "test", "-f", "/home/e2e/canaryRO/file", + }, + postRun: checkHostDir(filepath.Join(hostHomeDir, "canaryRO")), + exit: 0, + }, + // For the --mount variants we are really just verifying the CLI + // acceptance of one or more --mount flags. Translation from --mount + // strings to BindPath structs is checked in unit tests. The + // functionality of bind mounts of various kinds is already checked + // above, with --bind flags. No need to duplicate all of these. + { + name: "MountSingle", + args: []string{ + "--mount", canaryFileMount, + imageRef, + "test", "-f", contCanaryFile, + }, + exit: 0, + }, + { + name: "MountNested", + args: []string{ + "--mount", canaryDirMount, + "--mount", "source=" + hostCanaryFile + ",destination=" + filepath.Join(contCanaryDir, "file3"), + imageRef, + "test", "-f", "/canary/file3", + }, + postRun: checkHostFile(filepath.Join(hostCanaryDir, "file3")), + exit: 0, + }, + } + + for _, profile := range e2e.OCIProfiles { + profile := profile + createWorkspaceDirs(t) + + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.PostRun(tt.postRun), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), + ) + } + }) + } +} + +func (c actionTests) actionOciCdi(t *testing.T) { + // Grab the reference OCI archive we're going to use + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + // Set up a custom subtestWorkspace object that will holds the collection of temporary directories (nested under the main temporary directory, mainDir) that each test will use. + type subtestWorkspace struct { + mainDir string + jsonsDir string + mountDirs []string + } + + // Create a function to create a fresh subtestWorkspace, with distinct temporary directories, that each individual subtest will use + setupIndivSubtestWorkspace := func(t *testing.T, numMountDirs int) *subtestWorkspace { + stws := subtestWorkspace{} + mainDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "", "") + t.Cleanup(func() { + if !t.Failed() { + e2e.Privileged(cleanup) + } + }) + stws.mainDir = mainDir + + // No need to do anything with the cleanup functions returned here, because the directories created are all going to be children of (tw.)mainDir, whose cleanup was already registered above. + stws.jsonsDir, _ = e2e.MakeTempDir(t, stws.mainDir, "cdi-jsons-", "") + stws.mountDirs = make([]string, 0, numMountDirs) + for len(stws.mountDirs) < numMountDirs { + dir, _ := e2e.MakeTempDir(t, stws.mainDir, fmt.Sprintf("mount-dir-%d-", len(stws.mountDirs)+1), "") + // Make writable to all, due to current nested userns mapping restrictions. + // Will work without this once crun-specific single mapping is present. + os.Chmod(dir, 0o777) + stws.mountDirs = append(stws.mountDirs, dir) + } + + return &stws + } + + // Set up the JSON template that we're going to populate on a per-subtest basis with particular CDI spec values + e2eMountTemplateFilename := "cditemplate.json.tpl" + cdiJSONTemplateFilePath := filepath.Join("..", "test", "cdi", e2eMountTemplateFilename) + funcMap := template.FuncMap{ + // The name "title" is what the function will be called in the template text. + "tojson": func(o any) string { + s, _ := json.Marshal(o) + return string(s) + }, + } + cdiJSONTemplate, err := template.New(e2eMountTemplateFilename).Funcs(funcMap).ParseFiles(cdiJSONTemplateFilePath) + if err != nil { + t.Errorf("Could not read JSON template for CDI e2e tests from file %#v", cdiJSONTemplateFilePath) + return + } + + // The set of actual subtests + var wantUID uint32 = 1000 + var wantGID uint32 = 1000 + tests := []struct { + name string + devices []string + wantExit int + postRun func(t *testing.T) + DeviceNodes []cdispecs.DeviceNode + Mounts []cdispecs.Mount + Env []string + }{ + { + name: "ValidMounts", + devices: []string{ + "apptainertesting.sylabs.io/device=TesterDevice", + }, + wantExit: 0, + DeviceNodes: []cdispecs.DeviceNode{}, + Mounts: []cdispecs.Mount{ + { + ContainerPath: "/tmp/mount1", + Options: []string{"rw", "bind", "users"}, + }, + { + ContainerPath: "/tmp/mount3", + Options: []string{"rw", "bind", "users"}, + }, + { + ContainerPath: "/tmp/mount13", + Options: []string{"rw", "bind", "users"}, + }, + { + ContainerPath: "/tmp/mount17", + Options: []string{"rw", "bind", "users"}, + }, + }, + Env: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + }, + }, + { + name: "InvalidDevice", + devices: []string{ + "apptainertesting.sylabs.io/device=DoesNotExist", + }, + wantExit: 255, + DeviceNodes: []cdispecs.DeviceNode{}, + Mounts: []cdispecs.Mount{}, + Env: []string{}, + }, + { + name: "KmsgDevice", + devices: []string{ + "apptainertesting.sylabs.io/device=TesterDevice", + }, + wantExit: 0, + DeviceNodes: []cdispecs.DeviceNode{ + { + HostPath: "/dev/kmsg", + Path: "/dev/kmsg", + Permissions: "rw", + Type: "c", + UID: &wantUID, + GID: &wantGID, + }, + }, + }, + } + + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stws := setupIndivSubtestWorkspace(t, len(tt.Mounts)) + + // Populate the HostPath values we're going to feed into the CDI JSON template, based on the subtestWorkspace we just created + for i, d := range stws.mountDirs { + tt.Mounts[i].HostPath = d + } + + // Inject this subtest's values into the template to create the CDI JSON file + cdiJSONFilePath := filepath.Join(stws.jsonsDir, fmt.Sprintf("%s-cdi.json", tt.name)) + cdiJSONFile, err := os.OpenFile(cdiJSONFilePath, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Errorf("could not create file %#v for writing CDI JSON: %v", cdiJSONFilePath, err) + } + if err = cdiJSONTemplate.Execute(cdiJSONFile, tt); err != nil { + t.Errorf("error executing template %#v to create CDI JSON: %v", cdiJSONTemplateFilePath, err) + return + } + cdiJSONFile.Close() + + // Create a list of test strings, each of which will be echoed into a separate file in a separate mount in the container. + testfileStrings := make([]string, 0, len(tt.Mounts)) + for i := range tt.Mounts { + testfileStrings = append(testfileStrings, fmt.Sprintf("test_string_for_mount_%d_in_test_%s", i, tt.name)) + } + + // Generate the command to be executed in the container + // Start by printing all environment variables, to test using e2e.ContainMatch conditions later + execCmd := "/usr/bin/env" + + // Add commands to test the presence of mapped devices. + for _, d := range tt.DeviceNodes { + testFlag := "-f" + switch d.Type { + case "c": + testFlag = "-c" + } + execCmd += fmt.Sprintf(" && test %s %s", testFlag, d.Path) + } + + // Add commands to test the presence, and functioning, of mounts. + for i, m := range tt.Mounts { + // Add a separate teststring echo statement for each mount + execCmd += fmt.Sprintf(" && echo %s > %s/testfile_%d", testfileStrings[i], m.ContainerPath, i) + } + + // Create a postRun function to check that the testfiles written to the container mounts made their way to the right host temporary directories + testMountsAndEnv := func(t *testing.T) { + for i, m := range tt.Mounts { + testfileFilename := filepath.Join(m.HostPath, fmt.Sprintf("testfile_%d", i)) + b, err := os.ReadFile(testfileFilename) + if err != nil { + t.Errorf("could not read testfile %s", testfileFilename) + return + } + + s := string(b) + if s != testfileStrings[i]+"\n" { + t.Errorf("mismatched testfileString; expected %#v, got %#v (mount: %#v)", s, testfileStrings[i], m) + } + } + } + + // Create a set of e2e.ApptainerCmdResultOp objects to test that environment variables have been correctly injected into the container + envExpects := make([]e2e.ApptainerCmdResultOp, 0, len(tt.Env)) + for _, e := range tt.Env { + envExpects = append(envExpects, e2e.ExpectOutput(e2e.ContainMatch, e)) + } + + // Run the subtest. + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithCommand("exec"), + e2e.WithArgs( + "--device", + strings.Join(tt.devices, ","), + "--cdi-dirs", + stws.jsonsDir, + imageRef, + "/bin/sh", "-c", execCmd), + e2e.WithProfile(profile), + e2e.ExpectExit(tt.wantExit, envExpects...), + e2e.PostRun(tt.postRun), + e2e.PostRun(testMountsAndEnv), + ) + }) + } + }) + } +} + +// Check that both root via fakeroot and user without fakeroot are mapped to +// uid/gid on host, by writing a file out to host and checking ownership. +func (c actionTests) actionOciIDMaps(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + bindDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "usermap", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { + t.Run(profile.String(), func(t *testing.T) { + cmdArgs := []string{ + "-B", fmt.Sprintf("%s:/test", bindDir), + imageRef, + "/bin/touch", fmt.Sprintf("/test/%s", profile.String()), + } + c.env.RunApptainer( + t, + e2e.AsSubtest(profile.String()), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(cmdArgs...), + e2e.ExpectExit(0), + e2e.PostRun(func(t *testing.T) { + fp := filepath.Join(bindDir, profile.String()) + expectUID := profile.HostUser(t).UID + expectGID := profile.HostUser(t).GID + if !fs.IsOwner(fp, expectUID) { + t.Errorf("%s not owned by uid %d", fp, expectUID) + } + if !fs.IsGroup(fp, expectGID) { + t.Errorf("%s not owned by gid %d", fp, expectGID) + } + }), + ) + }) + } +} + +// actionOCICompat checks that the --oci mode has the behavior that the native mode gains from the --compat flag. +// Must be run in sequential section as it modifies host process umask. +func (c actionTests) actionOciCompat(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + type test struct { + name string + args []string + exitCode int + expect e2e.ApptainerCmdResultOp + } + + tests := []test{ + { + name: "containall", + args: []string{imageRef, "sh", "-c", "ls -lah $HOME"}, + exitCode: 0, + expect: e2e.ExpectOutput(e2e.ContainMatch, "total 0"), + }, + { + name: "writable-tmpfs", + args: []string{imageRef, "sh", "-c", "touch /test"}, + exitCode: 0, + }, + { + name: "no-init", + args: []string{imageRef, "sh", "-c", "ps"}, + exitCode: 0, + expect: e2e.ExpectOutput(e2e.UnwantedContainMatch, "sinit"), + }, + { + name: "no-umask", + args: []string{imageRef, "sh", "-c", "umask"}, + exitCode: 0, + expect: e2e.ExpectOutput(e2e.ContainMatch, "0022"), + }, + } + + oldUmask := syscall.Umask(0) + defer syscall.Umask(oldUmask) + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit( + tt.exitCode, + tt.expect, + ), + ) + } +} + +// actionOciOverlay checks that --overlay functions correctly in OCI mode. +// +//nolint:maintidx +func (c actionTests) actionOciOverlay(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + for _, profile := range e2e.OCIProfiles { + testDir, err := fs.MakeTmpDir(c.env.TestDir, "overlaytestdir", 0o755) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(testDir) + } + }) + + // Create a few writable overlay subdirs under testDir + for i := 0; i < 3; i++ { + dirName := fmt.Sprintf("my_rw_ol_dir%d", i) + fullPath := filepath.Join(testDir, dirName) + dirs.MkdirOrFatal(t, fullPath, 0o755) + upperPath := filepath.Join(fullPath, "upper") + dirs.MkdirOrFatal(t, upperPath, 0o777) + dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777) + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(fullPath) + } + }) + } + + // Create a few read-only overlay subdirs under testDir + for i := 0; i < 3; i++ { + dirName := fmt.Sprintf("my_ro_ol_dir%d", i) + fullPath := filepath.Join(testDir, dirName) + dirs.MkdirOrFatal(t, fullPath, 0o755) + upperPath := filepath.Join(fullPath, "upper") + dirs.MkdirOrFatal(t, upperPath, 0o777) + dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777) + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(fullPath) + } + }) + if err = os.WriteFile( + filepath.Join(upperPath, fmt.Sprintf("testfile.%d", i)), + []byte(fmt.Sprintf("test_string_%d\n", i)), + 0o644); err != nil { + t.Fatal(err) + } + if err = os.WriteFile( + filepath.Join(upperPath, "maskable_testfile"), + []byte(fmt.Sprintf("maskable_string_%d\n", i)), + 0o644); err != nil { + t.Fatal(err) + } + } + + // Create a copy of the extfs test image to be used for testing readonly + // extfs image overlays + readonlyExtfsImgPath := filepath.Join(testDir, "readonly-extfs.img") + err = fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err) + } + + // Create a copy of the extfs test image to be used for testing writable + // extfs image overlays + writableExtfsImgPath := filepath.Join(testDir, "writable-extfs.img") + err = fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err) + } + + tests := []struct { + name string + args []string + exitCode int + requiredCmds []string + wantOutputs []e2e.ApptainerCmdResultOp + }{ + { + name: "ExistRWDir", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + exitCode: 0, + }, + { + name: "ExistRWDirRevisit", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/my_test_file"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "ExistRWDirRevisitAsRO", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0:ro"), imageRef, "cat", "/my_test_file"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "RWOverlayMissing", + args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "echo", "hi"}, + exitCode: 255, + }, + { + name: "ROOverlayMissing", + args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent:ro"), imageRef, "echo", "hi"}, + exitCode: 255, + }, + { + name: "AutoAddTmpfs", + args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "sh", "-c", "echo this_should_disappear > /my_test_file"}, + exitCode: 0, + }, + { + name: "SeveralRODirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", + }, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + }, + }, + { + name: "AllTypesAtOnce", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", readonlyExtfsImgPath + ":ro", + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse", "fuse2fs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + e2e.ExpectOutput(e2e.ContainMatch, extfsTestString), + }, + }, + { + name: "SquashfsAndDirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + e2e.ExpectOutput(e2e.ContainMatch, squashfsTestString), + }, + }, + { + name: "ExtfsAndDirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", readonlyExtfsImgPath + ":ro", + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"fuse2fs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + e2e.ExpectOutput(e2e.ContainMatch, extfsTestString), + }, + }, + { + name: "SquashfsAndDirsAndMissingRO", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "something_nonexistent:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse", "fusermount"}, + exitCode: 255, + }, + { + name: "SquashfsAndDirsAndMissingRW", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "something_nonexistent"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse", "fusermount"}, + exitCode: 255, + }, + { + name: "TwoWritables", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir1"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + exitCode: 255, + }, + { + name: "ThreeWritables", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir1"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir2"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + exitCode: 255, + }, + { + name: "WritableExtfs", + args: []string{"--overlay", writableExtfsImgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + }, + { + name: "WritableExtfsRevisit", + args: []string{"--overlay", writableExtfsImgPath, imageRef, "cat", "/my_test_file"}, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsRevisitAsRO", + args: []string{"--overlay", writableExtfsImgPath + ":ro", imageRef, "cat", "/my_test_file"}, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsWithDirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", writableExtfsImgPath, + imageRef, "cat", "/my_test_file", + }, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsWithMix", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", readonlyExtfsImgPath + ":ro", + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", writableExtfsImgPath, + imageRef, "cat", "/my_test_file", + }, + exitCode: 0, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsWithAll", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", readonlyExtfsImgPath + ":ro", + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", writableExtfsImgPath, + imageRef, "cat", "/my_test_file", + }, + exitCode: 0, + requiredCmds: []string{"squashfuse", "fuse2fs", "fuse-overlayfs", "fusermount"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + } + + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + if !haveAllCommands(t, tt.requiredCmds) { + continue + } + + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit( + tt.exitCode, + tt.wantOutputs..., + ), + ) + } + }) + } +} + +func haveAllCommands(t *testing.T, cmds []string) bool { + for _, c := range cmds { + if _, err := exec.LookPath(c); err != nil { + return false + } + } + + return true +} + +// actionOciOverlayTeardown checks that OCI-mode overlays are correctly +// unmounted even in root mode (i.e., when user namespaces are not involved). +func (c actionTests) actionOciOverlayTeardown(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + const mountInfoPath string = "/proc/self/mountinfo" + numMountLinesPre, err := countLines(mountInfoPath) + if err != nil { + t.Fatal(err) + } + + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "oci_overlay_teardown-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "upper"), 0o777) + dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "work"), 0o777) + + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.OCIRootProfile), + e2e.WithCommand("exec"), + e2e.WithArgs("--overlay", tmpDir+":ro", imageRef, "/bin/true"), + e2e.ExpectExit(0), + ) + + numMountLinesPost, err := countLines(mountInfoPath) + if err != nil { + t.Fatal(err) + } + + assert.Equal( + t, numMountLinesPost, numMountLinesPre, + "Number of mounts after running in OCI-mode with overlays (%d) does not match the number before the run (%d)", numMountLinesPost, numMountLinesPre) +} + +func countLines(path string) (int, error) { + file, err := os.Open(path) + if err != nil { + return -1, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + lines := 0 + for scanner.Scan() { + lines++ + } + + return lines, nil +} + +// Check that write permissions are indeed available for writable FUSE-mounted +// extfs image overlays. +func (c actionTests) actionOciOverlayExtfsPerms(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + + for _, profile := range e2e.OCIProfiles { + // First, create a writable extfs overlay with `apptainer overlay create`. + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "oci_overlay_extfs_perms-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + imgPath := filepath.Join(tmpDir, "extfs-perms-test.img") + + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("overlay"), + e2e.WithArgs("create", "--size", "64", imgPath), + e2e.ExpectExit(0), + ) + + // Now test whether we can write to, and subsequently read from, the image + // we created. + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + tests := []struct { + name string + args []string + exitCode int + wantOutputs []e2e.ApptainerCmdResultOp + }{ + { + name: "FirstWrite", + args: []string{"--overlay", imgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + exitCode: 0, + }, + { + name: "ThenRead", + args: []string{"--overlay", imgPath, imageRef, "cat", "/my_test_file"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + } + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit( + tt.exitCode, + tt.wantOutputs..., + ), + ) + } + }) + } +} + +// Make sure --workdir and --scratch work together nicely even when workdir is a +// relative path. Test needs to be run in non-parallel mode, because it changes +// the current working directory of the host. +func (c actionTests) ociRelWorkdirScratch(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + testdir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "persistent-overlay-", "") + t.Cleanup(func() { + if !t.Failed() { + e2e.Privileged(cleanup) + } + }) + + const subdirName string = "mysubdir" + if err := os.Mkdir(filepath.Join(testdir, subdirName), 0o777); err != nil { + t.Fatalf("could not create subdirectory %q in %q: %s", subdirName, testdir, err) + } + + // Change current working directory, with deferred undoing of change. + prevCwd, err := os.Getwd() + if err != nil { + t.Fatalf("could not get current working directory: %s", err) + } + defer os.Chdir(prevCwd) + if err = os.Chdir(testdir); err != nil { + t.Fatalf("could not change cwd to %q: %s", testdir, err) + } + + profiles := e2e.OCIProfiles + + for _, p := range profiles { + c.env.RunApptainer( + t, + e2e.AsSubtest(p.String()), + e2e.WithProfile(p), + e2e.WithCommand("exec"), + e2e.WithArgs("--workdir", "./"+subdirName, "--scratch", "/myscratch", imageRef, "true"), + e2e.ExpectExit(0), + ) + } +} + +// ociSTDPipe tests pipe stdin/stdout to apptainer actions cmd +func (c actionTests) ociSTDPipe(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + stdinTests := []struct { + name string + command string + argv []string + input string + exit int + }{ + { + name: "TrueSTDIN", + command: "exec", + argv: []string{imageRef, "grep", "hi"}, + input: "hi", + exit: 0, + }, + { + name: "FalseSTDIN", + command: "exec", + argv: []string{imageRef, "grep", "hi"}, + input: "bye", + exit: 1, + }, + } + + var input bytes.Buffer + + for _, tt := range stdinTests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand(tt.command), + e2e.WithArgs(tt.argv...), + e2e.WithStdin(&input), + e2e.PreRun(func(t *testing.T) { + input.WriteString(tt.input) + }), + e2e.ExpectExit(tt.exit), + ) + input.Reset() + } + + user := e2e.CurrentUser(t) + stdoutTests := []struct { + name string + command string + argv []string + output string + exit int + }{ + { + name: "CwdPath", + command: "exec", + argv: []string{"--cwd", "/etc", imageRef, "pwd"}, + output: "/etc", + exit: 0, + }, + { + name: "PwdPath", + command: "exec", + argv: []string{"--pwd", "/etc", imageRef, "pwd"}, + output: "/etc", + exit: 0, + }, + { + name: "id", + command: "exec", + argv: []string{imageRef, "id", "-un"}, + output: user.Name, + exit: 0, + }, + } + for _, tt := range stdoutTests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand(tt.command), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit( + tt.exit, + e2e.ExpectOutput(e2e.ExactMatch, tt.output), + ), + ) + } +} diff --git a/e2e/cgroups/cgroups.go b/e2e/cgroups/cgroups.go index 2538c3106f..885c818b3b 100644 --- a/e2e/cgroups/cgroups.go +++ b/e2e/cgroups/cgroups.go @@ -255,9 +255,7 @@ func (c *ctx) instanceStatsRootless(t *testing.T) { c.instanceStats(t, e2e.UserProfile) } -func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { - e2e.EnsureImage(t, c.env) - +func (c *ctx) actionApply(t *testing.T, profile e2e.Profile, imageRef string) { tests := []struct { name string args []string @@ -265,64 +263,92 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { expectErrorOut string rootfull bool rootless bool + skipOCI bool + onlyOCI bool }{ { name: "nonexistent toml", - args: []string{"--apply-cgroups", "testdata/cgroups/doesnotexist.toml", c.env.ImagePath, "/bin/sleep", "5"}, + args: []string{"--apply-cgroups", "testdata/cgroups/doesnotexist.toml", imageRef, "/bin/sleep", "5"}, expectErrorCode: 255, expectErrorOut: "no such file or directory", rootfull: true, rootless: true, + skipOCI: false, + onlyOCI: false, }, { name: "invalid toml", - args: []string{"--apply-cgroups", "testdata/cgroups/invalid.toml", c.env.ImagePath, "/bin/sleep", "5"}, + args: []string{"--apply-cgroups", "testdata/cgroups/invalid.toml", imageRef, "/bin/sleep", "5"}, expectErrorCode: 255, expectErrorOut: "toml: expected character", rootfull: true, rootless: true, + skipOCI: false, + onlyOCI: false, }, { name: "memory limit", - args: []string{"--apply-cgroups", "testdata/cgroups/memory_limit.toml", c.env.ImagePath, "/bin/sleep", "5"}, + args: []string{"--apply-cgroups", "testdata/cgroups/memory_limit.toml", imageRef, "/bin/sleep", "5"}, expectErrorCode: 137, rootfull: true, rootless: true, + skipOCI: true, + onlyOCI: false, + }, + { + name: "memory limit oci", + args: []string{"--apply-cgroups", "testdata/cgroups/memory_limit.toml", imageRef, "/bin/sleep", "5"}, + // crun returns a 1 when the OOM kill happens. + expectErrorCode: 1, + rootfull: true, + rootless: true, + skipOCI: false, + onlyOCI: true, }, { name: "cpu success", - args: []string{"--apply-cgroups", "testdata/cgroups/cpu_success.toml", c.env.ImagePath, "/bin/true"}, + args: []string{"--apply-cgroups", "testdata/cgroups/cpu_success.toml", imageRef, "/bin/true"}, expectErrorCode: 0, rootfull: true, // This currently fails in the e2e scenario due to the way we are using a mount namespace. // It *does* work if you test it, directly calling the apptainer CLI. // Reason is believed to be: https://github.com/opencontainers/runc/issues/3026 rootless: false, + skipOCI: false, + onlyOCI: false, }, // Device access is allowed by default. { name: "device allow default", - args: []string{"--apply-cgroups", "testdata/cgroups/null.toml", c.env.ImagePath, "cat", "/dev/null"}, + args: []string{"--apply-cgroups", "testdata/cgroups/null.toml", imageRef, "cat", "/dev/null"}, expectErrorCode: 0, rootfull: true, rootless: true, + skipOCI: false, + onlyOCI: false, }, // Device limits are properly applied only in rootful mode. Rootless will ignore them with a warning. { name: "device deny", - args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", c.env.ImagePath, "cat", "/dev/null"}, + args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", imageRef, "cat", "/dev/null"}, expectErrorCode: 1, expectErrorOut: "Operation not permitted", rootfull: true, rootless: false, + // runc/crun always allow /dev/null access + skipOCI: true, + onlyOCI: false, }, { name: "device ignored", - args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", c.env.ImagePath, "cat", "/dev/null"}, + args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", imageRef, "cat", "/dev/null"}, expectErrorCode: 0, expectErrorOut: "Device limits will not be applied with rootless cgroups", rootfull: false, rootless: true, + // runc/crun silently ignore in rootless + skipOCI: true, + onlyOCI: false, }, } @@ -334,6 +360,13 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { if !profile.Privileged() && !tt.rootless { t.Skip() } + if profile.OCI() && tt.skipOCI { + t.Skip() + } + if !profile.OCI() && tt.onlyOCI { + t.Skip() + } + exitFunc := []e2e.ApptainerCmdResultOp{} if tt.expectErrorOut != "" { exitFunc = []e2e.ApptainerCmdResultOp{e2e.ExpectError(e2e.ContainMatch, tt.expectErrorOut)} @@ -350,13 +383,27 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { } func (c *ctx) actionApplyRoot(t *testing.T) { - c.actionApply(t, e2e.RootProfile) + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) + t.Run(e2e.RootProfile.String(), func(t *testing.T) { + c.actionApply(t, e2e.RootProfile, c.env.ImagePath) + }) + t.Run(e2e.OCIRootProfile.String(), func(t *testing.T) { + c.actionApply(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIArchivePath) + }) } func (c *ctx) actionApplyRootless(t *testing.T) { + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) for _, profile := range []e2e.Profile{e2e.UserProfile, e2e.UserNamespaceProfile, e2e.FakerootProfile} { t.Run(profile.String(), func(t *testing.T) { - c.actionApply(t, profile) + c.actionApply(t, profile, c.env.ImagePath) + }) + } + for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { + t.Run(profile.String(), func(t *testing.T) { + c.actionApply(t, profile, "oci-archive:"+c.env.OCIArchivePath) }) } } @@ -499,21 +546,21 @@ var resourceFlagTests = []resourceFlagTest{ }, } -func (c *ctx) actionFlags(t *testing.T, profile e2e.Profile) { +func (c *ctx) actionFlags(t *testing.T, profile e2e.Profile, imageRef string) { e2e.EnsureImage(t, c.env) for _, tt := range resourceFlagTests { t.Run(tt.name, func(t *testing.T) { if cgroups.IsCgroup2UnifiedMode() { - c.actionFlagV2(t, tt, profile) + c.actionFlagV2(t, tt, profile, imageRef) return } - c.actionFlagV1(t, tt, profile) + c.actionFlagV1(t, tt, profile, imageRef) }) } } -func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profile) { +func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profile, imageRef string) { // Don't try to test a resource that doesn't exist in our caller cgroup. // E.g. some systems don't have memory.memswp, and might not have blkio.bfq require.CgroupsResourceExists(t, tt.controllerV1, tt.resourceV1) @@ -530,7 +577,7 @@ func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profil } args := tt.args - args = append(args, "-B", "/sys/fs/cgroup", c.env.ImagePath, "/bin/sh", "-c", shellCmd) + args = append(args, "-B", "/sys/fs/cgroup", imageRef, "/bin/sh", "-c", shellCmd) c.env.RunApptainer( t, @@ -541,7 +588,7 @@ func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profil ) } -func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profile) { +func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profile, imageRef string) { if tt.skipV2 { t.Skip() } @@ -566,7 +613,7 @@ func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profil shellCmd := fmt.Sprintf("cat /sys/fs/cgroup$(cat /proc/self/cgroup | grep '^0::' | cut -d ':' -f 3)/%s", tt.resourceV2) args := tt.args - args = append(args, "-B", "/sys/fs/cgroup", c.env.ImagePath, "/bin/sh", "-c", shellCmd) + args = append(args, "-B", "/sys/fs/cgroup", imageRef, "/bin/sh", "-c", shellCmd) c.env.RunApptainer( t, @@ -578,13 +625,27 @@ func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profil } func (c *ctx) actionFlagsRoot(t *testing.T) { - c.actionFlags(t, e2e.RootProfile) + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) + t.Run(e2e.RootProfile.String(), func(t *testing.T) { + c.actionFlags(t, e2e.RootProfile, c.env.ImagePath) + }) + t.Run(e2e.OCIRootProfile.String(), func(t *testing.T) { + c.actionFlags(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIArchivePath) + }) } func (c *ctx) actionFlagsRootless(t *testing.T) { + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) for _, profile := range []e2e.Profile{e2e.UserProfile, e2e.UserNamespaceProfile, e2e.FakerootProfile} { t.Run(profile.String(), func(t *testing.T) { - c.actionFlags(t, profile) + c.actionFlags(t, profile, c.env.ImagePath) + }) + } + for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { + t.Run(profile.String(), func(t *testing.T) { + c.actionFlags(t, profile, "oci-archive:"+c.env.OCIArchivePath) }) } } diff --git a/e2e/config/config.go b/e2e/config/config.go index 8d58cf203e..0b365877b8 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -1003,5 +1003,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "config file": c.configFile, // test --config file option "config global": np(c.configGlobal), // test various global configuration "config global combination": np(c.configGlobalCombination), // test various global configuration with combination + "oci config global": np(c.ociConfigGlobal), // test various global configuration for OCI mode } } diff --git a/e2e/config/oci.go b/e2e/config/oci.go new file mode 100644 index 0000000000..0e5295c5d0 --- /dev/null +++ b/e2e/config/oci.go @@ -0,0 +1,294 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package config + +import ( + "fmt" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" +) + +//nolint:maintidx +func (c configTests) ociConfigGlobal(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + archiveRef := "oci-archive:" + c.env.OCIArchivePath + + setDirective := func(t *testing.T, directive, value string) { + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("config global"), + e2e.WithArgs("--set", directive, value), + e2e.ExpectExit(0), + ) + } + resetDirective := func(t *testing.T, directive string) { + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("config global"), + e2e.WithArgs("--reset", directive), + e2e.ExpectExit(0), + ) + } + + tests := []struct { + name string + argv []string + profile e2e.Profile + addRequirementsFn func(*testing.T) + cwd string + directive string + directiveValue string + exit int + resultOp e2e.ApptainerCmdResultOp + }{ + // { + // name: "AllowPidNsNo", + // argv: []string{"--pid", "--no-init", archiveRef, "/bin/sh", "-c", "echo $$"}, + // profile: e2e.OCIUserProfile, + // directive: "allow pid ns", + // directiveValue: "no", + // exit: 0, + // resultOp: e2e.ExpectOutput(e2e.UnwantedExactMatch, "1"), + // }, + // { + // name: "AllowPidNsYes", + // argv: []string{"--pid", "--no-init", archiveRef, "/bin/sh", "-c", "echo $$"}, + // profile: e2e.OCIUserProfile, + // directive: "allow pid ns", + // directiveValue: "yes", + // exit: 0, + // resultOp: e2e.ExpectOutput(e2e.ExactMatch, "1"), + // }, + { + name: "ConfigPasswdNo", + argv: []string{ + archiveRef, "grep", + fmt.Sprintf("%s:x:%d", e2e.OCIUserProfile.ContainerUser(t).Name, e2e.OCIUserProfile.ContainerUser(t).UID), + "/etc/passwd", + }, + profile: e2e.OCIUserProfile, + directive: "config passwd", + directiveValue: "no", + exit: 1, + }, + { + name: "ConfigPasswdYes", + argv: []string{ + archiveRef, "grep", + fmt.Sprintf("%s:x:%d", e2e.OCIUserProfile.ContainerUser(t).Name, e2e.OCIUserProfile.ContainerUser(t).UID), + "/etc/passwd", + }, + profile: e2e.OCIUserProfile, + directive: "config passwd", + directiveValue: "yes", + exit: 0, + }, + { + name: "ConfigGroupNo", + argv: []string{ + archiveRef, "grep", + fmt.Sprintf("x:%d:%s", e2e.OCIUserProfile.ContainerUser(t).GID, e2e.OCIUserProfile.ContainerUser(t).Name), + "/etc/group", + }, + profile: e2e.OCIUserProfile, + directive: "config group", + directiveValue: "no", + exit: 1, + }, + { + name: "ConfigGroupYes", + argv: []string{ + archiveRef, "grep", + fmt.Sprintf("x:%d:%s", e2e.OCIUserProfile.ContainerUser(t).GID, e2e.OCIUserProfile.ContainerUser(t).Name), + "/etc/group", + }, + profile: e2e.OCIUserProfile, + directive: "config group", + directiveValue: "yes", + exit: 0, + }, + // Test container doesn't have an /etc/resolv.conf, so presence check is okay here. + { + name: "ConfigResolvConfNo", + argv: []string{archiveRef, "test", "-f", "/etc/resolv.conf"}, + profile: e2e.OCIUserProfile, + directive: "config resolv_conf", + directiveValue: "no", + exit: 1, + }, + { + name: "ConfigResolvConfYes", + argv: []string{archiveRef, "test", "-f", "/etc/resolv.conf"}, + profile: e2e.OCIUserProfile, + directive: "config resolv_conf", + directiveValue: "yes", + exit: 0, + }, + { + name: "MountProcNo", + argv: []string{archiveRef, "test", "-d", "/proc/self"}, + profile: e2e.OCIUserProfile, + directive: "mount proc", + directiveValue: "no", + exit: 1, + }, + { + name: "MountProcYes", + argv: []string{archiveRef, "test", "-d", "/proc/self"}, + profile: e2e.OCIUserProfile, + directive: "mount proc", + directiveValue: "yes", + exit: 0, + }, + { + name: "MountSysNo", + argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, + profile: e2e.OCIUserProfile, + directive: "mount sys", + directiveValue: "no", + exit: 1, + }, + { + name: "MountSysYes", + argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, + profile: e2e.OCIUserProfile, + directive: "mount sys", + directiveValue: "yes", + exit: 0, + }, + // + // mount dev is not currently honored. We are mimicking --compat in the + // native runtime, which implies `minimal` here. Using `no` isn't an + // option, as the OCI runtime spec requires certain devices: + // https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#default-devices + // + // { + // name: "MountDevNo", + // argv: []string{archiveRef, "test", "-d", "/dev/pts"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "no", + // exit: 1, + // }, { + // name: "MountDevMinimal", + // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "minimal", + // exit: 1, + // }, { + // name: "MountDevYes", + // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "yes", + // exit: 0, + // }, // just test 'mount devpts = no' as yes depends of kernel version + // { + // name: "MountDevPtsNo", + // argv: []string{"-C", archiveRef, "test", "-d", "/dev/pts"}, + // profile: e2e.OCIUserProfile, + // directive: "mount devpts", + // directiveValue: "no", + // exit: 1, + // }, + // + // We have to check for a mount of $HOME, rather than presence of dir, + // as runc/crun will create the dir in the container fs if it doesn't + // exist. + { + name: "MountHomeNo", + argv: []string{archiveRef, "grep", e2e.OCIUserProfile.ContainerUser(t).Dir, "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + cwd: "/", + directive: "mount home", + directiveValue: "no", + exit: 1, + }, + { + name: "MountHomeYes", + argv: []string{archiveRef, "grep", e2e.OCIUserProfile.ContainerUser(t).Dir, "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + cwd: "/", + directive: "mount home", + directiveValue: "yes", + exit: 0, + }, + { + name: "MountTmpNo", + argv: []string{archiveRef, "grep", " /tmp ", "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + directive: "mount tmp", + directiveValue: "no", + exit: 1, + }, + { + name: "MountTmpYes", + argv: []string{archiveRef, "grep", " /tmp ", "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + directive: "mount tmp", + directiveValue: "yes", + exit: 0, + }, + // + // bind path isn't supported at present because we are mimicking + // --compat behavior in the native runtime. However, we should revisit + // what makes most sense for users here before 4.0. + // + // { + // name: "BindPathPasswd", + // argv: []string{archiveRef, "test", "-f", "/passwd"}, + // profile: e2e.OCIUserProfile, + // directive: "bind path", + // directiveValue: "/etc/passwd:/passwd", + // exit: 0, + // }, + { + name: "UserBindControlNo", + argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, + profile: e2e.OCIUserProfile, + directive: "user bind control", + directiveValue: "no", + exit: 1, + }, + { + name: "UserBindControlYes", + argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, + profile: e2e.OCIUserProfile, + directive: "user bind control", + directiveValue: "yes", + exit: 0, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(tt.profile), + e2e.WithDir(tt.cwd), + e2e.PreRun(func(t *testing.T) { + if tt.addRequirementsFn != nil { + tt.addRequirementsFn(t) + } + setDirective(t, tt.directive, tt.directiveValue) + }), + e2e.PostRun(func(t *testing.T) { + resetDirective(t, tt.directive) + }), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit(tt.exit, tt.resultOp), + ) + } +} diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index b4f27e7c8d..1d5d0195c5 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -208,49 +208,53 @@ func (c ctx) testDockerHost(t *testing.T) { }, } - t.Run("exec", func(t *testing.T) { - for _, tt := range tests { - cmdOps := []e2e.ApptainerCmdOp{ - e2e.WithProfile(e2e.RootProfile), - e2e.AsSubtest(tt.name), - e2e.WithCommand("exec"), - e2e.WithArgs("--disable-cache", dockerURI, "/bin/true"), - e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), - e2e.ExpectExit(tt.exit), - } - c.env.RunApptainer(t, cmdOps...) - } - }) - - t.Run("pull", func(t *testing.T) { - for _, tt := range tests { - cmdOps := []e2e.ApptainerCmdOp{ - e2e.WithProfile(e2e.RootProfile), - e2e.AsSubtest(tt.name), - e2e.WithCommand("pull"), - e2e.WithArgs("--force", "--disable-cache", dockerURI), - e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), - e2e.WithDir(tmpPath), - e2e.ExpectExit(tt.exit), - } - c.env.RunApptainer(t, cmdOps...) - } - }) - - t.Run("build", func(t *testing.T) { - for _, tt := range tests { - cmdOps := []e2e.ApptainerCmdOp{ - e2e.WithProfile(e2e.RootProfile), - e2e.AsSubtest(tt.name), - e2e.WithCommand("build"), - e2e.WithArgs("--force", "--disable-cache", "test.sif", dockerURI), - e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), - e2e.WithDir(tmpPath), - e2e.ExpectExit(tt.exit), - } - c.env.RunApptainer(t, cmdOps...) - } - }) + for _, profile := range []e2e.Profile{e2e.RootProfile, e2e.OCIRootProfile} { + t.Run(profile.String(), func(t *testing.T) { + t.Run("exec", func(t *testing.T) { + for _, tt := range tests { + cmdOps := []e2e.ApptainerCmdOp{ + e2e.WithProfile(profile), + e2e.AsSubtest(profile.String() + "/" + tt.name), + e2e.WithCommand("exec"), + e2e.WithArgs("--disable-cache", dockerURI, "/bin/true"), + e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), + e2e.ExpectExit(tt.exit), + } + c.env.RunApptainer(t, cmdOps...) + } + }) + + t.Run("pull", func(t *testing.T) { + for _, tt := range tests { + cmdOps := []e2e.ApptainerCmdOp{ + e2e.WithProfile(profile), + e2e.AsSubtest(tt.name), + e2e.WithCommand("pull"), + e2e.WithArgs("--force", "--disable-cache", dockerURI), + e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), + e2e.WithDir(tmpPath), + e2e.ExpectExit(tt.exit), + } + c.env.RunApptainer(t, cmdOps...) + } + }) + + t.Run("build", func(t *testing.T) { + for _, tt := range tests { + cmdOps := []e2e.ApptainerCmdOp{ + e2e.WithProfile(profile), + e2e.AsSubtest(tt.name), + e2e.WithCommand("build"), + e2e.WithArgs("--force", "--disable-cache", "test.sif", dockerURI), + e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), + e2e.WithDir(tmpPath), + e2e.ExpectExit(tt.exit), + } + c.env.RunApptainer(t, cmdOps...) + } + }) + }) + } // Clean up docker image e2e.Privileged(func(t *testing.T) { @@ -488,28 +492,28 @@ func (c ctx) testDockerRegistry(t *testing.T) { dfd e2e.DefFileDetails }{ { - name: "BusyBox", + name: "Alpine", exit: 0, dfd: e2e.DefFileDetails{ Bootstrap: "docker", - From: fmt.Sprintf("%s/my-busybox", c.env.TestRegistry), + From: fmt.Sprintf("%s/my-alpine", c.env.TestRegistry), }, }, { - name: "BusyBoxRegistry", + name: "AlpineRegistry", exit: 0, dfd: e2e.DefFileDetails{ Bootstrap: "docker", - From: "my-busybox", + From: "my-alpine", Registry: c.env.TestRegistry, }, }, { - name: "BusyBoxNamespace", + name: "AlpineNamespace", exit: 255, dfd: e2e.DefFileDetails{ Bootstrap: "docker", - From: "my-busybox", + From: "my-alpine", Registry: c.env.TestRegistry, Namespace: "not-a-namespace", }, @@ -880,6 +884,103 @@ func (c ctx) testDockerCMDENTRYPOINT(t *testing.T) { } } +// Check that the USER in a docker container is honored under --oci mode +func (c ctx) testDockerUSER(t *testing.T) { + dockerURI := "docker://ghcr.io/apptainer/docker-user" + tests := []struct { + name string + cmd string + args []string + expectOutputs []e2e.ApptainerCmdResultOp + profile e2e.Profile + expectExit int + }{ + // Sanity check apptainer native engine... no support for USER + { + name: "default", + cmd: "run", + profile: e2e.UserProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, fmt.Sprintf("uid=%d(%s) gid=%d", + e2e.UserProfile.ContainerUser(t).UID, + e2e.UserProfile.ContainerUser(t).Name, + e2e.UserProfile.ContainerUser(t).GID, + )), + }, + }, + // `--oci` modes (USER honored by default) + { + name: "OCIUser", + cmd: "run", + profile: e2e.OCIUserProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, `uid=2000(testuser) gid=2000(testgroup)`), + }, + }, + { + name: "OCIFakeroot", + profile: e2e.OCIFakerootProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, `uid=0(root) gid=0(root)`), + }, + }, + { + name: "OCIRoot", + cmd: "run", + profile: e2e.OCIRootProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, `uid=2000(testuser) gid=2000(testgroup)`), + }, + }, + // `--oci` modes: check that we don't override container-user's home directory + { + name: "OrigHomeOCIUser", + cmd: "exec", + profile: e2e.OCIUserProfile, + args: []string{dockerURI, "env"}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/home/testuser\b`), + }, + expectExit: 0, + }, + { + name: "OrigHomeOCIFakeroot", + cmd: "exec", + profile: e2e.OCIFakerootProfile, + args: []string{dockerURI, "env"}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/root\b`), + }, + expectExit: 0, + }, + { + name: "OrigHomeOCIRoot", + cmd: "exec", + profile: e2e.OCIRootProfile, + args: []string{dockerURI, "env"}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/home/testuser\b`), + }, + expectExit: 0, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(tt.profile), + e2e.WithCommand("run"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(tt.expectExit, tt.expectOutputs...), + ) + } +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := ctx{ @@ -901,8 +1002,13 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("entrypoint", c.testDockerENTRYPOINT) t.Run("cmdentrypoint", c.testDockerCMDENTRYPOINT) t.Run("cmd quotes", c.testDockerCMDQuotes) + t.Run("user", c.testDockerUSER) // Regressions t.Run("issue 4524", c.issue4524) + t.Run("issue 1286", c.issue1286) + t.Run("issue 1528", c.issue1528) + t.Run("issue 1586", c.issue1586) + t.Run("issue 1670", c.issue1670) }, // Tests that are especially slow, or run against a local docker // registry, can be run in parallel, with `--disable-cache` used within diff --git a/e2e/docker/regressions.go b/e2e/docker/regressions.go index ccf410d1d6..c3dcdeb932 100644 --- a/e2e/docker/regressions.go +++ b/e2e/docker/regressions.go @@ -11,6 +11,7 @@ package docker import ( "fmt" + "io" "os" "path" "path/filepath" @@ -83,7 +84,7 @@ func (c ctx) issue4943(t *testing.T) { func (c ctx) issue5172(t *testing.T) { // create $HOME/.config/containers/registries.conf - regImage := fmt.Sprintf("docker://%s/my-busybox", c.env.TestRegistry) + regImage := fmt.Sprintf("docker://%s/my-alpine", c.env.TestRegistry) imagePath := filepath.Join(c.env.TestDir, "issue-5172") c.env.RunApptainer( @@ -210,3 +211,133 @@ func (c ctx) issue1704(t *testing.T) { e2e.ExpectExit(0, e2e.ExpectOutput(e2e.ContainMatch, strings.TrimSpace(defFileContents))), ) } + +// https://github.com/sylabs/singularity/issues/1286 +// Ensure the bare docker://hello-world image runs in all modes +func (c ctx) issue1286(t *testing.T) { + for _, profile := range e2e.AllProfiles() { + c.env.RunApptainer( + t, + e2e.AsSubtest(profile.String()), + e2e.WithProfile(profile), + e2e.WithCommand("run"), + e2e.WithArgs("docker://hello-world"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "Hello from Docker!"), + ), + ) + } +} + +// https://github.com/sylabs/singularity/issues/1528 +// Check that host's TERM value gets passed to OCI container. +func (c ctx) issue1528(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + + imageRef := "oci-archive:" + c.env.OCIArchivePath + + _, wasHostTermSet := os.LookupEnv("TERM") + if !wasHostTermSet { + if err := os.Setenv("TERM", "xterm"); err != nil { + t.Errorf("could not set TERM environment variable on host") + } + defer os.Unsetenv("TERM") + } + + singEnvTermPrevious, wasHostSingEnvTermSet := os.LookupEnv("APPTAINERENV_TERM") + if wasHostSingEnvTermSet { + if err := os.Unsetenv("APPTAINERENV_TERM"); err != nil { + t.Errorf("could not unset APPTAINERENV_TERM environment variable on host") + } + defer os.Setenv("APPTAINERENV_TERM", singEnvTermPrevious) + } else { + defer os.Unsetenv("APPTAINERENV_TERM") + } + + envTerm := os.Getenv("TERM") + wantTermString := fmt.Sprintf("TERM=%s\n", envTerm) + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + c.env.RunApptainer( + t, + e2e.AsSubtest("issue1528"), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(imageRef, "env"), + e2e.ExpectExit(0, e2e.ExpectOutput(e2e.ContainMatch, wantTermString)), + ) + }) + } + + singEnvTerm := envTerm + "testsuffix" + if err := os.Setenv("APPTAINERENV_TERM", singEnvTerm); err != nil { + t.Errorf("could not set APPTAINERENV_TERM environment variable on host") + } + wantTermString = fmt.Sprintf("TERM=%s\n", singEnvTerm) + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + c.env.RunApptainer( + t, + e2e.AsSubtest("issue1528override"), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(imageRef, "env"), + e2e.ExpectExit(0, e2e.ExpectOutput(e2e.ContainMatch, wantTermString)), + ) + }) + } +} + +// https://github.com/sylabs/singularity/issues/1586 +// In OCI mode, ensure that nothing is left in TMPDIR from a docker:// image with restrictive file permissions. +func (c ctx) issue1586(t *testing.T) { + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "issue1586-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithArgs("docker://almalinux:9.1-minimal-20230407", "/bin/true"), + e2e.WithEnv(append(os.Environ(), "TMPDIR="+tmpDir)), + e2e.ExpectExit(0, + e2e.ExpectError(e2e.UnwantedContainMatch, "permission denied"), + ), + ) + + d, err := os.Open(tmpDir) + if err != nil { + t.Errorf("Couldn't open TMPDIR %s: %v", tmpDir, err) + } + defer d.Close() + if _, err = d.Readdir(1); err != io.EOF { + t.Errorf("TMPDIR is not empty after apptainer exited") + } +} + +// https://github.com/sylabs/singularity/issues/1670 +// Check that runc/crun can add directories the rootfs before entering the +// container, by running a container based on busybox that lacks, e.g., /proc +func (c ctx) issue1670(t *testing.T) { + for _, profile := range e2e.OCIProfiles { + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, fmt.Sprintf("issue1670-%s-", profile.String()), "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + c.env.RunApptainer( + t, + e2e.AsSubtest(profile.String()), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs("--overlay", fmt.Sprintf("%s:ro", tmpDir), "docker://busybox", "echo", "hi"), + e2e.ExpectExit(0), + ) + } +} diff --git a/e2e/env/env.go b/e2e/env/env.go index 0c928bc747..b4509dc523 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -645,5 +645,11 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "issue 5426": c.issue5426, // https://github.com/apptainer/singularity/issues/5426 "issue 43": c.issue43, // https://github.com/sylabs/singularity/issues/43 "issue 1263": c.issue1263, // https://github.com/sylabs/singularity/issues/1263 + // + // --oci mode + // + "oci environment apptainerenv": c.ociApptainerEnv, + "oci environment option": c.ociEnvOption, + "oci environment file": c.ociEnvFile, } } diff --git a/e2e/env/oci.go b/e2e/env/oci.go new file mode 100644 index 0000000000..9d7610dcde --- /dev/null +++ b/e2e/env/oci.go @@ -0,0 +1,300 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package apptainerenv + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" +) + +func (c ctx) ociApptainerEnv(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIArchivePath + + // Append or prepend this path. + partialPath := "/foo" + + // Overwrite the path with this one. + overwrittenPath := "/usr/bin:/bin" + + // A path with a trailing comma + trailingCommaPath := "/usr/bin:/bin," + + tests := []struct { + name string + image string + path string + env []string + }{ + { + name: "DefaultPath", + image: defaultImage, + path: defaultPath, + env: []string{}, + }, + { + name: "AppendToDefaultPath", + image: defaultImage, + path: defaultPath + ":" + partialPath, + env: []string{"APPTAINERENV_APPEND_PATH=/foo"}, + }, + { + name: "PrependToDefaultPath", + image: defaultImage, + path: partialPath + ":" + defaultPath, + env: []string{"APPTAINERENV_PREPEND_PATH=/foo"}, + }, + { + name: "OverwriteDefaultPath", + image: defaultImage, + path: overwrittenPath, + env: []string{"APPTAINERENV_PATH=" + overwrittenPath}, + }, + { + name: "OverwriteTrailingCommaPath", + image: defaultImage, + path: trailingCommaPath, + env: []string{"APPTAINERENV_PATH=" + trailingCommaPath}, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithEnv(tt.env), + e2e.WithRootlessEnv(), + e2e.WithArgs(tt.image, "/bin/sh", "-c", "echo $PATH"), + e2e.ExpectExit( + 0, + e2e.ExpectOutput(e2e.ExactMatch, tt.path), + ), + ) + } +} + +func (c ctx) ociEnvOption(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIArchivePath + + tests := []struct { + name string + image string + envOpt []string + hostEnv []string + matchEnv string + matchVal string + }{ + { + name: "DefaultPath", + image: defaultImage, + matchEnv: "PATH", + matchVal: defaultPath, + }, + { + name: "DefaultPathOverride", + image: defaultImage, + envOpt: []string{"PATH=/"}, + matchEnv: "PATH", + matchVal: "/", + }, + { + name: "AppendDefaultPath", + image: defaultImage, + envOpt: []string{"APPEND_PATH=/foo"}, + matchEnv: "PATH", + matchVal: defaultPath + ":/foo", + }, + { + name: "PrependDefaultPath", + image: defaultImage, + envOpt: []string{"PREPEND_PATH=/foo"}, + matchEnv: "PATH", + matchVal: "/foo:" + defaultPath, + }, + { + name: "TestMultiLine", + image: defaultImage, + envOpt: []string{"MULTI=Hello\nWorld"}, + matchEnv: "MULTI", + matchVal: "Hello\nWorld", + }, + { + name: "TestEscapedNewline", + image: defaultImage, + envOpt: []string{"ESCAPED=Hello\\nWorld"}, + matchEnv: "ESCAPED", + matchVal: "Hello\\nWorld", + }, + { + name: "TestInvalidKey", + image: defaultImage, + envOpt: []string{"BASH_FUNC_ml%%=TEST"}, + matchEnv: "BASH_FUNC_ml%%", + matchVal: "", + }, + { + name: "TestDefaultLdLibraryPath", + image: defaultImage, + matchEnv: "LD_LIBRARY_PATH", + matchVal: apptainerLibs, + }, + { + name: "TestCustomTrailingCommaPath", + image: defaultImage, + envOpt: []string{"LD_LIBRARY_PATH=/foo,"}, + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo,:" + apptainerLibs, + }, + { + name: "TestCustomLdLibraryPath", + image: defaultImage, + envOpt: []string{"LD_LIBRARY_PATH=/foo"}, + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo:" + apptainerLibs, + }, + { + name: "APPTAINER_NAME", + image: defaultImage, + matchEnv: "APPTAINER_NAME", + matchVal: defaultImage, + }, + } + + for _, tt := range tests { + args := make([]string, 0) + if tt.envOpt != nil { + args = append(args, "--env", strings.Join(tt.envOpt, ",")) + } + args = append(args, tt.image, "/bin/sh", "-c", "echo \"${"+tt.matchEnv+"}\"") + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithEnv(tt.hostEnv), + e2e.WithRootlessEnv(), + e2e.WithArgs(args...), + e2e.ExpectExit( + 0, + e2e.ExpectOutput(e2e.ExactMatch, tt.matchVal), + ), + ) + } +} + +func (c ctx) ociEnvFile(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIArchivePath + + dir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "envfile-", "") + defer cleanup(t) + p := filepath.Join(dir, "env.file") + + tests := []struct { + name string + image string + envFile string + envOpt []string + hostEnv []string + matchEnv string + matchVal string + }{ + { + name: "DefaultPathOverride", + image: defaultImage, + envFile: "PATH=/", + matchEnv: "PATH", + matchVal: "/", + }, + { + name: "DefaultPathOverrideEnvOptionPrecedence", + image: defaultImage, + envOpt: []string{"PATH=/etc"}, + envFile: "PATH=/", + matchEnv: "PATH", + matchVal: "/etc", + }, + { + name: "DefaultPathOverrideEnvOptionPrecedence", + image: defaultImage, + envOpt: []string{"PATH=/etc"}, + envFile: "PATH=/", + matchEnv: "PATH", + matchVal: "/etc", + }, + { + name: "AppendDefaultPath", + image: defaultImage, + envFile: "APPEND_PATH=/", + matchEnv: "PATH", + matchVal: defaultPath + ":/", + }, + { + name: "PrependDefaultPath", + image: defaultImage, + envFile: "PREPEND_PATH=/", + matchEnv: "PATH", + matchVal: "/:" + defaultPath, + }, + { + name: "DefaultLdLibraryPath", + image: defaultImage, + matchEnv: "LD_LIBRARY_PATH", + matchVal: apptainerLibs, + }, + { + name: "CustomLdLibraryPath", + image: defaultImage, + envFile: "LD_LIBRARY_PATH=/foo", + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo:" + apptainerLibs, + }, + { + name: "CustomTrailingCommaPath", + image: defaultImage, + envFile: "LD_LIBRARY_PATH=/foo,", + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo,:" + apptainerLibs, + }, + } + + for _, tt := range tests { + args := make([]string, 0) + if tt.envOpt != nil { + args = append(args, "--env", strings.Join(tt.envOpt, ",")) + } + if tt.envFile != "" { + os.WriteFile(p, []byte(tt.envFile), 0o644) + args = append(args, "--env-file", p) + } + args = append(args, tt.image, "/bin/sh", "-c", "echo $"+tt.matchEnv) + + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithEnv(tt.hostEnv), + e2e.WithRootlessEnv(), + e2e.WithArgs(args...), + e2e.ExpectExit( + 0, + e2e.ExpectOutput(e2e.ExactMatch, tt.matchVal), + ), + ) + } +} diff --git a/e2e/gpu/gpu.go b/e2e/gpu/gpu.go index 02f5fd3cc5..d9f73374aa 100644 --- a/e2e/gpu/gpu.go +++ b/e2e/gpu/gpu.go @@ -107,6 +107,44 @@ func (c ctx) testNvidiaLegacy(t *testing.T) { } } +func (c ctx) ociTestNvidiaLegacy(t *testing.T) { + require.Nvidia(t) + + imageURL := "docker://ubuntu:20.04" + + // Basic test that we can run the bound in `nvidia-smi` which *should* be on the PATH + tests := []struct { + profile e2e.Profile + args []string + env []string + }{ + { + profile: e2e.OCIUserProfile, + args: []string{"--nv", imageURL, "nvidia-smi"}, + }, + { + profile: e2e.OCIFakerootProfile, + args: []string{"--nv", imageURL, "nvidia-smi"}, + }, + { + profile: e2e.OCIRootProfile, + args: []string{"--nv", imageURL, "nvidia-smi"}, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.profile.String()), + e2e.WithProfile(tt.profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.WithEnv(tt.env), + e2e.ExpectExit(0), + ) + } +} + func (c ctx) testNvCCLI(t *testing.T) { require.Nvidia(t) require.NvCCLI(t) @@ -289,6 +327,52 @@ func (c ctx) testRocm(t *testing.T) { } } +func (c ctx) ociTestRocm(t *testing.T) { + require.Rocm(t) + + require.Command(t, "lsmod") + + // rocminfo now needs lsmod - do a brittle bind in for simplicity. + lsmod, err := exec.LookPath("lsmod") + if err != nil { + t.Fatalf("while finding lsmod: %v", err) + } + + // Use Ubuntu 22.04 as this is the most recent distro officially supported by ROCm. + // We can't use our test image as it's alpine based and we need a compatible glibc. + imageURL := "docker://ubuntu:22.04" + + // Basic test that we can run the bound in `rocminfo` which *should* be on the PATH + tests := []struct { + profile e2e.Profile + args []string + }{ + { + profile: e2e.OCIUserProfile, + args: []string{"-B", lsmod, "--rocm", imageURL, "rocminfo"}, + }, + { + profile: e2e.OCIFakerootProfile, + args: []string{"-B", lsmod, "--rocm", imageURL, "rocminfo"}, + }, + { + profile: e2e.OCIRootProfile, + args: []string{"-B", lsmod, "--rocm", imageURL, "rocminfo"}, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.profile.String()), + e2e.WithProfile(tt.profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(0), + ) + } +} + //nolint:dupl func (c ctx) testBuildNvidiaLegacy(t *testing.T) { require.Nvidia(t) @@ -555,5 +639,8 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "build nvidia": c.testBuildNvidiaLegacy, "build nvccli": c.testBuildNvCCLI, "build rocm": c.testBuildRocm, + // oci mode + "oci nvidia": c.ociTestNvidiaLegacy, + "oci rocm": c.ociTestRocm, } } diff --git a/e2e/imgbuild/imgbuild.go b/e2e/imgbuild/imgbuild.go index 3e881b0852..8032de8efd 100644 --- a/e2e/imgbuild/imgbuild.go +++ b/e2e/imgbuild/imgbuild.go @@ -1811,7 +1811,7 @@ func (c imgBuildTests) buildLibraryHost(t *testing.T) { ) } -// testWritableTmpfs checks that we can run the build using a writeable tmpfs in the %test step +// testWritableTmpfs checks that we can run the build using a writable tmpfs in the %test step func (c imgBuildTests) testWritableTmpfs(t *testing.T) { e2e.EnsureImage(t, c.env) diff --git a/e2e/internal/e2e/apptainercmd.go b/e2e/internal/e2e/apptainercmd.go index 3b36416a04..23c6bda17c 100644 --- a/e2e/internal/e2e/apptainercmd.go +++ b/e2e/internal/e2e/apptainercmd.go @@ -108,7 +108,7 @@ func (r *ApptainerCmdResult) expectMatch(mt MatchType, stream streamType, patter // get rid of the trailing newline if strings.TrimSuffix(output, "\n") != pattern { return errors.Errorf( - "Command %q:\nExpect %s stream exact match:\n%s\nCommand %s output:\n%s", + "Command %q:\nExpect %s stream exact match:\n%s\nCommand %s stream:\n%s", r.FullCmd, streamName, pattern, streamName, output, ) } @@ -122,7 +122,7 @@ func (r *ApptainerCmdResult) expectMatch(mt MatchType, stream streamType, patter case UnwantedExactMatch: if strings.TrimSuffix(output, "\n") == pattern { return errors.Errorf( - "Command %q:\nExpect %s stream not matching:\n%s\nCommand %s output:\n%s", + "Command %q:\nExpect %s stream not matching:\n%s\nCommand %s stream:\n%s", r.FullCmd, streamName, pattern, streamName, output, ) } @@ -136,7 +136,7 @@ func (r *ApptainerCmdResult) expectMatch(mt MatchType, stream streamType, patter } if !matched { return errors.Errorf( - "Command %q:\nExpect %s stream match regular expression:\n%s\nCommand %s output:\n%s", + "Command %q:\nExpect %s stream match regular expression:\n%s\nCommand %s stream:\n%s", r.FullCmd, streamName, pattern, streamName, output, ) } @@ -496,8 +496,8 @@ func (env TestEnv) RunApptainer(t *testing.T, cmdOps ...ApptainerCmdOp) { // a profile is required if s.profile.name == "" { i := 0 - availableProfiles := make([]string, len(Profiles)) - for profile := range Profiles { + availableProfiles := make([]string, len(NativeProfiles)) + for profile := range NativeProfiles { availableProfiles[i] = profile i++ } @@ -535,6 +535,20 @@ func (env TestEnv) RunApptainer(t *testing.T, cmdOps ...ApptainerCmdOp) { cmd.Env = os.Environ() } + // Clear user-specific DBUS / XDG vars when we are using a priv profile, + // as they don't make sense for the root user... and wouldn't be set in a + // real root user session. + if privileged { + i := 0 + for _, e := range cmd.Env { + if !(strings.HasPrefix(e, "DBUS_SESSION_BUS_ADDRESS=") || strings.HasPrefix(e, "XDG_RUNTIME_DIR=")) { + cmd.Env[i] = e + i++ + } + } + cmd.Env = cmd.Env[:i] + } + // By default, each E2E command shares a temporary image cache // directory. If a test is directly testing the cache, or depends on // specific ordered cache behavior then diff --git a/e2e/internal/e2e/env.go b/e2e/internal/e2e/env.go index da4fec7d6f..9a99c9cafd 100644 --- a/e2e/internal/e2e/env.go +++ b/e2e/internal/e2e/env.go @@ -18,7 +18,9 @@ type TestEnv struct { SingularityImagePath string // Path to a Singularity image for legacy tests DebianImagePath string // Path to an image containing a Debian distribution with libc compatible to the host libc OrasTestImage string // URI to SIF image pushed into local registry with ORAS + OCIArchivePath string // Path to test OCI archive tar file TestDir string // Path to the directory from which an Apptainer command needs to be executed + DockerArchivePath string // Path to test Docker archive tar file TestRegistry string // Host:Port of local registry TestRegistryImage string // URI to OCI image pushed into local registry HomeDir string // HomeDir sets the home directory that will be used for the execution of a command diff --git a/e2e/internal/e2e/image.go b/e2e/internal/e2e/image.go index eb171c55af..ad2d602209 100644 --- a/e2e/internal/e2e/image.go +++ b/e2e/internal/e2e/image.go @@ -13,6 +13,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -285,6 +286,78 @@ func BusyboxSIF(t *testing.T) string { return busyboxSIF } +func DownloadFile(url string, path string) error { + dl, err := os.Create(path) + if err != nil { + return err + } + defer dl.Close() + + r, err := http.Get(url) + if err != nil { + return err + } + defer r.Body.Close() + + _, err = io.Copy(dl, r.Body) + if err != nil { + return err + } + return nil +} + +// EnsureImage checks if e2e OCI test archive is available, and fetches +// it otherwise. +func EnsureOCIArchive(t *testing.T, env TestEnv) { + ensureMutex.Lock() + defer ensureMutex.Unlock() + + switch _, err := os.Stat(env.OCIArchivePath); { + case err == nil: + // OK: file exists, return + return + + case os.IsNotExist(err): + // OK: file does not exist, continue + + default: + // FATAL: something else is wrong + t.Fatalf("Failed when checking image %q: %+v\n", + env.OCIArchivePath, + err) + } + + // Prepare oci-archive source + t.Logf("Copying %s to %s", env.TestRegistryImage, "oci-archive:"+env.OCIArchivePath) + CopyImage(t, env.TestRegistryImage, "oci-archive:"+env.OCIArchivePath, true, false) +} + +// EnsureDockerArchive checks if e2e Docker test archive is available, and fetches +// it otherwise. +func EnsureDockerArchive(t *testing.T, env TestEnv) { + ensureMutex.Lock() + defer ensureMutex.Unlock() + + switch _, err := os.Stat(env.DockerArchivePath); { + case err == nil: + // OK: file exists, return + return + + case os.IsNotExist(err): + // OK: file does not exist, continue + + default: + // FATAL: something else is wrong + t.Fatalf("Failed when checking image %q: %+v\n", + env.DockerArchivePath, + err) + } + + // Prepare oci-archive source + t.Logf("Copying %s to %s", env.TestRegistryImage, "docker-archive:"+env.DockerArchivePath) + CopyImage(t, env.TestRegistryImage, "docker-archive:"+env.DockerArchivePath, true, false) +} + func parseRef(refString string) (ref types.ImageReference, err error) { parts := strings.SplitN(refString, ":", 2) if len(parts) < 2 { diff --git a/e2e/internal/e2e/profile.go b/e2e/internal/e2e/profile.go index 3fe4868ce8..930c86e782 100644 --- a/e2e/internal/e2e/profile.go +++ b/e2e/internal/e2e/profile.go @@ -24,19 +24,29 @@ const ( fakerootProfile = "FakerootProfile" userNamespaceProfile = "UserNamespaceProfile" rootUserNamespaceProfile = "RootUserNamespaceProfile" + ociUserProfile = "OCIUserProfile" + ociRootProfile = "OCIRootProfile" + ociFakerootProfile = "OCIFakerootProfile" ) var ( - // UserProfile is the execution profile for a regular user. - UserProfile = Profiles[userProfile] - // RootProfile is the execution profile for root. - RootProfile = Profiles[rootProfile] - // FakerootProfile is the execution profile for fakeroot. - FakerootProfile = Profiles[fakerootProfile] - // UserNamespaceProfile is the execution profile for a regular user and a user namespace. - UserNamespaceProfile = Profiles[userNamespaceProfile] - // RootUserNamespaceProfile is the execution profile for root and a user namespace. - RootUserNamespaceProfile = Profiles[rootUserNamespaceProfile] + // UserProfile is the execution profile for a regular user, using the Apptainer native runtime. + UserProfile = NativeProfiles[userProfile] + // RootProfile is the execution profile for root, using the Apptainer native runtime. + RootProfile = NativeProfiles[rootProfile] + // FakerootProfile is the execution profile for fakeroot, using the Apptainer native runtime. + FakerootProfile = NativeProfiles[fakerootProfile] + // UserNamespaceProfile is the execution profile for a regular user and a user namespace, using the Apptainer native runtime. + UserNamespaceProfile = NativeProfiles[userNamespaceProfile] + // RootUserNamespaceProfile is the execution profile for root and a user namespace, using the Apptainer native runtime. + RootUserNamespaceProfile = NativeProfiles[rootUserNamespaceProfile] + // OCIUserProfile is the execution profile for a regular user, using the Apptainer native runtime. + OCIUserProfile = OCIProfiles[ociUserProfile] + // RootProfile is the execution profile for root, using the Apptainer native runtime. + OCIRootProfile = OCIProfiles[ociRootProfile] + // FakerootProfile is the execution profile for fakeroot, using the Apptainer native runtime. + OCIFakerootProfile = OCIProfiles[ociFakerootProfile] + // UserNamespaceProfile is the execution profile for a regular user and a user namespace, using the Apptainer native runtime. ) // Profile represents various properties required to run an E2E test @@ -58,10 +68,11 @@ type Profile struct { requirementsFn func(*testing.T) // function checking requirements for the profile apptainerOption string // option added to apptainer command for the profile optionForCommands []string // apptainer commands concerned by the option to be added + oci bool // whether the profile uses the OCI low-level runtime } -// Profiles defines all available profiles. -var Profiles = map[string]Profile{ +// NativeProfiles defines all available profiles for the native apptainer runtime +var NativeProfiles = map[string]Profile{ userProfile: { name: "User", privileged: false, @@ -71,6 +82,7 @@ var Profiles = map[string]Profile{ requirementsFn: nil, apptainerOption: "", optionForCommands: []string{}, + oci: false, }, rootProfile: { name: "Root", @@ -81,6 +93,7 @@ var Profiles = map[string]Profile{ requirementsFn: nil, apptainerOption: "", optionForCommands: []string{}, + oci: false, }, fakerootProfile: { name: "Fakeroot", @@ -91,6 +104,7 @@ var Profiles = map[string]Profile{ requirementsFn: fakerootRequirements, apptainerOption: "--fakeroot", optionForCommands: []string{"shell", "exec", "run", "test", "instance start", "build"}, + oci: false, }, userNamespaceProfile: { name: "UserNamespace", @@ -101,6 +115,7 @@ var Profiles = map[string]Profile{ requirementsFn: require.UserNamespace, apptainerOption: "--userns", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: false, }, rootUserNamespaceProfile: { name: "RootUserNamespace", @@ -111,15 +126,70 @@ var Profiles = map[string]Profile{ requirementsFn: require.UserNamespace, apptainerOption: "--userns", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: false, }, } +// OCIProfiles defines all available profiles for the OCI runtime +var OCIProfiles = map[string]Profile{ + ociUserProfile: { + name: "OCIUser", + privileged: false, + hostUID: origUID, + containerUID: origUID, + defaultCwd: "", + requirementsFn: ociRequirements, + apptainerOption: "--oci", + optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: true, + }, + ociRootProfile: { + name: "OCIRoot", + privileged: true, + hostUID: 0, + containerUID: 0, + defaultCwd: "", + requirementsFn: ociRequirements, + apptainerOption: "--oci", + optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: true, + }, + ociFakerootProfile: { + name: "OCIFakeroot", + privileged: false, + hostUID: origUID, + containerUID: 0, + defaultCwd: "", + requirementsFn: ociRequirements, + apptainerOption: "--oci --fakeroot", + optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: true, + }, +} + +// AllProfiles is initialized to the union of NativeProfiles and OCIProfiles +func AllProfiles() map[string]Profile { + ap := map[string]Profile{} + for k, p := range NativeProfiles { + ap[k] = p + } + for k, p := range OCIProfiles { + ap[k] = p + } + return ap +} + // Privileged returns whether the test should be executed with // elevated privileges or not. func (p Profile) Privileged() bool { return p.privileged } +// OCI returns whether the profile is using an OCI runtime, rather than the apptainer native runtime. +func (p Profile) OCI() bool { + return p.oci +} + // Requirements calls the different require.* functions // necessary for running an E2E test under this profile. func (p Profile) Requirements(t *testing.T) { @@ -205,3 +275,26 @@ func fakerootRequirements(t *testing.T) { t.Fatalf("fakeroot configuration error: %s", err) } } + +// ociRequirements ensures requirements are satisfied to correctly execute +// commands with the OCI runtime / profile. +func ociRequirements(t *testing.T) { + require.UserNamespace(t) + require.Command(t, "runc") + + uid := uint32(origUID) + + // check that current user has valid mappings in /etc/subuid + if _, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid); err != nil { + t.Fatalf("fakeroot configuration error: %s", err) + } + + // check that current user has valid mappings in /etc/subgid; + // since that file contains the group mappings for a given user + // *name*, it is keyed by user name, not by group name. This + // means that even if we are requesting the *group* mappings, we + // need to pass the *user* ID. + if _, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, uid); err != nil { + t.Fatalf("fakeroot configuration error: %s", err) + } +} diff --git a/e2e/oci/oci.go b/e2e/oci/oci.go index a89f913c74..8845910653 100644 --- a/e2e/oci/oci.go +++ b/e2e/oci/oci.go @@ -93,6 +93,7 @@ func genericOciMount(t *testing.T, c *ctx) (string, func()) { } c.env.RunApptainer( t, + e2e.AsSubtest("mount"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci mount"), e2e.WithArgs(c.env.ImagePath, bundleDir), @@ -102,6 +103,7 @@ func genericOciMount(t *testing.T, c *ctx) (string, func()) { cleanup := func() { c.env.RunApptainer( t, + e2e.AsSubtest("umount"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci umount"), e2e.WithArgs(bundleDir), @@ -158,6 +160,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("create"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci create"), e2e.WithArgs("-b", bundleDir, containerID), @@ -176,6 +179,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("start"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -189,6 +193,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("attach"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci attach"), e2e.WithArgs(containerID), @@ -207,6 +212,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("delete"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci delete"), e2e.WithArgs(containerID), @@ -225,6 +231,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("create"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci create"), e2e.WithArgs("-b", bundleDir, containerID), @@ -243,6 +250,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("start"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -256,6 +264,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("pause"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci pause"), e2e.WithArgs(containerID), @@ -273,6 +282,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("resume"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci resume"), e2e.WithArgs(containerID), @@ -290,6 +300,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("start again"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -298,17 +309,20 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("exec"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci exec"), - e2e.WithArgs(containerID, "id"), - e2e.ExpectExit(0), + e2e.WithArgs(containerID, "hostname"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "apptainer")), ) c.env.RunApptainer( t, + e2e.AsSubtest("kill"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci kill"), - e2e.WithArgs("-t", "2", containerID, "KILL"), + e2e.WithArgs(containerID, "KILL"), e2e.PostRun(func(t *testing.T) { if !t.Failed() { c.checkOciState(t, containerID, ociruntime.Stopped) @@ -319,6 +333,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("delete"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci delete"), e2e.WithArgs(containerID), @@ -327,6 +342,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("state fail"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci state"), e2e.WithArgs(containerID), @@ -334,6 +350,7 @@ func (c ctx) testOciBasic(t *testing.T) { ) c.env.RunApptainer( t, + e2e.AsSubtest("kill fail"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci kill"), e2e.WithArgs(containerID), @@ -341,6 +358,7 @@ func (c ctx) testOciBasic(t *testing.T) { ) c.env.RunApptainer( t, + e2e.AsSubtest("start fail"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -432,7 +450,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("basic", c.testOciBasic) t.Run("attach", c.testOciAttach) t.Run("run", c.testOciRun) - t.Run("help", c.testOciHelp) })), + "help": c.testOciHelp, } } diff --git a/e2e/suite.go b/e2e/suite.go index 0f14e17541..42328d7f91 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -170,9 +170,9 @@ func Run(t *testing.T) { testenv.OrasTestImage = fmt.Sprintf("oras://%s/oras_test_sif:latest", testenv.TestRegistry) // Provision local registry - testenv.TestRegistryImage = fmt.Sprintf("docker://%s/my-busybox:latest", testenv.TestRegistry) + testenv.TestRegistryImage = fmt.Sprintf("docker://%s/my-alpine:latest", testenv.TestRegistry) - // Copy small test image (busybox:latest) into local registry from DockerHub + // Copy small test image (alpine:latest) into local registry from DockerHub insecureSource := false insecureValue := os.Getenv("E2E_DOCKER_MIRROR_INSECURE") if insecureValue != "" { @@ -181,22 +181,35 @@ func Run(t *testing.T) { t.Fatalf("could not convert E2E_DOCKER_MIRROR_INSECURE=%s: %s", insecureValue, err) } } - e2e.CopyImage(t, "docker://busybox:latest", testenv.TestRegistryImage, insecureSource, true) + e2e.CopyImage(t, "docker://alpine:latest", testenv.TestRegistryImage, insecureSource, true) // SIF base test path, built on demand by e2e.EnsureImage imagePath := path.Join(name, "test.sif") t.Log("Path to test image:", imagePath) testenv.ImagePath = imagePath + // OCI Archive test image path, built on demand by e2e.EnsureOCIArchive + ociArchivePath := path.Join(name, "oci.tar") + t.Log("Path to test OCI archive:", ociArchivePath) + testenv.OCIArchivePath = ociArchivePath + + // Docker Archive test image path, built on demand by e2e.EnsureDockerArhive + dockerArchivePath := path.Join(name, "docker.tar") + t.Log("Path to test Docker archive:", dockerArchivePath) + testenv.DockerArchivePath = dockerArchivePath + // Local registry ORAS SIF image, built on demand by e2e.EnsureORASImage testenv.OrasTestImage = fmt.Sprintf("oras://%s/oras_test_sif:latest", testenv.TestRegistry) t.Cleanup(func() { - os.Remove(imagePath) + if !t.Failed() { + os.Remove(imagePath) + os.Remove(ociArchivePath) + os.Remove(dockerArchivePath) + } }) suite := testhelper.NewSuite(t, testenv) - suite.AddGroup("ACTIONS", actions.E2ETests) suite.AddGroup("BUILDCFG", e2ebuildcfg.E2ETests) suite.AddGroup("BUILD", imgbuild.E2ETests) diff --git a/e2e/testdata/help/help-oci-create.txt b/e2e/testdata/help/help-oci-create.txt index f71d62c5c7..682843d962 100644 --- a/e2e/testdata/help/help-oci-create.txt +++ b/e2e/testdata/help/help-oci-create.txt @@ -8,17 +8,13 @@ Description: bundle directory Options: - -b, --bundle string specify the OCI bundle path (required) - --empty-process run container without executing container - process (eg: for POD container) - -h, --help help for create - --log-format string specify the log file format. Available - formats are basic, kubernetes and json - (default "kubernetes") - -l, --log-path string specify the log file path - --pid-file string specify the pid file - -s, --sync-socket string specify the path to unix socket for state - synchronization + -b, --bundle string specify the OCI bundle path (required) + -h, --help help for create + --log-format string specify the log file format. Available formats + are basic, kubernetes and json (default + "kubernetes") + -l, --log-path string specify the log file path + --pid-file string specify the pid file Examples: diff --git a/e2e/testdata/help/help-oci-kill.txt b/e2e/testdata/help/help-oci-kill.txt index 1cab71a963..9eab494f27 100644 --- a/e2e/testdata/help/help-oci-kill.txt +++ b/e2e/testdata/help/help-oci-kill.txt @@ -8,10 +8,9 @@ Description: identified by container ID. Options: - -f, --force kill container process with SIGKILL - -h, --help help for kill - -s, --signal string signal sent to the container (default "SIGTERM") - -t, --timeout uint32 timeout in second before killing container + -f, --force kill container process with SIGKILL + -h, --help help for kill + -s, --signal string signal sent to the container (default "SIGTERM") Examples: diff --git a/e2e/testdata/help/help-oci-run.txt b/e2e/testdata/help/help-oci-run.txt index 6a095ea8d9..062e5c75e5 100644 --- a/e2e/testdata/help/help-oci-run.txt +++ b/e2e/testdata/help/help-oci-run.txt @@ -7,15 +7,13 @@ Description: Run will invoke equivalent of create/start/attach/delete commands in a row. Options: - -b, --bundle string specify the OCI bundle path (required) - -h, --help help for run - --log-format string specify the log file format. Available - formats are basic, kubernetes and json - (default "kubernetes") - -l, --log-path string specify the log file path - --pid-file string specify the pid file - -s, --sync-socket string specify the path to unix socket for state - synchronization + -b, --bundle string specify the OCI bundle path (required) + -h, --help help for run + --log-format string specify the log file format. Available formats + are basic, kubernetes and json (default + "kubernetes") + -l, --log-path string specify the log file path + --pid-file string specify the pid file Examples: diff --git a/e2e/testdata/help/help-oci-state.txt b/e2e/testdata/help/help-oci-state.txt index 7f7807d137..19ca7590c6 100644 --- a/e2e/testdata/help/help-oci-state.txt +++ b/e2e/testdata/help/help-oci-state.txt @@ -8,9 +8,7 @@ Description: container identified by container ID. Options: - -h, --help help for state - -s, --sync-socket string specify the path to unix socket for state - synchronization + -h, --help help for state Examples: diff --git a/go.mod b/go.mod index d465273efd..9b78730459 100644 --- a/go.mod +++ b/go.mod @@ -14,11 +14,13 @@ require ( github.com/buger/goterm v1.0.4 github.com/buger/jsonparser v1.1.1 github.com/cenkalti/backoff/v4 v4.2.1 + github.com/container-orchestrated-devices/container-device-interface v0.5.4 github.com/containerd/containerd v1.7.2 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 + github.com/containers/common v0.47.5 github.com/containers/image/v5 v5.26.1 - github.com/creack/pty v1.1.18 + github.com/creack/pty v1.1.18 // indirect github.com/cyphar/filepath-securejoin v0.2.3 github.com/docker/docker v24.0.2+incompatible github.com/docker/go-units v0.5.0 @@ -26,6 +28,7 @@ require ( github.com/go-log/log v0.2.0 github.com/google/uuid v1.3.0 github.com/gosimple/slug v1.13.1 + github.com/moby/term v0.5.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc4 github.com/opencontainers/runc v1.1.7 @@ -35,6 +38,7 @@ require ( github.com/opencontainers/umoci v0.4.7 github.com/pelletier/go-toml/v2 v2.0.8 github.com/pkg/errors v0.9.1 + github.com/samber/lo v1.38.1 github.com/seccomp/containers-golang v0.6.0 github.com/seccomp/libseccomp-golang v0.10.0 github.com/shopspring/decimal v1.3.1 @@ -93,6 +97,7 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -136,7 +141,6 @@ require ( github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect @@ -152,7 +156,6 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/safchain/ethtool v0.3.0 // indirect github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect - github.com/sergi/go-diff v1.2.0 // indirect github.com/sigstore/fulcio v1.3.1 // indirect github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12 // indirect github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect @@ -182,6 +185,7 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) replace oras.land/oras-go => github.com/sylabs/oras-go v1.2.4-0.20230628133146-a64659fc0454 diff --git a/go.sum b/go.sum index 0ac66e539e..2dab1f4fa4 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,138 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210312213058-32f4d319f0d2/go.mod h1:VPevheIvXETHZT/ddjwarP3POR5p/cnH9Hy5yoFnQjc= github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210319161527-f761c2329661 h1:LxxqfxscKXL1kv7QNh4nggNf4Ais8B0ME8zWMCAsttY= github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210319161527-f761c2329661/go.mod h1:VPevheIvXETHZT/ddjwarP3POR5p/cnH9Hy5yoFnQjc= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20210920160938-87db9fbc61c7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/adigunhammedolalekan/registry-auth v0.0.0-20200730122110-8cde180a3a60 h1:1IG6ye8dellBRE2uqvG0EzQScRqjsH/n5xOw+n0OGec= github.com/adigunhammedolalekan/registry-auth v0.0.0-20200730122110-8cde180a3a60/go.mod h1:DcXj4IQOoib2b4G2b8JU3VGV3ljXYbIq+PH4CcoAQTI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= @@ -44,69 +146,231 @@ github.com/apptainer/container-library-client v1.4.5 h1:zESpm9aLqELb0zr3isDspYU+ github.com/apptainer/container-library-client v1.4.5/go.mod h1:EA5bDsL/dxzAxFh3zBszSdVpB25wIwv/66M1qLOzrNU= github.com/apptainer/sif/v2 v2.11.5 h1:EhSvg+eTDwlp5FNGdF7WBtr6LlHEIYmunewaTKimmrQ= github.com/apptainer/sif/v2 v2.11.5/go.mod h1:xSD5/qc/M+tS0K20RMexmTbqM4EOQMjN+hcPW6vJYg4= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/bugsnag-go v1.5.1 h1:NnfkWPiRGJlUg6s5mRlsbudWcW/B/eGFSad98JxitaU= github.com/bugsnag/bugsnag-go v1.5.1/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= +github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/containerd v1.7.2 h1:UF2gdONnxO8I6byZXDi5sXWiWvlW3D/sci7dTQimEJo= github.com/containerd/containerd v1.7.2/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= +github.com/containerd/stargz-snapshotter/estargz v0.11.0/go.mod h1:/KsZXsJRllMbTKFfG0miFQWViQKdI9+9aSXs+HN0+ac= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM= github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0= +github.com/containers/common v0.47.5 h1:Qm9o+wVPO9sbggTKubN3xYMtPRaPv7dmcrJQgongHHw= +github.com/containers/common v0.47.5/go.mod h1:HgX0mFXyB0Tbe2REEIp9x9CxET6iSzmHfwR6S/t2LZc= +github.com/containers/image/v5 v5.19.1/go.mod h1:ewoo3u+TpJvGmsz64XgzbyTHwHtM94q7mgK/pX+v2SE= github.com/containers/image/v5 v5.26.1 h1:8y3xq8GO/6y8FR+nAedHPsAFiAtOrab9qHTBpbqaX8g= github.com/containers/image/v5 v5.26.1/go.mod h1:IwlOGzTkGnmfirXxt0hZeJlzv1zVukE03WZQ203Z9GA= +github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U= github.com/containers/ocicrypt v1.1.7/go.mod h1:7CAhjcj2H8AYp5YvEie7oVSK2AhBY8NscCYRawuDNtw= +github.com/containers/storage v1.38.2/go.mod h1:INP0RPLHWBxx+pTsO5uiHlDUGHDFvWZPWprAbAlQWPQ= github.com/containers/storage v1.48.0 h1:wiPs8J2xiFoOEAhxHDRtP6A90Jzj57VqzLRXOqeizns= github.com/containers/storage v1.48.0/go.mod h1:pRp3lkRo2qodb/ltpnudoXggrviRmaCmU5a5GhTBae0= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -120,55 +384,115 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S github.com/d2g/dhcp4client v1.0.0 h1:suYBsYZIkSlUMEz4TAYCczKf62IA2UWC+O8+KtdOhCo= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 h1:+CpLbZIeUn94m02LdEKPcgErLJ347NUwxPKs5u8ieiY= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v24.0.2+incompatible h1:QdqR7znue1mtkXIJ+ruQMGQhpw2JzMJLRXp6zpzF6tM= github.com/docker/cli v24.0.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg= github.com/docker/docker v24.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg= github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 h1:LofdAjjjqCSXMwLGgOgnE+rdPuvX9DxCqaHwKy7i/ko= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-log/log v0.2.0 h1:z8i91GBudxD5L3RmF0KVpetCbcGWAV7q1Tw1eRwQM9Q= github.com/go-log/log v0.2.0/go.mod h1:xzCnwajcues/6w7lne3yK2QU7DBPW7kqbgPGG5AF65U= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -182,9 +506,14 @@ github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= @@ -193,6 +522,8 @@ github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8en github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= @@ -202,6 +533,8 @@ github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrC github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -237,20 +570,48 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -260,132 +621,287 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= +github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/honeycombio/beeline-go v1.10.0 h1:cUDe555oqvw8oD76BQJ8alk7FP0JZ/M/zXpNvOEDLDc= github.com/honeycombio/libhoney-go v1.16.0 h1:kPpqoz6vbOzgp7jC6SR7SkNj7rua7rgxvznI6M3KdHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548 h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6 h1:unJdfS94Y3k85TKy+mvKzjW5R9rIC+Lv4KGbE7uNu0I= github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6/go.mod h1:PUgW5vI9ANEaV6qv9a6EKu8gAySgwf0xrzG9xIB/CK0= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -401,103 +917,197 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/networkplumbing/go-nft v0.3.0 h1:IIc6yHjN85KyJx21p3ZEsO0iBMYHNXux22rc9Q8TfFw= github.com/networkplumbing/go-nft v0.3.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84/go.mod h1:Qnt1q4cjDNQI9bT832ziho5Iw2BhK8o1KwLOwW56VP4= github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.1.7 h1:y2EZDS8sNng4Ksf0GUYNhKbTShZJPJg1FiXJNH/uoCk= github.com/opencontainers/runc v1.1.7/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0-rc.3 h1:l04uafi6kxByhbxev7OWiuUv0LZxEsYUfDWZ6bztAuU= github.com/opencontainers/runtime-spec v1.1.0-rc.3/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/opencontainers/umoci v0.4.7 h1:mbIbtMpZ3v9oMpKaLopnWoLykgmnixeLzq51EzAX5nQ= github.com/opencontainers/umoci v0.4.7/go.mod h1:lgJ4bnwJezsN1o/5d7t/xdRPvmf8TvBko5kKYJsYvgo= +github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/proglottis/gpgme v0.1.1/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4= github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/containers-golang v0.6.0 h1:VWPMMIDr8pAtNjCX0WvLEEK9EQi5lAm4HtJbDtAtFvQ= github.com/seccomp/containers-golang v0.6.0/go.mod h1:Dd9mONHvW4YdbSzdm23yf2CFw0iqvqLhO0mEFvPIvm4= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA= github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -509,6 +1119,8 @@ github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12 h1:x/WnxasgR40qGY github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12/go.mod h1:8c+a8Yo7r8gKuYbIaz+c3oOdw9iMXx+tMdOg2+b+2jQ= github.com/sigstore/sigstore v1.7.1 h1:fCATemikcBK0cG4+NcM940MfoIgmioY1vC6E66hXxks= github.com/sigstore/sigstore v1.7.1/go.mod h1:0PmMzfJP2Y9+lugD0wer4e7TihR5tM7NcIs3bQNk5xg= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -520,21 +1132,44 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -546,13 +1181,19 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/sylabs/json-resp v0.8.2 h1:k2dgtMXL+nztYtxrI24Zck/sfexyly4D1+X504eZTKQ= github.com/sylabs/json-resp v0.8.2/go.mod h1:Q9X4wRlZNPv3x76KaL8vTCBO4aC/DP2gh13xdtEqd1g= github.com/sylabs/oras-go v1.2.4-0.20230628133146-a64659fc0454 h1:mYW7NTm96PhI8MLJ9Sp0cE8evQf1FL4q64P4lVqNtvI= github.com/sylabs/oras-go v1.2.4-0.20230628133146-a64659fc0454/go.mod h1:H1q/Fxq/+StNafSx4Svb30ozrUEgeo5yBwKMXGnZV+w= +github.com/sylabs/release-tools v0.1.0/go.mod h1:pqP/z/11/rYMQ0OM/Nn7TxGijw7KfZwW9UolD/J1TUo= +github.com/sylabs/sif/v2 v2.3.1/go.mod h1:NnvveH62GiibimL00MrI6YYcZfb7DnZMcRo/40giY+0= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA= github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -566,48 +1207,86 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/vbatts/go-mtree v0.5.0 h1:dM+5XZdqH0j9CSZeerhoN/tAySdwnmevaZHO1XGW2Vc= github.com/vbatts/go-mtree v0.5.0/go.mod h1:7JbaNHyBMng+RP8C3Q4E+4Ca8JnGQA2R/MB+jb4tSOk= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/vbauerster/mpb/v7 v7.3.2/go.mod h1:wfxIZcOJq/bG1/lAtfzMXcOiSvbqVi/5GX5WCSi+IsA= github.com/vbauerster/mpb/v8 v8.4.0 h1:Jq2iNA7T6SydpMVOwaT+2OBWlXS9Th8KEvBqeu5eeTo= github.com/vbauerster/mpb/v8 v8.4.0/go.mod h1:vjp3hSTuCtR+x98/+2vW3eZ8XzxvGoP8CPseHMhiPyc= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 h1:p7OofyZ509h8DmPLh8Hn+EIIZm/xYhdZHJ9GnXHdr6U= github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -616,55 +1295,172 @@ go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5queth go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opentelemetry.io/otel v1.15.0 h1:NIl24d4eiLJPM0vKn4HjLYM+UZf6gSfi9Z+NmCxkWbk= go.opentelemetry.io/otel v1.15.0/go.mod h1:qfwLEbWhLPk5gyWrne4XnF0lC8wtywbuJbgfAE3zbek= go.opentelemetry.io/otel/trace v1.15.0 h1:5Fwje4O2ooOxkfyqI/kJwxWotggDLix4BSAvpE1wlpo= go.opentelemetry.io/otel/trace v1.15.0/go.mod h1:CUsmE2Ht1CRkvE8OsMESvraoZrrcgD1J2W8GV1ev0Y4= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -672,99 +1468,379 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211001092434-39dca1131b70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -777,26 +1853,43 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-jose/go-jose.v2 v2.6.1 h1:qEzJlIDmG9q5VO0M/o8tGS65QMHMS1w01TQJB1VPJ4U= gopkg.in/go-jose/go-jose.v2 v2.6.1/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -808,9 +1901,60 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= mvdan.cc/sh/v3 v3.6.1-0.20221221181323-d3feb15bed3a h1:AK8d/j//vNToLhsO0B15DybhgHIQ27GbmvcTXv7S7VY= mvdan.cc/sh/v3 v3.6.1-0.20221221181323-d3feb15bed3a/go.mod h1:U4mhtBLZ32iWhif5/lD+ygy1zrgaQhUu+XFy7C8+TTA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/app/apptainer/oci_attach_linux.go b/internal/app/apptainer/oci_attach_linux.go deleted file mode 100644 index c800ebc233..0000000000 --- a/internal/app/apptainer/oci_attach_linux.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net" - "os" - osignal "os/signal" - "sync" - "syscall" - - "github.com/creack/pty" - - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/unix" - specs "github.com/opencontainers/runtime-spec/specs-go" - "golang.org/x/term" -) - -func resize(controlSocket string, oversized bool) { - ctrl := &ociruntime.Control{} - ctrl.ConsoleSize = &specs.Box{} - - c, err := unix.Dial(controlSocket) - if err != nil { - sylog.Errorf("failed to connect to control socket") - return - } - defer c.Close() - - rows, cols, err := pty.Getsize(os.Stdin) - if err != nil { - sylog.Errorf("terminal resize error: %s", err) - return - } - - ctrl.ConsoleSize.Height = uint(rows) - ctrl.ConsoleSize.Width = uint(cols) - - if oversized { - ctrl.ConsoleSize.Height++ - ctrl.ConsoleSize.Width++ - } - - enc := json.NewEncoder(c) - if enc == nil { - sylog.Errorf("cannot instantiate JSON encoder") - return - } - - if err := enc.Encode(ctrl); err != nil { - sylog.Errorf("%s", err) - return - } -} - -func attach(engineConfig *oci.EngineConfig, run bool) error { - var ostate *term.State - var conn net.Conn - var wg sync.WaitGroup - - state := &engineConfig.State - - if state.AttachSocket == "" { - return fmt.Errorf("attach socket not available, container state: %s", state.Status) - } - if state.ControlSocket == "" { - return fmt.Errorf("control socket not available, container state: %s", state.Status) - } - - hasTerminal := engineConfig.OciConfig.Process.Terminal - if hasTerminal && !term.IsTerminal(0) { - return fmt.Errorf("attach requires a terminal when terminal config is set to true") - } - - var err error - conn, err = unix.Dial(state.AttachSocket) - if err != nil { - return err - } - defer conn.Close() - - if hasTerminal { - ostate, _ = term.MakeRaw(0) - resize(state.ControlSocket, true) - resize(state.ControlSocket, false) - } - - wg.Add(1) - - go func() { - // catch SIGWINCH signal for terminal resize - signals := make(chan os.Signal, 1) - pid := state.Pid - osignal.Notify(signals) - - for { - s := <-signals - switch s { - case syscall.SIGWINCH: - if hasTerminal { - resize(state.ControlSocket, false) - } - default: - syscall.Kill(pid, s.(syscall.Signal)) - } - } - }() - - if hasTerminal || !run { - // Pipe session to bash and visa-versa - go func() { - io.Copy(os.Stdout, conn) - wg.Done() - }() - go func() { - io.Copy(conn, os.Stdin) - }() - wg.Wait() - - if hasTerminal { - fmt.Printf("\r") - return term.Restore(0, ostate) - } - return nil - } - - io.Copy(io.Discard, conn) - return nil -} - -// OciAttach attaches console to a running container -func OciAttach(ctx context.Context, containerID string) error { - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return err - } - if engineConfig.GetState().Status != ociruntime.Running { - return fmt.Errorf("could not attach to %s: not in running state", containerID) - } - - defer exitContainer(ctx, containerID, false) - - return attach(engineConfig, false) -} diff --git a/internal/app/apptainer/oci_create_linux.go b/internal/app/apptainer/oci_create_linux.go deleted file mode 100644 index d19c6b4c7b..0000000000 --- a/internal/app/apptainer/oci_create_linux.go +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "encoding/json" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/internal/pkg/util/starter" - "github.com/apptainer/apptainer/pkg/runtime/engine/config" -) - -// OciCreate creates a container from an OCI bundle -func OciCreate(containerID string, args *OciArgs) error { - _, err := getState(containerID) - if err == nil { - return fmt.Errorf("%s already exists", containerID) - } - - os.Clearenv() - - absBundle, err := filepath.Abs(args.BundlePath) - if err != nil { - return fmt.Errorf("failed to determine bundle absolute path: %s", err) - } - - if err := os.Chdir(absBundle); err != nil { - return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) - } - - engineConfig := oci.NewConfig() - generator := generate.New(&engineConfig.OciConfig.Spec) - engineConfig.SetBundlePath(absBundle) - engineConfig.SetLogPath(args.LogPath) - engineConfig.SetLogFormat(args.LogFormat) - engineConfig.SetPidFile(args.PidFile) - - // load config.json from bundle path - configJSON := filepath.Join(absBundle, "config.json") - fb, err := os.Open(configJSON) - if err != nil { - return fmt.Errorf("oci specification file %q is missing or cannot be read", configJSON) - } - - data, err := io.ReadAll(fb) - if err != nil { - return fmt.Errorf("failed to read OCI specification file %s: %s", configJSON, err) - } - - fb.Close() - - if err := json.Unmarshal(data, generator.Config); err != nil { - return fmt.Errorf("failed to parse OCI specification file %s: %s", configJSON, err) - } - - engineConfig.EmptyProcess = args.EmptyProcess - engineConfig.SyncSocket = args.SyncSocketPath - - commonConfig := &config.Common{ - ContainerID: containerID, - EngineName: oci.Name, - EngineConfig: engineConfig, - } - - procName := fmt.Sprintf("Apptainer OCI %s", containerID) - return starter.Run( - procName, - commonConfig, - starter.WithStdin(os.Stdin), - starter.WithStderr(os.Stderr), - starter.WithStdout(os.Stdout), - ) -} diff --git a/internal/app/apptainer/oci_delete_linux.go b/internal/app/apptainer/oci_delete_linux.go deleted file mode 100644 index 4f59396f99..0000000000 --- a/internal/app/apptainer/oci_delete_linux.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "context" - "fmt" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/util/exec" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciDelete deletes container resources -func OciDelete(ctx context.Context, containerID string) error { - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return err - } - - switch engineConfig.State.Status { - case ociruntime.Running: - return fmt.Errorf("cannot delete '%s', the state of the container must be created or stopped", containerID) - case ociruntime.Stopped: - case ociruntime.Created: - if err := OciKill(containerID, "SIGTERM", 2); err != nil { - return err - } - engineConfig, err = getEngineConfig(containerID) - if err != nil { - return err - } - } - - hooks := engineConfig.OciConfig.Hooks - if hooks != nil { - for _, h := range hooks.Poststop { - if err := exec.Hook(ctx, &h, &engineConfig.State.State); err != nil { - sylog.Warningf("%s", err) - } - } - } - - // remove instance files - file, err := instance.Get(containerID, instance.OciSubDir) - if err != nil { - return err - } - return file.Delete() -} diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go deleted file mode 100644 index a94b404d52..0000000000 --- a/internal/app/apptainer/oci_exec_linux.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "fmt" - "os" - "strings" - - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/internal/pkg/util/starter" - "github.com/apptainer/apptainer/pkg/ociruntime" -) - -// OciExec executes a command in a container -func OciExec(containerID string, cmdArgs []string) error { - commonConfig, err := getCommonConfig(containerID) - if err != nil { - return fmt.Errorf("%s doesn't exist", containerID) - } - - engineConfig := commonConfig.EngineConfig.(*oci.EngineConfig) - - switch engineConfig.GetState().Status { - case ociruntime.Running, ociruntime.Paused: - default: - args := strings.Join(cmdArgs, " ") - return fmt.Errorf("cannot execute command %q, container '%s' is not running", args, containerID) - } - - engineConfig.Exec = true - engineConfig.OciConfig.SetProcessArgs(cmdArgs) - - os.Clearenv() - - procName := fmt.Sprintf("Apptainer OCI %s", containerID) - return starter.Exec(procName, commonConfig) -} diff --git a/internal/app/apptainer/oci_kill_linux.go b/internal/app/apptainer/oci_kill_linux.go deleted file mode 100644 index ae1fbdd743..0000000000 --- a/internal/app/apptainer/oci_kill_linux.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "fmt" - "io" - "syscall" - "time" - - "github.com/apptainer/apptainer/internal/pkg/util/signal" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/util/unix" -) - -// OciKill kills container process -func OciKill(containerID string, killSignal string, killTimeout int) error { - // send signal to the instance - state, err := getState(containerID) - if err != nil { - return err - } - - if state.Status != ociruntime.Created && state.Status != ociruntime.Running { - return fmt.Errorf("cannot kill '%s', the state of the container must be created or running", containerID) - } - - sig := syscall.SIGTERM - - if killSignal != "" { - sig, err = signal.Convert(killSignal) - if err != nil { - return err - } - } - - if killTimeout > 0 { - c, err := unix.Dial(state.ControlSocket) - if err != nil { - return fmt.Errorf("failed to connect to control socket") - } - defer c.Close() - - killed := make(chan bool, 1) - - go func() { - // wait runtime close socket connection for ACK - d := make([]byte, 1) - if _, err := c.Read(d); err == io.EOF { - killed <- true - } - }() - - if err := syscall.Kill(state.Pid, sig); err != nil { - return err - } - - select { - case <-killed: - case <-time.After(time.Duration(killTimeout) * time.Second): - return syscall.Kill(state.Pid, syscall.SIGKILL) - } - } else { - return syscall.Kill(state.Pid, sig) - } - - return nil -} diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index f603ddc97a..9c654a21c0 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -2,90 +2,180 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + package apptainer import ( "context" - "encoding/json" "fmt" - "os" - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/runtime/engine/config" - "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" + ocibundle "github.com/apptainer/apptainer/pkg/ocibundle/sif" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/namespaces" + lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" ) // OciArgs contains CLI arguments type OciArgs struct { - BundlePath string - LogPath string - LogFormat string - SyncSocketPath string - PidFile string - FromFile string - KillSignal string - KillTimeout uint32 - EmptyProcess bool - ForceKill bool + BundlePath string + OverlayPaths []string + LogPath string + LogFormat string + PidFile string + FromFile string + KillSignal string + KillTimeout uint32 + EmptyProcess bool + ForceKill bool } -func getCommonConfig(containerID string) (*config.Common, error) { - commonConfig := config.Common{ - EngineConfig: &oci.EngineConfig{}, +// OciRun runs a container (equivalent to create/start/delete) +func OciRun(ctx context.Context, containerID string, args *OciArgs) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err } + return oci.Run(ctx, containerID, args.BundlePath, args.PidFile, systemdCgroups) +} - file, err := instance.Get(containerID, instance.OciSubDir) +// OciRun runs a container via the OCI runtime, wrapped with prep / cleanup steps. +func OciRunWrapped(ctx context.Context, containerID string, args *OciArgs) error { + systemdCgroups, err := systemdCgroups() if err != nil { - return nil, fmt.Errorf("no container found with name %s", containerID) + return err } - if err := json.Unmarshal(file.Config, &commonConfig); err != nil { - return nil, fmt.Errorf("failed to read %s container configuration: %s", containerID, err) + return oci.RunWrapped(ctx, containerID, args.BundlePath, args.PidFile, args.OverlayPaths, systemdCgroups) +} + +// OciCreate creates a container from an OCI bundle +func OciCreate(containerID string, args *OciArgs) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err } + return oci.Create(containerID, args.BundlePath, systemdCgroups) +} - return &commonConfig, nil +// OciStart starts a previously create container +func OciStart(containerID string) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Start(containerID, systemdCgroups) } -func getEngineConfig(containerID string) (*oci.EngineConfig, error) { - commonConfig, err := getCommonConfig(containerID) +// OciDelete deletes container resources +func OciDelete(ctx context.Context, containerID string) error { + systemdCgroups, err := systemdCgroups() if err != nil { - return nil, err + return err } - return commonConfig.EngineConfig.(*oci.EngineConfig), nil + return oci.Delete(ctx, containerID, systemdCgroups) } -func getState(containerID string) (*ociruntime.State, error) { - engineConfig, err := getEngineConfig(containerID) +// OciExec executes a command in a container +func OciExec(containerID string, cmdArgs []string) error { + systemdCgroups, err := systemdCgroups() if err != nil { - return nil, err + return err } - return &engineConfig.State, nil + return oci.Exec(containerID, cmdArgs, systemdCgroups) } -func exitContainer(ctx context.Context, containerID string, delete bool) { - state, err := getState(containerID) +// OciKill kills container process +func OciKill(containerID string, killSignal string) error { + systemdCgroups, err := systemdCgroups() if err != nil { - if !delete { - sylog.Errorf("%s", err) - os.Exit(1) - } - return + return err } + return oci.Kill(containerID, killSignal, systemdCgroups) +} + +// OciPause pauses processes in a container +func OciPause(containerID string) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Pause(containerID, systemdCgroups) +} - if state.ExitCode != nil { - defer os.Exit(*state.ExitCode) +// OciResume pauses processes in a container +func OciResume(containerID string) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err } + return oci.Resume(containerID, systemdCgroups) +} - if delete { - if err := OciDelete(ctx, containerID); err != nil { - sylog.Errorf("%s", err) +// OciState queries container state +func OciState(containerID string, args *OciArgs) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.State(containerID, systemdCgroups) +} + +// OciUpdate updates container cgroups resources +func OciUpdate(containerID string, args *OciArgs) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Update(containerID, args.FromFile, systemdCgroups) +} + +// OciMount mount a SIF image to create an OCI bundle +func OciMount(ctx context.Context, image string, bundle string) error { + d, err := ocibundle.FromSif(image, bundle, true) + if err != nil { + return err + } + return d.Create(ctx, nil) +} + +// OciUmount umount SIF and delete OCI bundle +func OciUmount(bundle string) error { + d, err := ocibundle.FromSif("", bundle, true) + if err != nil { + return err + } + return d.Delete() +} + +func systemdCgroups() (use bool, err error) { + cfg := apptainerconf.GetCurrentConfig() + if cfg == nil { + cfg, err = apptainerconf.Parse(buildcfg.APPTAINER_CONF_FILE) + if err != nil { + return false, fmt.Errorf("unable to parse apptainer configuration file: %w", err) } } + + useSystemd := cfg.SystemdCgroups + + // As non-root, we need cgroups v2 unified mode for systemd support. + // Fall back to cgroupfs if this is not available. + hostUID, err := namespaces.HostUID() + if err != nil { + return false, fmt.Errorf("while finding host uid: %w", err) + } + if hostUID != 0 && !lccgroups.IsCgroup2UnifiedMode() { + useSystemd = false + } + + return useSystemd, nil } diff --git a/internal/app/apptainer/oci_mount_linux.go b/internal/app/apptainer/oci_mount_linux.go deleted file mode 100644 index d7f0e758ac..0000000000 --- a/internal/app/apptainer/oci_mount_linux.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - ocibundle "github.com/apptainer/apptainer/pkg/ocibundle/sif" -) - -// OciMount mount a SIF image to create an OCI bundle -func OciMount(image string, bundle string) error { - d, err := ocibundle.FromSif(image, bundle, true) - if err != nil { - return err - } - return d.Create(nil) -} - -// OciUmount umount SIF and delete OCI bundle -func OciUmount(bundle string) error { - d, err := ocibundle.FromSif("", bundle, true) - if err != nil { - return err - } - return d.Delete() -} diff --git a/internal/app/apptainer/oci_pause_linux.go b/internal/app/apptainer/oci_pause_linux.go deleted file mode 100644 index 74a529fd81..0000000000 --- a/internal/app/apptainer/oci_pause_linux.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/util/unix" -) - -// OciPauseResume pauses/resumes processes in a container -func OciPauseResume(containerID string, pause bool) error { - state, err := getState(containerID) - if err != nil { - return err - } - - if state.ControlSocket == "" { - return fmt.Errorf("can't find control socket") - } - - if pause && state.Status != ociruntime.Running { - return fmt.Errorf("container %s is not running", containerID) - } else if !pause && state.Status != ociruntime.Paused { - return fmt.Errorf("container %s is not paused", containerID) - } - - ctrl := &ociruntime.Control{} - if pause { - ctrl.Pause = true - } else { - ctrl.Resume = true - } - - c, err := unix.Dial(state.ControlSocket) - if err != nil { - return fmt.Errorf("failed to connect to control socket") - } - defer c.Close() - - enc := json.NewEncoder(c) - if enc == nil { - return fmt.Errorf("cannot instantiate new JSON encoder") - } - - if err := enc.Encode(ctrl); err != nil { - return err - } - - // wait runtime close socket connection for ACK - d := make([]byte, 1) - if _, err := c.Read(d); err != io.EOF { - return err - } - - // check status - state, err = getState(containerID) - if err != nil { - return err - } - if pause && state.Status != ociruntime.Paused { - return fmt.Errorf("bad status %s returned instead of paused", state.Status) - } else if !pause && state.Status != ociruntime.Running { - return fmt.Errorf("bad status %s returned instead of running", state.Status) - } - - return nil -} diff --git a/internal/app/apptainer/oci_run_linux.go b/internal/app/apptainer/oci_run_linux.go deleted file mode 100644 index 1869b756f0..0000000000 --- a/internal/app/apptainer/oci_run_linux.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/unix" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// OciRun runs a container (equivalent to create/start/delete) -func OciRun(ctx context.Context, containerID string, args *OciArgs) error { - dir, err := instance.GetDir(containerID, instance.OciSubDir) - if err != nil { - return err - } - if err := os.MkdirAll(dir, 0o700); err != nil { - return err - } - args.SyncSocketPath = filepath.Join(dir, "run.sock") - - l, err := unix.CreateSocket(args.SyncSocketPath) - if err != nil { - os.Remove(args.SyncSocketPath) - return err - } - - defer l.Close() - - status := make(chan string, 1) - - if err := OciCreate(containerID, args); err != nil { - defer os.Remove(args.SyncSocketPath) - if _, err1 := getState(containerID); err1 != nil { - return err - } - if err := OciDelete(ctx, containerID); err != nil { - sylog.Warningf("can't delete container %s", containerID) - } - return err - } - - defer exitContainer(ctx, containerID, true) - defer os.Remove(args.SyncSocketPath) - - go func() { - var state specs.State - - for { - c, err := l.Accept() - if err != nil { - status <- err.Error() - return - } - - dec := json.NewDecoder(c) - if err := dec.Decode(&state); err != nil { - status <- err.Error() - return - } - - c.Close() - - switch state.Status { - case ociruntime.Created: - // ignore error there and wait for stopped status - OciStart(containerID) - case ociruntime.Running: - status <- string(state.Status) - case ociruntime.Stopped: - status <- string(state.Status) - } - } - }() - - // wait running status - s := <-status - if s != ociruntime.Running { - return fmt.Errorf("%s", s) - } - - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return err - } - - if err := attach(engineConfig, true); err != nil { - // kill container before deletion - sylog.Errorf("%s", err) - OciKill(containerID, "SIGKILL", 1) - return err - } - - // wait stopped status - s = <-status - if s != ociruntime.Stopped { - return fmt.Errorf("%s", s) - } - - return nil -} diff --git a/internal/app/apptainer/oci_start_linux.go b/internal/app/apptainer/oci_start_linux.go deleted file mode 100644 index 7e324bd673..0000000000 --- a/internal/app/apptainer/oci_start_linux.go +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "encoding/json" - "fmt" - "io" - - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/util/unix" -) - -// OciStart starts a previously create container -func OciStart(containerID string) error { - state, err := getState(containerID) - if err != nil { - return err - } - - if state.Status != ociruntime.Created { - return fmt.Errorf("cannot start '%s', the state of the container must be %s", containerID, ociruntime.Created) - } - - if state.ControlSocket == "" { - return fmt.Errorf("can't find control socket") - } - - ctrl := &ociruntime.Control{} - ctrl.StartContainer = true - - c, err := unix.Dial(state.ControlSocket) - if err != nil { - return fmt.Errorf("failed to connect to control socket") - } - defer c.Close() - - enc := json.NewEncoder(c) - if enc == nil { - return fmt.Errorf("cannot instantiate new JSON encoder") - } - - if err := enc.Encode(ctrl); err != nil { - return err - } - - // wait runtime close socket connection for ACK - d := make([]byte, 1) - if _, err := c.Read(d); err != io.EOF { - return err - } - - return nil -} diff --git a/internal/app/apptainer/oci_state_linux.go b/internal/app/apptainer/oci_state_linux.go deleted file mode 100644 index 8026e61168..0000000000 --- a/internal/app/apptainer/oci_state_linux.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "encoding/json" - "fmt" - - "github.com/apptainer/apptainer/pkg/util/unix" -) - -// OciState query container state -func OciState(containerID string, args *OciArgs) error { - // query instance files and returns state - state, err := getState(containerID) - if err != nil { - return err - } - if args.SyncSocketPath != "" { - data, err := json.Marshal(state) - if err != nil { - return fmt.Errorf("failed to marshal state data: %s", err) - } else if err := unix.WriteSocket(args.SyncSocketPath, data); err != nil { - return err - } - } else { - c, err := json.MarshalIndent(state, "", "\t") - if err != nil { - return err - } - fmt.Println(string(c)) - } - return nil -} diff --git a/internal/app/apptainer/oci_update_linux.go b/internal/app/apptainer/oci_update_linux.go deleted file mode 100644 index 02949b294e..0000000000 --- a/internal/app/apptainer/oci_update_linux.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "encoding/json" - "fmt" - "io" - "os" - - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/pkg/ociruntime" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// OciUpdate updates container cgroups resources -func OciUpdate(containerID string, args *OciArgs) error { - var reader io.Reader - - state, err := getState(containerID) - if err != nil { - return err - } - - if state.State.Status != ociruntime.Running && state.State.Status != ociruntime.Created { - return fmt.Errorf("container %s is neither running nor created", containerID) - } - - if args.FromFile == "" { - return fmt.Errorf("you must specify --from-file") - } - - resources := &specs.LinuxResources{} - manager, err := cgroups.GetManagerForPid(state.State.Pid) - if err != nil { - return fmt.Errorf("failed to get cgroups manager: %v", err) - } - - if args.FromFile == "-" { - reader = os.Stdin - } else { - f, err := os.Open(args.FromFile) - if err != nil { - return err - } - reader = f - } - - data, err := io.ReadAll(reader) - if err != nil { - return fmt.Errorf("failed to read cgroups config file: %s", err) - } - - if err := json.Unmarshal(data, resources); err != nil { - return err - } - - return manager.UpdateFromSpec(resources) -} diff --git a/internal/app/apptainer/overlay_create.go b/internal/app/apptainer/overlay_create.go index 23a03f0dde..3e9dd1789c 100644 --- a/internal/app/apptainer/overlay_create.go +++ b/internal/app/apptainer/overlay_create.go @@ -18,6 +18,7 @@ import ( "runtime" "strings" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/image" @@ -232,7 +233,7 @@ func OverlayCreate(size int, imgPath string, overlaySparse bool, isFakeroot bool // the fakeroot command (in suid flow with no user // namespaces), using the --fakeroot option here // prevents overlay from working, most unfortunately. - err = fakeroot.UnshareRootMapped([]string{"/bin/true"}) + err = fakefake.UnshareRootMapped([]string{"/bin/true"}) if err != nil { sylog.Debugf("UnshareRootMapped failed: %v", err) if isFakeroot { @@ -248,7 +249,7 @@ func OverlayCreate(size int, imgPath string, overlaySparse bool, isFakeroot bool if isFakeroot { sylog.Debugf("Trying root-mapped namespace") - err = fakeroot.UnshareRootMapped(os.Args) + err = fakefake.UnshareRootMapped(os.Args) if err == nil { // everything was done by the child os.Exit(0) diff --git a/internal/pkg/build/stage.go b/internal/pkg/build/stage.go index 18cad349a3..2bc8c74122 100644 --- a/internal/pkg/build/stage.go +++ b/internal/pkg/build/stage.go @@ -19,7 +19,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/build/files" "github.com/apptainer/apptainer/internal/pkg/buildcfg" - "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/pkg/build/types" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -99,7 +99,7 @@ func (s *stage) runPostScript(sessionResolv, sessionHosts string) error { // the nested apptainer will run fakeroot if it isn't // started and pass down the components and environment // to nested apptainers. - fakerootBinds, err = fakeroot.GetFakeBinds(s.b.Opts.FakerootPath) + fakerootBinds, err = fakefake.GetFakeBinds(s.b.Opts.FakerootPath) if err != nil { return fmt.Errorf("while getting fakeroot bindpoints: %v", err) } diff --git a/internal/pkg/cgroups/manager_linux.go b/internal/pkg/cgroups/manager_linux.go index e630b93b6e..8dc9717c9e 100644 --- a/internal/pkg/cgroups/manager_linux.go +++ b/internal/pkg/cgroups/manager_linux.go @@ -14,7 +14,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "github.com/apptainer/apptainer/internal/pkg/util/env" @@ -344,15 +343,8 @@ func NewManagerWithSpec(spec *specs.LinuxResources, pid int, group string, syste if pid == 0 { return nil, fmt.Errorf("a pid is required to create a new cgroup") } - if group == "" && !systemd { - group = filepath.Join("/apptainer", strconv.Itoa(pid)) - } - if group == "" && systemd { - if os.Getuid() == 0 { - group = "system.slice:apptainer:" + strconv.Itoa(pid) - } else { - group = "user.slice:apptainer:" + strconv.Itoa(pid) - } + if group == "" { + group = DefaultPathForPid(systemd, pid) } sylog.Debugf("Creating cgroups manager for %s", group) diff --git a/internal/pkg/cgroups/util.go b/internal/pkg/cgroups/util.go index 2c27fdcf82..96ddc7a340 100644 --- a/internal/pkg/cgroups/util.go +++ b/internal/pkg/cgroups/util.go @@ -11,6 +11,9 @@ package cgroups import ( "fmt" + "os" + "path/filepath" + "strconv" lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" ) @@ -51,3 +54,24 @@ func pidToPath(pid int) (path string, err error) { } return path, nil } + +// DefaultPathForPid returns a default cgroup path for a given PID. +func DefaultPathForPid(systemd bool, pid int) (group string) { + // Default naming is pid of first process added to cgroup. + strPid := strconv.Itoa(pid) + // Request is for an empty cgroup... name it for the requestor's (our) pid. + if pid == -1 { + strPid = "parent-" + strconv.Itoa(os.Getpid()) + } + + if systemd { + if os.Getuid() == 0 { + group = "system.slice:apptainer:" + strPid + } else { + group = "user.slice:apptainer:" + strPid + } + } else { + group = filepath.Join("/apptainer", strPid) + } + return group +} diff --git a/internal/pkg/checkpoint/dmtcp/checkpoint.go b/internal/pkg/checkpoint/dmtcp/checkpoint.go index 2f46df3c31..bbe186fadf 100644 --- a/internal/pkg/checkpoint/dmtcp/checkpoint.go +++ b/internal/pkg/checkpoint/dmtcp/checkpoint.go @@ -14,7 +14,7 @@ import ( "os" "path/filepath" - apptainerConfig "github.com/apptainer/apptainer/pkg/runtime/engine/apptainer/config" + "github.com/apptainer/apptainer/pkg/util/bind" ) type Entry struct { @@ -38,11 +38,11 @@ func (e *Entry) CoordinatorPort() (string, error) { return s.Text(), nil } -func (e *Entry) BindPath() apptainerConfig.BindPath { - return apptainerConfig.BindPath{ +func (e *Entry) BindPath() bind.Path { + return bind.Path{ Source: e.path, Destination: containerStatepath, - Options: map[string]*apptainerConfig.BindOption{ + Options: map[string]*bind.Option{ "rw": {}, }, } diff --git a/internal/pkg/confgen/testdata/test_1.out.correct b/internal/pkg/confgen/testdata/test_1.out.correct index 0cfec2ec2d..7fbea721a8 100644 --- a/internal/pkg/confgen/testdata/test_1.out.correct +++ b/internal/pkg/confgen/testdata/test_1.out.correct @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_2.in b/internal/pkg/confgen/testdata/test_2.in index e42de0ba5d..110e0fe7c2 100644 --- a/internal/pkg/confgen/testdata/test_2.in +++ b/internal/pkg/confgen/testdata/test_2.in @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs, +# and the equivalent data in --oci mode. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_2.out.correct b/internal/pkg/confgen/testdata/test_2.out.correct index e9c044bcaf..4c980da8ff 100644 --- a/internal/pkg/confgen/testdata/test_2.out.correct +++ b/internal/pkg/confgen/testdata/test_2.out.correct @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_3.in b/internal/pkg/confgen/testdata/test_3.in index e2c70b9066..c61f15800f 100644 --- a/internal/pkg/confgen/testdata/test_3.in +++ b/internal/pkg/confgen/testdata/test_3.in @@ -137,9 +137,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs, +# and the equivalent data in --oci mode. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_3.out.correct b/internal/pkg/confgen/testdata/test_3.out.correct index e9c044bcaf..4c980da8ff 100644 --- a/internal/pkg/confgen/testdata/test_3.out.correct +++ b/internal/pkg/confgen/testdata/test_3.out.correct @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_default.tmpl b/internal/pkg/confgen/testdata/test_default.tmpl index d13b50c400..4579403e2d 100644 --- a/internal/pkg/confgen/testdata/test_default.tmpl +++ b/internal/pkg/confgen/testdata/test_default.tmpl @@ -148,9 +148,10 @@ mount slave = {{ if eq .MountSlave true }}yes{{ else }}no{{ end }} # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = {{ .SessiondirMaxSize }} diff --git a/internal/pkg/fakeroot/fakefake.go b/internal/pkg/fakefake/fakefake.go similarity index 99% rename from internal/pkg/fakeroot/fakefake.go rename to internal/pkg/fakefake/fakefake.go index 10816aff23..2a9e642b6e 100644 --- a/internal/pkg/fakeroot/fakefake.go +++ b/internal/pkg/fakefake/fakefake.go @@ -9,7 +9,7 @@ // This file is for "fake fakeroot", that is, root-mapped unprivileged // user namespaces (unshare -r) and the fakeroot command -package fakeroot +package fakefake import ( "bufio" diff --git a/internal/pkg/instance/instance_linux.go b/internal/pkg/instance/instance_linux.go index c44fce0196..f0c343c1f8 100644 --- a/internal/pkg/instance/instance_linux.go +++ b/internal/pkg/instance/instance_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -24,8 +24,6 @@ import ( ) const ( - // OciSubDir represents directory where OCI instance files are stored - OciSubDir = "oci" // AppSubDir represents directory where Apptainer instance files are stored AppSubDir = "app" // LogSubDir represents directory where Apptainer instance log files are stored diff --git a/internal/pkg/runtime/engine/apptainer/prepare_linux.go b/internal/pkg/runtime/engine/apptainer/prepare_linux.go index 33ba28a29e..95868811b5 100644 --- a/internal/pkg/runtime/engine/apptainer/prepare_linux.go +++ b/internal/pkg/runtime/engine/apptainer/prepare_linux.go @@ -25,6 +25,7 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cgroups" + "github.com/apptainer/apptainer/internal/pkg/fakefake" fakerootutil "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/image/driver" "github.com/apptainer/apptainer/internal/pkg/instance" @@ -98,7 +99,7 @@ func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error { if fakerootPath := e.EngineConfig.GetFakerootPath(); fakerootPath != "" { // look for fakeroot again because the PATH used is // more restricted at this point than it was earlier - newPath, err := fakerootutil.FindFake() + newPath, err := fakefake.FindFake() if err != nil { return fmt.Errorf("error finding fakeroot in privileged PATH: %v", err) } diff --git a/internal/pkg/runtime/engine/apptainer/process_linux.go b/internal/pkg/runtime/engine/apptainer/process_linux.go index f0dce42fa9..26ccf07a49 100644 --- a/internal/pkg/runtime/engine/apptainer/process_linux.go +++ b/internal/pkg/runtime/engine/apptainer/process_linux.go @@ -32,7 +32,7 @@ import ( "unsafe" "github.com/apptainer/apptainer/internal/pkg/checkpoint/dmtcp" - "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/instance" "github.com/apptainer/apptainer/internal/pkg/plugin" "github.com/apptainer/apptainer/internal/pkg/security" @@ -889,7 +889,7 @@ func runActionScript(engineConfig *apptainerConfig.EngineConfig) ([]string, []st } } - fakeargs := fakeroot.GetFakeArgs() + fakeargs := fakefake.GetFakeArgs() fakerootPath := fakeargs[0] _, err = os.Stat(fakerootPath) if err == nil && getEnvVal(penv, "FAKEROOTKEY") == "" { @@ -906,7 +906,7 @@ func runActionScript(engineConfig *apptainerConfig.EngineConfig) ([]string, []st if engineConfig.GetFakerootPath() == "" { // Must be joining an instance, so also set BIND // variables for nesting - fakebinds, _ := fakeroot.GetFakeBinds(fakerootPath) + fakebinds, _ := fakefake.GetFakeBinds(fakerootPath) bindval := strings.Join(fakebinds, ",") for _, pfx := range env.ApptainerPrefixes { bindvar := pfx + "BIND=" diff --git a/internal/pkg/runtime/engine/fakeroot/config/config.go b/internal/pkg/runtime/engine/fakeroot/config/config.go index 833d31d831..c7fede2169 100644 --- a/internal/pkg/runtime/engine/fakeroot/config/config.go +++ b/internal/pkg/runtime/engine/fakeroot/config/config.go @@ -19,4 +19,5 @@ type EngineConfig struct { Envs []string `json:"envs"` Home string `json:"home"` BuildEnv bool `json:"buildEnv"` + NoPIDNS bool `json:"NoPIDNS"` } diff --git a/internal/pkg/runtime/engine/fakeroot/engine_linux.go b/internal/pkg/runtime/engine/fakeroot/engine_linux.go index 7f2cd1d108..6951360d7f 100644 --- a/internal/pkg/runtime/engine/fakeroot/engine_linux.go +++ b/internal/pkg/runtime/engine/fakeroot/engine_linux.go @@ -101,7 +101,11 @@ func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error { g.AddOrReplaceLinuxNamespace(specs.UserNamespace, "") g.AddOrReplaceLinuxNamespace(specs.MountNamespace, "") - g.AddOrReplaceLinuxNamespace(specs.PIDNamespace, "") + + // If we enter a PID NS in the --oci action -> oci run flow, then crun / runc will fail. + if !e.EngineConfig.NoPIDNS { + g.AddOrReplaceLinuxNamespace(specs.PIDNamespace, "") + } uid := uint32(os.Getuid()) gid := uint32(os.Getgid()) diff --git a/internal/pkg/runtime/engine/oci/cleanup_linux.go b/internal/pkg/runtime/engine/oci/cleanup_linux.go deleted file mode 100644 index f15b5a9794..0000000000 --- a/internal/pkg/runtime/engine/oci/cleanup_linux.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "context" - "fmt" - "os" - "syscall" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// CleanupContainer is called from master after the MonitorContainer returns. -// It is responsible for ensuring that the container has been properly torn down. -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Specifically in oci engine, no additional privileges are gained here. However, -// most likely this still will be executed as root since `apptainer oci` -// command set requires privileged execution. -func (e *EngineOperations) CleanupContainer(ctx context.Context, fatal error, status syscall.WaitStatus) error { - if e.EngineConfig.Cgroups != nil { - if err := e.EngineConfig.Cgroups.Destroy(); err != nil { - sylog.Warningf("failed to remove cgroup configuration: %v", err) - } - } - - pidFile := e.EngineConfig.GetPidFile() - if pidFile != "" { - os.Remove(pidFile) - } - - // if container wasn't created, delete instance files - if e.EngineConfig.State.Status == ociruntime.Creating { - name := e.CommonConfig.ContainerID - file, err := instance.Get(name, instance.OciSubDir) - if err != nil { - sylog.Warningf("no instance files found for %s: %s", name, err) - return nil - } - if err := file.Delete(); err != nil { - sylog.Warningf("failed to delete instance files: %s", err) - } - return nil - } - - exitCode := 0 - desc := "" - - if fatal != nil { - exitCode = 255 - desc = fatal.Error() - } else if status.Signaled() { - s := status.Signal() - exitCode = int(s) + 128 - desc = fmt.Sprintf("interrupted by signal %s", s.String()) - } else { - exitCode = status.ExitStatus() - desc = fmt.Sprintf("exited with code %d", status.ExitStatus()) - } - - e.EngineConfig.State.ExitCode = &exitCode - e.EngineConfig.State.ExitDesc = desc - - if err := e.updateState(ociruntime.Stopped); err != nil { - return err - } - - if e.EngineConfig.State.AttachSocket != "" { - os.Remove(e.EngineConfig.State.AttachSocket) - } - if e.EngineConfig.State.ControlSocket != "" { - os.Remove(e.EngineConfig.State.ControlSocket) - } - - return nil -} diff --git a/internal/pkg/runtime/engine/oci/config_linux.go b/internal/pkg/runtime/engine/oci/config_linux.go deleted file mode 100644 index c88660690c..0000000000 --- a/internal/pkg/runtime/engine/oci/config_linux.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "sync" - - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" - "github.com/apptainer/apptainer/pkg/ociruntime" -) - -// Name of the engine. -const Name = "oci" - -// EngineConfig is the config for the OCI engine. -type EngineConfig struct { - BundlePath string `json:"bundlePath"` - LogPath string `json:"logPath"` - LogFormat string `json:"logFormat"` - PidFile string `json:"pidFile"` - OciConfig *oci.Config `json:"ociConfig"` - MasterPts int `json:"masterPts"` - SlavePts int `json:"slavePts"` - OutputStreams [2]int `json:"outputStreams"` - ErrorStreams [2]int `json:"errorStreams"` - InputStreams [2]int `json:"inputStreams"` - SyncSocket string `json:"syncSocket"` - EmptyProcess bool `json:"emptyProcess"` - Exec bool `json:"exec"` - SystemdCgroups bool `json:"systemdCgroups"` - Cgroups *cgroups.Manager `json:"-"` - - sync.Mutex `json:"-"` - State ociruntime.State `json:"state"` -} - -// NewConfig returns an oci.EngineConfig. -func NewConfig() *EngineConfig { - ret := &EngineConfig{ - OciConfig: &oci.Config{}, - } - - return ret -} - -// SetBundlePath sets the container bundle path. -func (e *EngineConfig) SetBundlePath(path string) { - e.BundlePath = path -} - -// GetBundlePath returns the container bundle path. -func (e *EngineConfig) GetBundlePath() string { - return e.BundlePath -} - -// SetState sets the container state as defined by OCI state specification. -func (e *EngineConfig) SetState(state *ociruntime.State) { - e.State = *state -} - -// GetState returns the container state as defined by OCI state specification. -func (e *EngineConfig) GetState() *ociruntime.State { - return &e.State -} - -// SetLogPath sets the container log path. -func (e *EngineConfig) SetLogPath(path string) { - e.LogPath = path -} - -// GetLogPath returns the container log path. -func (e *EngineConfig) GetLogPath() string { - return e.LogPath -} - -// SetLogFormat sets the container log format. -func (e *EngineConfig) SetLogFormat(format string) { - e.LogFormat = format -} - -// GetLogFormat returns the container log format. -func (e *EngineConfig) GetLogFormat() string { - return e.LogFormat -} - -// SetPidFile sets the pid file path. -func (e *EngineConfig) SetPidFile(path string) { - e.PidFile = path -} - -// GetPidFile gets the pid file path. -func (e *EngineConfig) GetPidFile() string { - return e.PidFile -} - -// SetSystemdCgroups sets whether to manage cgroups with systemd. -func (e *EngineConfig) SetSystemdCgroups(systemd bool) { - e.SystemdCgroups = systemd -} - -// SetSystemdCgroups gets whether to manage cgroups with systemd. -func (e *EngineConfig) GetSystemdCgroups() bool { - return e.SystemdCgroups -} diff --git a/internal/pkg/runtime/engine/oci/create_linux.go b/internal/pkg/runtime/engine/oci/create_linux.go deleted file mode 100644 index 97d3325a27..0000000000 --- a/internal/pkg/runtime/engine/oci/create_linux.go +++ /dev/null @@ -1,991 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "net" - "net/rpc" - "os" - "path/filepath" - "strings" - "syscall" - "time" - - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc/client" - "github.com/apptainer/apptainer/internal/pkg/util/fs" - "github.com/apptainer/apptainer/internal/pkg/util/fs/mount" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/fs/proc" - "github.com/apptainer/apptainer/pkg/util/namespaces" - "github.com/apptainer/apptainer/pkg/util/sysctl" - "github.com/apptainer/apptainer/pkg/util/unix" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -var symlinkDevices = []struct { - old string - new string -}{ - {"/proc/self/fd", "/dev/fd"}, - {"/proc/kcore", "/dev/core"}, - {"pts/ptmx", "/dev/ptmx"}, - {"/proc/self/fd/0", "/dev/stdin"}, - {"/proc/self/fd/1", "/dev/stdout"}, - {"/proc/self/fd/2", "/dev/stderr"}, -} - -type device struct { - major int64 - minor int64 - path string - mode os.FileMode - uid int - gid int -} - -var devices = []device{ - {1, 7, "/dev/full", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 3, "/dev/null", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 8, "/dev/random", syscall.S_IFCHR | 0o666, 0, 0}, - {5, 0, "/dev/tty", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 9, "/dev/urandom", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 5, "/dev/zero", syscall.S_IFCHR | 0o666, 0, 0}, -} - -var cgroupDevices = []specs.LinuxDeviceCgroup{ - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(7), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(3), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(8), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(5), - Minor: cgroups.Int64ptr(0), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(9), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(5), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(136), - Minor: cgroups.Int64ptr(-1), - Access: "rwm", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(5), - Minor: cgroups.Int64ptr(1), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(5), - Minor: cgroups.Int64ptr(2), - Access: "rw", - }, -} - -type container struct { - engine *EngineOperations - rpcOps *client.RPC - rootfs string - rpcRoot string - userNS bool - utsNS bool - mntNS bool - devIndex int - cgroupV1MountIndex int -} - -var statusChan = make(chan string, 1) - -// CreateContainer is called from master process to prepare container -// environment, e.g. perform mount operations, etc. -// -// Additional privileges required for setup may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Specifically in oci engine, no additional privileges are gained. Container -// setup (e.g. mount operations) where privileges may be required is performed -// by calling RPC server methods (see internal/app/starter/rpc_linux.go for details). -// -// However, most likely this still will be executed as root since `apptainer oci` -// command set requires privileged execution. -// -//nolint:maintidx -func (e *EngineOperations) CreateContainer(ctx context.Context, pid int, rpcConn net.Conn) error { - var err error - - if e.CommonConfig.EngineName != Name { - return fmt.Errorf("engineName configuration doesn't match runtime name") - } - - rpcOps := &client.RPC{} - rpcOps.Client = rpc.NewClient(rpcConn) - rpcOps.Name = e.CommonConfig.EngineName - - if rpcOps.Client == nil { - return fmt.Errorf("failed to initialize RPC client") - } - - if err := e.createState(pid); err != nil { - return err - } - - rootfs := e.EngineConfig.OciConfig.Root.Path - - if !filepath.IsAbs(rootfs) { - rootfs = filepath.Join(e.EngineConfig.GetBundlePath(), rootfs) - } - - resolvedRootfs, err := filepath.EvalSymlinks(rootfs) - if err != nil { - return fmt.Errorf("failed to resolve %s path: %s", rootfs, err) - } - - c := &container{ - engine: e, - rpcOps: rpcOps, - rootfs: resolvedRootfs, - rpcRoot: fmt.Sprintf("/proc/%d/root", pid), - cgroupV1MountIndex: -1, - devIndex: -1, - } - - for _, ns := range e.EngineConfig.OciConfig.Linux.Namespaces { - switch ns.Type { - case specs.UserNamespace: - c.userNS = true - case specs.UTSNamespace: - c.utsNS = true - case specs.MountNamespace: - c.mntNS = true - } - } - - p := &mount.Points{} - if e.EngineConfig.OciConfig.Linux.MountLabel != "" { - if err := p.SetContext(e.EngineConfig.OciConfig.Linux.MountLabel); err != nil { - return err - } - } - - system := &mount.System{Points: p, Mount: c.mount} - - for i, point := range e.EngineConfig.OciConfig.Config.Mounts { - // A cgroup v1 mount point will be intercepted and handled separately in c.addCgroups(...) - if point.Type == "cgroup" { - c.cgroupV1MountIndex = i - continue - } - // dev creation - if point.Destination == "/dev" && point.Type == "tmpfs" { - c.devIndex = i - } - } - - if err := c.addDevices(system); err != nil { - return err - } - - if err := c.addCgroups(pid, system); err != nil { - return err - } - - // import OCI mount spec - if err := system.Points.ImportFromSpec(e.EngineConfig.OciConfig.Config.Mounts); err != nil { - return err - } - - if err := c.addRootfsMount(system); err != nil { - return err - } - - if err := system.RunAfterTag(mount.KernelTag, c.addDefaultDevices); err != nil { - return err - } - - if err := system.RunAfterTag(mount.KernelTag, c.addAllPaths); err != nil { - return err - } - - if err := proc.SetOOMScoreAdj(pid, e.EngineConfig.OciConfig.Process.OOMScoreAdj); err != nil { - return err - } - - if err := namespaces.Enter(pid, "ipc"); err != nil { - return err - } - if err := namespaces.Enter(pid, "net"); err != nil { - return err - } - - for key, value := range e.EngineConfig.OciConfig.Linux.Sysctl { - if err := sysctl.Set(key, value); err != nil { - return err - } - } - - if err := namespaces.Enter(os.Getpid(), "ipc"); err != nil { - return err - } - if err := namespaces.Enter(os.Getpid(), "net"); err != nil { - return err - } - - sylog.Debugf("Mount all") - if err := system.MountAll(); err != nil { - return err - } - - if c.utsNS && e.EngineConfig.OciConfig.Hostname != "" { - if _, err := rpcOps.SetHostname(e.EngineConfig.OciConfig.Hostname); err != nil { - return err - } - } - - // update namespaces configuration path - namespaces := []struct { - nstype string - ns specs.LinuxNamespaceType - checkEnabled bool - }{ - {"pid", specs.PIDNamespace, false}, - {"uts", specs.UTSNamespace, false}, - {"ipc", specs.IPCNamespace, false}, - {"mnt", specs.MountNamespace, false}, - {"cgroup", specs.CgroupNamespace, false}, - {"net", specs.NetworkNamespace, false}, - {"user", specs.UserNamespace, true}, - } - - path := fmt.Sprintf("/proc/%d/ns", pid) - - for _, n := range namespaces { - has, err := proc.HasNamespace(pid, n.nstype) - if err == nil && (has || n.checkEnabled) { - enabled := false - if n.checkEnabled { - if e.EngineConfig.OciConfig.Linux != nil { - for _, namespace := range e.EngineConfig.OciConfig.Linux.Namespaces { - if n.ns == namespace.Type { - enabled = true - break - } - } - } - } - if has || enabled { - nspath := filepath.Join(path, n.nstype) - e.EngineConfig.OciConfig.AddOrReplaceLinuxNamespace(n.ns, nspath) - } - } else if err != nil { - return fmt.Errorf("failed to check %s root and container namespace: %s", n.ns, err) - } - } - - method := "pivot" - if !c.mntNS { - method = "chroot" - } - - _, err = rpcOps.Chroot(c.rootfs, method) - if err != nil { - return fmt.Errorf("chroot failed: %s", err) - } - - if e.EngineConfig.SlavePts != -1 { - if err := syscall.Close(e.EngineConfig.SlavePts); err != nil { - return fmt.Errorf("failed to close slave part: %s", err) - } - } - if e.EngineConfig.OutputStreams[0] != -1 { - if err := syscall.Close(e.EngineConfig.OutputStreams[1]); err != nil { - return fmt.Errorf("failed to close write output stream: %s", err) - } - } - if e.EngineConfig.ErrorStreams[0] != -1 { - if err := syscall.Close(e.EngineConfig.ErrorStreams[1]); err != nil { - return fmt.Errorf("failed to close write error stream: %s", err) - } - } - if e.EngineConfig.InputStreams[0] != -1 { - if err := syscall.Close(e.EngineConfig.InputStreams[1]); err != nil { - return fmt.Errorf("failed to close write input stream: %s", err) - } - } - - return nil -} - -func (e *EngineOperations) createState(pid int) error { - e.EngineConfig.Lock() - defer e.EngineConfig.Unlock() - - name := e.CommonConfig.ContainerID - - file, err := instance.Add(name, instance.OciSubDir) - if err != nil { - return err - } - - e.EngineConfig.State.Version = specs.Version - e.EngineConfig.State.Bundle = e.EngineConfig.GetBundlePath() - e.EngineConfig.State.ID = e.CommonConfig.ContainerID - e.EngineConfig.State.Pid = pid - e.EngineConfig.State.Status = ociruntime.Creating - e.EngineConfig.State.Annotations = e.EngineConfig.OciConfig.Annotations - - file.Config, err = json.Marshal(e.CommonConfig) - if err != nil { - return err - } - - file.User = "root" - file.Pid = pid - file.PPid = os.Getpid() - file.Image = filepath.Join(e.EngineConfig.GetBundlePath(), e.EngineConfig.OciConfig.Root.Path) - - if err := file.Update(); err != nil { - return err - } - - socketPath := e.EngineConfig.SyncSocket - - if socketPath != "" { - data, err := json.Marshal(e.EngineConfig.State) - if err != nil { - sylog.Warningf("failed to marshal state data: %s", err) - } else if err := unix.WriteSocket(socketPath, data); err != nil { - sylog.Warningf("%s", err) - } - } - - return nil -} - -func (e *EngineOperations) updateState(status string) error { - e.EngineConfig.Lock() - defer e.EngineConfig.Unlock() - - file, err := instance.Get(e.CommonConfig.ContainerID, instance.OciSubDir) - if err != nil { - return err - } - // do nothing if already stopped - if e.EngineConfig.State.Status == ociruntime.Stopped { - return nil - } - oldStatus := e.EngineConfig.State.Status - e.EngineConfig.State.Status = specs.ContainerState(status) - - t := time.Now().UnixNano() - - switch status { - case ociruntime.Created: - if e.EngineConfig.State.CreatedAt == nil { - e.EngineConfig.State.CreatedAt = &t - } - case ociruntime.Running: - if e.EngineConfig.State.StartedAt == nil { - e.EngineConfig.State.StartedAt = &t - } - case ociruntime.Stopped: - if e.EngineConfig.State.FinishedAt == nil { - e.EngineConfig.State.FinishedAt = &t - } - } - - file.Config, err = json.Marshal(e.CommonConfig) - if err != nil { - return err - } - - if err := file.Update(); err != nil { - return err - } - - socketPath := e.EngineConfig.SyncSocket - - if socketPath != "" { - data, err := json.Marshal(e.EngineConfig.State) - if err != nil { - sylog.Warningf("failed to marshal state data: %s", err) - } else if err := unix.WriteSocket(socketPath, data); err != nil { - sylog.Warningf("%s", err) - } - } - - // send running or stopped status right after container creation - // to notify that container process started - if statusChan != nil && oldStatus == ociruntime.Created && - (status == ociruntime.Running || status == ociruntime.Stopped) { - statusChan <- status - } - return nil -} - -// one shot function to wait on running or stopped status -func (e *EngineOperations) waitStatusUpdate() { - if statusChan == nil { - return - } - // block until status update is sent - <-statusChan - // close channel and set it to nil - close(statusChan) - statusChan = nil -} - -func (c *container) addCgroups(pid int, system *mount.System) error { - name := c.engine.CommonConfig.ContainerID - resources := c.engine.EngineConfig.OciConfig.Linux.Resources - systemd := c.engine.EngineConfig.GetSystemdCgroups() - cgroupsPath := c.engine.EngineConfig.OciConfig.Linux.CgroupsPath - - if !systemd && !filepath.IsAbs(cgroupsPath) { - if cgroupsPath == "" { - cgroupsPath = filepath.Join("/apptainer-oci", name) - } else { - cgroupsPath = filepath.Join("/", cgroupsPath) - } - } - - if systemd && cgroupsPath == "" { - cgroupsPath = "system.slice:apptainer-oci:" + name - } - - c.engine.EngineConfig.OciConfig.Linux.CgroupsPath = cgroupsPath - - manager, err := cgroups.NewManagerWithSpec(resources, pid, cgroupsPath, systemd) - if err != nil { - return fmt.Errorf("failed to apply cgroups resources restriction: %s", err) - } - - // If a mount point exists for a cgroup v1 hierarchy we will handle it here. - // This is not necessary for cgroups v2 - as the unified hierarchy will be handled with a simple bind. - if c.cgroupV1MountIndex >= 0 { - m := c.engine.EngineConfig.OciConfig.Config.Mounts[c.cgroupV1MountIndex] - c.engine.EngineConfig.OciConfig.Config.Mounts = append( - c.engine.EngineConfig.OciConfig.Config.Mounts[:c.cgroupV1MountIndex], - c.engine.EngineConfig.OciConfig.Config.Mounts[c.cgroupV1MountIndex+1:]..., - ) - - cgroupRootPath, err := manager.GetCgroupRootPath() - if err != nil { - return fmt.Errorf("failed to determine cgroup root path: %w", err) - } - - flags, opt := mount.ConvertOptions(m.Options) - options := strings.Join(opt, ",") - - readOnly := false - if flags&syscall.MS_RDONLY != 0 { - readOnly = true - flags &^= uintptr(syscall.MS_RDONLY) - } - - hasMode := false - for _, o := range opt { - if strings.HasPrefix(o, "mode=") { - hasMode = true - break - } - } - if !hasMode { - options += ",mode=755" - } - - if err := system.Points.AddFS(mount.OtherTag, m.Destination, "tmpfs", flags, options); err != nil { - return err - } - - createSymlinks := func(*mount.System) error { - cgroupPath := filepath.Join(c.rpcRoot, c.rootfs, m.Destination) - if _, err := os.Stat(filepath.Join(cgroupPath, "cpu")); err != nil && os.IsNotExist(err) { - if err := c.rpcOps.Symlink("cpu,cpuacct", filepath.Join(c.rootfs, m.Destination, "cpu")); err != nil { - return err - } - if err := c.rpcOps.Symlink("cpu,cpuacct", filepath.Join(c.rootfs, m.Destination, "cpuacct")); err != nil { - return err - } - } - - if _, err := os.Stat(filepath.Join(cgroupPath, "net_cls")); err != nil && os.IsNotExist(err) { - if err := c.rpcOps.Symlink("net_cls,net_prio", filepath.Join(c.rootfs, m.Destination, "net_cls")); err != nil { - return err - } - if err := c.rpcOps.Symlink("net_cls,net_prio", filepath.Join(c.rootfs, m.Destination, "net_prio")); err != nil { - return err - } - } - return nil - } - - if err := system.RunAfterTag(mount.OtherTag, createSymlinks); err != nil { - return err - } - - f, err := os.Open(fmt.Sprintf("/proc/%d/cgroup", pid)) - if err != nil { - return err - } - defer f.Close() - - flags |= uintptr(syscall.MS_BIND) - if readOnly { - flags |= syscall.MS_RDONLY - } - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - cgroupLine := strings.Split(scanner.Text(), ":") - if strings.HasPrefix(cgroupLine[1], "name=") { - cgroupLine[1] = strings.Replace(cgroupLine[1], "name=", "", 1) - } - if cgroupLine[1] != "" { - source := filepath.Join(cgroupRootPath, cgroupLine[1], cgroupLine[2]) - dest := filepath.Join(m.Destination, cgroupLine[1]) - if err := system.Points.AddBind(mount.OtherTag, source, dest, flags); err != nil { - return err - } - if readOnly { - if err := system.Points.AddRemount(mount.OtherTag, dest, flags); err != nil { - return err - } - } - } - } - - if readOnly { - if err := system.Points.AddRemount(mount.FinalTag, m.Destination, flags); err != nil { - return err - } - } - } - - c.engine.EngineConfig.Cgroups = manager - - return nil -} - -func (c *container) addAllPaths(system *mount.System) error { - // add masked path - if err := c.addMaskedPathsMount(system); err != nil { - return err - } - - // add read-only path - if !c.userNS { - if err := c.addReadonlyPathsMount(system); err != nil { - return err - } - } - - return nil -} - -func (c *container) addRootfsMount(system *mount.System) error { - flags := uintptr(syscall.MS_BIND) - - if c.engine.EngineConfig.OciConfig.Root.Readonly { - sylog.Debugf("Mounted read-only") - flags |= syscall.MS_RDONLY - } - - parentRootfs, err := proc.ParentMount(c.rootfs) - if err != nil { - return err - } - - sylog.Debugf("Parent rootfs: %s", parentRootfs) - - if err := c.rpcOps.Mount("", parentRootfs, "", syscall.MS_PRIVATE, ""); err != nil { - return err - } - if err := system.Points.AddBind(mount.RootfsTag, c.rootfs, c.rootfs, flags); err != nil { - return err - } - if flags&syscall.MS_RDONLY != 0 { - return system.Points.AddRemount(mount.FinalTag, c.rootfs, flags) - } - - return nil -} - -func (c *container) addDefaultDevices(system *mount.System) error { - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - rootfsPath := filepath.Join(c.rpcRoot, c.rootfs) - - devPath := filepath.Join(rootfsPath, fs.EvalRelative("/dev", rootfsPath)) - if _, err := os.Lstat(devPath); os.IsNotExist(err) { - if err := os.Mkdir(devPath, 0o755); err != nil { - return err - } - } - - for _, symlink := range symlinkDevices { - path := filepath.Join(rootfsPath, symlink.new) - if _, err := os.Lstat(path); os.IsNotExist(err) { - if c.userNS { - path = filepath.Join(c.rootfs, symlink.new) - if err := c.rpcOps.Symlink(symlink.old, path); err != nil { - return err - } - } else { - if err := os.Symlink(symlink.old, path); err != nil { - return err - } - } - } - } - - if c.engine.EngineConfig.OciConfig.Process.Terminal { - path := filepath.Join(rootfsPath, "dev", "console") - if _, err := os.Lstat(path); os.IsNotExist(err) { - if c.userNS { - if _, err := c.rpcOps.Touch(filepath.Join(c.rootfs, "dev", "console")); err != nil { - return err - } - } else { - if err := fs.Touch(path); err != nil { - return err - } - } - path = fmt.Sprintf("/proc/self/fd/%d", c.engine.EngineConfig.SlavePts) - console, err := os.Readlink(path) - if err != nil { - return err - } - if err := system.Points.AddBind(mount.OtherTag, console, "/dev/console", syscall.MS_BIND); err != nil { - return err - } - } - } - - for _, device := range devices { - dev := int((device.major << 8) | (device.minor & 0xff) | ((device.minor & 0xfff00) << 12)) - path := filepath.Join(rootfsPath, device.path) - if _, err := os.Lstat(path); os.IsNotExist(err) { - if c.userNS { - path = filepath.Join(c.rootfs, device.path) - if _, err := os.Stat(device.path); os.IsNotExist(err) { - sylog.Debugf("skipping mount, %s doesn't exists", device.path) - continue - } - dirpath := filepath.Dir(path) - if _, err := c.rpcOps.MkdirAll(dirpath, 0o755); err != nil { - return fmt.Errorf("could not create parent directory %s: %s", dirpath, err) - } - if _, err := c.rpcOps.Touch(path); err != nil { - return fmt.Errorf("could not create file %s: %s", path, err) - } - if err := c.rpcOps.Mount(device.path, path, "", syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("could not mount %s to %s: %s", device.path, path, err) - } - } else { - dirpath := filepath.Dir(path) - if err := os.MkdirAll(dirpath, 0o755); err != nil { - return fmt.Errorf("could not create parent directory %s: %s", dirpath, err) - } - if err := syscall.Mknod(path, uint32(device.mode), dev); err != nil { - return fmt.Errorf("could not create device %s: %s", path, err) - } - if device.uid != 0 || device.gid != 0 { - if err := os.Chown(path, device.uid, device.gid); err != nil { - return fmt.Errorf("could not change %s owner: %s", path, err) - } - } - } - } - } - - return nil -} - -func (c *container) addDevices(system *mount.System) error { - for _, d := range c.engine.EngineConfig.OciConfig.Linux.Devices { - var dev device - - if d.Path == "" { - return fmt.Errorf("device path required") - } - dev.path = d.Path - - if d.FileMode != nil { - dev.mode = *d.FileMode - } else { - dev.mode = 0o644 - } - - switch d.Type { - case "c", "u": - dev.mode |= syscall.S_IFCHR - dev.major = d.Major - dev.minor = d.Minor - case "b": - dev.mode |= syscall.S_IFBLK - dev.major = d.Major - dev.minor = d.Minor - case "p": - dev.mode |= syscall.S_IFIFO - default: - return fmt.Errorf("device type unknown for %s", d.Path) - } - - if d.UID != nil { - dev.uid = int(*d.UID) - } - if d.GID != nil { - dev.gid = int(*d.GID) - } - - devices = append(devices, dev) - } - - if c.devIndex >= 0 { - m := &c.engine.EngineConfig.OciConfig.Config.Mounts[c.devIndex] - - flags, _ := mount.ConvertOptions(m.Options) - - flags |= uintptr(syscall.MS_BIND) - if flags&syscall.MS_RDONLY != 0 { - if err := system.Points.AddRemount(mount.FinalTag, m.Destination, flags); err != nil { - return err - } - for i := len(m.Options) - 1; i >= 0; i-- { - if m.Options[i] == "ro" { - m.Options = append(m.Options[:i], m.Options[i+1:]...) - break - } - } - } - - if c.engine.EngineConfig.OciConfig.Linux.Resources == nil { - c.engine.EngineConfig.OciConfig.Linux.Resources = &specs.LinuxResources{} - } - - // cgroupDevices are essential for operation, so must be allowed *prior* to a configured wildcard deny. - c.engine.EngineConfig.OciConfig.Linux.Resources.Devices = append(cgroupDevices, c.engine.EngineConfig.OciConfig.Linux.Resources.Devices...) - } - - return nil -} - -func (c *container) addMaskedPathsMount(system *mount.System) error { - paths := c.engine.EngineConfig.OciConfig.Linux.MaskedPaths - - dir, err := instance.GetDir(c.engine.CommonConfig.ContainerID, instance.OciSubDir) - if err != nil { - return err - } - nullPath := filepath.Join(dir, "null") - - if _, err := os.Stat(nullPath); os.IsNotExist(err) { - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - if err := os.Mkdir(nullPath, 0o755); err != nil { - return err - } - } - - for _, path := range paths { - relativePath := filepath.Join(c.rootfs, path) - rpcPath := filepath.Join(c.rpcRoot, relativePath) - fi, err := os.Stat(rpcPath) - if err != nil { - sylog.Debugf("ignoring masked path %s: %s", path, err) - continue - } - if fi.IsDir() { - if err := system.Points.AddBind(mount.OtherTag, nullPath, relativePath, syscall.MS_BIND); err != nil { - return err - } - } else if err := system.Points.AddBind(mount.OtherTag, "/dev/null", relativePath, syscall.MS_BIND); err != nil { - return err - } - } - return nil -} - -func (c *container) addReadonlyPathsMount(system *mount.System) error { - paths := c.engine.EngineConfig.OciConfig.Linux.ReadonlyPaths - - for _, path := range paths { - relativePath := filepath.Join(c.rootfs, path) - rpcPath := filepath.Join(c.rpcRoot, relativePath) - _, err := os.Stat(rpcPath) - if err != nil { - sylog.Debugf("ignoring read-only path %s: %s", path, err) - continue - } - if err := system.Points.AddBind(mount.OtherTag, relativePath, relativePath, syscall.MS_BIND|syscall.MS_RDONLY); err != nil { - return err - } - if err := system.Points.AddRemount(mount.OtherTag, relativePath, syscall.MS_BIND|syscall.MS_RDONLY); err != nil { - return err - } - } - return nil -} - -func (c *container) mount(point *mount.Point, system *mount.System) error { - source := point.Source - dest := point.Destination - flags, opts := mount.ConvertOptions(point.Options) - optsString := strings.Join(opts, ",") - ignore := false - - if flags&syscall.MS_REMOUNT != 0 { - ignore = true - } - - if !strings.HasPrefix(dest, c.rootfs) { - rootfsPath := filepath.Join(c.rpcRoot, c.rootfs) - relativeDest := fs.EvalRelative(dest, c.rootfs) - procDest := filepath.Join(rootfsPath, relativeDest) - - dest = filepath.Join(c.rootfs, relativeDest) - - sylog.Debugf("Checking if %s exists", procDest) - if _, err := os.Stat(procDest); os.IsNotExist(err) && !ignore { - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - if point.Type != "" { - sylog.Debugf("Creating %s", procDest) - if c.userNS { - if _, err := c.rpcOps.MkdirAll(dest, 0o755); err != nil { - return err - } - } else { - if err := os.MkdirAll(procDest, 0o755); err != nil { - return err - } - } - } else { - var st syscall.Stat_t - - dir := filepath.Dir(procDest) - if _, err := os.Stat(dir); os.IsNotExist(err) { - sylog.Debugf("Creating parent %s", dir) - if c.userNS { - if err := c.rpcOps.Mkdir(filepath.Dir(dest), 0o755); err != nil { - return err - } - } else { - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - } - } - - if err := syscall.Stat(source, &st); err != nil { - sylog.Debugf("Ignoring %s: %s", source, err) - return nil - } - switch st.Mode & syscall.S_IFMT { - case syscall.S_IFDIR: - sylog.Debugf("Creating dir %s", filepath.Base(procDest)) - if c.userNS { - if err := c.rpcOps.Mkdir(dest, 0o755); err != nil { - return err - } - } else { - if err := os.Mkdir(procDest, 0o755); err != nil { - return err - } - } - case syscall.S_IFREG, - syscall.S_IFBLK, - syscall.S_IFCHR, - syscall.S_IFIFO, - syscall.S_IFSOCK: - sylog.Debugf("Creating file %s", filepath.Base(procDest)) - if c.userNS { - if _, err := c.rpcOps.Touch(dest); err != nil { - return err - } - } else { - if err := fs.Touch(procDest); err != nil { - return err - } - } - } - } - } - } else { - procDest := filepath.Join(c.rpcRoot, dest) - - sylog.Debugf("Checking if %s exists", procDest) - if _, err := os.Stat(procDest); os.IsNotExist(err) { - sylog.Warningf("destination %s doesn't exist", dest) - return nil - } - } - - if ignore { - sylog.Debugf("(re)mount %s", dest) - } else { - sylog.Debugf("Mount %s to %s : %s [%s]", source, dest, point.Type, optsString) - } - - err := c.rpcOps.Mount(source, dest, point.Type, flags, optsString) - if err != nil { - sylog.Debugf("RPC mount error: %s", err) - } - - return err -} diff --git a/internal/pkg/runtime/engine/oci/engine_linux.go b/internal/pkg/runtime/engine/oci/engine_linux.go deleted file mode 100644 index cb099a4125..0000000000 --- a/internal/pkg/runtime/engine/oci/engine_linux.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "github.com/apptainer/apptainer/internal/pkg/runtime/engine" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc/server" - ociServer "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc/server" - "github.com/apptainer/apptainer/pkg/runtime/engine/config" -) - -// EngineOperations is an Apptainer OCI runtime engine that implements engine.Operations. -// Basically, this is the core of `apptainer oci` commands. -type EngineOperations struct { - CommonConfig *config.Common `json:"-"` - EngineConfig *EngineConfig `json:"engineConfig"` -} - -// InitConfig stores the parsed config.Common inside the engine. -// -// Since this method simply stores config.Common, it does not matter -// whether or not there are any elevated privileges during this call. -func (e *EngineOperations) InitConfig(cfg *config.Common, privStageOne bool) { - e.CommonConfig = cfg -} - -// Config returns a pointer to EngineConfig literal as a config.EngineConfig -// interface. This pointer gets stored in the Engine.Common field. -// -// Since this method simply returns a zero value of the concrete -// EngineConfig, it does not matter whether or not there are any elevated -// privileges during this call. -func (e *EngineOperations) Config() config.EngineConfig { - return e.EngineConfig -} - -func init() { - engine.RegisterOperations( - Name, - &EngineOperations{ - EngineConfig: &EngineConfig{}, - }, - ) - - ocimethods := new(ociServer.Methods) - ocimethods.Methods = new(server.Methods) - engine.RegisterRPCMethods( - Name, - ocimethods, - ) -} diff --git a/internal/pkg/runtime/engine/oci/monitor_linux.go b/internal/pkg/runtime/engine/oci/monitor_linux.go deleted file mode 100644 index 41fb666151..0000000000 --- a/internal/pkg/runtime/engine/oci/monitor_linux.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "fmt" - "os" - "syscall" -) - -// MonitorContainer is called from master once the container has -// been spawned. It will block until the container exists. -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Particularly here no additional privileges are gained as monitor does -// not need them for wait4 and kill syscalls. However, most likely this -// still will be executed as root since `apptainer oci` command set requires -// privileged execution. -func (e *EngineOperations) MonitorContainer(pid int, signals chan os.Signal) (syscall.WaitStatus, error) { - var status syscall.WaitStatus - - for { - s := <-signals - switch s { - case syscall.SIGCHLD: - if wpid, err := syscall.Wait4(pid, &status, syscall.WNOHANG, nil); err != nil { - return status, fmt.Errorf("error while waiting child: %s", err) - } else if wpid != pid { - continue - } - return status, nil - case syscall.SIGURG: - // Ignore SIGURG, which is used for non-cooperative goroutine - // preemption starting with Go 1.14. For more information, see - // https://github.com/golang/go/issues/24543. - break - default: - if err := syscall.Kill(pid, s.(syscall.Signal)); err != nil { - return status, fmt.Errorf("interrupted by signal %s", s.String()) - } - } - } -} diff --git a/internal/pkg/runtime/engine/oci/prepare_linux.go b/internal/pkg/runtime/engine/oci/prepare_linux.go deleted file mode 100644 index dd3df77bdc..0000000000 --- a/internal/pkg/runtime/engine/oci/prepare_linux.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "fmt" - "os" - - "github.com/apptainer/apptainer/internal/pkg/buildcfg" - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/starter" - "github.com/apptainer/apptainer/internal/pkg/util/fs" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/apptainerconf" - "github.com/apptainer/apptainer/pkg/util/capabilities" - "github.com/creack/pty" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// make master/slave as global variable to avoid GC close file descriptor -var ( - master *os.File - slave *os.File -) - -// PrepareConfig is called during stage1 to validate and prepare -// container configuration. It is responsible for reading capabilities, -// checking what namespaces are required, opening streams for attach and -// exec, etc. -// -// No additional privileges can be gained as any of them are already -// dropped by the time PrepareConfig is called. However, most likely this -// still will be executed as root since `apptainer oci` command set -// requires privileged execution. -// -//nolint:maintidx -func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error { - if e.CommonConfig.EngineName != Name { - return fmt.Errorf("incorrect engine") - } - - if e.EngineConfig.OciConfig.Generator.Config != &e.EngineConfig.OciConfig.Spec { - return fmt.Errorf("bad engine configuration provided") - } - - if starterConfig.GetIsSUID() { - return fmt.Errorf("suid workflow disabled by administrator") - } - - if e.EngineConfig.OciConfig.Process == nil { - return fmt.Errorf("empty OCI process configuration") - } - - if e.EngineConfig.OciConfig.Linux == nil { - return fmt.Errorf("empty OCI linux configuration") - } - - // TODO - investigate whether this is the highest place to pull this value from apptainer.conf - if !fs.IsOwner(buildcfg.APPTAINER_CONF_FILE, 0) { - return fmt.Errorf("%s must be owned by root", buildcfg.APPTAINER_CONF_FILE) - } - sConf, err := apptainerconf.Parse(buildcfg.APPTAINER_CONF_FILE) - if err != nil { - return fmt.Errorf("unable to parse apptainer.conf file: %s", err) - } - e.EngineConfig.SystemdCgroups = sConf.SystemdCgroups - - // reset state config that could be passed to engine - e.EngineConfig.State = ociruntime.State{} - - user := &e.EngineConfig.OciConfig.Process.User - gids := make([]int, 0, len(user.AdditionalGids)+1) - - uid := int(user.UID) - gid := user.GID - - gids = append(gids, int(gid)) - for _, g := range user.AdditionalGids { - gids = append(gids, int(g)) - } - - starterConfig.SetTargetUID(uid) - starterConfig.SetTargetGID(gids) - - if !e.EngineConfig.Exec { - starterConfig.SetInstance(true) - } - - userNS := false - for _, ns := range e.EngineConfig.OciConfig.Linux.Namespaces { - if ns.Type == specs.UserNamespace { - userNS = true - break - } - } - - starterConfig.SetNsFlagsFromSpec(e.EngineConfig.OciConfig.Linux.Namespaces) - if err := starterConfig.SetNsPathFromSpec(e.EngineConfig.OciConfig.Linux.Namespaces); err != nil { - return err - } - - if userNS { - if len(e.EngineConfig.OciConfig.Linux.UIDMappings) == 0 { - return fmt.Errorf("user namespace invoked without uid mapping") - } - if len(e.EngineConfig.OciConfig.Linux.GIDMappings) == 0 { - return fmt.Errorf("user namespace invoked without gid mapping") - } - if err := starterConfig.AddUIDMappings(e.EngineConfig.OciConfig.Linux.UIDMappings); err != nil { - return err - } - if err := starterConfig.AddGIDMappings(e.EngineConfig.OciConfig.Linux.GIDMappings); err != nil { - return err - } - } - - if e.EngineConfig.OciConfig.Linux.RootfsPropagation != "" { - starterConfig.SetMountPropagation(e.EngineConfig.OciConfig.Linux.RootfsPropagation) - } else { - starterConfig.SetMountPropagation("private") - } - - starterConfig.SetNoNewPrivs(e.EngineConfig.OciConfig.Process.NoNewPrivileges) - - if e.EngineConfig.OciConfig.Process.Capabilities != nil { - if err := e.checkCapabilities(); err != nil { - return err - } - - // force cap_sys_admin for seccomp and no_new_priv flag - caps := append(e.EngineConfig.OciConfig.Process.Capabilities.Effective, "CAP_SYS_ADMIN") - starterConfig.SetCapabilities(capabilities.Effective, caps) - - caps = append(e.EngineConfig.OciConfig.Process.Capabilities.Permitted, "CAP_SYS_ADMIN") - starterConfig.SetCapabilities(capabilities.Permitted, caps) - - starterConfig.SetCapabilities(capabilities.Inheritable, e.EngineConfig.OciConfig.Process.Capabilities.Inheritable) - starterConfig.SetCapabilities(capabilities.Bounding, e.EngineConfig.OciConfig.Process.Capabilities.Bounding) - starterConfig.SetCapabilities(capabilities.Ambient, e.EngineConfig.OciConfig.Process.Capabilities.Ambient) - } - - e.EngineConfig.MasterPts = -1 - e.EngineConfig.SlavePts = -1 - e.EngineConfig.OutputStreams = [2]int{-1, -1} - e.EngineConfig.ErrorStreams = [2]int{-1, -1} - e.EngineConfig.InputStreams = [2]int{-1, -1} - - if e.EngineConfig.GetLogFormat() == "" { - sylog.Debugf("No log format specified, setting kubernetes log format by default") - e.EngineConfig.SetLogFormat("kubernetes") - } - - if !e.EngineConfig.Exec { - if e.EngineConfig.OciConfig.Process.Terminal { - var err error - - master, slave, err = pty.Open() - if err != nil { - return err - } - consoleSize := e.EngineConfig.OciConfig.Process.ConsoleSize - if consoleSize != nil { - var size pty.Winsize - - size.Cols = uint16(consoleSize.Width) - size.Rows = uint16(consoleSize.Height) - if err := pty.Setsize(slave, &size); err != nil { - return err - } - } - e.EngineConfig.MasterPts = int(master.Fd()) - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.MasterPts); err != nil { - return err - } - e.EngineConfig.SlavePts = int(slave.Fd()) - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.SlavePts); err != nil { - return err - } - } else { - r, w, err := os.Pipe() - if err != nil { - return err - } - e.EngineConfig.OutputStreams = [2]int{int(r.Fd()), int(w.Fd())} - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.OutputStreams[0]); err != nil { - return err - } - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.OutputStreams[1]); err != nil { - return err - } - - r, w, err = os.Pipe() - if err != nil { - return err - } - e.EngineConfig.ErrorStreams = [2]int{int(r.Fd()), int(w.Fd())} - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.ErrorStreams[0]); err != nil { - return err - } - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.ErrorStreams[1]); err != nil { - return err - } - - r, w, err = os.Pipe() - if err != nil { - return err - } - e.EngineConfig.InputStreams = [2]int{int(w.Fd()), int(r.Fd())} - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.InputStreams[0]); err != nil { - return err - } - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.InputStreams[1]); err != nil { - return err - } - } - } else { - starterConfig.SetNamespaceJoinOnly(true) - cPath := e.EngineConfig.OciConfig.Linux.CgroupsPath - if cPath == "" { - return nil - } - ppid := os.Getppid() - - sylog.Debugf("Adding process %d to instance cgroup %q", ppid, cPath) - manager, err := cgroups.GetManagerForGroup(cPath) - if err != nil { - return fmt.Errorf("couldn't create cgroup manager: %v", err) - } - if err := manager.AddProc(ppid); err != nil { - return fmt.Errorf("couldn't add process to instance cgroup: %v", err) - } - } - - return nil -} - -func (e *EngineOperations) checkCapabilities() error { - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Permitted { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Effective { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Inheritable { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Bounding { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Ambient { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - return nil -} diff --git a/internal/pkg/runtime/engine/oci/process_linux.go b/internal/pkg/runtime/engine/oci/process_linux.go deleted file mode 100644 index 09820a2d4f..0000000000 --- a/internal/pkg/runtime/engine/oci/process_linux.go +++ /dev/null @@ -1,497 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net" - "os" - osexec "os/exec" - "os/signal" - "path/filepath" - "strconv" - "strings" - "syscall" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/security" - "github.com/apptainer/apptainer/internal/pkg/util/exec" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/copy" - "github.com/apptainer/apptainer/pkg/util/rlimit" - "github.com/apptainer/apptainer/pkg/util/unix" - "github.com/creack/pty" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// StartProcess is called during stage2 after RPC server finished -// environment preparation. This is the container process itself. -// -// No additional privileges can be gained during this call (unless container -// is executed as root intentionally) as starter will set uid/euid/suid -// to the targetUID (PrepareConfig will set it by calling starter.Config.SetTargetUID). -func (e *EngineOperations) StartProcess(masterConnFd int) error { - cwd := e.EngineConfig.OciConfig.Process.Cwd - - if cwd == "" { - cwd = "/" - } - - if !filepath.IsAbs(cwd) { - return fmt.Errorf("cwd property must be an absolute path") - } - - if err := os.Chdir(cwd); err != nil { - return fmt.Errorf("can't enter in current working directory: %s", err) - } - - if err := setRlimit(e.EngineConfig.OciConfig.Process.Rlimits); err != nil { - return err - } - - comm := os.NewFile(uintptr(masterConnFd), "master-socket") - masterConn, err := net.FileConn(comm) - comm.Close() - if err != nil { - return fmt.Errorf("failed to copy master unix socket descriptor: %s", err) - } - - if e.EngineConfig.EmptyProcess { - return e.emptyProcess(masterConn) - } - - args := e.EngineConfig.OciConfig.Process.Args - env := e.EngineConfig.OciConfig.Process.Env - - for _, e := range e.EngineConfig.OciConfig.Process.Env { - if strings.HasPrefix(e, "PATH=") { - os.Setenv("PATH", e[5:]) - } - } - - bpath, err := osexec.LookPath(args[0]) - if err != nil { - return fmt.Errorf("%s", err) - } - args[0] = bpath - - if e.EngineConfig.MasterPts != -1 { - slaveFd := e.EngineConfig.SlavePts - if err := syscall.Dup3(slaveFd, int(os.Stdin.Fd()), 0); err != nil { - return err - } - if err := syscall.Dup3(slaveFd, int(os.Stdout.Fd()), 0); err != nil { - return err - } - if err := syscall.Dup3(slaveFd, int(os.Stderr.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.MasterPts); err != nil { - return err - } - if err := syscall.Close(slaveFd); err != nil { - return err - } - if _, err := syscall.Setsid(); err != nil { - return err - } - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, os.Stdin.Fd(), uintptr(syscall.TIOCSCTTY), 1); err != 0 { - return fmt.Errorf("failed to set crontrolling terminal: %s", err.Error()) - } - } else if e.EngineConfig.OutputStreams[1] != -1 { - if err := syscall.Dup3(e.EngineConfig.OutputStreams[1], int(os.Stdout.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.OutputStreams[1]); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.OutputStreams[0]); err != nil { - return err - } - - if err := syscall.Dup3(e.EngineConfig.ErrorStreams[1], int(os.Stderr.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.ErrorStreams[1]); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.ErrorStreams[0]); err != nil { - return err - } - - if err := syscall.Dup3(e.EngineConfig.InputStreams[1], int(os.Stdin.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.InputStreams[1]); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.InputStreams[0]); err != nil { - return err - } - } - - // trigger pre-start process - if _, err := masterConn.Write([]byte("t")); err != nil { - return fmt.Errorf("failed to pause process: %s", err) - } - if !e.EngineConfig.Exec { - // block on read start given - data := make([]byte, 1) - if _, err := masterConn.Read(data); err != nil { - return fmt.Errorf("failed to receive start signal: %s", err) - } - } - - if err := security.Configure(&e.EngineConfig.OciConfig.Spec); err != nil { - return fmt.Errorf("failed to apply security configuration: %s", err) - } - - err = syscall.Exec(args[0], args, env) - return fmt.Errorf("exec %s failed: %s", args[0], err) -} - -// PreStartProcess is called from master after before container startup. -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -func (e *EngineOperations) PreStartProcess(ctx context.Context, pid int, masterConn net.Conn, fatalChan chan error) error { - if e.EngineConfig.Exec { - return nil - } - - file, err := instance.Get(e.CommonConfig.ContainerID, instance.OciSubDir) - if err != nil { - return err - } - e.EngineConfig.State.AttachSocket = filepath.Join(filepath.Dir(file.Path), "attach.sock") - - attach, err := unix.CreateSocket(e.EngineConfig.State.AttachSocket) - if err != nil { - return err - } - - e.EngineConfig.State.ControlSocket = filepath.Join(filepath.Dir(file.Path), "control.sock") - - control, err := unix.CreateSocket(e.EngineConfig.State.ControlSocket) - if err != nil { - return err - } - - logPath := e.EngineConfig.GetLogPath() - if logPath == "" { - containerID := e.CommonConfig.ContainerID - dir, err := instance.GetDir(containerID, instance.OciSubDir) - if err != nil { - return err - } - logPath = filepath.Join(dir, containerID+".log") - } - - format := e.EngineConfig.GetLogFormat() - formatter, ok := instance.LogFormats[format] - if !ok { - return fmt.Errorf("log format %s is not supported", format) - } - - logger, err := instance.NewLogger(logPath, formatter) - if err != nil { - return err - } - - pidFile := e.EngineConfig.GetPidFile() - if pidFile != "" { - if err := os.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0o644); err != nil { - return err - } - } - - if err := e.updateState(ociruntime.Created); err != nil { - return err - } - - start := make(chan bool, 1) - - go e.handleControl(masterConn, attach, control, logger, start, fatalChan) - - hooks := e.EngineConfig.OciConfig.Hooks - if hooks != nil { - for _, h := range hooks.Prestart { - if err := exec.Hook(ctx, &h, &e.EngineConfig.State.State); err != nil { - return err - } - } - } - - // detach process - syscall.Kill(os.Getppid(), syscall.SIGUSR1) - - // block until start event received - <-start - close(start) - - return nil -} - -// PostStartProcess is called from master after successful -// execution of the container process. It will execute OCI -// post start hooks (if any). -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Here, however, oci engine does not escalate privileges, which means -// OCI hooks will be executed on behalf of a user who spawned a container -// (but not the one who runs it as targetUID may be arbitrary). -// -// Most likely this still will be executed as root since `apptainer oci` -// command set requires privileged execution. -func (e *EngineOperations) PostStartProcess(ctx context.Context, pid int) error { - if err := e.updateState(ociruntime.Running); err != nil { - return err - } - hooks := e.EngineConfig.OciConfig.Hooks - if hooks != nil { - for _, h := range hooks.Poststart { - if err := exec.Hook(ctx, &h, &e.EngineConfig.State.State); err != nil { - sylog.Warningf("%s", err) - } - } - } - return nil -} - -func setRlimit(rlimits []specs.POSIXRlimit) error { - resources := make(map[string]struct{}) - - for _, rl := range rlimits { - if err := rlimit.Set(rl.Type, rl.Soft, rl.Hard); err != nil { - return err - } - if _, found := resources[rl.Type]; found { - return fmt.Errorf("%s was already set", rl.Type) - } - resources[rl.Type] = struct{}{} - } - - return nil -} - -func (e *EngineOperations) emptyProcess(masterConn net.Conn) error { - // pause process on next read - if _, err := masterConn.Write([]byte("t")); err != nil { - return fmt.Errorf("failed to pause process: %s", err) - } - - // block on read start given - data := make([]byte, 1) - if _, err := masterConn.Read(data); err != nil { - return fmt.Errorf("failed to receive ack from master: %s", err) - } - - var status syscall.WaitStatus - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGCHLD, syscall.SIGINT, syscall.SIGTERM) - - if err := security.Configure(&e.EngineConfig.OciConfig.Spec); err != nil { - return fmt.Errorf("failed to apply security configuration: %s", err) - } - - masterConn.Close() - - for { - s := <-signals - switch s { - case syscall.SIGCHLD: - for { - if pid, _ := syscall.Wait4(-1, &status, syscall.WNOHANG, nil); pid <= 0 { - break - } - } - case syscall.SIGINT, syscall.SIGTERM: - os.Exit(0) - } - } -} - -func (e *EngineOperations) handleStream(l net.Listener, logger *instance.Logger, fatalChan chan error) { - var stdout io.ReadWriteCloser - var stderr io.ReadCloser - var stdin io.WriteCloser - var outputWriters *copy.MultiWriter - var errorWriters *copy.MultiWriter - var inputWriters *copy.MultiWriter - var tbuf *copy.TerminalBuffer - - hasTerminal := e.EngineConfig.OciConfig.Process.Terminal - - inputWriters = ©.MultiWriter{} - outputWriters = ©.MultiWriter{} - outWriter, _ := logger.NewWriter("stdout", true) - outputWriters.Add(outWriter) - - if hasTerminal { - stdout = os.NewFile(uintptr(e.EngineConfig.MasterPts), "stream-master-pts") - tbuf = copy.NewTerminalBuffer() - outputWriters.Add(tbuf) - inputWriters.Add(stdout) - } else { - outputStream := os.NewFile(uintptr(e.EngineConfig.OutputStreams[0]), "stdout-stream") - errorStream := os.NewFile(uintptr(e.EngineConfig.ErrorStreams[0]), "error-stream") - inputStream := os.NewFile(uintptr(e.EngineConfig.InputStreams[0]), "input-stream") - stdout = outputStream - stderr = errorStream - stdin = inputStream - outputWriters.Add(os.Stdout) - inputWriters.Add(stdin) - } - - if stderr != nil { - errorWriters = ©.MultiWriter{} - errWriter, _ := logger.NewWriter("stderr", true) - errorWriters.Add(errWriter) - errorWriters.Add(os.Stderr) - } - - go func() { - for { - c, err := l.Accept() - if err != nil { - fatalChan <- err - return - } - - go func() { - outputWriters.Add(c) - if stderr != nil { - errorWriters.Add(c) - } - - if tbuf != nil { - c.Write(tbuf.Line()) - } - - io.Copy(inputWriters, c) - - outputWriters.Del(c) - if stderr != nil { - errorWriters.Del(c) - } - c.Close() - }() - } - }() - - go func() { - io.Copy(outputWriters, stdout) - stdout.Close() - }() - - if stderr != nil { - go func() { - io.Copy(errorWriters, stderr) - stderr.Close() - }() - } - if stdin != nil { - go func() { - io.Copy(inputWriters, os.Stdin) - stdin.Close() - }() - } -} - -func (e *EngineOperations) handleControl(masterConn net.Conn, attach, control net.Listener, logger *instance.Logger, start chan bool, fatalChan chan error) { - var master *os.File - started := false - - if e.EngineConfig.OciConfig.Process.Terminal { - master = os.NewFile(uintptr(e.EngineConfig.MasterPts), "control-master-pts") - } - - for { - c, err := control.Accept() - if err != nil { - fatalChan <- err - return - } - dec := json.NewDecoder(c) - ctrl := &ociruntime.Control{} - if err := dec.Decode(ctrl); err != nil { - fatalChan <- err - return - } - - if ctrl.StartContainer && !started { - started = true - - e.handleStream(attach, logger, fatalChan) - - // since container process block on read, send it an - // ACK so when it will receive data, the container - // process will be executed - if _, err := masterConn.Write([]byte("s")); err != nil { - fatalChan <- fmt.Errorf("failed to send ACK to start process: %s", err) - return - } - - // send start event - start <- true - - // wait status update - e.waitStatusUpdate() - } - if ctrl.ConsoleSize != nil && master != nil { - size := &pty.Winsize{ - Cols: uint16(ctrl.ConsoleSize.Width), - Rows: uint16(ctrl.ConsoleSize.Height), - } - if err := pty.Setsize(master, size); err != nil { - fatalChan <- err - return - } - } - if ctrl.ReopenLog { - if err := logger.ReOpenFile(); err != nil { - fatalChan <- err - return - } - } - if ctrl.Pause { - if err := e.EngineConfig.Cgroups.Freeze(); err != nil { - fatalChan <- err - return - } - if err := e.updateState(ociruntime.Paused); err != nil { - fatalChan <- err - return - } - } - if ctrl.Resume { - if err := e.updateState(ociruntime.Running); err != nil { - fatalChan <- err - return - } - if err := e.EngineConfig.Cgroups.Thaw(); err != nil { - fatalChan <- err - return - } - } - - c.Close() - } -} diff --git a/internal/pkg/runtime/engine/oci/rpc/args.go b/internal/pkg/runtime/engine/oci/rpc/args.go deleted file mode 100644 index 6b0aca349c..0000000000 --- a/internal/pkg/runtime/engine/oci/rpc/args.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package rpc - -// TouchArgs defines the arguments to touch. -type TouchArgs struct { - Path string -} diff --git a/internal/pkg/runtime/engine/oci/rpc/client/client.go b/internal/pkg/runtime/engine/oci/rpc/client/client.go deleted file mode 100644 index 9ee0c05936..0000000000 --- a/internal/pkg/runtime/engine/oci/rpc/client/client.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package client - -import ( - "os" - - args "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc" - client "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc/client" - ociargs "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc" -) - -// RPC holds the state necessary for remote procedure calls. -type RPC struct { - client.RPC -} - -// MkdirAll calls the mkdir RPC using the supplied arguments. -func (t *RPC) MkdirAll(path string, perm os.FileMode) (int, error) { - arguments := &args.MkdirArgs{ - Path: path, - Perm: perm, - } - var reply int - err := t.Client.Call(t.Name+".MkdirAll", arguments, &reply) - return reply, err -} - -// Touch calls the touch RPC using the supplied arguments. -func (t *RPC) Touch(path string) (int, error) { - arguments := &ociargs.TouchArgs{ - Path: path, - } - var reply int - err := t.Client.Call(t.Name+".Touch", arguments, &reply) - return reply, err -} diff --git a/internal/pkg/runtime/engine/oci/rpc/server/server_linux.go b/internal/pkg/runtime/engine/oci/rpc/server/server_linux.go deleted file mode 100644 index fa2bfdfead..0000000000 --- a/internal/pkg/runtime/engine/oci/rpc/server/server_linux.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package server - -import ( - "os" - "syscall" - - "github.com/apptainer/apptainer/internal/pkg/util/fs" - - args "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc" - server "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc/server" - ociargs "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc" - "github.com/apptainer/apptainer/internal/pkg/util/mainthread" -) - -// Methods is a receiver type. -type Methods struct { - *server.Methods -} - -// MkdirAll performs a mkdir with the specified arguments. -func (t *Methods) MkdirAll(arguments *args.MkdirArgs, reply *int) (err error) { - mainthread.Execute(func() { - oldmask := syscall.Umask(0) - err = os.MkdirAll(arguments.Path, arguments.Perm) - syscall.Umask(oldmask) - }) - return err -} - -// Touch performs a touch with the specified arguments. -func (t *Methods) Touch(arguments *ociargs.TouchArgs, reply *int) (err error) { - return fs.Touch(arguments.Path) -} diff --git a/internal/pkg/runtime/launcher/launcher.go b/internal/pkg/runtime/launcher/launcher.go new file mode 100644 index 0000000000..812349835d --- /dev/null +++ b/internal/pkg/runtime/launcher/launcher.go @@ -0,0 +1,33 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package launcher is responsible for implementing launchers, which can start a +// container, with configuration passed from the CLI layer. +// +// The package currently implements a single native.Launcher, with an Exec +// method that constructs a runtime configuration and calls the Apptainer +// runtime starter binary to start the container. +// +// TODO - the launcher package will be extended to support launching containers +// via the OCI runc/crun runtime, in addition to the current Apptainer runtime +// starter, by adding an oci.Launcher. +package launcher + +import "context" + +// Launcher is responsible for configuring and launching a container image. +// It will execute a runtime, such as Apptainer's native runtime (via the starter +// binary), or an external OCI runtime (e.g. runc). +type Launcher interface { + // Exec will execute the container image 'image', starting 'process', and + // passing arguments 'args'. If instanceName is specified, the container + // must be launched as a background instance, otherwise it must run + // interactively, attached to the console. + Exec(ctx context.Context, image string, process string, args []string, instanceName string) error +} diff --git a/internal/pkg/runtime/launch/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go similarity index 94% rename from internal/pkg/runtime/launch/launcher_linux.go rename to internal/pkg/runtime/launcher/native/launcher_linux.go index 42fb49a1b4..e260f9de68 100644 --- a/internal/pkg/runtime/launch/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -7,7 +7,9 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package launch +// Package native implements a Launcher that will configure and launch a +// container with Apptainer's own (native) runtime. +package native import ( "context" @@ -24,6 +26,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cgroups" "github.com/apptainer/apptainer/internal/pkg/checkpoint/dmtcp" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/image/driver" "github.com/apptainer/apptainer/internal/pkg/image/unpacker" @@ -31,6 +34,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/plugin" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/internal/pkg/security" "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/internal/pkg/util/env" @@ -47,6 +51,7 @@ import ( "github.com/apptainer/apptainer/pkg/runtime/engine/config" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/bind" "github.com/apptainer/apptainer/pkg/util/capabilities" "github.com/apptainer/apptainer/pkg/util/cryptkey" "github.com/apptainer/apptainer/pkg/util/fs/proc" @@ -57,13 +62,31 @@ import ( "golang.org/x/sys/unix" ) -func NewLauncher(opts ...Option) (*Launcher, error) { - lo := launchOptions{} +// Launcher will holds configuration for, and will launch a container using +// Apptainer's own (native) runtime. +type Launcher struct { + uid uint32 + gid uint32 + cfg launcher.Options + engineConfig *apptainerConfig.EngineConfig + generator *generate.Generator +} + +// NewLauncher returns a native.Launcher with an initial configuration set by opts. +func NewLauncher(opts ...launcher.Option) (*Launcher, error) { + lo := launcher.Options{} for _, opt := range opts { if err := opt(&lo); err != nil { return nil, fmt.Errorf("%w", err) } } + if len(lo.Devices) > 0 { + return nil, fmt.Errorf("CDI device mappings unsupported in native launcher") + } + + if len(lo.CdiDirs) > 0 { + return nil, fmt.Errorf("CDI device mappings unsupported in native launcher") + } // Initialize empty default Apptainer Engine and OCI configuration engineConfig := apptainerConfig.NewConfig() @@ -93,7 +116,7 @@ func NewLauncher(opts ...Option) (*Launcher, error) { // This includes interactive containers, instances, and joining an existing instance. // //nolint:maintidx -func (l *Launcher) Exec(ctx context.Context, image string, args []string, instanceName string) error { +func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { var err error var fakerootPath string @@ -120,7 +143,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan if l.cfg.IgnoreFakerootCmd { err = errors.New("fakeroot command is ignored because of --ignore-fakeroot-command") } else { - fakerootPath, err = fakeroot.FindFake() + fakerootPath, err = fakefake.FindFake() } if err != nil { sylog.Infof("fakeroot command not found, using only root-mapped namespace") @@ -134,7 +157,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan if l.cfg.IgnoreUserns { err = errors.New("could not start root-mapped namespace because --ignore-userns is set") } else { - err = fakeroot.UnshareRootMapped(os.Args) + err = fakefake.UnshareRootMapped(os.Args) } if err == nil { // All good @@ -144,7 +167,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan if l.cfg.IgnoreFakerootCmd { err = errors.New("fakeroot command is ignored because of --ignore-fakeroot-command") } else { - fakerootPath, err = fakeroot.FindFake() + fakerootPath, err = fakefake.FindFake() } if err != nil { sylog.Fatalf("--fakeroot requires either being in %v, unprivileged user namespaces, or the fakeroot command", fakeroot.SubUIDFile) @@ -168,6 +191,9 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan } } + // Native runtime expects command to execute as arg[0] + args = append([]string{process}, args...) + // Set arguments to pass to contained process. l.generator.SetProcessArgs(args) @@ -275,7 +301,11 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan // If user wants to set a hostname, it requires the UTS namespace. if l.cfg.Hostname != "" { - l.cfg.Namespaces.UTS = true + // This is a sanity-check; actionPreRun in actions.go should have prevented this scenario from arising. + if !l.cfg.Namespaces.UTS { + return fmt.Errorf("internal error: trying to set hostname without UTS namespace") + } + l.engineConfig.SetHostname(l.cfg.Hostname) } @@ -289,7 +319,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan l.engineConfig.SetUseBuildConfig(l.cfg.UseBuildConfig) // When running as root, the user can optionally allow setuid with container. - err = withPrivilege(l.uid, l.cfg.AllowSUID, "--allow-setuid", func() error { + err = launcher.WithPrivilege(l.uid, l.cfg.AllowSUID, "--allow-setuid", func() error { l.engineConfig.SetAllowSUID(l.cfg.AllowSUID) return nil }) @@ -298,7 +328,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan } // When running as root, the user can optionally keep all privs in the container. - err = withPrivilege(l.uid, l.cfg.KeepPrivs, "--keep-privs", func() error { + err = launcher.WithPrivilege(l.uid, l.cfg.KeepPrivs, "--keep-privs", func() error { l.engineConfig.SetKeepPrivs(l.cfg.KeepPrivs) return nil }) @@ -327,7 +357,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan l.setCgroups(instanceName) // --boot flag requires privilege, so check for this. - err = withPrivilege(l.uid, l.cfg.Boot, "--boot", func() error { return nil }) + err = launcher.WithPrivilege(l.uid, l.cfg.Boot, "--boot", func() error { return nil }) if err != nil { sylog.Fatalf("Could not configure --boot: %s", err) } @@ -350,7 +380,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan l.engineConfig.SetInstance(true) l.engineConfig.SetBootInstance(l.cfg.Boot) - if useSuid && !l.cfg.Namespaces.User && hidepidProc() { + if useSuid && !l.cfg.Namespaces.User && launcher.HidepidProc() { return fmt.Errorf("hidepid option set on /proc mount, require 'hidepid=0' to start instance with setuid workflow") } @@ -362,7 +392,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan if l.cfg.Boot { l.cfg.Namespaces.UTS = true l.cfg.Namespaces.Net = true - if l.cfg.Hostname == "" { + if len(l.cfg.Hostname) < 1 { l.engineConfig.SetHostname(instanceName) } if !l.cfg.KeepPrivs { @@ -452,7 +482,7 @@ func (l *Launcher) setTargetIDs(useSuid bool) (err error) { } // If a target uid was requested, and we are root or non-suid, handle that. - err = withPrivilege(pseudoRoot, uidParam != "", "uid security feature with suid mode", func() error { + err = launcher.WithPrivilege(pseudoRoot, uidParam != "", "uid security feature with suid mode", func() error { u, err := strconv.ParseUint(uidParam, 10, 32) if err != nil { return fmt.Errorf("failed to parse provided UID: %w", err) @@ -468,7 +498,7 @@ func (l *Launcher) setTargetIDs(useSuid bool) (err error) { } // If any target gids were requested, and we are root or non-suid, handle that. - err = withPrivilege(pseudoRoot, gidParam != "", "gid security feature with suid mode", func() error { + err = launcher.WithPrivilege(pseudoRoot, gidParam != "", "gid security feature with suid mode", func() error { gids := strings.Split(gidParam, ":") for _, id := range gids { g, err := strconv.ParseUint(id, 10, 32) @@ -636,14 +666,14 @@ func (l *Launcher) useSuid(insideUserNs bool) (useSuid bool) { // setBinds sets engine configuration for requested bind mounts. func (l *Launcher) setBinds(fakerootPath string) error { // First get binds from -B/--bind and env var - binds, err := apptainerConfig.ParseBindPath(l.cfg.BindPaths) + binds, err := bind.ParseBindPath(l.cfg.BindPaths) if err != nil { return fmt.Errorf("while parsing bind path: %w", err) } // Now add binds from one or more --mount and env var. // Note that these do not get exported for nested containers for _, m := range l.cfg.Mounts { - bps, err := apptainerConfig.ParseMountString(m) + bps, err := bind.ParseMountString(m) if err != nil { return fmt.Errorf("while parsing mount %q: %w", m, err) } @@ -653,11 +683,11 @@ func (l *Launcher) setBinds(fakerootPath string) error { if fakerootPath != "" { l.engineConfig.SetFakerootPath(fakerootPath) // Add binds for fakeroot command - fakebindPaths, err := fakeroot.GetFakeBinds(fakerootPath) + fakebindPaths, err := fakefake.GetFakeBinds(fakerootPath) if err != nil { return fmt.Errorf("while getting fakeroot bindpoints: %w", err) } - fakebinds, err := apptainerConfig.ParseBindPath(fakebindPaths) + fakebinds, err := bind.ParseBindPath(fakebindPaths) if err != nil { return fmt.Errorf("while parsing fakeroot bind paths: %w", err) } @@ -763,8 +793,8 @@ func (l *Launcher) setHome() error { // Handle any user request to override the home directory source/dest homeSlice := strings.Split(l.cfg.HomeDir, ":") - if len(homeSlice) > 2 || len(homeSlice) == 0 { - return fmt.Errorf("home argument has incorrect number of elements: %v", len(homeSlice)) + if len(homeSlice) < 1 || len(homeSlice) > 2 { + return fmt.Errorf("home argument has incorrect number of elements: %v", homeSlice) } l.engineConfig.SetHomeSource(homeSlice[0]) if len(homeSlice) == 1 { @@ -966,7 +996,7 @@ func (l *Launcher) setEnvVars(ctx context.Context, args []string) error { content, err := os.ReadFile(l.cfg.EnvFile) if err != nil { - return fmt.Errorf("could not read %q environment file: %w", l.cfg.EnvFile, err) + return fmt.Errorf("could not read environment file %q: %w", l.cfg.EnvFile, err) } envvars, err := interpreter.EvaluateEnv(ctx, content, args, currentEnv) @@ -981,7 +1011,7 @@ func (l *Launcher) setEnvVars(ctx context.Context, args []string) error { for _, envar := range envvars { e := strings.SplitN(envar, "=", 2) if len(e) != 2 { - sylog.Warningf("Ignore environment variable %q: '=' is missing", envar) + sylog.Warningf("Ignored environment variable %q: '=' is missing", envar) continue } // Don't attempt to overwrite bash builtin readonly vars @@ -991,7 +1021,7 @@ func (l *Launcher) setEnvVars(ctx context.Context, args []string) error { } // Ensure we don't overwrite --env variables with environment file if _, ok := l.cfg.Env[e[0]]; ok { - sylog.Warningf("Ignore environment variable %s from %s: override from --env", e[0], l.cfg.EnvFile) + sylog.Warningf("Ignored environment variable %s from %s: override from --env", e[0], l.cfg.EnvFile) } else { l.cfg.Env[e[0]] = e[1] } @@ -1227,38 +1257,6 @@ func runPluginCallbacks(cfg *config.Common) error { return nil } -// withPrivilege calls fn if cond is satisfied, and we are uid 0 -func withPrivilege(uid uint32, cond bool, desc string, fn func() error) error { - if !cond { - return nil - } - if uid != 0 { - return fmt.Errorf("%s requires root privileges", desc) - } - return fn() -} - -// hidepidProc checks if hidepid is set on /proc mount point, when this -// option is an instance started with setuid workflow could not even be -// joined later or stopped correctly. -func hidepidProc() bool { - entries, err := proc.GetMountInfoEntry("/proc/self/mountinfo") - if err != nil { - sylog.Warningf("while reading /proc/self/mountinfo: %s", err) - return false - } - for _, e := range entries { - if e.Point == "/proc" { - for _, o := range e.SuperOptions { - if strings.HasPrefix(o, "hidepid=") { - return true - } - } - } - } - return false -} - // convertImage extracts the image found at filename to directory dir within a temporary directory // tempDir. If the unsquashfs binary is not located, the binary at unsquashfsPath is used. It is // the caller's responsibility to remove rootfsDir when no longer needed. diff --git a/internal/pkg/runtime/launcher/oci/README.md b/internal/pkg/runtime/launcher/oci/README.md new file mode 100644 index 0000000000..1c3275108d --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/README.md @@ -0,0 +1,141 @@ +# internal/pkg/runtime/launcher/oci + +This package contains routines that configure and launch a container in an OCI +bundle format, using a low-level OCI runtime, either `crun` or `runc` at this +time. `crun` is currently preferred. `runc` is used where `crun` is not +available. + +**Note** - at present, all functionality works with either `crun` or `runc`. +However, in future `crun` may be required for all functionality, as `runc` does +not support some limited ID mappings etc. that may be beneficial in an HPC +scenario. + +The package contrasts with `internal/pkg/runtime/launcher/native` which executes +Apptainer format containers (SIF/Sandbox/squashfs/ext3), using one of our own +runtime engines (`internal/pkg/runtime/engine/*`). + +There are two flows that are implemented here. + +* Basic OCI runtime operations agains an existing bundle, which will be executed + via the `apptainer oci` command group. These are not widely used by + end-users of apptainer. +* A `Launcher`, that implements an `Exec` function that will be called by + 'actions' (run/shell/exec) in `--oci` mode, and will: + * Prepare an OCI bundle according to `launcher.Options` passed through from + the CLI layer. + * Execute the bundle, interactively, via the OCI Run operation. + +**Note** - this area of code is under heavy development for experimental. +It is likely that it will be heavily refactored, and split, in future. + +## Basic OCI Operations + +The following files implement basic OCI operations on a runtime bundle: + +### `oci_linux.go` + +Defines constants, path resolution, and minimal bundle locking functions. + +### `oci_runc_linux.go` + +Holds implementations of the Run / Start / Exec / Kill / Delete / Pause / Resume +/ State OCI runtime operations. + +See + + +These functions are thin wrappers around the `runc`/`crun` operations of the +same name. + +### `oci_conmon_linux.go` + +Hold an implementation of the Create OCI runtime operation. This calls out to +`conmon`, which in turn calls `crun` or `runc`. + +`conmon` is used to manage logging and console streams for containers that are +started backgrounded, so we don't have to do that ourselves. + +### `oci_attach_linux.go` + +Implements an `Attach` function, which can attach the user's console to the +streams of a container running in the background, which is being monitored by +conmon. + +### Testing + +End-to-end flows of basic OCI operations on an existing bundle are tested in the +OCI group of the e2e suite, `e2e/oci`. + +## Launcher Flow + +The `Launcher` type connects the standard apptainer CLI actions +(run/shell/exec), to execution of an OCI container in a native bundle. Invoked +with the `--oci` flag, this is in contrast to running a Apptainer format +container, with Apptainer's own runtime engine. + +### `spec_linux.go` + +Provides a minimal OCI runtime spec, that will form the basis of container +execution that is roughly comparable to running a native apptainer container +with `--compat` (`--containall`). + +### `mounts_linux.go` + +Provides code handling the addition of required mounts to the OCI runtime spec. + +### `process_linux.go` + +Provides code handling configuration of container process execution, including +user mapping. + +### `launcher_linux.go` + +Implements `Launcher.Exec`, which is called from the CLI layer. It will: + +* Create a temporary bundle directory. +* Use `pkg/ocibundle/native` to retrieve the specified image, and extract it in + the temporary bundle. +* Configure the container by creating an appropriate runtime spec. +* Call the interactive OCI Run function to execute the container with `crun` or + `runc`. + +### Namespace Considerations + +An OCI container started via `Launch.Exec` as a non-root user always uses at +least one user namespace. + +The user namespace is created *prior to* calling `runc` or `crun`, so we'll call +it an *outer* user namespace. + +Creation of this outer user namespace is via using the `RunNS` function, instead +of `Run`. The `RunNS` function executes the Apptainer `starter` binary, with a +minimal configuration of the fakeroot engine ( +`internal/pkg/runtime/engine/fakeroot/config`). + +The `starter` will create a user namespace and ID mapping, and will then execute +`apptainer oci run` to perform the basic OCI Run operation against the bundle +that the `Launcher.Exec` function has prepared. + +The outer user namespace from which `runc` or `crun` is called *always* maps the +host user id to root inside the userns. + +When a container is run in `--fakeroot` mode, the outer user namespace is the +only user namespace. The OCI runtime config does not request any additional +userns or ID mapping be performed by `crun` / `runc`. + +When a container is **not** run in `--fakeroot` mode, the OCI runtime config for +the bundle requests that `crun` / `runc`: + +* Create another, inner, user namespace for the container. +* Apply an ID mapping which reverses the 'fakeroot' outer ID mapping. + +I.E. when a container runs without `--fakeroot`, the ID mapping is: + +* User ID on host (1001) +* Root in outer user namespace (0) +* User ID in container (1001) + +### Testing + +End-to-end testing of the launcher flow is via the `e2e/actions` suite. Tests +prefixed `oci`. diff --git a/internal/pkg/runtime/launcher/oci/cdi_linux.go b/internal/pkg/runtime/launcher/oci/cdi_linux.go new file mode 100644 index 0000000000..436edb715f --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/cdi_linux.go @@ -0,0 +1,60 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "fmt" + "sync" + + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// A container to hold the CDI registry, plus a sync.Once object to ensure we only have to ask for it once +var regSyncContainer struct { + reg cdi.Registry + initOnce sync.Once + err error +} + +// addCDIDevices adds an array of CDI devices to an existing spec. +// Accepts optional, variable number of cdi.Option arguments (to which cdi.WithAutoRefresh(false) will be prepended). Note that due to the use of a sync.Once initialization strategy, these options will only have an effect if this is the first call made to addCDIDevices(). +func addCDIDevices(spec *specs.Spec, cdiDevices []string, cdiRegOptions ...cdi.Option) error { + regSyncContainer.initOnce.Do(func() { + // Get the CDI registry, passing a cdi.WithAutoRefresh(false) option so that CDI registry files are not scanned asynchronously. (We are about to call a manual refresh, below.) + realCDIOptions := append([]cdi.Option{cdi.WithAutoRefresh(false)}, cdiRegOptions...) + regSyncContainer.reg = cdi.GetRegistry(realCDIOptions...) + regSyncContainer.err = regSyncContainer.reg.Refresh() + }) + + if regSyncContainer.err != nil { + return fmt.Errorf("Error encountered refreshing the CDI registry during initialization: %v", regSyncContainer.err) + } + + for _, cdiDevice := range cdiDevices { + if !isCDIDevice(cdiDevice) { + return fmt.Errorf("string %#v does not represent a valid CDI device", cdiDevice) + } + } + + if _, err := regSyncContainer.reg.InjectDevices(spec, cdiDevices...); err != nil { + return fmt.Errorf("Error encountered setting up CDI devices: %w", err) + } + + return nil +} + +// isCDIDevice checks whether a string is a valid CDI device selector. +func isCDIDevice(str string) bool { + return cdi.IsQualifiedName(str) +} diff --git a/internal/pkg/runtime/launcher/oci/cdi_linux_test.go b/internal/pkg/runtime/launcher/oci/cdi_linux_test.go new file mode 100644 index 0000000000..39858c8023 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/cdi_linux_test.go @@ -0,0 +1,254 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/apptainer/apptainer/pkg/util/slice" + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/opencontainers/runtime-spec/specs-go" +) + +var specDirs = []string{filepath.Join("..", "..", "..", "..", "..", "test", "cdi")} + +type mountsList []specs.Mount + +func (a mountsList) Len() int { return len(a) } +func (a mountsList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a mountsList) Less(i, j int) bool { return a[i].Destination < a[j].Destination } + +func Test_addCDIDevice(t *testing.T) { + var wantUID uint32 = 1000 + var wantGID uint32 = 1000 + tests := []struct { + name string + devices []string + wantDevices []specs.LinuxDevice + wantMounts mountsList + wantErr bool + wantEnv []string + }{ + { + name: "ValidOneDeviceKmsg", + devices: []string{ + "apptainertesting.sylabs.io/device=kmsgDevice", + }, + wantDevices: []specs.LinuxDevice{ + { + Path: "/dev/kmsg", + Type: "c", + Major: 1, + Minor: 11, + FileMode: nil, + UID: &wantUID, + GID: &wantGID, + }, + }, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmountforkmsg", + Type: "", + Options: []string{"rw"}, + }, + }, + wantEnv: []string{ + "FOO=VALID_SPEC", + "BAR=BARVALUE1", + }, + }, + { + name: "ValidTmpDevices", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice17", + "apptainertesting.sylabs.io/device=tmpmountDevice1", + }, + wantDevices: []specs.LinuxDevice{}, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmount1", + Type: "", + Options: []string{"ro"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount3", + Type: "", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount13", + Type: "", + Options: []string{"rw"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount17", + Type: "", + Options: []string{"r"}, + }, + }, + wantEnv: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + "FOO=VALID_SPEC", + "BAR=BARVALUE1", + }, + }, + { + name: "ValidTmpDevicesFromOneJSON", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice1", + }, + wantDevices: []specs.LinuxDevice{}, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmount1", + Type: "", + Options: []string{"ro"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount3", + Type: "", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount13", + Type: "", + Options: []string{"rw"}, + }, + }, + wantEnv: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + }, + }, + { + name: "ValidMixedDevices", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice17", + "apptainertesting.sylabs.io/device=kmsgDevice", + "apptainertesting.sylabs.io/device=tmpmountDevice1", + }, + wantDevices: []specs.LinuxDevice{ + { + Path: "/dev/kmsg", + Type: "c", + Major: 1, + Minor: 11, + FileMode: nil, + UID: &wantUID, + GID: &wantGID, + }, + }, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmount1", + Type: "", + Options: []string{"ro"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount3", + Type: "", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount13", + Type: "", + Options: []string{"rw"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount17", + Type: "", + Options: []string{"r"}, + }, + { + Source: "/tmp", + Destination: "/tmpmountforkmsg", + Type: "", + Options: []string{"rw"}, + }, + }, + wantEnv: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + "FOO=VALID_SPEC", + "BAR=BARVALUE1", + }, + }, + { + name: "InvalidNameOneDevice", + devices: []string{ + "apptainertesting.sylabs.io/device=noSuchDevice", + }, + wantErr: true, + }, + { + name: "InvalidNameSeveralDevices", + devices: []string{ + "apptainertesting.sylabs.io/device=noSuchDevice", + "apptainertesting.sylabs.io/device=noSuchDeviceEither", + }, + wantErr: true, + }, + { + name: "InvalidNameAmongValids", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice17", + "apptainertesting.sylabs.io/device=noSuchDevice", + "apptainertesting.sylabs.io/device=tmpmountDevice1", + "apptainertesting.sylabs.io/device=kmsgDevice", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := minimalSpec() + err := addCDIDevices(&spec, tt.devices, cdi.WithSpecDirs(specDirs...)) + if (err != nil) != tt.wantErr { + t.Errorf("addCDIDevices() mismatched error values; expected %v, got %v.", tt.wantErr, err) + } + + // We need this if-statement because the comparison below is done with reflection, and so a nil array and a non-nil but zero-length array will be considered different (which is not what we want here) + if (len(tt.wantMounts) > 0) || (len(spec.Mounts) > 0) { + // Note that the current implementation of OCI/CDI sorts the mounts generated by the set of mapped devices, therefore we compare against a sorted list. + sort.Sort(tt.wantMounts) + if !reflect.DeepEqual(mountsList(spec.Mounts), tt.wantMounts) { + t.Errorf("addCDIDevices() mismatched mounts; expected %v, got %v.", tt.wantMounts, spec.Mounts) + } + } + + envMissing := slice.Subtract(tt.wantEnv, spec.Process.Env) + if len(envMissing) > 0 { + t.Errorf("addCDIDevices() mismatched environment variables; expected, but did not find, the following environment variables: %v", envMissing) + } + }) + } +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go new file mode 100644 index 0000000000..5b2ebd23e9 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -0,0 +1,541 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/cgroups" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/internal/pkg/util/fs/files" + "github.com/apptainer/apptainer/pkg/ocibundle" + "github.com/apptainer/apptainer/pkg/ocibundle/native" + "github.com/apptainer/apptainer/pkg/ocibundle/tools" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/google/uuid" + lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runtime-spec/specs-go" +) + +var ( + ErrUnsupportedOption = errors.New("not supported by OCI launcher") + ErrNotImplemented = errors.New("not implemented by OCI launcher") +) + +// Launcher will holds configuration for, and will launch a container using an +// OCI runtime. +type Launcher struct { + cfg launcher.Options + apptainerConf *apptainerconf.File +} + +// NewLauncher returns a oci.Launcher with an initial configuration set by opts. +func NewLauncher(opts ...launcher.Option) (*Launcher, error) { + lo := launcher.Options{} + for _, opt := range opts { + if err := opt(&lo); err != nil { + return nil, fmt.Errorf("%w", err) + } + } + + if err := checkOpts(lo); err != nil { + return nil, err + } + + c := apptainerconf.GetCurrentConfig() + if c == nil { + return nil, fmt.Errorf("apptainer configuration is not initialized") + } + + return &Launcher{cfg: lo, apptainerConf: c}, nil +} + +// checkOpts ensures that options set are supported by the oci.Launcher. +// +// nolint:maintidx +func checkOpts(lo launcher.Options) error { + badOpt := []string{} + + if lo.Writable { + badOpt = append(badOpt, "Writable") + } + if lo.WritableTmpfs { + sylog.Infof("--oci mode uses --writable-tmpfs by default") + } + if lo.NoHome { + badOpt = append(badOpt, "NoHome") + } + + if len(lo.FuseMount) > 0 { + badOpt = append(badOpt, "FuseMount") + } + + if len(lo.NoMount) > 0 { + badOpt = append(badOpt, "NoMount") + } + + if lo.NvCCLI { + badOpt = append(badOpt, "NvCCLI") + } + + if len(lo.ContainLibs) > 0 { + badOpt = append(badOpt, "ContainLibs") + } + + if lo.CleanEnv { + badOpt = append(badOpt, "CleanEnv") + } + if lo.NoEval { + badOpt = append(badOpt, "NoEval") + } + + // Network always set in CLI layer even if network namespace not requested. + // We only support isolation at present + if lo.Namespaces.Net && lo.Network != "none" { + badOpt = append(badOpt, "Network (except none)") + } + + if len(lo.NetworkArgs) > 0 { + badOpt = append(badOpt, "NetworkArgs") + } + + if lo.AddCaps != "" { + badOpt = append(badOpt, "AddCaps") + } + if lo.DropCaps != "" { + badOpt = append(badOpt, "DropCaps") + } + if lo.AllowSUID { + badOpt = append(badOpt, "AllowSUID") + } + if lo.KeepPrivs { + badOpt = append(badOpt, "KeepPrivs") + } + if lo.NoPrivs { + badOpt = append(badOpt, "NoPrivs") + } + if len(lo.SecurityOpts) > 0 { + badOpt = append(badOpt, "SecurityOpts") + } + if lo.NoUmask { + badOpt = append(badOpt, "NoUmask") + } + + // ConfigFile always set by CLI. We should support only the default from build time. + if lo.ConfigFile != "" && lo.ConfigFile != buildcfg.APPTAINER_CONF_FILE { + badOpt = append(badOpt, "ConfigFile") + } + + if lo.ShellPath != "" { + badOpt = append(badOpt, "ShellPath") + } + + if lo.Boot { + badOpt = append(badOpt, "Boot") + } + if lo.NoInit { + badOpt = append(badOpt, "NoInit") + } + if lo.Contain { + badOpt = append(badOpt, "Contain") + } + if lo.ContainAll { + badOpt = append(badOpt, "ContainAll") + } + + if lo.AppName != "" { + badOpt = append(badOpt, "AppName") + } + + if lo.KeyInfo != nil { + badOpt = append(badOpt, "KeyInfo") + } + + if lo.SIFFUSE { + badOpt = append(badOpt, "SIFFUSE") + } + + if len(badOpt) > 0 { + return fmt.Errorf("%w: %s", ErrUnsupportedOption, strings.Join(badOpt, ",")) + } + + return nil +} + +// createSpec creates an initial OCI runtime specification, suitable to launch a +// container. This spec excludes the Process config, as this has to be computed +// where the image config is available, to account for the image's CMD / +// ENTRYPOINT / ENV / USER. See finalizeSpec() function. +func (l *Launcher) createSpec() (*specs.Spec, error) { + spec := minimalSpec() + + spec = addNamespaces(spec, l.cfg.Namespaces) + + if len(l.cfg.Hostname) > 0 { + // This is a sanity-check; actionPreRun in actions.go should have prevented this scenario from arising. + if !l.cfg.Namespaces.UTS { + return nil, fmt.Errorf("internal error: trying to set hostname without UTS namespace") + } + + spec.Hostname = l.cfg.Hostname + } + + mounts, err := l.getMounts() + if err != nil { + return nil, err + } + spec.Mounts = mounts + + cgPath, resources, err := l.getCgroup() + if err != nil { + return nil, err + } + if cgPath != "" { + spec.Linux.CgroupsPath = cgPath + spec.Linux.Resources = resources + } + + return &spec, nil +} + +// finalizeSpec updates the bundle config, filling in Process config that depends on the image spec. +func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *specs.Spec, image string, process string, args []string) (err error) { + imgSpec := b.ImageSpec() + if imgSpec == nil { + return fmt.Errorf("bundle has no image spec") + } + + // In the absence of a USER in the OCI image config, we will run the + // container process as our current user / group. + currentUID := uint32(os.Getuid()) + currentGID := uint32(os.Getgid()) + targetUID := currentUID + targetGID := currentGID + containerUser := false + + // If the OCI image config specifies a USER we will: + // * When unprivileged - run as that user, via nested subuid/gid mappings (host user -> userns root -> OCI USER) + // * When privileged - directly run as that user, as a host uid/gid. + if imgSpec.Config.User != "" { + imgUser, err := tools.BundleUser(b.Path(), imgSpec.Config.User) + if err != nil { + return err + } + imgUID, err := strconv.ParseUint(imgUser.Uid, 10, 32) + if err != nil { + return err + } + imgGID, err := strconv.ParseUint(imgUser.Gid, 10, 32) + if err != nil { + return err + } + targetUID = uint32(imgUID) + targetGID = uint32(imgGID) + containerUser = true + sylog.Debugf("Running as USER specified in OCI image config %d:%d", targetUID, targetGID) + } + + // Fakeroot always overrides to give us root in the container (via userns & idmap if unprivileged). + if l.cfg.Fakeroot { + targetUID = 0 + targetGID = 0 + } + + if targetUID != 0 && currentUID != 0 { + uidMap, gidMap, err := getReverseUserMaps(currentUID, targetUID, targetGID) + if err != nil { + return err + } + spec.Linux.UIDMappings = uidMap + spec.Linux.GIDMappings = gidMap + // Must add userns to the runc/crun applied config for the inner reverse uid/gid mapping to work. + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UserNamespace}, + ) + } + + u := specs.User{ + UID: targetUID, + GID: targetGID, + } + + specProcess, err := l.getProcess(ctx, *imgSpec, image, b.Path(), process, args, u) + if err != nil { + return err + } + spec.Process = specProcess + + if len(l.cfg.CdiDirs) > 0 { + err = addCDIDevices(spec, l.cfg.Devices, cdi.WithSpecDirs(l.cfg.CdiDirs...)) + } else { + err = addCDIDevices(spec, l.cfg.Devices) + } + if err != nil { + return err + } + + if err := b.Update(ctx, spec); err != nil { + return err + } + + // Prepare DNS settings for the container. + if err := l.prepareResolvConf(tools.RootFs(b.Path()).Path()); err != nil { + return err + } + + // If we are entering as root, or a USER defined in the container, then passwd/group + // information should be present already. + if targetUID == 0 || containerUser { + return nil + } + // Otherwise, add to the passwd and group files in the container. + if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path(), targetUID, targetGID); err != nil { + return err + } + + return nil +} + +func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { + if os.Getuid() == 0 || l.cfg.Fakeroot { + return nil + } + + containerPasswd := filepath.Join(rootfs, "etc", "passwd") + containerGroup := filepath.Join(rootfs, "etc", "group") + + if l.apptainerConf.ConfigPasswd { + sylog.Debugf("Updating passwd file: %s", containerPasswd) + content, err := files.Passwd(containerPasswd, l.cfg.HomeDir, int(uid), nil) + if err != nil { + sylog.Warningf("%s", err) + } else if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { + return fmt.Errorf("while writing passwd file: %w", err) + } + } else { + sylog.Debugf("Skipping update of %s due to apptainer.conf", containerPasswd) + } + + if l.apptainerConf.ConfigGroup { + sylog.Debugf("Updating group file: %s", containerGroup) + content, err := files.Group(containerGroup, int(uid), []int{int(gid)}, nil) + if err != nil { + sylog.Warningf("%s", err) + } else if err := os.WriteFile(containerGroup, content, 0o755); err != nil { + return fmt.Errorf("while writing passwd file: %w", err) + } + } else { + sylog.Debugf("Skipping update of %s due to apptainer.conf", containerGroup) + } + + return nil +} + +func (l *Launcher) prepareResolvConf(rootfs string) error { + hostResolvConfPath := "/etc/resolv.conf" + containerEtc := filepath.Join(rootfs, "etc") + containerResolvConfPath := filepath.Join(rootfs, "etc", "resolv.conf") + + if !l.apptainerConf.ConfigResolvConf { + sylog.Debugf("Skipping update of %s due to apptainer.conf", containerResolvConfPath) + return nil + } + + var resolvConfData []byte + var err error + if len(l.cfg.DNS) > 0 { + dns := strings.Replace(l.cfg.DNS, " ", "", -1) + ips := strings.Split(dns, ",") + for _, ip := range ips { + if net.ParseIP(ip) == nil { + return fmt.Errorf("DNS nameserver %v is not a valid IP address", ip) + } + line := fmt.Sprintf("nameserver %s\n", ip) + resolvConfData = append(resolvConfData, line...) + } + } else { + resolvConfData, err = os.ReadFile(hostResolvConfPath) + if err != nil { + return fmt.Errorf("could not read host's resolv.conf file: %w", err) + } + } + + stat, err := os.Stat(containerEtc) + if os.IsNotExist(err) || !stat.IsDir() { + sylog.Warningf("container does not contain an /etc directory; skipping resolv.conf configuration") + return nil + } + + if err := os.WriteFile(containerResolvConfPath, resolvConfData, 0o755); err != nil { + return fmt.Errorf("while writing container's resolv.conf file: %v", err) + } + + return nil +} + +// Exec will interactively execute a container via the runc low-level runtime. +// image is a reference to an OCI image, e.g. docker://ubuntu or oci:/tmp/mycontainer +func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { + if instanceName != "" { + return fmt.Errorf("%w: instanceName", ErrNotImplemented) + } + + if l.cfg.SysContext == nil { + return fmt.Errorf("launcher SysContext must be set for OCI image handling") + } + + // If we need to, enter a new cgroup to workaround an issue with crun container cgroup creation (#1538). + if err := l.crunNestCgroup(); err != nil { + return fmt.Errorf("while applying crun cgroup workaround: %w", err) + } + + bundleDir, err := os.MkdirTemp("", "oci-bundle") + if err != nil { + return nil + } + defer func() { + sylog.Debugf("Removing OCI bundle at: %s", bundleDir) + if err := fs.ForceRemoveAll(bundleDir); err != nil { + sylog.Errorf("Couldn't remove OCI bundle %s: %v", bundleDir, err) + } + }() + + sylog.Debugf("Creating OCI bundle at: %s", bundleDir) + + var imgCache *cache.Handle + if !l.cfg.CacheDisabled { + imgCache, err = cache.New(cache.Config{ + ParentDir: os.Getenv(cache.DirEnv), + }) + if err != nil { + return err + } + } + + // Create OCI runtime spec, excluding the Process settings which must consider the image spec. + spec, err := l.createSpec() + if err != nil { + return fmt.Errorf("while creating OCI spec: %w", err) + } + + // Create a bundle - obtain and extract the image. + b, err := native.New( + native.OptBundlePath(bundleDir), + native.OptImageRef(image), + native.OptSysCtx(l.cfg.SysContext), + native.OptImgCache(imgCache), + ) + if err != nil { + return err + } + if err := b.Create(ctx, spec); err != nil { + return err + } + + // With reference to the bundle's image spec, now set the process configuration. + if err := l.finalizeSpec(ctx, b, spec, image, process, args); err != nil { + return err + } + + id, err := uuid.NewRandom() + if err != nil { + return fmt.Errorf("while generating container id: %w", err) + } + + if os.Getuid() == 0 { + // Execution of runc/crun run, wrapped with prep / cleanup. + err = RunWrapped(ctx, id.String(), b.Path(), "", l.cfg.OverlayPaths, l.apptainerConf.SystemdCgroups) + } else { + // Reexec apptainer oci run in a userns with mappings. + // Note - the oci run command will pull out the SystemdCgroups setting from config. + err = RunWrappedNS(ctx, id.String(), b.Path(), l.cfg.OverlayPaths) + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + return err +} + +// getCgroup will return a cgroup path and resources for the runtime to create. +func (l *Launcher) getCgroup() (path string, resources *specs.LinuxResources, err error) { + if l.cfg.CGroupsJSON == "" { + return "", nil, nil + } + path = cgroups.DefaultPathForPid(l.apptainerConf.SystemdCgroups, -1) + resources, err = cgroups.UnmarshalJSONResources(l.cfg.CGroupsJSON) + if err != nil { + return "", nil, err + } + return path, resources, nil +} + +// crunNestCgroup will check whether we are using crun, and enter a cgroup if +// running as a non-root user under cgroups v2, with systemd. This is required +// to satisfy a common user-owned ancestor cgroup requirement on e.g. bare ssh +// logins. See: https://github.com/sylabs/singularity/issues/1538 +func (l *Launcher) crunNestCgroup() error { + r, err := runtime() + if err != nil { + return err + } + + // No workaround required for runc. + if filepath.Base(r) == "runc" { + return nil + } + + // No workaround required if we are run as root. + if os.Getuid() == 0 { + return nil + } + + // We can only create a new cgroup under cgroups v2 with systemd as manager. + // Generally we won't hit the issue that needs a workaround under cgroups v1, so no-op instead of a warning here. + if !(lccgroups.IsCgroup2UnifiedMode() && l.apptainerConf.SystemdCgroups) { + return nil + } + + // We are running crun as a user. Enter a cgroup now. + pid := os.Getpid() + sylog.Debugf("crun workaround - adding process %d to sibling cgroup", pid) + manager, err := cgroups.NewManagerWithSpec(&specs.LinuxResources{}, pid, "", l.apptainerConf.SystemdCgroups) + if err != nil { + return fmt.Errorf("couldn't create cgroup manager: %w", err) + } + cgPath, _ := manager.GetCgroupRelPath() + sylog.Debugf("In sibling cgroup: %s", cgPath) + + return nil +} + +func mergeMap(a map[string]string, b map[string]string) map[string]string { + for k, v := range b { + a[k] = v + } + return a +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go new file mode 100644 index 0000000000..7549b1be6e --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go @@ -0,0 +1,70 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "reflect" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/test" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" +) + +func TestNewLauncher(t *testing.T) { + test.DropPrivilege(t) + defer test.ResetPrivilege(t) + + sc, err := apptainerconf.GetConfig(nil) + if err != nil { + t.Fatalf("while initializing apptainerconf: %s", err) + } + apptainerconf.SetCurrentConfig(sc) + + tests := []struct { + name string + opts []launcher.Option + want *Launcher + wantErr bool + }{ + { + name: "default", + want: &Launcher{apptainerConf: sc}, + wantErr: false, + }, + { + name: "validOption", + opts: []launcher.Option{ + launcher.OptHome("/home/test", false, false), + }, + want: &Launcher{cfg: launcher.Options{HomeDir: "/home/test"}, apptainerConf: sc}, + }, + { + name: "unsupportedOption", + opts: []launcher.Option{ + launcher.OptSecurity([]string{"seccomp:example.json"}), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewLauncher(tt.opts...) + if (err != nil) != tt.wantErr { + t.Errorf("NewLauncher() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewLauncher() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go new file mode 100644 index 0000000000..3ecb969cc0 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -0,0 +1,605 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/internal/pkg/util/gpu" + "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/bind" + "github.com/opencontainers/runtime-spec/specs-go" +) + +const containerLibDir = "/.singularity.d/libs" + +// getMounts returns a mount list for the container's OCI runtime spec. +func (l *Launcher) getMounts() ([]specs.Mount, error) { + mounts := &[]specs.Mount{} + l.addProcMount(mounts) + l.addSysMount(mounts) + if err := l.addDevMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring devpts mount: %w", err) + } + if err := l.addTmpMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring tmp mounts: %w", err) + } + if err := l.addHomeMount(mounts); err != nil { + return nil, fmt.Errorf("while configuring home mount: %w", err) + } + if err := l.addScratchMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring scratch mount(s): %w", err) + } + if err := l.addBindMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring bind mount(s): %w", err) + } + if (l.cfg.Rocm || l.apptainerConf.AlwaysUseRocm) && !l.cfg.NoRocm { + if err := l.addRocmMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring ROCm mount(s): %w", err) + } + } + if (l.cfg.Nvidia || l.apptainerConf.AlwaysUseNv) && !l.cfg.NoNvidia { + if err := l.addNvidiaMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring Nvidia mount(s): %w", err) + } + } + + return *mounts, nil +} + +// addTmpMounts adds tmpfs mounts for /tmp and /var/tmp in the container. +func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) error { + const ( + tmpDest = "/tmp" + vartmpDest = "/var/tmp" + ) + + if !l.apptainerConf.MountTmp { + sylog.Debugf("Skipping mount of /tmp due to apptainer.conf") + return nil + } + + if len(l.cfg.WorkDir) > 0 { + sylog.Debugf("WorkDir specification provided: %s", l.cfg.WorkDir) + const ( + tmpSrcSubdir = "tmp" + vartmpSrcSubdir = "var_tmp" + ) + + workdir, err := filepath.Abs(filepath.Clean(l.cfg.WorkDir)) + if err != nil { + return fmt.Errorf("can't determine absolute path of workdir %s: %s", workdir, err) + } + + tmpSrc := filepath.Join(workdir, tmpSrcSubdir) + vartmpSrc := filepath.Join(workdir, vartmpSrcSubdir) + + if err := fs.Mkdir(tmpSrc, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", tmpSrc, err) + } + if err := fs.Mkdir(vartmpSrc, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", vartmpSrc, err) + } + + *mounts = append(*mounts, + + specs.Mount{ + Destination: tmpDest, + Type: "none", + Source: tmpSrc, + Options: []string{ + "rbind", + "nosuid", + "relatime", + "mode=777", + }, + }, + specs.Mount{ + Destination: vartmpDest, + Type: "none", + Source: vartmpSrc, + Options: []string{ + "rbind", + "nosuid", + "relatime", + "mode=777", + }, + }, + ) + + return nil + } + + sylog.Debugf(("No workdir specification provided. Proceeding with tmpfs mounts for /tmp and /var/tmp")) + *mounts = append(*mounts, + + specs.Mount{ + Destination: tmpDest, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "mode=777", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + specs.Mount{ + Destination: vartmpDest, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "mode=777", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + ) + + return nil +} + +// addDevMounts adds mounts to assemble a minimal /dev in the container. +func (l *Launcher) addDevMounts(mounts *[]specs.Mount) error { + ptsMount := specs.Mount{ + Destination: "/dev/pts", + Type: "devpts", + Source: "devpts", + Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}, + } + + if os.Getuid() == 0 { + group, err := user.GetGrNam("tty") + if err != nil { + return fmt.Errorf("while identifying tty gid: %w", err) + } + ptsMount.Options = append(ptsMount.Options, fmt.Sprintf("gid=%d", group.GID)) + } + + *mounts = append(*mounts, + specs.Mount{ + Destination: "/dev", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "strictatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + ptsMount, + specs.Mount{ + Destination: "/dev/shm", + Type: "tmpfs", + Source: "shm", + Options: []string{ + "nosuid", + "noexec", + "nodev", + "mode=1777", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + specs.Mount{ + Destination: "/dev/mqueue", + Type: "mqueue", + Source: "mqueue", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + ) + + return nil +} + +// addProcMount adds the /proc tree in the container. +func (l *Launcher) addProcMount(mounts *[]specs.Mount) { + if !l.apptainerConf.MountProc { + sylog.Debugf("Skipping mount of /proc due to apptainer.conf") + return + } + + *mounts = append(*mounts, + specs.Mount{ + Source: "proc", + Destination: "/proc", + Type: "proc", + }) +} + +// addSysMount adds the /sys tree in the container. +func (l *Launcher) addSysMount(mounts *[]specs.Mount) { + if !l.apptainerConf.MountSys { + sylog.Debugf("Skipping mount of /sys due to apptainer.conf") + return + } + + if os.Getuid() == 0 { + *mounts = append(*mounts, + specs.Mount{ + Source: "sysfs", + Destination: "/sys", + Type: "sysfs", + Options: []string{"nosuid", "noexec", "nodev", "ro"}, + }) + } else { + *mounts = append(*mounts, + specs.Mount{ + Source: "/sys", + Destination: "/sys", + Type: "none", + Options: []string{"rbind", "nosuid", "noexec", "nodev", "ro"}, + }) + } +} + +// addHomeMount adds a user home directory as a tmpfs mount. We are currently +// emulating `--compat` / `--containall`, so the user must specifically bind in +// their home directory from the host for it to be available. +func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { + if !l.apptainerConf.MountHome { + sylog.Debugf("Skipping mount of $HOME due to apptainer.conf") + return nil + } + + // Get the host user's data + pw, err := user.CurrentOriginal() + if err != nil { + return err + } + + if l.cfg.CustomHome { + // Handle any user request to override the home directory source/dest + homeSlice := strings.Split(l.cfg.HomeDir, ":") + if len(homeSlice) < 1 || len(homeSlice) > 2 { + return fmt.Errorf("home argument has incorrect number of elements: %v", homeSlice) + } + homeSrc := homeSlice[0] + l.cfg.HomeDir = homeSrc + + // User requested more than just a custom home dir; they want to bind-mount a directory in the host to this custom home dir. + // This means the home dir, as viewed from inside the container, is actually the second member of the ":"-separated slice. The first member is actually just the source portion of the requested bind-mount. + if len(homeSlice) > 1 { + homeDest := homeSlice[1] + l.cfg.HomeDir = homeDest + + // Since the home dir is a bind-mount in this case, we don't have to mount a tmpfs directory for the in-container home dir, and we can just do the bind-mount & return. + return addBindMount(mounts, bind.Path{ + Source: homeSrc, + Destination: homeDest, + }) + } + } else { + // If we're running in fake-root mode (and we haven't requested a custom home dir), we do need to create a tmpfs mount for the home dir, but it's a special case (because of its location & permissions), so we handle that here & return. + if l.cfg.Fakeroot { + l.cfg.HomeDir = "/root" + *mounts = append(*mounts, + specs.Mount{ + Destination: "/root", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }) + + return nil + } + + // No fakeroot and no custom home dir - so the in-container home dir will be named the same as the host user's home dir. (Though note that it'll still be a tmpfs mount! It'll get mounted by the catch-all mount append, below.) + l.cfg.HomeDir = pw.Dir + } + + // If we've not hit a special case (bind-mounted custom home dir, or fakeroot), then create a tmpfs mount as a home dir in the requested location (whether it's custom or not; by this point, l.cfg.HomeDir will reflect the right value). + *mounts = append(*mounts, + specs.Mount{ + Destination: l.cfg.HomeDir, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + fmt.Sprintf("uid=%d", pw.UID), + fmt.Sprintf("gid=%d", pw.GID), + }, + }) + + return nil +} + +// addScratchMounts adds tmpfs mounts for scratch directories in the container. +func (l *Launcher) addScratchMounts(mounts *[]specs.Mount) error { + const scratchContainerDirName = "/scratch" + + if len(l.cfg.WorkDir) > 0 { + workdir, err := filepath.Abs(filepath.Clean(l.cfg.WorkDir)) + if err != nil { + return fmt.Errorf("can't determine absolute path of workdir %s: %s", workdir, err) + } + scratchContainerDirPath := filepath.Join(workdir, scratchContainerDirName) + if err := fs.Mkdir(scratchContainerDirPath, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", scratchContainerDirPath, err) + } + + for _, s := range l.cfg.ScratchDirs { + scratchDirPath := filepath.Join(scratchContainerDirPath, s) + if err := fs.Mkdir(scratchDirPath, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", scratchDirPath, err) + } + + *mounts = append(*mounts, + specs.Mount{ + Destination: s, + Type: "", + Source: scratchDirPath, + Options: []string{ + "rbind", + "nosuid", + "relatime", + "nodev", + }, + }, + ) + } + } else { + for _, s := range l.cfg.ScratchDirs { + *mounts = append(*mounts, + specs.Mount{ + Destination: s, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "nodev", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + ) + } + } + + return nil +} + +func (l *Launcher) addBindMounts(mounts *[]specs.Mount) error { + // First get binds from -B/--bind and env var + binds, err := bind.ParseBindPath(l.cfg.BindPaths) + if err != nil { + return fmt.Errorf("while parsing bind path: %w", err) + } + // Now add binds from one or more --mount and env var. + for _, m := range l.cfg.Mounts { + bps, err := bind.ParseMountString(m) + if err != nil { + return fmt.Errorf("while parsing mount %q: %w", m, err) + } + binds = append(binds, bps...) + } + + for _, b := range binds { + if !l.apptainerConf.UserBindControl { + sylog.Warningf("Ignoring bind mount request: user bind control disabled by system administrator") + return nil + } + if err := addBindMount(mounts, b); err != nil { + return fmt.Errorf("while adding mount %q: %w", b.Source, err) + } + } + return nil +} + +func addBindMount(mounts *[]specs.Mount, b bind.Path) error { + if b.ID() != "" || b.ImageSrc() != "" { + return fmt.Errorf("image binds are not yet supported by the OCI runtime") + } + + opts := []string{"rbind", "nosuid", "nodev"} + if b.Readonly() { + opts = append(opts, "ro") + } + + absSource, err := filepath.Abs(b.Source) + if err != nil { + return fmt.Errorf("cannot determine absolute path of %s: %w", b.Source, err) + } + if _, err := os.Stat(absSource); err != nil { + return fmt.Errorf("cannot stat bind source %s: %w", b.Source, err) + } + + if !filepath.IsAbs(b.Destination) { + return fmt.Errorf("bind destination %s must be an absolute path", b.Destination) + } + + sylog.Debugf("Adding bind of %s to %s, with options %v", absSource, b.Destination, opts) + + *mounts = append(*mounts, + specs.Mount{ + Source: absSource, + Destination: b.Destination, + Type: "none", + Options: opts, + }) + return nil +} + +func addDevBindMount(mounts *[]specs.Mount, b bind.Path) error { + opts := []string{"bind", "nosuid"} + if b.Readonly() { + opts = append(opts, "ro") + } + + b.Source = filepath.Clean(b.Source) + if !strings.HasPrefix(b.Source, "/dev") { + return fmt.Errorf("device bind source must be an absolute path under /dev: %s", b.Source) + } + if b.Source != b.Destination { + return fmt.Errorf("device bind source %s must be the same as destination %s", b.Source, b.Destination) + } + if _, err := os.Stat(b.Source); err != nil { + return fmt.Errorf("cannot stat bind source %s: %w", b.Source, err) + } + + sylog.Debugf("Adding device bind of %s to %s, with options %v", b.Source, b.Destination, opts) + + *mounts = append(*mounts, + specs.Mount{ + Source: b.Source, + Destination: b.Destination, + Type: "none", + Options: opts, + }) + return nil +} + +func (l *Launcher) addRocmMounts(mounts *[]specs.Mount) error { + gpuConfFile := filepath.Join(buildcfg.APPTAINER_CONFDIR, "rocmliblist.conf") + + libs, bins, err := gpu.RocmPaths(gpuConfFile) + if err != nil { + sylog.Warningf("While finding ROCm bind points: %v", err) + } + if len(libs) == 0 { + sylog.Warningf("Could not find any ROCm libraries on this host!") + } + + devs, err := gpu.RocmDevices() + if err != nil { + sylog.Warningf("While finding ROCm devices: %v", err) + } + if len(devs) == 0 { + sylog.Warningf("Could not find any ROCm devices on this host!") + } + + for _, binary := range bins { + containerBinary := filepath.Join("/usr/bin", filepath.Base(binary)) + bind := bind.Path{ + Source: binary, + Destination: containerBinary, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, lib := range libs { + containerLib := filepath.Join(containerLibDir, filepath.Base(lib)) + bind := bind.Path{ + Source: lib, + Destination: containerLib, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, dev := range devs { + bind := bind.Path{ + Source: dev, + Destination: dev, + } + if err := addDevBindMount(mounts, bind); err != nil { + return err + } + } + + return nil +} + +func (l *Launcher) addNvidiaMounts(mounts *[]specs.Mount) error { + if l.apptainerConf.UseNvCCLI { + sylog.Warningf("--nvccli not yet supported with --oci. Falling back to legacy --nv support.") + } + + gpuConfFile := filepath.Join(buildcfg.APPTAINER_CONFDIR, "nvliblist.conf") + libs, bins, err := gpu.NvidiaPaths(gpuConfFile) + if err != nil { + sylog.Warningf("While finding Nvidia bind points: %v", err) + } + if len(libs) == 0 { + sylog.Warningf("Could not find any Nvidia libraries on this host!") + } + + ipcs, err := gpu.NvidiaIpcsPath() + if err != nil { + sylog.Warningf("While finding Nvidia IPCs: %v", err) + } + + devs, err := gpu.NvidiaDevices(true) + if err != nil { + sylog.Warningf("While finding Nvidia devices: %v", err) + } + if len(devs) == 0 { + sylog.Warningf("Could not find any ROCm devices on this host!") + } + + for _, binary := range bins { + containerBinary := filepath.Join("/usr/bin", filepath.Base(binary)) + bind := bind.Path{ + Source: binary, + Destination: containerBinary, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, lib := range libs { + containerLib := filepath.Join(containerLibDir, filepath.Base(lib)) + bind := bind.Path{ + Source: lib, + Destination: containerLib, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, ipc := range ipcs { + bind := bind.Path{ + Source: ipc, + Destination: ipc, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, dev := range devs { + bind := bind.Path{ + Source: dev, + Destination: dev, + } + if err := addDevBindMount(mounts, bind); err != nil { + return err + } + } + + return nil +} diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux_test.go b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go new file mode 100644 index 0000000000..b4f4b3f54d --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go @@ -0,0 +1,286 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "reflect" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/bind" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func Test_addBindMount(t *testing.T) { + tests := []struct { + name string + b bind.Path + wantMounts *[]specs.Mount + wantErr bool + }{ + { + name: "Valid", + b: bind.Path{ + Source: "/tmp", + Destination: "/tmp", + }, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/tmp", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + }, + { + name: "ValidRO", + b: bind.Path{ + Source: "/tmp", + Destination: "/tmp", + Options: map[string]*bind.Option{"ro": {}}, + }, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/tmp", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev", "ro"}, + }, + }, + }, + { + name: "BadSource", + b: bind.Path{ + Source: "doesnotexist!", + Destination: "/mnt", + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "RelDest", + b: bind.Path{ + Source: "/tmp", + Destination: "relative", + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "ImageID", + b: bind.Path{ + Source: "/myimage.sif", + Destination: "/mnt", + Options: map[string]*bind.Option{"id": {Value: "4"}}, + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "ImageSrc", + b: bind.Path{ + Source: "/myimage.sif", + Destination: "/mnt", + Options: map[string]*bind.Option{"img-src": {Value: "/test"}}, + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mounts := &[]specs.Mount{} + err := addBindMount(mounts, tt.b) + if (err != nil) != tt.wantErr { + t.Errorf("addBindMount() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(mounts, tt.wantMounts) { + t.Errorf("addBindMount() want %v, got %v", tt.wantMounts, mounts) + } + }) + } +} + +func TestLauncher_addBindMounts(t *testing.T) { + tests := []struct { + name string + cfg launcher.Options + userbind bool + wantMounts *[]specs.Mount + wantErr bool + }{ + { + name: "Disabled", + cfg: launcher.Options{ + BindPaths: []string{"/tmp"}, + }, + wantMounts: &[]specs.Mount{}, + wantErr: false, + }, + { + name: "ValidBindSrc", + cfg: launcher.Options{ + BindPaths: []string{"/tmp"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/tmp", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + wantErr: false, + }, + { + name: "ValidBindSrcDst", + cfg: launcher.Options{ + BindPaths: []string{"/tmp:/mnt"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + wantErr: false, + }, + { + name: "ValidBindRO", + cfg: launcher.Options{ + BindPaths: []string{"/tmp:/mnt:ro"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev", "ro"}, + }, + }, + wantErr: false, + }, + { + name: "InvalidBindSrc", + cfg: launcher.Options{ + BindPaths: []string{"!doesnotexist"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "RelBindDst", + cfg: launcher.Options{ + BindPaths: []string{"/tmp:relative"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "UnsupportedBindID", + cfg: launcher.Options{ + BindPaths: []string{"my.sif:/mnt:id=2"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "UnsupportedBindImgSrc", + cfg: launcher.Options{ + BindPaths: []string{"my.sif:/mnt:img-src=/test"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "ValidMount", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=/tmp,destination=/mnt"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + wantErr: false, + }, + { + name: "ValidMountRO", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=/tmp,destination=/mnt,ro"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev", "ro"}, + }, + }, + wantErr: false, + }, + { + name: "UnsupportedMountID", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=my.sif,destination=/mnt,id=2"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "UnsupportedMountImgSrc", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=my.sif,destination=/mnt,image-src=/test"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := &Launcher{ + cfg: tt.cfg, + apptainerConf: &apptainerconf.File{}, + } + if tt.userbind { + l.apptainerConf.UserBindControl = true + } + mounts := &[]specs.Mount{} + err := l.addBindMounts(mounts) + if (err != nil) != tt.wantErr { + t.Errorf("addBindMount() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(mounts, tt.wantMounts) { + t.Errorf("addBindMount() want %v, got %v", tt.wantMounts, mounts) + } + }) + } +} diff --git a/internal/pkg/runtime/launcher/oci/oci_attach_linux.go b/internal/pkg/runtime/launcher/oci/oci_attach_linux.go new file mode 100644 index 0000000000..1b32deb7d4 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_attach_linux.go @@ -0,0 +1,250 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "os" + "path/filepath" + + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/containers/common/pkg/config" + "github.com/moby/term" + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +var ErrDetach = errors.New("detached from container") + +// attachStreams contains streams that will be attached to the container +type attachStreams struct { + // OutputStream will be attached to container's STDOUT + OutputStream io.Writer + // ErrorStream will be attached to container's STDERR + ErrorStream io.Writer + // InputStream will be attached to container's STDIN + InputStream io.Reader + // AttachOutput is whether to attach to STDOUT + // If false, stdout will not be attached + AttachOutput bool + // AttachError is whether to attach to STDERR + // If false, stdout will not be attached + AttachError bool + // AttachInput is whether to attach to STDIN + // If false, stdout will not be attached + AttachInput bool +} + +/* Sync with stdpipe_t in conmon.c */ +const ( + AttachPipeStdin = 1 + AttachPipeStdout = 2 + AttachPipeStderr = 3 +) + +// Attach attaches the console to a running container +func Attach(ctx context.Context, containerID string) error { + streams := attachStreams{ + OutputStream: os.Stdout, + ErrorStream: os.Stderr, + InputStream: bufio.NewReader(os.Stdin), + AttachOutput: true, + AttachError: true, + AttachInput: true, + } + + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + attachSock := filepath.Join(sd, bundleLink, attachSocket) + conn, err := openUnixSocket(attachSock) + if err != nil { + return fmt.Errorf("while connecting to attach socket: %w", err) + } + defer func() { + if err := conn.Close(); err != nil { + sylog.Errorf("while closing attach socket: %v", err) + } + }() + + detachKeys, err := processDetachKeys(config.DefaultDetachKeys) + if err != nil { + return fmt.Errorf("invalid detach key sequence: %w", err) + } + + receiveStdoutError, stdinDone := setupStdioChannels(streams, conn, detachKeys) + + return readStdio(conn, streams, receiveStdoutError, stdinDone) +} + +// The following utility functions are taken from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +func openUnixSocket(path string) (*net.UnixConn, error) { + fd, err := unix.Open(path, unix.O_PATH, 0) + if err != nil { + return nil, err + } + defer unix.Close(fd) + return net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: fmt.Sprintf("/proc/self/fd/%d", fd), Net: "unixpacket"}) +} + +func setupStdioChannels(streams attachStreams, conn *net.UnixConn, detachKeys []byte) (chan error, chan error) { + receiveStdoutError := make(chan error) + go func() { + receiveStdoutError <- redirectResponseToOutputStreams(streams.OutputStream, streams.ErrorStream, streams.AttachOutput, streams.AttachError, conn) + }() + + stdinDone := make(chan error) + go func() { + var err error + if streams.AttachInput { + _, err = copyDetachable(conn, streams.InputStream, detachKeys) + } + stdinDone <- err + }() + + return receiveStdoutError, stdinDone +} + +func redirectResponseToOutputStreams(outputStream, errorStream io.Writer, writeOutput, writeError bool, conn io.Reader) error { + var err error + buf := make([]byte, 8192+1) /* Sync with conmon STDIO_BUF_SIZE */ + for { + nr, er := conn.Read(buf) + if nr > 0 { + var dst io.Writer + var doWrite bool + switch buf[0] { + case AttachPipeStdout: + dst = outputStream + doWrite = writeOutput + case AttachPipeStderr: + dst = errorStream + doWrite = writeError + default: + sylog.Infof("Received unexpected attach type %+d", buf[0]) + } + if dst == nil { + return errors.New("output destination cannot be nil") + } + + if doWrite { + nw, ew := dst.Write(buf[1:nr]) + if ew != nil { + err = ew + break + } + if nr != nw+1 { + err = io.ErrShortWrite + break + } + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return err +} + +func readStdio(conn *net.UnixConn, streams attachStreams, receiveStdoutError, stdinDone chan error) error { + var err error + select { + case err = <-receiveStdoutError: + conn.CloseWrite() + return err + case err = <-stdinDone: + if err == ErrDetach { + conn.CloseWrite() + return err + } + if err == nil { + // copy stdin is done, close it + if connErr := conn.CloseWrite(); connErr != nil { + sylog.Errorf("Unable to close conn: %v", connErr) + } + } + if streams.AttachOutput || streams.AttachError { + return <-receiveStdoutError + } + } + return nil +} + +func copyDetachable(dst io.Writer, src io.Reader, keys []byte) (written int64, err error) { + buf := make([]byte, 32*1024) + for { + nr, er := src.Read(buf) + if nr > 0 { + preservBuf := []byte{} + for i, key := range keys { + preservBuf = append(preservBuf, buf[0:nr]...) + if nr != 1 || buf[0] != key { + break + } + if i == len(keys)-1 { + return 0, ErrDetach + } + nr, er = src.Read(buf) + } + var nw int + var ew error + if len(preservBuf) > 0 { + nw, ew = dst.Write(preservBuf) + nr = len(preservBuf) + } else { + nw, ew = dst.Write(buf[0:nr]) + } + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return written, err +} + +func processDetachKeys(keys string) ([]byte, error) { + // Check the validity of the provided keys first + if len(keys) == 0 { + return []byte{}, nil + } + detachKeys, err := term.ToBytes(keys) + if err != nil { + return nil, fmt.Errorf("invalid detach keys: %w", err) + } + return detachKeys, nil +} diff --git a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go new file mode 100644 index 0000000000..2b3a9c261c --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go @@ -0,0 +1,254 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "syscall" + "time" + + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/google/uuid" + "golang.org/x/sys/unix" +) + +type ociError struct { + Level string `json:"level,omitempty"` + Time string `json:"time,omitempty"` + Msg string `json:"msg,omitempty"` +} + +// Create creates a container from an OCI bundle +func Create(containerID, bundlePath string, systemdCgroups bool) error { + conmon, err := bin.FindBin("conmon") + if err != nil { + return err + } + runtimeBin, err := runtime() + if err != nil { + return err + } + // chdir to bundle and lock it, so another oci create cannot use the same bundle + absBundle, err := filepath.Abs(bundlePath) + if err != nil { + return fmt.Errorf("failed to determine bundle absolute path: %w", err) + } + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %w", absBundle, err) + } + if err := lockBundle(absBundle); err != nil { + return fmt.Errorf("while locking bundle: %w", err) + } + + // Create our own state location for conmon and apptainer related files + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + err = os.MkdirAll(sd, 0o700) + if err != nil { + return fmt.Errorf("while creating state directory: %w", err) + } + containerUUID, err := uuid.NewRandom() + if err != nil { + return err + } + + // Pipes for sync and start communication with conmon + syncFds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("could not create sync socket pair: %w", err) + } + syncChild := os.NewFile(uintptr(syncFds[0]), "sync_child") + syncParent := os.NewFile(uintptr(syncFds[1]), "sync_parent") + defer syncParent.Close() + + startFds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("could not create sync socket pair: %w", err) + } + startChild := os.NewFile(uintptr(startFds[0]), "start_child") + startParent := os.NewFile(uintptr(startFds[1]), "start_parent") + defer startParent.Close() + + apptainerBin := filepath.Join(buildcfg.BINDIR, "apptainer") + + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + cmdArgs := []string{ + "--api-version", "1", + "--cid", containerID, + "--name", containerID, + "--cuuid", containerUUID.String(), + "--runtime", runtimeBin, + "--conmon-pidfile", path.Join(sd, conmonPidFile), + "--container-pidfile", path.Join(sd, containerPidFile), + "--log-path", path.Join(sd, containerLogFile), + "--runtime-arg", "--root", + "--runtime-arg", rsd, + "--runtime-arg", "--log", + "--runtime-arg", path.Join(sd, runcLogFile), + "--full-attach", + "--terminal", + "--bundle", absBundle, + "--exit-command", apptainerBin, + "--exit-command-arg", "--debug", + "--exit-command-arg", "oci", + "--exit-command-arg", "cleanup", + "--exit-command-arg", containerID, + } + + if systemdCgroups { + cmdArgs = append(cmdArgs, "--systemd-cgroup") + } + + cmd := exec.Command(conmon, cmdArgs...) + cmd.Dir = absBundle + cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3), fmt.Sprintf("_OCI_STARTPIPE=%d", 4)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + cmd.ExtraFiles = append(cmd.ExtraFiles, syncChild, startChild) + + // Run conmon and close it's end of the pipes in our parent process + sylog.Debugf("Starting conmon with args %v", cmdArgs) + if err := cmd.Start(); err != nil { + if err2 := releaseBundle(absBundle); err2 != nil { + sylog.Errorf("while releasing bundle: %v", err) + } + return fmt.Errorf("while starting conmon: %w", err) + } + syncChild.Close() + startChild.Close() + + // No other setup at present... just signal conmon to start work + writeConmonPipeData(startParent) + // After conmon receives from start pipe it should start container and exit + // without error. + err = cmd.Wait() + if err != nil { + if err2 := releaseBundle(absBundle); err2 != nil { + sylog.Errorf("while releasing bundle: %v", err) + } + return fmt.Errorf("while starting conmon: %w", err) + } + + // We check for errors from runc (which conmon invokes) via the sync pipe + pid, err := readConmonPipeData(syncParent, path.Join(sd, runcLogFile)) + if err != nil { + if err2 := Delete(context.TODO(), containerID, systemdCgroups); err2 != nil { + sylog.Errorf("Removing container %s from runtime after creation failed", containerID) + } + return err + } + + // Create a symlink from the state dir to the bundle, so it's easy to find later on. + bundleLink := path.Join(sd, "bundle") + if err := os.Symlink(absBundle, bundleLink); err != nil { + return fmt.Errorf("could not link attach socket: %w", err) + } + + sylog.Infof("Container %s created with PID %d", containerID, pid) + return nil +} + +// The following utility functions are taken from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +func readConmonPipeData(pipe *os.File, ociLog string) (int, error) { + // syncInfo is used to return data from monitor process to daemon + type syncInfo struct { + Data int `json:"data"` + Message string `json:"message,omitempty"` + } + + // Wait to get container pid from conmon + type syncStruct struct { + si *syncInfo + err error + } + ch := make(chan syncStruct) + go func() { + var si *syncInfo + rdr := bufio.NewReader(pipe) + b, err := rdr.ReadBytes('\n') + if err != nil { + ch <- syncStruct{err: err} + } + if err := json.Unmarshal(b, &si); err != nil { + ch <- syncStruct{err: err} + return + } + ch <- syncStruct{si: si} + }() + + data := -1 + select { + case ss := <-ch: + if ss.err != nil { + if ociLog != "" { + ociLogData, err := os.ReadFile(ociLog) + if err == nil { + var ociErr ociError + if err := json.Unmarshal(ociLogData, &ociErr); err == nil { + return -1, fmt.Errorf("runc error: %s", ociErr.Msg) + } + } + } + return -1, fmt.Errorf("container create failed (no logs from conmon): %w", ss.err) + } + sylog.Debugf("Received: %d", ss.si.Data) + if ss.si.Data < 0 { + if ociLog != "" { + ociLogData, err := os.ReadFile(ociLog) + if err == nil { + var ociErr ociError + if err := json.Unmarshal(ociLogData, &ociErr); err == nil { + return ss.si.Data, fmt.Errorf("runc error: %s", ociErr.Msg) + } + } + } + // If we failed to parse the JSON errors, then print the output as it is + if ss.si.Message != "" { + return ss.si.Data, fmt.Errorf("runc error: %s", ss.si.Message) + } + return ss.si.Data, fmt.Errorf("container creation failed") + } + data = ss.si.Data + case <-time.After(createTimeout): + return -1, fmt.Errorf("container creation timeout") + } + return data, nil +} + +// writeConmonPipeData writes nonce data to a pipe +func writeConmonPipeData(pipe *os.File) error { + someData := []byte{0} + _, err := pipe.Write(someData) + return err +} diff --git a/internal/pkg/runtime/launcher/oci/oci_linux.go b/internal/pkg/runtime/launcher/oci/oci_linux.go new file mode 100644 index 0000000000..45cc4c560a --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_linux.go @@ -0,0 +1,144 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "fmt" + "os" + "path" + "path/filepath" + "time" + + "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/fs/lock" + securejoin "github.com/cyphar/filepath-securejoin" +) + +const ( + // Relative path inside ~/.apptainer for conmon and apptainer state + ociPath = "oci" + // State directory files + containerPidFile = "container.pid" + containerLogFile = "container.log" + runcLogFile = "runc.log" + conmonPidFile = "conmon.pid" + bundleLink = "bundle" + // Files in the OCI bundle root + bundleLock = ".apptainer-oci.lock" + attachSocket = "attach" + // Timeouts + createTimeout = 30 * time.Second +) + +// runtime returns path to the OCI runtime - crun (preferred), or runc. +func runtime() (path string, err error) { + path, err = bin.FindBin("crun") + if err == nil { + return + } + sylog.Debugf("While finding crun: %s", err) + sylog.Debugf("Falling back to runc as OCI runtime.") + return bin.FindBin("runc") +} + +// runtimeStateDir returns path to use for crun/runc's state handling. +func runtimeStateDir() (path string, err error) { + // Ensure we get correct uid for host if we were re-exec'd in id mapped userns + pw, err := user.CurrentOriginal() + if err != nil { + return "", err + } + if pw.UID == 0 { + return "/run/apptainer-oci", nil + } + return fmt.Sprintf("/run/user/%d/apptainer-oci", pw.UID), nil +} + +// stateDir returns the path to container state handled by conmon/apptainer +// (as opposed to runc's state in RuncStateDir) +func stateDir(containerID string) (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + u, err := user.CurrentOriginal() + if err != nil { + return "", err + } + + configDir, err := syfs.ConfigDirForUsername(u.Name) + if err != nil { + return "", err + } + + rootPath := filepath.Join(configDir, ociPath) + containerPath := filepath.Join(hostname, containerID) + path, err := securejoin.SecureJoin(rootPath, containerPath) + if err != nil { + return "", err + } + return path, err +} + +// lockBundle creates a lock file in a bundle directory +func lockBundle(bundlePath string) error { + bl := path.Join(bundlePath, bundleLock) + _, err := os.Stat(bl) + if err == nil { + return fmt.Errorf("bundle is locked by another process") + } + if !os.IsNotExist(err) { + return fmt.Errorf("while stat-ing lock file: %w", err) + } + + fd, err := lock.Exclusive(bundlePath) + if err != nil { + return fmt.Errorf("while acquiring directory lock: %w", err) + } + defer lock.Release(fd) + + err = fs.EnsureFileWithPermission(bl, 0o600) + if err != nil { + return fmt.Errorf("while creating lock file: %w", err) + } + return nil +} + +// releaseBundle removes a lock file in a bundle directory +func releaseBundle(bundlePath string) error { + bl := path.Join(bundlePath, bundleLock) + _, err := os.Stat(bl) + if os.IsNotExist(err) { + return fmt.Errorf("bundle is not locked") + } + if err != nil { + return fmt.Errorf("while stat-ing lock file: %w", err) + } + + fd, err := lock.Exclusive(bundlePath) + if err != nil { + return fmt.Errorf("while acquiring directory lock: %w", err) + } + defer lock.Release(fd) + + err = os.Remove(bl) + if err != nil { + return fmt.Errorf("while removing lock file: %w", err) + } + return nil +} diff --git a/internal/pkg/runtime/launcher/oci/oci_overlay.go b/internal/pkg/runtime/launcher/oci/oci_overlay.go new file mode 100644 index 0000000000..518d83c94d --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_overlay.go @@ -0,0 +1,116 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/util/fs/overlay" + "github.com/apptainer/apptainer/pkg/ocibundle/tools" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" +) + +// WrapWithWritableTmpFs runs a function wrapped with prep / cleanup steps for a writable tmpfs. +func WrapWithWritableTmpFs(f func() error, bundleDir string) error { + // TODO: --oci mode always emulating --compat, which uses --writable-tmpfs. + // Provide a way of disabling this, for a read only rootfs. + overlayDir, err := prepareWritableTmpfs(bundleDir) + sylog.Debugf("Done with prepareWritableTmpfs; overlayDir is: %q", overlayDir) + if err != nil { + return err + } + + err = f() + + // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. + if cleanupErr := cleanupWritableTmpfs(bundleDir, overlayDir); cleanupErr != nil { + sylog.Errorf("While cleaning up writable tmpfs: %v", cleanupErr) + } + + // Return any error from the actual container payload - preserve exit code. + return err +} + +// WrapWithOverlays runs a function wrapped with prep / cleanup steps for overlays. +func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) error { + writableOverlayFound := false + s := overlay.Set{} + for _, p := range overlayPaths { + item, err := overlay.NewItemFromString(p) + if err != nil { + return err + } + + item.SetParentDir(bundleDir) + + if item.Writable && writableOverlayFound { + return fmt.Errorf("you can't specify more than one writable overlay; %#v has already been specified as a writable overlay; use '--overlay %s:ro' instead", s.WritableOverlay, item.SourcePath) + } + if item.Writable { + writableOverlayFound = true + s.WritableOverlay = item + } else { + s.ReadonlyOverlays = append(s.ReadonlyOverlays, item) + } + } + + rootFsDir := tools.RootFs(bundleDir).Path() + err := s.Mount(rootFsDir) + if err != nil { + return err + } + + if writableOverlayFound { + err = f() + } else { + err = WrapWithWritableTmpFs(f, bundleDir) + } + + // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. + if cleanupErr := s.Unmount(rootFsDir); cleanupErr != nil { + sylog.Errorf("While unmounting rootfs overlay: %v", cleanupErr) + } + + // Return any error from the actual container payload - preserve exit code. + return err +} + +func prepareWritableTmpfs(bundleDir string) (string, error) { + sylog.Debugf("Configuring writable tmpfs overlay for %s", bundleDir) + c := apptainerconf.GetCurrentConfig() + if c == nil { + return "", fmt.Errorf("apptainer configuration is not initialized") + } + return tools.CreateOverlayTmpfs(bundleDir, int(c.SessiondirMaxSize)) +} + +func cleanupWritableTmpfs(bundleDir, overlayDir string) error { + sylog.Debugf("Cleaning up writable tmpfs overlay for %s", bundleDir) + return tools.DeleteOverlayTmpfs(bundleDir, overlayDir) +} + +// absOverlay takes an overlay description string (a path, optionally followed by a colon with an option string, like ":ro" or ":rw"), and replaces any relative path in the description string with an absolute one. +func absOverlay(desc string) (string, error) { + splitted := strings.SplitN(desc, ":", 2) + barePath := splitted[0] + absBarePath, err := filepath.Abs(barePath) + if err != nil { + return "", err + } + absDesc := absBarePath + if len(splitted) > 1 { + absDesc += ":" + splitted[1] + } + + return absDesc, nil +} diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go new file mode 100644 index 0000000000..f091904178 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -0,0 +1,360 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + fakerootConfig "github.com/apptainer/apptainer/internal/pkg/runtime/engine/fakeroot/config" + "github.com/apptainer/apptainer/internal/pkg/util/starter" + "github.com/apptainer/apptainer/pkg/runtime/engine/config" + "github.com/apptainer/apptainer/pkg/sylog" +) + +// Delete deletes container resources +func Delete(ctx context.Context, containerID string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "delete", containerID) + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + err = cmd.Run() + if err != nil { + return fmt.Errorf("while calling runc delete: %w", err) + } + + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + + bLink := filepath.Join(sd, bundleLink) + bundle, err := filepath.EvalSymlinks(bLink) + if err != nil { + return fmt.Errorf("while finding bundle directory: %w", err) + } + + sylog.Debugf("Removing bundle symlink") + if err := os.Remove(bLink); err != nil { + return fmt.Errorf("while removing bundle symlink: %w", err) + } + + sylog.Debugf("Releasing bundle lock") + return releaseBundle(bundle) +} + +// Exec executes a command in a container +func Exec(containerID string, cmdArgs []string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "exec", containerID) + runtimeArgs = append(runtimeArgs, cmdArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// Kill kills container process +func Kill(containerID string, killSignal string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + "kill", + containerID, + killSignal, + } + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// Pause pauses processes in a container +func Pause(containerID string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "pause", containerID) + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// Resume un-pauses processes in a container +func Resume(containerID string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "resume", containerID) + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// Run runs a container (equivalent to create/start/delete) +func Run(ctx context.Context, containerID, bundlePath, pidFile string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + absBundle, err := filepath.Abs(bundlePath) + if err != nil { + return fmt.Errorf("failed to determine bundle absolute path: %s", err) + } + + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) + } + + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "run", "-b", absBundle) + if pidFile != "" { + runtimeArgs = append(runtimeArgs, "--pid-file="+pidFile) + } + + runtimeArgs = append(runtimeArgs, containerID) + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// RunWrapped runs a container via the OCI runtime, wrapped with prep / cleanup steps. +func RunWrapped(ctx context.Context, containerID, bundlePath, pidFile string, overlayPaths []string, systemdCgroups bool) error { + runFunc := func() error { + return Run(ctx, containerID, bundlePath, pidFile, systemdCgroups) + } + + if len(overlayPaths) > 0 { + return WrapWithOverlays(runFunc, bundlePath, overlayPaths) + } + + return WrapWithWritableTmpFs(runFunc, bundlePath) +} + +// RunWrappedNS reexecs apptainer in a user namespace, with supplied uid/gid mapping, calling oci run. +func RunWrappedNS(ctx context.Context, containerID, bundlePath string, overlayPaths []string) error { + absBundle, err := filepath.Abs(bundlePath) + if err != nil { + return fmt.Errorf("failed to determine bundle absolute path: %s", err) + } + + args := []string{ + filepath.Join(buildcfg.BINDIR, "apptainer"), + "oci", + "run-wrapped", + "-b", absBundle, + } + for _, p := range overlayPaths { + absPath, err := absOverlay(p) + if err != nil { + return fmt.Errorf("could not convert %q to absolute path: %w", p, err) + } + + args = append(args, "--overlay", absPath) + } + args = append(args, containerID) + + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) + } + + sylog.Debugf("Calling fakeroot engine to execute %q", strings.Join(args, " ")) + + cfg := &config.Common{ + EngineName: fakerootConfig.Name, + ContainerID: "fakeroot", + EngineConfig: &fakerootConfig.EngineConfig{ + Envs: os.Environ(), + Args: args, + NoPIDNS: true, + }, + } + + return starter.Run( + "Apptainer oci userns", + cfg, + starter.WithStdin(os.Stdin), + starter.WithStdout(os.Stdout), + starter.WithStderr(os.Stderr), + ) +} + +// Start starts a previously created container +func Start(containerID string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "start", containerID) + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// State queries container state +func State(containerID string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "state", containerID) + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} + +// Update updates container cgroups resources +func Update(containerID, cgFile string, systemdCgroups bool) error { + runtimeBin, err := runtime() + if err != nil { + return err + } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + + runtimeArgs := []string{ + "--root", rsd, + } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "update", "-r", cgFile, containerID) + + cmd := exec.Command(runtimeBin, runtimeArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) + return cmd.Run() +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go new file mode 100644 index 0000000000..f9b36b5600 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -0,0 +1,335 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "context" + "fmt" + "os" + "strings" + "syscall" + + "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/internal/pkg/util/env" + "github.com/apptainer/apptainer/internal/pkg/util/shell/interpreter" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/term" +) + +const apptainerLibs = "/.singularity.d/libs" + +func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, image, bundle, process string, args []string, u specs.User) (*specs.Process, error) { + // Assemble the runtime & user-requested environment, which will be merged + // with the image ENV and set in the container at runtime. + rtEnv := defaultEnv(image, bundle) + + // Propagate TERM from host. Doing this here means it can be overridden by APPTAINERENV_TERM. + hostTerm, isHostTermSet := os.LookupEnv("TERM") + if isHostTermSet { + rtEnv["TERM"] = hostTerm + } + + // APPTAINERENV_ has lowest priority + rtEnv = mergeMap(rtEnv, apptainerEnvMap()) + // --env-file can override APPTAINERENV_ + if l.cfg.EnvFile != "" { + e, err := envFileMap(ctx, l.cfg.EnvFile) + if err != nil { + return nil, err + } + rtEnv = mergeMap(rtEnv, e) + } + // --env flag can override --env-file and APPTAINERENV_ + rtEnv = mergeMap(rtEnv, l.cfg.Env) + + // Ensure HOME points to the required home directory, even if it is a custom one, unless the container explicitly specifies its USER, in which case we don't want to touch HOME. + if imgSpec.Config.User == "" { + rtEnv["HOME"] = l.cfg.HomeDir + } + + cwd, err := l.getProcessCwd() + if err != nil { + return nil, err + } + + p := specs.Process{ + Args: getProcessArgs(imgSpec, process, args), + Cwd: cwd, + Env: getProcessEnv(imgSpec, rtEnv), + User: u, + Terminal: getProcessTerminal(), + } + + return &p, nil +} + +// getProcessTerminal determines whether the container process should run with a terminal. +func getProcessTerminal() bool { + // Sets the default Process.Terminal to false if our stdin is not a terminal. + return term.IsTerminal(syscall.Stdin) +} + +// getProcessArgs returns the process args for a container, with reference to the OCI Image Spec. +// The process and image parameters may override the image CMD and/or ENTRYPOINT. +func getProcessArgs(imageSpec imgspecv1.Image, process string, args []string) []string { + var processArgs []string + + if process != "" { + processArgs = []string{process} + } else { + processArgs = imageSpec.Config.Entrypoint + } + + if len(args) > 0 { + processArgs = append(processArgs, args...) + } else { + if process == "" { + processArgs = append(processArgs, imageSpec.Config.Cmd...) + } + } + + return processArgs +} + +// getProcessCwd computes the Cwd that the container process should start in. +// Currently this is the user's tmpfs home directory (see --containall). +// Because this is called after mounts have already been computed, we can count on l.cfg.HomeDir containing the right value, incorporating any custom home dir overrides (i.e., --home). +func (l *Launcher) getProcessCwd() (dir string, err error) { + if len(l.cfg.CwdPath) > 0 { + return l.cfg.CwdPath, nil + } + + return l.cfg.HomeDir, nil +} + +// getReverseUserMaps returns uid and gid mappings that re-map container uid to target +// uid. This 'reverses' the host user to container root mapping in the initial +// userns from which the OCI runtime is launched. +// +// e.g. host 1001 -> fakeroot userns 0 -> container targetUID +func getReverseUserMaps(hostUID, targetUID, targetGID uint32) (uidMap, gidMap []specs.LinuxIDMapping, err error) { + // Get user's configured subuid & subgid ranges + subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, hostUID) + if err != nil { + return nil, nil, err + } + // We must always be able to map at least 0->65535 inside the container, so we cover 'nobody'. + if subuidRange.Size < 65536 { + return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subuidRange.Size) + } + subgidRange, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, hostUID) + if err != nil { + return nil, nil, err + } + if subgidRange.Size < 65536 { + return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subgidRange.Size) + } + + uidMap, gidMap = reverseMapByRange(targetUID, targetGID, *subuidRange, *subgidRange) + return uidMap, gidMap, nil +} + +func reverseMapByRange(targetUID, targetGID uint32, subuidRange, subgidRange specs.LinuxIDMapping) (uidMap, gidMap []specs.LinuxIDMapping) { + if targetUID < subuidRange.Size { + uidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 1, + Size: targetUID, + }, + { + ContainerID: targetUID, + HostID: 0, + Size: 1, + }, + { + ContainerID: targetUID + 1, + HostID: targetUID + 1, + Size: subuidRange.Size - targetUID, + }, + } + } else { + uidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 1, + Size: subuidRange.Size, + }, + { + ContainerID: targetUID, + HostID: 0, + Size: 1, + }, + } + } + + if targetGID < subgidRange.Size { + gidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 1, + Size: targetGID, + }, + { + ContainerID: targetGID, + HostID: 0, + Size: 1, + }, + { + ContainerID: targetGID + 1, + HostID: targetGID + 1, + Size: subgidRange.Size - targetGID, + }, + } + } else { + gidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 1, + Size: subgidRange.Size, + }, + { + ContainerID: targetGID, + HostID: 0, + Size: 1, + }, + } + } + + return uidMap, gidMap +} + +// getProcessEnv combines the image config ENV with the ENV requested at runtime. +// APPEND_PATH and PREPEND_PATH are honored as with the native apptainer runtime. +// LD_LIBRARY_PATH is modified to always include the apptainer lib bind directory. +func getProcessEnv(imageSpec imgspecv1.Image, runtimeEnv map[string]string) []string { + path := "" + appendPath := "" + prependPath := "" + ldLibraryPath := "" + + // Start with the environment from the image config. + g := generate.New(nil) + g.Config.Process = &specs.Process{Env: imageSpec.Config.Env} + + // Obtain PATH, and LD_LIBRARY_PATH if set in the image config, for special handling. + for _, env := range imageSpec.Config.Env { + e := strings.SplitN(env, "=", 2) + if len(e) < 2 { + continue + } + if e[0] == "PATH" { + path = e[1] + } + if e[0] == "LD_LIBRARY_PATH" { + ldLibraryPath = e[1] + } + } + + // Apply env vars from runtime, except PATH and LD_LIBRARY_PATH releated. + for k, v := range runtimeEnv { + switch k { + case "PATH": + path = v + case "APPEND_PATH": + appendPath = v + case "PREPEND_PATH": + prependPath = v + case "LD_LIBRARY_PATH": + ldLibraryPath = v + default: + g.SetProcessEnv(k, v) + } + } + + // Compute and set optionally APPEND-ed / PREPEND-ed PATH. + if appendPath != "" { + path = path + ":" + appendPath + } + if prependPath != "" { + path = prependPath + ":" + path + } + if path != "" { + g.SetProcessEnv("PATH", path) + } + + // Ensure LD_LIBRARY_PATH always contains apptainer lib binding dir. + if !strings.Contains(ldLibraryPath, apptainerLibs) { + ldLibraryPath = strings.TrimPrefix(ldLibraryPath+":"+apptainerLibs, ":") + } + g.SetProcessEnv("LD_LIBRARY_PATH", ldLibraryPath) + + return g.Config.Process.Env +} + +// defaultEnv returns default environment variables set in the container. +func defaultEnv(image, bundle string) map[string]string { + return map[string]string{ + env.ApptainerPrefix + "CONTAINER": bundle, + env.ApptainerPrefix + "NAME": image, + } +} + +// apptainerEnvMap returns a map of APPTAINERENV_ prefixed env vars to their values. +func apptainerEnvMap() map[string]string { + apptainerEnv := map[string]string{} + + for _, envVar := range os.Environ() { + if !strings.HasPrefix(envVar, env.ApptainerEnvPrefix) { + continue + } + parts := strings.SplitN(envVar, "=", 2) + if len(parts) < 2 { + continue + } + key := strings.TrimPrefix(parts[0], env.ApptainerEnvPrefix) + apptainerEnv[key] = parts[1] + } + + return apptainerEnv +} + +// envFileMap returns a map of KEY=VAL env vars from an environment file +func envFileMap(ctx context.Context, f string) (map[string]string, error) { + envMap := map[string]string{} + + content, err := os.ReadFile(f) + if err != nil { + return envMap, fmt.Errorf("could not read environment file %q: %w", f, err) + } + + // Use the embedded shell interpreter to evaluate the env file, with an empty starting environment. + // Shell takes care of comments, quoting etc. for us and keeps compatibility with native runtime. + env, err := interpreter.EvaluateEnv(ctx, content, []string{}, []string{}) + if err != nil { + return envMap, fmt.Errorf("while processing %s: %w", f, err) + } + + for _, envVar := range env { + parts := strings.SplitN(envVar, "=", 2) + if len(parts) < 2 { + continue + } + // Strip out the runtime env vars set by the shell interpreter + if parts[0] == "GID" || + parts[0] == "HOME" || + parts[0] == "IFS" || + parts[0] == "OPTIND" || + parts[0] == "PWD" || + parts[0] == "UID" { + continue + } + envMap[parts[0]] = parts[1] + } + + return envMap, nil +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux_test.go b/internal/pkg/runtime/launcher/oci/process_linux_test.go new file mode 100644 index 0000000000..e2251881a6 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/process_linux_test.go @@ -0,0 +1,442 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "context" + "os" + "path/filepath" + "reflect" + "testing" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestApptainerEnvMap(t *testing.T) { + tests := []struct { + name string + setEnv map[string]string + want map[string]string + }{ + { + name: "None", + setEnv: map[string]string{}, + want: map[string]string{}, + }, + { + name: "NonPrefixed", + setEnv: map[string]string{"FOO": "bar"}, + want: map[string]string{}, + }, + { + name: "PrefixedSingle", + setEnv: map[string]string{"APPTAINERENV_FOO": "bar"}, + want: map[string]string{"FOO": "bar"}, + }, + { + name: "PrefixedMultiple", + setEnv: map[string]string{ + "APPTAINERENV_FOO": "bar", + "APPTAINERENV_ABC": "123", + }, + want: map[string]string{ + "FOO": "bar", + "ABC": "123", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.setEnv { + os.Setenv(k, v) + t.Cleanup(func() { + os.Unsetenv(k) + }) + } + if got := apptainerEnvMap(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("apptainerEnvMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestEnvFileMap(t *testing.T) { + tests := []struct { + name string + envFile string + want map[string]string + wantErr bool + }{ + { + name: "EmptyFile", + envFile: "", + want: map[string]string{ + "EUID": "0", + }, + wantErr: false, + }, + { + name: "Simple", + envFile: `FOO=BAR + ABC=123`, + want: map[string]string{ + "EUID": "0", + "FOO": "BAR", + "ABC": "123", + }, + wantErr: false, + }, + { + name: "DoubleQuote", + envFile: `FOO="FOO BAR"`, + want: map[string]string{ + "EUID": "0", + "FOO": "FOO BAR", + }, + wantErr: false, + }, + { + name: "SingleQuote", + envFile: `FOO='FOO BAR'`, + want: map[string]string{ + "EUID": "0", + "FOO": "FOO BAR", + }, + wantErr: false, + }, + { + name: "MultiLine", + envFile: "FOO=\"FOO\nBAR\"", + want: map[string]string{ + "EUID": "0", + "FOO": "FOO\nBAR", + }, + wantErr: false, + }, + { + name: "Invalid", + envFile: "!!!@@NOTAVAR", + want: map[string]string{}, + wantErr: true, + }, + } + + tmpDir := t.TempDir() + envFile := filepath.Join(tmpDir, "env-file") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := os.WriteFile(envFile, []byte(tt.envFile), 0o755); err != nil { + t.Fatalf("Could not write test env-file: %v", err) + } + + got, err := envFileMap(context.Background(), envFile) + if (err != nil) != tt.wantErr { + t.Errorf("envFileMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("envFileMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetProcessArgs(t *testing.T) { + tests := []struct { + name string + imgEntrypoint []string + imgCmd []string + bundleProcess string + bundleArgs []string + expectProcessArgs []string + }{ + { + name: "imageEntrypointOnly", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"ENTRYPOINT"}, + }, + { + name: "imageCmdOnly", + imgEntrypoint: []string{}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"CMD"}, + }, + { + name: "imageEntrypointCMD", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"ENTRYPOINT", "CMD"}, + }, + { + name: "ProcessOnly", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "ArgsOnly", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ARGS"}, + }, + { + name: "ProcessArgs", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"PROCESS", "ARGS"}, + }, + { + name: "overrideEntrypointOnlyProcess", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "overrideCmdOnlyArgs", + imgEntrypoint: []string{}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ARGS"}, + }, + { + name: "overrideBothProcess", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "overrideBothArgs", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ENTRYPOINT", "ARGS"}, + }, + { + name: "overrideBothProcessArgs", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "PROCESS", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"PROCESS", "ARGS"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := imgspecv1.Image{ + Config: imgspecv1.ImageConfig{ + Entrypoint: tt.imgEntrypoint, + Cmd: tt.imgCmd, + }, + } + args := getProcessArgs(i, tt.bundleProcess, tt.bundleArgs) + if !reflect.DeepEqual(args, tt.expectProcessArgs) { + t.Errorf("Expected: %v, Got: %v", tt.expectProcessArgs, args) + } + }) + } +} + +func TestGetProcessEnv(t *testing.T) { + tests := []struct { + name string + imageEnv []string + bundleEnv map[string]string + wantEnv []string + }{ + { + name: "Default", + imageEnv: []string{}, + bundleEnv: map[string]string{}, + wantEnv: []string{"LD_LIBRARY_PATH=/.singularity.d/libs"}, + }, + { + name: "ImagePath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "PATH=/foo", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "OverridePath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"PATH": "/bar"}, + wantEnv: []string{ + "PATH=/bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "AppendPath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"APPEND_PATH": "/bar"}, + wantEnv: []string{ + "PATH=/foo:/bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "PrependPath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"PREPEND_PATH": "/bar"}, + wantEnv: []string{ + "PATH=/bar:/foo", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageLdLibraryPath", + imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", + }, + }, + { + name: "BundleLdLibraryPath", + imageEnv: []string{}, + bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/foo"}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", + }, + }, + { + name: "OverrideLdLibraryPath", + imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, + bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/bar"}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/bar:/.singularity.d/libs", + }, + }, + { + name: "ImageVar", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "FOO=bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageOverride", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{"FOO": "baz"}, + wantEnv: []string{ + "FOO=baz", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageAdditional", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{"ABC": "123"}, + wantEnv: []string{ + "FOO=bar", + "ABC=123", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + imgSpec := imgspecv1.Image{ + Config: imgspecv1.ImageConfig{Env: tt.imageEnv}, + } + + env := getProcessEnv(imgSpec, tt.bundleEnv) + + if !reflect.DeepEqual(env, tt.wantEnv) { + t.Errorf("want: %v, got: %v", tt.wantEnv, env) + } + }) + } +} + +func TestLauncher_reverseMapByRange(t *testing.T) { + tests := []struct { + name string + targetUID uint32 + targetGID uint32 + subUIDMap specs.LinuxIDMapping + subGIDMap specs.LinuxIDMapping + wantUIDMap []specs.LinuxIDMapping + wantGIDMap []specs.LinuxIDMapping + wantErr bool + }{ + { + // TargetID is smaller than size of subuid/subgid map. + name: "LowTargetID", + targetUID: 1000, + targetGID: 2000, + subUIDMap: specs.LinuxIDMapping{HostID: 1000, ContainerID: 100000, Size: 65536}, + subGIDMap: specs.LinuxIDMapping{HostID: 2000, ContainerID: 200000, Size: 65536}, + wantUIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 1000}, + {ContainerID: 1000, HostID: 0, Size: 1}, + {ContainerID: 1001, HostID: 1001, Size: 64536}, + }, + wantGIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 2000}, + {ContainerID: 2000, HostID: 0, Size: 1}, + {ContainerID: 2001, HostID: 2001, Size: 63536}, + }, + }, + { + // TargetID is higher than size of subuid/subgid map. + name: "HighTargetID", + targetUID: 70000, + targetGID: 80000, + subUIDMap: specs.LinuxIDMapping{HostID: 1000, ContainerID: 100000, Size: 65536}, + subGIDMap: specs.LinuxIDMapping{HostID: 2000, ContainerID: 200000, Size: 65536}, + wantUIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 65536}, + {ContainerID: 70000, HostID: 0, Size: 1}, + }, + wantGIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 65536}, + {ContainerID: 80000, HostID: 0, Size: 1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotUIDMap, gotGIDMap := reverseMapByRange(tt.targetUID, tt.targetGID, tt.subUIDMap, tt.subGIDMap) + if !reflect.DeepEqual(gotUIDMap, tt.wantUIDMap) { + t.Errorf("Launcher.getReverseUserMaps() gotUidMap = %v, want %v", gotUIDMap, tt.wantUIDMap) + } + if !reflect.DeepEqual(gotGIDMap, tt.wantGIDMap) { + t.Errorf("Launcher.getReverseUserMaps() gotGidMap = %v, want %v", gotGIDMap, tt.wantGIDMap) + } + }) + } +} diff --git a/internal/pkg/runtime/launcher/oci/spec_linux.go b/internal/pkg/runtime/launcher/oci/spec_linux.go new file mode 100644 index 0000000000..42ba358898 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/spec_linux.go @@ -0,0 +1,122 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "os" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/pkg/sylog" + + "github.com/opencontainers/runtime-spec/specs-go" +) + +// defaultNamespaces matching native runtime with --compat / --containall. +var defaultNamespaces = []specs.LinuxNamespace{ + { + Type: specs.IPCNamespace, + }, + { + Type: specs.PIDNamespace, + }, + { + Type: specs.MountNamespace, + }, +} + +// minimalSpec returns an OCI runtime spec with a minimal OCI configuration that +// is a starting point for compatibility with Apptainer's native launcher in +// `--compat` mode. +func minimalSpec() specs.Spec { + config := specs.Spec{ + Version: specs.Version, + } + config.Root = &specs.Root{ + Path: "rootfs", + // TODO - support read-only. At present we always have a writable tmpfs overlay, like native runtime --compat. + Readonly: false, + } + config.Process = &specs.Process{ + Terminal: true, + // Default fallback to a shell at / - will generally be overwritten by + // the launcher. + Args: []string{"sh"}, + Cwd: "/", + } + config.Process.User = specs.User{} + config.Process.Env = []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + + // TODO - these are appropriate minimum for rootless. We need to tie into + // Apptainer's cap-add / cap-drop mechanism. + config.Process.Capabilities = &specs.LinuxCapabilities{ + Bounding: []string{ + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FOWNER", + "CAP_FSETID", + "CAP_KILL", + "CAP_NET_BIND_SERVICE", + "CAP_SETFCAP", + "CAP_SETGID", + "CAP_SETPCAP", + "CAP_SETUID", + "CAP_SYS_CHROOT", + }, + } + + // All mounts are added by the launcher, as it must handle flags. + config.Mounts = []specs.Mount{} + + config.Linux = &specs.Linux{Namespaces: defaultNamespaces} + return config +} + +// addNamespaces adds requested namespace, if appropriate, to an existing spec. +// It is assumed that spec contains at least the defaultNamespaces. +func addNamespaces(spec specs.Spec, ns launcher.Namespaces) specs.Spec { + if ns.IPC { + sylog.Infof("--oci runtime always uses an IPC namespace, ipc flag is redundant.") + } + + // Currently supports only `--network none`, i.e. isolated loopback only. + // Launcher.checkopts enforces this. + if ns.Net { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.NetworkNamespace}, + ) + } + + if ns.PID { + sylog.Infof("--oci runtime always uses a PID namespace, pid flag is redundant.") + } + + if ns.User { + if os.Getuid() == 0 { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UserNamespace}, + ) + } else { + sylog.Infof("--oci runtime always uses a user namespace when run as a non-root userns, user flag is redundant.") + } + } + + if ns.UTS { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UTSNamespace}, + ) + } + + return spec +} diff --git a/internal/pkg/runtime/launcher/oci/spec_linux_test.go b/internal/pkg/runtime/launcher/oci/spec_linux_test.go new file mode 100644 index 0000000000..7a1d4759dc --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/spec_linux_test.go @@ -0,0 +1,72 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "reflect" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/test" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func Test_addNamespaces(t *testing.T) { + test.DropPrivilege(t) + defer test.ResetPrivilege(t) + + tests := []struct { + name string + ns launcher.Namespaces + wantNS []specs.LinuxNamespace + }{ + { + name: "none", + ns: launcher.Namespaces{}, + wantNS: defaultNamespaces, + }, + { + name: "pid", + ns: launcher.Namespaces{PID: true}, + wantNS: defaultNamespaces, + }, + { + name: "ipc", + ns: launcher.Namespaces{IPC: true}, + wantNS: defaultNamespaces, + }, + { + name: "user", + ns: launcher.Namespaces{User: true}, + wantNS: defaultNamespaces, + }, + { + name: "net", + ns: launcher.Namespaces{Net: true}, + wantNS: append(defaultNamespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace}), + }, + { + name: "uts", + ns: launcher.Namespaces{UTS: true}, + wantNS: append(defaultNamespaces, specs.LinuxNamespace{Type: specs.UTSNamespace}), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := minimalSpec() + newSpec := addNamespaces(spec, tt.ns) + newNS := newSpec.Linux.Namespaces + if !reflect.DeepEqual(newNS, tt.wantNS) { + t.Errorf("addNamespaces() got %v, want %v", newNS, tt.wantNS) + } + }) + } +} diff --git a/internal/pkg/runtime/launch/options.go b/internal/pkg/runtime/launcher/options.go similarity index 80% rename from internal/pkg/runtime/launch/options.go rename to internal/pkg/runtime/launcher/options.go index 5e57689ed6..6ed9016a1b 100644 --- a/internal/pkg/runtime/launch/options.go +++ b/internal/pkg/runtime/launcher/options.go @@ -7,31 +7,29 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -// Package launcher is responsible for starting a container, with configuration -// passed to it from the CLI layer. -// -// The package currently implements a single Launcher, with an Exec method that -// constructs a runtime configuration and calls the Apptainer runtime starter -// binary to start the container. -// -// TODO - the launcher package will be extended to support launching containers -// via the OCI runc/crun runtime, in addition to the current Apptainer runtime -// starter. -package launch +package launcher import ( - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" - apptainerConfig "github.com/apptainer/apptainer/pkg/runtime/engine/apptainer/config" "github.com/apptainer/apptainer/pkg/util/cryptkey" + "github.com/containers/image/v5/types" ) -// launchOptions accumulates configuration from passed functional options. Note -// that the launchOptions is modified heavily by logic during the Exec function -// call. -type launchOptions struct { +// Namespaces holds flags for the optional (non-mount) namespaces that can be +// requested for a container launch. +type Namespaces struct { + User bool + UTS bool + PID bool + IPC bool + Net bool +} + +// Options accumulates launch configuration from passed functional options. Note +// that the Options is modified heavily by logic during the Exec function call. +type Options struct { // Writable marks the container image itself as writable. Writable bool - // WriteableTmpfs applies an ephemeral writable overlay to the container. + // WritableTmpfs applies an ephemeral writable overlay to the container. WritableTmpfs bool // OverlayPaths holds paths to image or directory overlays to be applied. OverlayPaths []string @@ -151,31 +149,23 @@ type launchOptions struct { UseBuildConfig bool TmpDir string Underlay bool // whether prefer underlay over overlay -} -type Launcher struct { - uid uint32 - gid uint32 - cfg launchOptions - engineConfig *apptainerConfig.EngineConfig - generator *generate.Generator -} + // SysContext holds Docker/OCI image handling configuration. + // This will be used by a launcher handling OCI images directly. + SysContext *types.SystemContext -// Namespaces holds flags for the optional (non-mount) namespaces that can be -// requested for a container launch. -type Namespaces struct { - User bool - UTS bool - PID bool - IPC bool - Net bool + // Devices contains the list of device mappings (if any), e.g. CDI mappings. + Devices []string + + // CdiDirs contains the list of directories in which CDI should look for device definition JSON files + CdiDirs []string } -type Option func(co *launchOptions) error +type Option func(co *Options) error // OptWritable sets the container image to be writable. func OptWritable(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Writable = b return nil } @@ -183,7 +173,7 @@ func OptWritable(b bool) Option { // OptWritableTmpFs applies an ephemeral writable overlay to the container. func OptWritableTmpfs(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.WritableTmpfs = b return nil } @@ -191,7 +181,7 @@ func OptWritableTmpfs(b bool) Option { // OptOverlayPaths sets overlay images and directories to apply to the container. func OptOverlayPaths(op []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.OverlayPaths = op return nil } @@ -199,7 +189,7 @@ func OptOverlayPaths(op []string) Option { // OptScratchDirs sets temporary host directories to create and bind into the container. func OptScratchDirs(sd []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ScratchDirs = sd return nil } @@ -207,7 +197,7 @@ func OptScratchDirs(sd []string) Option { // OptWorkDir sets the parent path for scratch directories, and contained home/tmp on the host. func OptWorkDir(wd string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.WorkDir = wd return nil } @@ -219,7 +209,7 @@ func OptWorkDir(wd string) Option { // custom is a marker that this is user supplied, and must not be overridden. // disable will disable the home mount entirely, ignoring other options. func OptHome(homeDir string, custom bool, disable bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.HomeDir = homeDir lo.CustomHome = custom lo.NoHome = disable @@ -233,7 +223,7 @@ func OptHome(homeDir string, custom bool, disable bool) Option { // mounts lists bind mount specifications in Docker CSV processed format. // fuseMounts list FUSE mounts in : format. func OptMounts(binds []string, mounts []string, fuseMounts []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.BindPaths = binds lo.Mounts = mounts lo.FuseMount = fuseMounts @@ -243,7 +233,7 @@ func OptMounts(binds []string, mounts []string, fuseMounts []string) Option { // OptNoMount disables the specified bind mounts. func OptNoMount(nm []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoMount = nm return nil } @@ -253,7 +243,7 @@ func OptNoMount(nm []string) Option { // // nvccli sets whether to use the nvidia-container-runtime (true), or legacy bind mounts (false). func OptNvidia(nv bool, nvccli bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Nvidia = nv || nvccli lo.NvCCLI = nvccli return nil @@ -262,7 +252,7 @@ func OptNvidia(nv bool, nvccli bool) Option { // OptNoNvidia disables NVIDIA GPU support, even if enabled via apptainer.conf. func OptNoNvidia(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoNvidia = b return nil } @@ -270,7 +260,7 @@ func OptNoNvidia(b bool) Option { // OptRocm enable Rocm GPU support. func OptRocm(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Rocm = b return nil } @@ -278,7 +268,7 @@ func OptRocm(b bool) Option { // OptNoRocm disables Rocm GPU support, even if enabled via apptainer.conf. func OptNoRocm(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoRocm = b return nil } @@ -286,7 +276,7 @@ func OptNoRocm(b bool) Option { // OptContainLibs mounts specified libraries into the container .singularity.d/libs dir. func OptContainLibs(cl []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ContainLibs = cl return nil } @@ -298,7 +288,7 @@ func OptContainLibs(cl []string) Option { // env is a map of name=value env vars to set. // clean removes host variables from the container environment. func OptEnv(env map[string]string, envFile string, clean bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Env = env lo.EnvFile = envFile lo.CleanEnv = clean @@ -308,7 +298,7 @@ func OptEnv(env map[string]string, envFile string, clean bool) Option { // OptNoEval disables shell evaluation of args and env vars. func OptNoEval(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoEval = b return nil } @@ -316,7 +306,7 @@ func OptNoEval(b bool) Option { // OptNamespaces enable the individual kernel-support namespaces for the container. func OptNamespaces(n Namespaces) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Namespaces = n return nil } @@ -327,7 +317,7 @@ func OptNamespaces(n Namespaces) Option { // network is the name of the CNI configuration to enable. // args are arguments to pass to the CNI plugin. func OptNetwork(network string, args []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Network = network lo.NetworkArgs = args return nil @@ -336,7 +326,7 @@ func OptNetwork(network string, args []string) Option { // OptHostname sets a hostname for the container (infers/requires UTS namespace). func OptHostname(h string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Hostname = h return nil } @@ -344,7 +334,7 @@ func OptHostname(h string) Option { // OptDNS sets a DNS entry for the container resolv.conf. func OptDNS(d string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.DNS = d return nil } @@ -352,7 +342,7 @@ func OptDNS(d string) Option { // OptCaps sets capabilities to add and drop. func OptCaps(add, drop string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.AddCaps = add lo.DropCaps = drop return nil @@ -361,7 +351,7 @@ func OptCaps(add, drop string) Option { // OptAllowSUID permits setuid executables inside a container started by the root user. func OptAllowSUID(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.AllowSUID = b return nil } @@ -369,7 +359,7 @@ func OptAllowSUID(b bool) Option { // OptKeepPrivs keeps all privileges inside a container started by the root user. func OptKeepPrivs(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.KeepPrivs = b return nil } @@ -377,7 +367,7 @@ func OptKeepPrivs(b bool) Option { // OptNoPrivs drops all privileges inside a container. func OptNoPrivs(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoPrivs = b return nil } @@ -385,7 +375,7 @@ func OptNoPrivs(b bool) Option { // OptSecurity supplies a list of security options (selinux, apparmor, seccomp) to apply. func OptSecurity(s []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.SecurityOpts = s return nil } @@ -393,7 +383,7 @@ func OptSecurity(s []string) Option { // OptNoUmask disables propagation of the host umask into the container, using a default 0022. func OptNoUmask(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoUmask = b return nil } @@ -401,7 +391,7 @@ func OptNoUmask(b bool) Option { // OptCgroupsJSON sets a Cgroups resource limit configuration to apply to the container. func OptCgroupsJSON(cj string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.CGroupsJSON = cj return nil } @@ -409,7 +399,7 @@ func OptCgroupsJSON(cj string) Option { // OptConfigFile specifies an alternate apptainer.conf that will be used by unprivileged installations only. func OptConfigFile(c string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ConfigFile = c return nil } @@ -417,7 +407,7 @@ func OptConfigFile(c string) Option { // OptShellPath specifies a custom shell executable to be launched in the container. func OptShellPath(s string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ShellPath = s return nil } @@ -425,7 +415,7 @@ func OptShellPath(s string) Option { // OptCwdPath specifies the initial working directory in the container. func OptCwdPath(p string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.CwdPath = p return nil } @@ -433,7 +423,7 @@ func OptCwdPath(p string) Option { // OptFakeroot enables the fake root mode, using user namespaces and subuid / subgid mapping. func OptFakeroot(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Fakeroot = b return nil } @@ -441,7 +431,7 @@ func OptFakeroot(b bool) Option { // OptBoot enables execution of /sbin/init on startup of an instance container. func OptBoot(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Boot = b return nil } @@ -449,7 +439,7 @@ func OptBoot(b bool) Option { // OptNoInit disables shim process when PID namespace is used. func OptNoInit(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoInit = b return nil } @@ -457,7 +447,7 @@ func OptNoInit(b bool) Option { // OptContain starts the container with minimal /dev and empty home/tmp mounts. func OptContain(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Contain = b return nil } @@ -465,7 +455,7 @@ func OptContain(b bool) Option { // OptContainAll infers Contain, and adds PID, IPC namespaces, and CleanEnv. func OptContainAll(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ContainAll = b return nil } @@ -473,7 +463,7 @@ func OptContainAll(b bool) Option { // OptAppName sets a SCIF application name to run. func OptAppName(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.AppName = a return nil } @@ -481,15 +471,15 @@ func OptAppName(a string) Option { // OptKeyInfo sets encryption key material to use when accessing an encrypted container image. func OptKeyInfo(ki *cryptkey.KeyInfo) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.KeyInfo = ki return nil } } -// CacheDisabled indicates caching of images was disabled in the CLI. +// OptCacheDisabled indicates caching of images was disabled in the CLI. func OptCacheDisabled(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.CacheDisabled = b return nil } @@ -497,7 +487,7 @@ func OptCacheDisabled(b bool) Option { // OptDMTCPLaunch func OptDMTCPLaunch(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.DMTCPLaunch = a return nil } @@ -505,7 +495,7 @@ func OptDMTCPLaunch(a string) Option { // OptDMTCPRestart func OptDMTCPRestart(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.DMTCPRestart = a return nil } @@ -513,7 +503,7 @@ func OptDMTCPRestart(a string) Option { // OptUnsquash func OptUnsquash(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Unsquash = b return nil } @@ -521,7 +511,7 @@ func OptUnsquash(b bool) Option { // OptIgnoreSubuid func OptIgnoreSubuid(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.IgnoreSubuid = b return nil } @@ -529,7 +519,7 @@ func OptIgnoreSubuid(b bool) Option { // OptIgnoreFakerootCmd func OptIgnoreFakerootCmd(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.IgnoreFakerootCmd = b return nil } @@ -537,7 +527,7 @@ func OptIgnoreFakerootCmd(b bool) Option { // OptIgnoreUserns func OptIgnoreUserns(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.IgnoreUserns = b return nil } @@ -545,7 +535,7 @@ func OptIgnoreUserns(b bool) Option { // OptUseBuildConfig func OptUseBuildConfig(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.UseBuildConfig = b return nil } @@ -553,7 +543,7 @@ func OptUseBuildConfig(b bool) Option { // OptTmpDir func OptTmpDir(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.TmpDir = a return nil } @@ -561,8 +551,32 @@ func OptTmpDir(a string) Option { // OptUnderlay func OptUnderlay(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Underlay = b return nil } } + +// OptSysContext sets Docker/OCI image handling configuration. +func OptSysContext(sc *types.SystemContext) Option { + return func(lo *Options) error { + lo.SysContext = sc + return nil + } +} + +// OptDevice sets CDI device mappings to apply. +func OptDevice(op []string) Option { + return func(lo *Options) error { + lo.Devices = op + return nil + } +} + +// OptCdiDirs sets CDI spec search-directories to apply. +func OptCdiDirs(op []string) Option { + return func(lo *Options) error { + lo.CdiDirs = op + return nil + } +} diff --git a/internal/pkg/runtime/launcher/util.go b/internal/pkg/runtime/launcher/util.go new file mode 100644 index 0000000000..f294df8fa3 --- /dev/null +++ b/internal/pkg/runtime/launcher/util.go @@ -0,0 +1,50 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package launcher + +import ( + "fmt" + "strings" + + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/fs/proc" +) + +// WithPrivilege calls fn if cond is satisfied, and we are uid 0 +func WithPrivilege(uid uint32, cond bool, desc string, fn func() error) error { + if !cond { + return nil + } + if uid != 0 { + return fmt.Errorf("%s requires root privileges", desc) + } + return fn() +} + +// HidepidProc checks if hidepid is set on the /proc mount point. +// +// If this is set then an instance started in the with setuid workflow cannot be +func HidepidProc() bool { + entries, err := proc.GetMountInfoEntry("/proc/self/mountinfo") + if err != nil { + sylog.Warningf("while reading /proc/self/mountinfo: %s", err) + return false + } + for _, e := range entries { + if e.Point == "/proc" { + for _, o := range e.SuperOptions { + if strings.HasPrefix(o, "hidepid=") { + return true + } + } + } + } + return false +} diff --git a/internal/pkg/test/tool/dirs/mkdir.go b/internal/pkg/test/tool/dirs/mkdir.go new file mode 100644 index 0000000000..08f1d25059 --- /dev/null +++ b/internal/pkg/test/tool/dirs/mkdir.go @@ -0,0 +1,15 @@ +package dirs + +import ( + "os" + "testing" +) + +func MkdirOrFatal(t *testing.T, dir string, perm os.FileMode) { + if err := os.Mkdir(dir, perm); err != nil { + t.Fatalf("could not create %q: %s", dir, err) + } + if err := os.Chmod(dir, perm); err != nil { + t.Fatalf("could not chmod %q to %o: %s", dir, perm, err) + } +} diff --git a/internal/pkg/util/bin/bin.go b/internal/pkg/util/bin/bin.go index d11283c963..16f80d5cfd 100644 --- a/internal/pkg/util/bin/bin.go +++ b/internal/pkg/util/bin/bin.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -47,13 +47,16 @@ func FindBin(name string) (path string, err error) { return findOnPath(name, true) // All other executables // We will always search the user's PATH first for these - case "curl", + case "conmon", + "crun", + "curl", "debootstrap", "dnf", "fakeroot", "fakeroot-sysv", "fuse-overlayfs", "fuse2fs", + "fusermount", "go", "ldconfig", "mksquashfs", @@ -63,6 +66,7 @@ func FindBin(name string) (path string, err error) { "pacstrap", "rpm", "rpmkeys", + "runc", "squashfuse", "squashfuse_ll", "SUSEConnect", @@ -71,8 +75,9 @@ func FindBin(name string) (path string, err error) { "zypper", "gocryptfs": return findOnPath(name, false) + default: + return "", fmt.Errorf("executable name %q is not known to FindBin", name) } - return "", fmt.Errorf("unknown executable name %q", name) } // findOnPath performs a search on the configurated binary path for the diff --git a/internal/pkg/util/fs/helper.go b/internal/pkg/util/fs/helper.go index f44b5032e1..3af4f7d7b2 100644 --- a/internal/pkg/util/fs/helper.go +++ b/internal/pkg/util/fs/helper.go @@ -92,7 +92,7 @@ func IsLink(name string) bool { return info.Mode()&os.ModeSymlink != 0 } -// IsOwner check if name component is owned by user identified with uid. +// IsOwner checks if named file is owned by user identified with uid. func IsOwner(name string, uid uint32) bool { info, err := os.Stat(name) if err != nil { @@ -101,6 +101,15 @@ func IsOwner(name string, uid uint32) bool { return info.Sys().(*syscall.Stat_t).Uid == uid } +// IsGroup checks if named file is owned by group identified with gid. +func IsGroup(name string, gid uint32) bool { + info, err := os.Stat(name) + if err != nil { + return false + } + return info.Sys().(*syscall.Stat_t).Gid == gid +} + // IsExec check if name component has executable bit permission set. func IsExec(name string) bool { info, err := os.Stat(name) diff --git a/internal/pkg/util/fs/helper_linux_test.go b/internal/pkg/util/fs/helper_linux_test.go index fa07485722..4af90fe61d 100644 --- a/internal/pkg/util/fs/helper_linux_test.go +++ b/internal/pkg/util/fs/helper_linux_test.go @@ -167,7 +167,16 @@ func TestIsOwner(t *testing.T) { defer test.ResetPrivilege(t) if IsOwner("/etc/passwd", 0) != true { - t.Errorf("IsOwner returns false for /etc/passwd owner") + t.Errorf("IsOwner returns false for /etc/passwd root ownership") + } +} + +func TestIsGroup(t *testing.T) { + test.DropPrivilege(t) + defer test.ResetPrivilege(t) + + if IsGroup("/etc/passwd", 0) != true { + t.Errorf("IsGroup returns false for /etc/passwd root group ownership") } } diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux.go b/internal/pkg/util/fs/overlay/overlay_item_linux.go new file mode 100644 index 0000000000..4eabc08eb4 --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_item_linux.go @@ -0,0 +1,348 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/pkg/image" + "github.com/apptainer/apptainer/pkg/sylog" +) + +// Item represents information about a single overlay item (as specified, +// for example, in a single --overlay argument) +type Item struct { + // Type represents what type of overlay this is (from among the values in + // pkg/image) + Type int + + // Writable represents whether this is a writable overlay + Writable bool + + // SourcePath is the path of the overlay item stripped of any colon-prefixed + // options (like ":ro") + SourcePath string + + // StagingDir is the directory on which this overlay item is staged, to be + // used as a source for an overlayfs mount as part of an overlay.Set + StagingDir string + + // parentDir is the (optional) location of a secure parent-directory in + // which to create mount directories if needed. If empty, one will be + // created with os.MkdirTemp() + parentDir string +} + +// NewItemFromString takes a string argument, as passed to --overlay, and +// returns an Item struct describing the requested overlay. +func NewItemFromString(overlayString string) (*Item, error) { + item := Item{Writable: true} + + var err error + splitted := strings.SplitN(overlayString, ":", 2) + item.SourcePath, err = filepath.Abs(splitted[0]) + if err != nil { + return nil, fmt.Errorf("error while trying to convert overlay path %q to absolute path: %w", splitted[0], err) + } + + if len(splitted) > 1 { + if splitted[1] == "ro" { + item.Writable = false + } + } + + s, err := os.Stat(item.SourcePath) + if (err != nil) && os.IsNotExist(err) { + return nil, fmt.Errorf("specified overlay %q does not exist", item.SourcePath) + } + if err != nil { + return nil, err + } + + if s.IsDir() { + item.Type = image.SANDBOX + } else if err := item.analyzeImageFile(); err != nil { + return nil, fmt.Errorf("while examining image file %s: %w", item.SourcePath, err) + } + + return &item, nil +} + +// analyzeImageFile attempts to determine the format of an image file based on +// its header +func (i *Item) analyzeImageFile() error { + img, err := image.Init(i.SourcePath, false) + if err != nil { + return err + } + + switch img.Type { + case image.SQUASHFS: + i.Type = image.SQUASHFS + // squashfs image must be readonly + i.Writable = false + case image.EXT3: + i.Type = image.EXT3 + default: + return fmt.Errorf("image %s is of a type that is not currently supported as overlay", i.SourcePath) + } + + return nil +} + +// SetParentDir sets the parent-dir in which to create overlay-specific mount +// directories. +func (i *Item) SetParentDir(d string) { + i.parentDir = d +} + +// GetParentDir gets a parent-dir in which to create overlay-specific mount +// directories. If one has not been set using SetParentDir(), one will be +// created using os.MkdirTemp(). +func (i *Item) GetParentDir() (string, error) { + // Check if we've already been given a parentDir value; if not, create + // one using os.MkdirTemp() + if len(i.parentDir) > 0 { + return i.parentDir, nil + } + + d, err := os.MkdirTemp("", "overlay-parent-") + if err != nil { + return d, err + } + + i.parentDir = d + return i.parentDir, nil +} + +// Mount performs the necessary steps to mount an individual Item. Note that +// this method does not mount the assembled overlay itself. That happens in +// Set.Mount(). +func (i *Item) Mount() error { + var err error + switch i.Type { + case image.SANDBOX: + err = i.mountDir() + case image.SQUASHFS: + err = i.mountWithFuse("squashfuse") + case image.EXT3: + if i.Writable { + err = i.mountWithFuse("fuse2fs", "-o", "rw") + } else { + err = i.mountWithFuse("fuse2fs", "-o", "ro") + } + default: + return fmt.Errorf("internal error: unrecognized image type in overlay.Item.Mount() (type: %v)", i.Type) + } + + if err != nil { + return err + } + + if i.Writable { + return i.prepareWritableOverlay() + } + + return nil +} + +// GetMountDir returns the path to the directory that will actually be mounted +// for this overlay. For squashfs overlays, this is equivalent to the +// Item.StagingDir field. But for all other overlays, it is the "upper" +// subdirectory of Item.StagingDir. +func (i Item) GetMountDir() string { + switch i.Type { + case image.SQUASHFS: + return i.StagingDir + + case image.SANDBOX: + if i.Writable || fs.IsDir(i.Upper()) { + return i.Upper() + } + return i.StagingDir + + default: + return i.Upper() + } +} + +// mountDir mounts directory-based Items. This involves bind-mounting followed +// by remounting of the directory onto itself. This pattern of bind-mount +// followed by remount allows application of more restrictive mount flags than +// are in force on the underlying filesystem. +func (i *Item) mountDir() error { + var err error + if len(i.StagingDir) < 1 { + i.StagingDir = i.SourcePath + } + + if err = EnsureOverlayDir(i.StagingDir, false, 0); err != nil { + return fmt.Errorf("error accessing directory %s: %w", i.StagingDir, err) + } + + sylog.Debugf("Performing identity bind-mount of %q", i.StagingDir) + if err = syscall.Mount(i.StagingDir, i.StagingDir, "", syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to bind %s: %w", i.StagingDir, err) + } + + // Best effort to cleanup mount + defer func() { + if err != nil { + sylog.Debugf("Encountered error with current overlay set; attempting to unmount %q", i.StagingDir) + syscall.Unmount(i.StagingDir, syscall.MNT_DETACH) + } + }() + + // Try to perform remount + sylog.Debugf("Performing remount of %q", i.StagingDir) + if err = syscall.Mount("", i.StagingDir, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to remount %s: %w", i.StagingDir, err) + } + + return nil +} + +// mountWithFuse mounts an image to a temporary directory using a specified fuse +// tool. It also verifies that fusermount is present before performing the +// mount. +func (i *Item) mountWithFuse(fuseMountTool string, additionalArgs ...string) error { + var err error + fuseMountCmd, err := bin.FindBin(fuseMountTool) + if err != nil { + return fmt.Errorf("use of image %q as overlay requires %s to be installed: %w", i.SourcePath, fuseMountTool, err) + } + + // Even though fusermount is not needed for this step, we shouldn't perform + // the mount unless we have the necessary tools to eventually unmount it + _, err = bin.FindBin("fusermount") + if err != nil { + return fmt.Errorf("use of image %q as overlay requires fusermount to be installed: %w", i.SourcePath, err) + } + + // Obtain parent directory in which to create overlay-related mount + // directories. See https://github.com/apptainer/singularity/pull/5575 for + // related discussion. + parentDir, err := i.GetParentDir() + if err != nil { + return fmt.Errorf("error while trying to create parent dir for overlay %q: %w", i.SourcePath, err) + } + fuseMountDir, err := os.MkdirTemp(parentDir, "overlay-mountpoint-") + if err != nil { + return fmt.Errorf("failed to create temporary dir %q for overlay %q: %w", fuseMountDir, i.SourcePath, err) + } + + // Best effort to cleanup temporary dir + defer func() { + if err != nil { + sylog.Debugf("Encountered error with current overlay set; attempting to remove %q", fuseMountDir) + os.Remove(fuseMountDir) + } + }() + + args := make([]string, 0, len(additionalArgs)+4) + + // TODO: Think through what makes sense for file ownership in FUSE-mounted + // images, vis a vis id-mappings and user-namespaces. + args = append(args, "-o") + args = append(args, "uid=0,gid=0") + + args = append(args, i.SourcePath) + args = append(args, fuseMountDir) + sylog.Debugf("Executing FUSE mount command: %s %s", fuseMountCmd, strings.Join(args, " ")) + execCmd := exec.Command(fuseMountCmd, args...) + execCmd.Stderr = os.Stderr + _, err = execCmd.Output() + if err != nil { + return fmt.Errorf("encountered error while trying to mount image %q as overlay at %s: %w", i.SourcePath, fuseMountDir, err) + } + i.StagingDir = fuseMountDir + + return nil +} + +// Unmount performs the necessary steps to unmount an individual Item. Note that +// this method does not unmount the overlay itself. That happens in +// Set.Unmount(). +func (i Item) Unmount() error { + switch i.Type { + case image.SANDBOX: + return i.unmountDir() + + case image.SQUASHFS: + fallthrough + case image.EXT3: + return i.unmountFuse() + + default: + return fmt.Errorf("internal error: unrecognized image type in overlay.Item.Unmount() (type: %v)", i.Type) + } +} + +// unmountDir unmounts directory-based Items. +func (i Item) unmountDir() error { + return DetachMount(i.StagingDir) +} + +// unmountFuse unmounts FUSE-based Items. +func (i Item) unmountFuse() error { + defer os.Remove(i.StagingDir) + err := UnmountWithFuse(i.StagingDir) + if err != nil { + return fmt.Errorf("error while trying to unmount image %q from %s: %w", i.SourcePath, i.StagingDir, err) + } + return nil +} + +// PrepareWritableOverlay ensures that the upper and work subdirs of a writable +// overlay dir exist, and if not, creates them. +func (i *Item) prepareWritableOverlay() error { + switch i.Type { + case image.SANDBOX: + i.StagingDir = i.SourcePath + fallthrough + case image.EXT3: + if err := EnsureOverlayDir(i.StagingDir, true, 0o755); err != nil { + return err + } + sylog.Debugf("Ensuring %q exists", i.Upper()) + if err := EnsureOverlayDir(i.Upper(), true, 0o755); err != nil { + sylog.Errorf("Could not create overlay upper dir. If using an overlay image ensure it contains 'upper' and 'work' directories") + return fmt.Errorf("err encountered while preparing upper subdir of overlay dir %q: %w", i.Upper(), err) + } + sylog.Debugf("Ensuring %q exists", i.Work()) + if err := EnsureOverlayDir(i.Work(), true, 0o700); err != nil { + sylog.Errorf("Could not create overlay work dir. If using an overlay image ensure it contains 'upper' and 'work' directories") + return fmt.Errorf("err encountered while preparing work subdir of overlay dir %q: %w", i.Work(), err) + } + default: + return fmt.Errorf("unsupported image type in prepareWritableOverlay() (type: %v)", i.Type) + } + + return nil +} + +// Upper returns the "upper"-subdir of the Item's DirToMount field. +// Useful for computing options strings for overlay-related mount system calls. +func (i Item) Upper() string { + return filepath.Join(i.StagingDir, "upper") +} + +// Work returns the "work"-subdir of the Item's DirToMount field. Useful +// for computing options strings for overlay-related mount system calls. +func (i Item) Work() string { + return filepath.Join(i.StagingDir, "work") +} diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go new file mode 100644 index 0000000000..bd7320bdf4 --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go @@ -0,0 +1,300 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/test/tool/dirs" + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/pkg/image" +) + +const ( + testFilePath string = "file-for-testing" + squashfsTestString string = "squashfs-test-string\n" + extfsTestString string = "extfs-test-string\n" +) + +var ( + imgsPath = filepath.Join("..", "..", "..", "..", "..", "test", "images") + squashfsImgPath = filepath.Join(imgsPath, "squashfs-for-overlay.img") + extfsImgPath = filepath.Join(imgsPath, "extfs-for-overlay.img") +) + +func mkTempDirOrFatal(t *testing.T) string { + tmpDir, err := os.MkdirTemp(t.TempDir(), "testoverlayitem-") + if err != nil { + t.Fatalf("failed to create temporary dir: %s", err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(tmpDir) + } + }) + + return tmpDir +} + +func mkTempOlDirOrFatal(t *testing.T) string { + tmpOlDir := mkTempDirOrFatal(t) + dirs.MkdirOrFatal(t, filepath.Join(tmpOlDir, "upper"), 0o777) + dirs.MkdirOrFatal(t, filepath.Join(tmpOlDir, "lower"), 0o777) + + return tmpOlDir +} + +func TestItemWritableField(t *testing.T) { + tmpOlDir := mkTempOlDirOrFatal(t) + rwOverlayStr := tmpOlDir + roOverlayStr := tmpOlDir + ":ro" + + rwItem, err := NewItemFromString(rwOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", rwOverlayStr, err) + } + roItem, err := NewItemFromString(roOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing roItem from string %q: %s", roOverlayStr, err) + } + + if !rwItem.Writable { + t.Errorf("Writable field of overlay.Item initialized with string %q should be true but is false", rwOverlayStr) + } + + if roItem.Writable { + t.Errorf("Writable field of overlay.Item initialized with string %q should be false but is true", roOverlayStr) + } +} + +func TestItemMissing(t *testing.T) { + const dir string = "/testoverlayitem-this_should_be_missing" + rwOverlayStr := dir + roOverlayStr := dir + ":ro" + + if _, err := NewItemFromString(rwOverlayStr); err == nil { + t.Errorf("unexpected success: initializing overlay.Item with missing file/dir (%q) should have failed", rwOverlayStr) + } + if _, err := NewItemFromString(roOverlayStr); err == nil { + t.Errorf("unexpected success: initializing overlay.Item with missing file/dir (%q) should have failed", roOverlayStr) + } +} + +func verifyAutoParentDir(t *testing.T, item *Item) { + const autoParentDirStr string = "overlay-parent-" + if parentDir, err := item.GetParentDir(); err != nil { + t.Fatalf("unexpected error while calling Item.GetParentDir(): %s", err) + } else if !strings.Contains(parentDir, autoParentDirStr) { + t.Errorf("auto-generated parent dir %q does not contain expected identifier string %q", parentDir, autoParentDirStr) + } else if !strings.HasPrefix(parentDir, "/tmp/") { + t.Errorf("auto-generated parent dir %q is not in expected location", parentDir) + } +} + +func TestAutofillParentDir(t *testing.T) { + tmpOlDir := mkTempOlDirOrFatal(t) + rwOverlayStr := tmpOlDir + roOverlayStr := tmpOlDir + ":ro" + + rwItem, err := NewItemFromString(rwOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", rwOverlayStr, err) + } + roItem, err := NewItemFromString(roOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing roItem from string %q: %s", roOverlayStr, err) + } + + verifyAutoParentDir(t, rwItem) + verifyAutoParentDir(t, roItem) +} + +func verifyExplicitParentDir(t *testing.T, item *Item, dir string) { + item.SetParentDir(dir) + if parentDir, err := item.GetParentDir(); err != nil { + t.Fatalf("unexpected error while calling Item.GetParentDir(): %s", err) + } else if parentDir != dir { + t.Errorf("item returned parent dir %q (expected: %q)", parentDir, dir) + } +} + +func TestExplicitParentDir(t *testing.T) { + tmpOlDir := mkTempOlDirOrFatal(t) + rwOverlayStr := tmpOlDir + roOverlayStr := tmpOlDir + ":ro" + + rwItem, err := NewItemFromString(rwOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", rwOverlayStr, err) + } + roItem, err := NewItemFromString(roOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing roItem from string %q: %s", roOverlayStr, err) + } + + verifyExplicitParentDir(t, rwItem, "/my-special-directory") + verifyExplicitParentDir(t, roItem, "/my-other-special-directory") +} + +func verifyDirExistsAndWritable(t *testing.T, dir string) { + s, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + t.Errorf("expected directory %q not found", dir) + } else { + t.Fatalf("unexpected error while looking for directory %q: %s", dir, err) + } + return + } + + if !s.IsDir() { + t.Fatalf("expected %q to be a directory but it is not", dir) + return + } + + file, err := os.CreateTemp(dir, "attempt-to-write-a-file") + if err != nil { + t.Fatalf("could not create a file inside %q, which should have been writable: %s", dir, err) + } + path := file.Name() + file.Close() + if err := os.Remove(path); err != nil { + t.Fatalf("unexpected error while trying to remove temporary file %q: %s", path, err) + } +} + +func TestUpperAndWorkCreation(t *testing.T) { + tmpDir := mkTempDirOrFatal(t) + + item, err := NewItemFromString(tmpDir) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", tmpDir, err) + } + + if err := item.prepareWritableOverlay(); err != nil { + t.Fatalf("unexpected error while calling prepareWritableOverlay(): %s", err) + } + + verifyDirExistsAndWritable(t, tmpDir+"/upper") + verifyDirExistsAndWritable(t, tmpDir+"/work") +} + +func dirMountUnmount(t *testing.T, olStr string) { + item, err := NewItemFromString(olStr) + if err != nil { + t.Fatalf("unexpected error while initializing overlay item from string %q: %s", olStr, err) + } + + if err := item.Mount(); err != nil { + t.Fatalf("while trying to mount dir %q: %s", olStr, err) + } + if err := item.Unmount(); err != nil { + t.Errorf("while trying to unmount dir %q: %s", olStr, err) + } +} + +func TestDirMounts(t *testing.T) { + dirMountUnmount(t, mkTempOlDirOrFatal(t)+":ro") + dirMountUnmount(t, mkTempOlDirOrFatal(t)) +} + +func tryImageRO(t *testing.T, olStr string, typeCode int, typeStr, expectStr string) { + item, err := NewItemFromString(olStr) + if err != nil { + t.Fatalf("failed to mount %s image at %q: %s", typeStr, olStr, err) + } + + if item.Type != typeCode { + t.Errorf("item.Type is %v (should be %v)", item.Type, typeStr) + } + + if err := item.Mount(); err != nil { + t.Fatalf("unable to mount %s image for reading: %s", typeStr, err) + } + t.Cleanup(func() { + item.Unmount() + }) + + testFileStagedPath := filepath.Join(item.GetMountDir(), testFilePath) + checkForStringInOverlay(t, typeStr, testFileStagedPath, expectStr) +} + +func TestSquashfsRO(t *testing.T) { + require.Command(t, "squashfuse") + require.Command(t, "fusermount") + tryImageRO(t, squashfsImgPath, image.SQUASHFS, "squashfs", squashfsTestString) +} + +func TestExtfsRO(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fusermount") + tmpDir := mkTempDirOrFatal(t) + readonlyExtfsImgPath := filepath.Join(tmpDir, "readonly-extfs.img") + if err := fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444); err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err) + } + tryImageRO(t, readonlyExtfsImgPath+":ro", image.EXT3, "extfs", extfsTestString) +} + +func TestExtfsRW(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + tmpDir := mkTempDirOrFatal(t) + + // Create a copy of the extfs test image to be used for testing writable + // extfs image overlays + writableExtfsImgPath := filepath.Join(tmpDir, "writable-extfs.img") + err := fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err) + } + + item, err := NewItemFromString(writableExtfsImgPath) + if err != nil { + t.Fatalf("failed to mount extfs image at %q: %s", writableExtfsImgPath, err) + } + + if item.Type != image.EXT3 { + t.Errorf("item.Type is %v (should be %v)", item.Type, image.EXT3) + } + + if err := item.Mount(); err != nil { + t.Fatalf("unable to mount extfs image for reading & writing: %s", err) + } + t.Cleanup(func() { + item.Unmount() + }) + + testFileStagedPath := filepath.Join(item.GetMountDir(), testFilePath) + checkForStringInOverlay(t, "extfs", testFileStagedPath, extfsTestString) + otherTestFileStagedPath := item.GetMountDir() + "_other" + otherExtfsTestString := "another string" + err = os.WriteFile(otherTestFileStagedPath, []byte(otherExtfsTestString), 0o755) + if err != nil { + t.Errorf("could not write to file %q in extfs image %q: %s", otherTestFileStagedPath, writableExtfsImgPath, err) + } + checkForStringInOverlay(t, "extfs", otherTestFileStagedPath, otherExtfsTestString) +} + +func checkForStringInOverlay(t *testing.T, typeStr, stagedPath, expectStr string) { + data, err := os.ReadFile(stagedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", stagedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in %s img: expected %q, but found: %q", typeStr, expectStr, foundStr) + } +} diff --git a/internal/pkg/util/fs/overlay/overlay_linux.go b/internal/pkg/util/fs/overlay/overlay_linux.go index d30c9ffcc4..5da3d3c678 100644 --- a/internal/pkg/util/fs/overlay/overlay_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_linux.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "os/exec" + "sync" "syscall" "github.com/apptainer/apptainer/internal/pkg/util/bin" @@ -34,7 +35,7 @@ const ( fuseDir ) -type fs struct { +type filesys struct { name string overlayDir dir } @@ -48,7 +49,7 @@ const ( Panfs int64 = 0xAAD7AAEA ) -var incompatibleFs = map[int64]fs{ +var incompatibleFilesys = map[int64]filesys{ // NFS filesystem Nfs: { name: "NFS", @@ -86,10 +87,10 @@ func check(path string, d dir) error { stfs := &unix.Statfs_t{} if err := statfs(path, stfs); err != nil { - return fmt.Errorf("could not retrieve underlying filesystem information for %s: %s", path, err) + return fmt.Errorf("could not retrieve underlying filesystem information for %s: %w", path, err) } - fs, ok := incompatibleFs[int64(stfs.Type)] + fs, ok := incompatibleFilesys[int64(stfs.Type)] if !ok || (ok && fs.overlayDir&d == 0) { return nil } @@ -152,7 +153,7 @@ var ErrNoRootlessOverlay = errors.New("rootless overlay not supported by kernel" func CheckRootless() error { mountBin, err := bin.FindBin("mount") if err != nil { - return fmt.Errorf("while looking for mount command: %s", err) + return fmt.Errorf("while looking for mount command: %w", err) } args := []string{ @@ -190,3 +191,104 @@ func CheckRootless() error { sylog.Debugf("Rootless overlay appears supported on this system.") return nil } + +// Info about kernel support for unprivileged overlays +var unprivOverlays struct { + kernelSupport bool + initOnce sync.Once + err error +} + +// UnprivOverlaysSupported checks whether there is kernel support for unprivileged overlays. The actual check is performed only once and cached in the unprivOverlays variable, above. +func UnprivOverlaysSupported() (bool, error) { + unprivOverlays.initOnce.Do(func() { + err := CheckRootless() + if err == nil { + unprivOverlays.kernelSupport = true + return + } + + if err == ErrNoRootlessOverlay { + unprivOverlays.kernelSupport = false + return + } + + unprivOverlays.err = err + }) + + if unprivOverlays.err != nil { + return false, unprivOverlays.err + } + + return unprivOverlays.kernelSupport, nil +} + +// ensureOverlayDir checks if a directory already exists; if it doesn't, and +// createIfMissing is true, it attempts to create it with the specified +// permissions. +func EnsureOverlayDir(dir string, createIfMissing bool, createPerm os.FileMode) error { + if len(dir) == 0 { + return fmt.Errorf("internal error: ensureOverlayDir() called with empty dir name") + } + + _, err := os.Stat(dir) + if err == nil { + return nil + } + + if !os.IsNotExist(err) { + return err + } + + if !createIfMissing { + return fmt.Errorf("missing overlay dir %q", dir) + } + + // Create the requested dir + if err := os.Mkdir(dir, createPerm); err != nil { + return fmt.Errorf("failed to create %q: %w", dir, err) + } + + return nil +} + +// detachAndDelete performs an unmount system call on the specified directory, +// followed by deletion of the directory and all of its contents. +func DetachAndDelete(overlayDir string) error { + sylog.Debugf("Detaching overlayDir %q", overlayDir) + if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to unmount %s: %w", overlayDir, err) + } + + sylog.Debugf("Removing overlayDir %q", overlayDir) + if err := os.RemoveAll(overlayDir); err != nil { + return fmt.Errorf("failed to remove %s: %w", overlayDir, err) + } + return nil +} + +// DetachMount performs an unmount system call on the specified directory. +func DetachMount(dir string) error { + sylog.Debugf("Calling syscall.Unmount() to detach %q", dir) + if err := syscall.Unmount(dir, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to detach %s: %w", dir, err) + } + + return nil +} + +// UnmountWithFuse performs an unmount on the specified directory using +// fusermount -u. +func UnmountWithFuse(dir string) error { + fusermountCmd, err := bin.FindBin("fusermount") + if err != nil { + // We should not be creating FUSE-based mounts in the first place + // without checking that fusermount is available. + return fmt.Errorf("fusermount not available while trying to perform unmount: %w", err) + } + sylog.Debugf("Executing FUSE unmount command: %s -u %s", fusermountCmd, dir) + execCmd := exec.Command(fusermountCmd, "-u", dir) + execCmd.Stderr = os.Stderr + _, err = execCmd.Output() + return err +} diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux.go b/internal/pkg/util/fs/overlay/overlay_set_linux.go new file mode 100644 index 0000000000..630e3257f1 --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_set_linux.go @@ -0,0 +1,200 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "fmt" + "os" + "os/exec" + "strings" + "syscall" + + "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/pkg/image" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/samber/lo" +) + +// Set represents a set of overlay directories which will be overlain on top of +// some filesystem mount point. The actual mount point atop which these +// directories will be overlain is not specified in the Set; it is left +// implicit, to be chosen by whichever function consumes a Set. A Set contains +// two types of directories: zero or more directories which will be mounted as +// read-only overlays atop the (implicit) mount point, and one directory which +// will be mounted as a writable overlay atop all the rest. An empty WritableLoc +// field indicates that no writable overlay is to be mounted. +type Set struct { + // ReadonlyOverlays is a list of directories to be mounted as read-only + // overlays. The mount point atop which these will be mounted is left + // implicit, to be chosen by whichever function consumes the Set. + ReadonlyOverlays []*Item + + // WritableOverlay is the directory to be mounted as a writable overlay. The + // mount point atop which this will be mounted is left implicit, to be + // chosen by whichever function consumes the Set. Empty value indicates no + // writable overlay is to be mounted. + WritableOverlay *Item +} + +// Mount prepares and mounts the entire Set onto the specified rootfs +// directory. +func (s Set) Mount(rootFsDir string) error { + // Perform identity mounts for this Set + dups := lo.FindDuplicatesBy(s.ReadonlyOverlays, func(item *Item) string { + return item.SourcePath + }) + if len(dups) > 0 { + return fmt.Errorf("duplicate overlays detected: %v", lo.Map(dups, func(item *Item, _ int) string { + return item.SourcePath + })) + } + + if err := s.performIndividualMounts(); err != nil { + return err + } + + // Perform actual overlay mount + return s.performFinalMount(rootFsDir) +} + +// UnmountOverlay ummounts a Set from a specified rootfs directory. +func (s Set) Unmount(rootFsDir string) error { + unprivOls, err := UnprivOverlaysSupported() + if err != nil { + return fmt.Errorf("while checking for unprivileged overlay support in kernel: %w", err) + } + + useKernelMount := unprivOls && !s.hasWritableExtfsImg() + if useKernelMount { + err = DetachMount(rootFsDir) + } else { + err = UnmountWithFuse(rootFsDir) + } + + if err != nil { + return err + } + + return s.detachIndividualMounts() +} + +// performIndividualMounts creates the mounts that furnish the individual +// elements of the Set. +func (s Set) performIndividualMounts() error { + overlaysToBind := s.ReadonlyOverlays + if s.WritableOverlay != nil { + overlaysToBind = append(overlaysToBind, s.WritableOverlay) + } + + // Try to do initial bind-mounts + for _, o := range overlaysToBind { + if err := o.Mount(); err != nil { + return err + } + } + + return nil +} + +// performFinalMount performs the final step in mounting a Set, namely mounting +// of the overlay with its full-fledged options string, representing all the +// individual Items (writable and read-only) that comprise the Set. +func (s Set) performFinalMount(rootFsDir string) error { + // Try to perform actual mount + options := s.options(rootFsDir) + unprivOls, err := UnprivOverlaysSupported() + if err != nil { + return fmt.Errorf("while checking for unprivileged overlay support in kernel: %w", err) + } + + useKernelMount := unprivOls && !s.hasWritableExtfsImg() + + if useKernelMount { + flags := uintptr(syscall.MS_NODEV) + sylog.Debugf("Mounting overlay (via syscall) with rootFsDir %q, options: %q, mount flags: %#v", rootFsDir, options, flags) + if err := syscall.Mount("overlay", rootFsDir, "overlay", flags, options); err != nil { + return fmt.Errorf("failed to mount %s: %w", rootFsDir, err) + } + } else { + fuseOlFsCmd, err := bin.FindBin("fuse-overlayfs") + if err != nil { + return fmt.Errorf("'fuse-overlayfs' must be used for this overlay specification, but is not available: %w", err) + } + + // Even though fusermount is not needed for this step, we shouldn't perform + // the mount unless we have the necessary tools to eventually unmount it + _, err = bin.FindBin("fusermount") + if err != nil { + return fmt.Errorf("'fuse-overlayfs' must be used for this overlay specification, and this also requires 'fusermount' to be installed: %w", err) + } + + sylog.Debugf("Mounting overlay (via fuse-overlayfs) with rootFsDir %q, options: %q", rootFsDir, options) + execCmd := exec.Command(fuseOlFsCmd, "-o", options, rootFsDir) + execCmd.Stderr = os.Stderr + _, err = execCmd.Output() + if err != nil { + return fmt.Errorf("failed to mount %s: %w", rootFsDir, err) + } + } + + return nil +} + +// options creates an options string to be used in an overlay mount, +// representing all the individual Items (writable and read-only) that comprise +// the Set. +func (s Set) options(rootFsDir string) string { + // Create lowerdir argument of options string + lowerDirs := lo.Map(s.ReadonlyOverlays, func(o *Item, _ int) string { + return o.GetMountDir() + }) + lowerDirJoined := strings.Join(append(lowerDirs, rootFsDir), ":") + + if s.WritableOverlay == nil { + return fmt.Sprintf("lowerdir=%s", lowerDirJoined) + } + + return fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", + lowerDirJoined, s.WritableOverlay.Upper(), s.WritableOverlay.Work()) +} + +func (s Set) hasWritableExtfsImg() bool { + if (s.WritableOverlay != nil) && (s.WritableOverlay.Type == image.EXT3) { + return true + } + + return false +} + +// detachIndividualMounts detaches the bind mounts & remounts created by +// performIndividualMounts, above. +func (s Set) detachIndividualMounts() error { + overlaysToDetach := s.ReadonlyOverlays + if s.WritableOverlay != nil { + overlaysToDetach = append(overlaysToDetach, s.WritableOverlay) + } + + // Don't stop on the first error; try to clean up as much as possible, and + // then return the first error encountered. + errors := []error{} + for _, overlay := range overlaysToDetach { + err := overlay.Unmount() + if err != nil { + sylog.Errorf("Error encountered trying to detach identity mount %s: %s", overlay.StagingDir, err) + errors = append(errors, err) + } + } + + if len(errors) > 0 { + return errors[0] + } + + return nil +} diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go new file mode 100644 index 0000000000..6dc114e374 --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go @@ -0,0 +1,249 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/apptainer/apptainer/internal/pkg/util/fs" +) + +func addROItemOrFatal(t *testing.T, s *Set, olStr string) *Item { + i, err := NewItemFromString(olStr) + if err != nil { + t.Fatalf("could not initialize overlay item from string %q: %s", olStr, err) + } + s.ReadonlyOverlays = append(s.ReadonlyOverlays, i) + + return i +} + +// wrapOverlayTest takes a testing function and wraps it in code that checks if +// the kernel has support for unprivileged overlays. If it does, the underlying +// function will be run twice, once with using kernel overlays and once using +// fuse-overlayfs (if present). Otherwise, only the latter option will be +// attempted. +func wrapOverlayTest(f func(t *testing.T)) func(t *testing.T) { + unprivOls, unprivOlsErr := UnprivOverlaysSupported() + return func(t *testing.T) { + if unprivOlsErr != nil { + t.Fatalf("while checking for unprivileged overlay support in kernel: %s", unprivOlsErr) + } + + fuseOverlayFunc := func(t *testing.T) { + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + f(t) + } + + if unprivOls { + t.Run("kerneloverlay", f) + unprivOverlays.kernelSupport = false + } + + t.Run("fuseoverlayfs", fuseOverlayFunc) + unprivOverlays.kernelSupport = unprivOls + } +} + +func TestAllTypesAtOnce(t *testing.T) { + wrapOverlayTest(func(t *testing.T) { + s := Set{} + + tmpRoOlDir := mkTempOlDirOrFatal(t) + addROItemOrFatal(t, &s, tmpRoOlDir+":ro") + + squashfsSupported := false + if _, err := exec.LookPath("squashfs"); err == nil { + squashfsSupported = true + addROItemOrFatal(t, &s, squashfsImgPath) + } + + extfsSupported := false + if _, err := exec.LookPath("fuse2fs"); err == nil { + extfsSupported = true + tmpDir := mkTempDirOrFatal(t) + readonlyExtfsImgPath := filepath.Join(tmpDir, "readonly-extfs.img") + if err := fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444); err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err) + } + addROItemOrFatal(t, &s, readonlyExtfsImgPath+":ro") + } + + tmpRwOlDir := mkTempOlDirOrFatal(t) + i, err := NewItemFromString(tmpRwOlDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRwOlDir, err) + } + s.WritableOverlay = i + + rootfsDir := mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + t.Cleanup(func() { + if t.Failed() { + s.Unmount(rootfsDir) + } + }) + + var expectStr string + if extfsSupported { + expectStr = extfsTestString + } else if squashfsSupported { + expectStr = squashfsTestString + } + + if squashfsSupported || extfsSupported { + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + } + + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("while trying to unmount overlay set: %s", err) + } + })(t) +} + +func TestPersistentWriteToDir(t *testing.T) { + wrapOverlayTest(func(t *testing.T) { + tmpRwOlDir := mkTempOlDirOrFatal(t) + i, err := NewItemFromString(tmpRwOlDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRwOlDir, err) + } + s := Set{WritableOverlay: i} + + performPersistentWriteTest(t, s) + })(t) +} + +func TestPersistentWriteToExtfsImg(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + tmpDir := mkTempDirOrFatal(t) + + // Create a copy of the extfs test image to be used for testing writable + // extfs image overlays + writableExtfsImgPath := filepath.Join(tmpDir, "writable-extfs.img") + err := fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err) + } + + i, err := NewItemFromString(writableExtfsImgPath) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", writableExtfsImgPath, err) + } + s := Set{WritableOverlay: i} + + performPersistentWriteTest(t, s) +} + +func performPersistentWriteTest(t *testing.T, s Set) { + rootfsDir := mkTempDirOrFatal(t) + + // This cleanup will serve adequately for both iterations of the overlay-set + // mounting, below. If it happens to get called while the set is not + // mounted, it should fail silently. + // Mount the overlay set, write a string to a file, and unmount. + // Mount the same set again, and check that we see the file with the + // expected contents. + t.Cleanup(func() { + if t.Failed() { + s.Unmount(rootfsDir) + } + }) + + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + expectStr := "my_test_string" + bytes := []byte(expectStr) + testFilePath := "my_test_file" + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + if err := os.WriteFile(testFileMountedPath, bytes, 0o644); err != nil { + t.Fatalf("while trying to write file inside mounted overlay-set: %s", err) + } + + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("while trying to unmount overlay set: %s", err) + } + + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("while trying to unmount overlay set: %s", err) + } +} + +func TestDuplicateItemsInSet(t *testing.T) { + wrapOverlayTest(func(t *testing.T) { + var rootfsDir string + var rwI *Item + var err error + + s := Set{} + + // First, test mounting of an overlay set with only readonly items, one of + // which is a duplicate of another. + addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") + roI2 := addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, roI2.SourcePath+":ro") + addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") + + rootfsDir = mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err == nil { + t.Errorf("unexpected success: Mounting overlay.Set with duplicate (%q) should have failed", roI2.SourcePath) + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + } + } + + // Next, test mounting of an overlay set with a writable item as well as + // several readonly items, one of which is a duplicate of another. + tmpRwOlDir := mkTempOlDirOrFatal(t) + rwI, err = NewItemFromString(tmpRwOlDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRwOlDir, err) + } + s.WritableOverlay = rwI + + rootfsDir = mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err == nil { + t.Errorf("unexpected success: Mounting overlay.Set with duplicate (%q) should have failed", roI2.SourcePath) + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + } + } + })(t) +} diff --git a/internal/pkg/util/passwdfile/passwdfile_unix.go b/internal/pkg/util/passwdfile/passwdfile_unix.go new file mode 100644 index 0000000000..5f70e2c2d3 --- /dev/null +++ b/internal/pkg/util/passwdfile/passwdfile_unix.go @@ -0,0 +1,180 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// This source code is an adaptation of: +// https://go.dev/src/os/user/lookup_unix.go +// to provide user lookup functionality against an arbitrary password file. + +package passwdfile + +import ( + "bufio" + "bytes" + "errors" + "io" + "os" + "os/user" + "strconv" + "strings" +) + +var colon = []byte(":") + +// lineFunc returns a value, an error, or (nil, nil) to skip the row. +type lineFunc func(line []byte) (v any, err error) + +// readColonFile parses r as an /etc/group or /etc/passwd style file, running +// fn for each row. readColonFile returns a value, an error, or (nil, nil) if +// the end of the file is reached without a match. +// +// readCols is the minimum number of colon-separated fields that will be passed +// to fn; in a long line additional fields may be silently discarded. +func readColonFile(r io.Reader, fn lineFunc, readCols int) (v any, err error) { + rd := bufio.NewReader(r) + + // Read the file line-by-line. + for { + var isPrefix bool + var wholeLine []byte + + // Read the next line. We do so in chunks (as much as reader's + // buffer is able to keep), check if we read enough columns + // already on each step and store final result in wholeLine. + for { + var line []byte + line, isPrefix, err = rd.ReadLine() + + if err != nil { + // We should return (nil, nil) if EOF is reached + // without a match. + if err == io.EOF { + err = nil + } + return nil, err + } + + // Simple common case: line is short enough to fit in a + // single reader's buffer. + if !isPrefix && len(wholeLine) == 0 { + wholeLine = line + break + } + + wholeLine = append(wholeLine, line...) + + // Check if we read the whole line (or enough columns) + // already. + if !isPrefix || bytes.Count(wholeLine, []byte{':'}) >= readCols { + break + } + } + + // There's no spec for /etc/passwd or /etc/group, but we try to follow + // the same rules as the glibc parser, which allows comments and blank + // space at the beginning of a line. + wholeLine = bytes.TrimSpace(wholeLine) + if len(wholeLine) == 0 || wholeLine[0] == '#' { + continue + } + v, err = fn(wholeLine) + if v != nil || err != nil { + return + } + + // If necessary, skip the rest of the line + for ; isPrefix; _, isPrefix, err = rd.ReadLine() { + if err != nil { + // We should return (nil, nil) if EOF is reached without a match. + if err == io.EOF { + err = nil + } + return nil, err + } + } + } +} + +// returns a *User for a row if that row's has the given value at the +// given index. +func matchUserIndexValue(value string, idx int) lineFunc { + var leadColon string + if idx > 0 { + leadColon = ":" + } + substr := []byte(leadColon + value + ":") + return func(line []byte) (v any, err error) { + if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 6 { + return + } + // kevin:x:1005:1006::/home/kevin:/usr/bin/zsh + parts := strings.SplitN(string(line), ":", 7) + if len(parts) < 6 || parts[idx] != value || parts[0] == "" || + parts[0][0] == '+' || parts[0][0] == '-' { + return + } + if _, err := strconv.Atoi(parts[2]); err != nil { + return nil, nil + } + if _, err := strconv.Atoi(parts[3]); err != nil { + return nil, nil + } + u := &user.User{ + Username: parts[0], + Uid: parts[2], + Gid: parts[3], + Name: parts[4], + HomeDir: parts[5], + } + // The pw_gecos field isn't quite standardized. Some docs + // say: "It is expected to be a comma separated list of + // personal data where the first item is the full name of the + // user." + u.Name, _, _ = strings.Cut(u.Name, ",") + return u, nil + } +} + +func findUserID(uid string, r io.Reader) (*user.User, error) { + i, e := strconv.Atoi(uid) + if e != nil { + return nil, errors.New("user: invalid userid " + uid) + } + if v, err := readColonFile(r, matchUserIndexValue(uid, 2), 6); err != nil { + return nil, err + } else if v != nil { + return v.(*user.User), nil + } + return nil, user.UnknownUserIdError(i) +} + +func findUsername(name string, r io.Reader) (*user.User, error) { + if v, err := readColonFile(r, matchUserIndexValue(name, 0), 6); err != nil { + return nil, err + } else if v != nil { + return v.(*user.User), nil + } + return nil, user.UnknownUserError(name) +} + +func LookupUserInFile(userFile, username string) (*user.User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUsername(username, f) +} + +func LookupUserIDInFile(userFile, uid string) (*user.User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUserID(uid, f) +} diff --git a/internal/pkg/util/starter/starter.go b/internal/pkg/util/starter/starter.go index 6b19ef6ec8..55c5d8b75c 100644 --- a/internal/pkg/util/starter/starter.go +++ b/internal/pkg/util/starter/starter.go @@ -118,7 +118,7 @@ func Run(name string, config *config.Common, ops ...CommandOp) error { cmd.Stderr = c.stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("while running %s: %s", c.path, err) + return fmt.Errorf("while running %s: %w", c.path, err) } return nil } diff --git a/mlocal/frags/go_common_opts.mk b/mlocal/frags/go_common_opts.mk index f8ccf9cbab..dd797a133b 100644 --- a/mlocal/frags/go_common_opts.mk +++ b/mlocal/frags/go_common_opts.mk @@ -1,6 +1,6 @@ # go tool default build options GO111MODULE := on -GO_TAGS := containers_image_openpgp sylog oci_engine apptainer_engine fakeroot_engine +GO_TAGS := containers_image_openpgp sylog apptainer_engine fakeroot_engine GO_TAGS_SUID := containers_image_openpgp sylog apptainer_engine fakeroot_engine GO_LDFLAGS := # Need to use non-pie build on ppc64le diff --git a/pkg/ocibundle/bundle.go b/pkg/ocibundle/bundle.go index 8777c56d1e..f96695f3a3 100644 --- a/pkg/ocibundle/bundle.go +++ b/pkg/ocibundle/bundle.go @@ -10,11 +10,17 @@ package ocibundle import ( + "context" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" ) // Bundle defines an OCI bundle interface to create/delete OCI bundles type Bundle interface { - Create(*specs.Spec) error + Create(context.Context, *specs.Spec) error + Update(context.Context, *specs.Spec) error + ImageSpec() *imgspecv1.Image Delete() error + Path() string } diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go new file mode 100644 index 0000000000..f3cb82390a --- /dev/null +++ b/pkg/ocibundle/native/bundle_linux.go @@ -0,0 +1,315 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package native + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + apexlog "github.com/apex/log" + "github.com/apptainer/apptainer/internal/pkg/build/oci" + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/pkg/ocibundle" + "github.com/apptainer/apptainer/pkg/ocibundle/tools" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker" + dockerarchive "github.com/containers/image/v5/docker/archive" + dockerdaemon "github.com/containers/image/v5/docker/daemon" + ociarchive "github.com/containers/image/v5/oci/archive" + ocilayout "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/types" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/umoci" + umocilayer "github.com/opencontainers/umoci/oci/layer" + "github.com/opencontainers/umoci/pkg/idtools" +) + +// Bundle is a native OCI bundle, created from imageRef. +type Bundle struct { + // imageRef is the reference to the OCI image source, e.g. docker://ubuntu:latest. + imageRef string + // imageSpec is the OCI image information, CMD, ENTRYPOINT, etc. + imageSpec *imgspecv1.Image + // bundlePath is the location where the OCI bundle will be created. + bundlePath string + // sysCtx provides containers/image transport configuration (auth etc.) + sysCtx *types.SystemContext + // imgCache is a Apptainer image cache, which OCI blobs are pulled through. + // Note that we only use the 'blob' cache section. The 'oci-tmp' cache section holds + // OCI->SIF conversions, which are not used here. + imgCache *cache.Handle + // process is the command to execute, which may override the image's ENTRYPOINT / CMD. + // Generic bundle properties + ocibundle.Bundle +} + +type Option func(b *Bundle) error + +// OptBundlePath sets the path that the bundle will be created at. +func OptBundlePath(bp string) Option { + return func(b *Bundle) error { + var err error + b.bundlePath, err = filepath.Abs(bp) + if err != nil { + return fmt.Errorf("failed to determine bundle path: %s", err) + } + return nil + } +} + +// OptImageRef sets the image source reference, from which the bundle will be created. +func OptImageRef(ref string) Option { + return func(b *Bundle) error { + b.imageRef = ref + return nil + } +} + +// OptSysCtx sets the OCI client SystemContext holding auth information etc. +func OptSysCtx(sc *types.SystemContext) Option { + return func(b *Bundle) error { + b.sysCtx = sc + return nil + } +} + +// OptImgCache sets the Apptainer image cache used to pull through OCI blobs. +func OptImgCache(ic *cache.Handle) Option { + return func(b *Bundle) error { + b.imgCache = ic + return nil + } +} + +// New returns a bundle interface to create/delete an OCI bundle from an OCI image ref. +func New(opts ...Option) (ocibundle.Bundle, error) { + b := Bundle{ + imageRef: "", + sysCtx: &types.SystemContext{}, + imgCache: nil, + } + + for _, opt := range opts { + if err := opt(&b); err != nil { + return nil, fmt.Errorf("while initializing bundle: %w", err) + } + } + + return &b, nil +} + +// Delete erases OCI bundle created an OCI image ref +func (b *Bundle) Delete() error { + return tools.DeleteBundle(b.bundlePath) +} + +// Create will created the on-disk structures for the OCI bundle, so that it is ready for execution. +func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { + // generate OCI bundle directory and config + g, err := tools.GenerateBundleConfig(b.bundlePath, ociConfig) + if err != nil { + return fmt.Errorf("failed to generate OCI bundle/config: %s", err) + } + // Due to our caching approach for OCI blobs, we need to pull blobs for the image + // out into a separate oci-layout directory. + tmpDir, err := os.MkdirTemp("", "oci-tmp") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + // Fetch into temp oci layout (will pull through cache if enabled) + if err := b.fetchImage(ctx, tmpDir); err != nil { + return err + } + // Extract from temp oci layout into bundle rootfs + if err := b.extractImage(ctx, tmpDir); err != nil { + return err + } + // Remove the temp oci layout. + if err := os.RemoveAll(tmpDir); err != nil { + return err + } + return b.writeConfig(g) +} + +// Update will update the OCI config for the OCI bundle, so that it is ready for execution. +func (b *Bundle) Update(ctx context.Context, ociConfig *specs.Spec) error { + // generate OCI bundle directory and config + g, err := tools.GenerateBundleConfig(b.bundlePath, ociConfig) + if err != nil { + return fmt.Errorf("failed to generate OCI bundle/config: %s", err) + } + return b.writeConfig(g) +} + +// ImageSpec returns the OCI image spec associated with the bundle. +func (b *Bundle) ImageSpec() (imgSpec *imgspecv1.Image) { + return b.imageSpec +} + +// Path returns the bundle's path on disk. +func (b *Bundle) Path() string { + return b.bundlePath +} + +func (b *Bundle) writeConfig(g *generate.Generator) error { + return tools.SaveBundleConfig(b.bundlePath, g) +} + +func (b *Bundle) fetchImage(ctx context.Context, tmpDir string) error { + if b.sysCtx == nil { + return fmt.Errorf("sysctx must be provided") + } + + policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} + policyCtx, err := signature.NewPolicyContext(policy) + if err != nil { + return err + } + + parts := strings.SplitN(b.imageRef, ":", 2) + if len(parts) < 2 { + return fmt.Errorf("could not parse image ref: %s", b.imageRef) + } + var srcRef types.ImageReference + + switch parts[0] { + case "docker": + srcRef, err = docker.ParseReference(parts[1]) + case "docker-archive": + srcRef, err = dockerarchive.ParseReference(parts[1]) + case "docker-daemon": + srcRef, err = dockerdaemon.ParseReference(parts[1]) + case "oci": + srcRef, err = ocilayout.ParseReference(parts[1]) + case "oci-archive": + srcRef, err = ociarchive.ParseReference(parts[1]) + default: + return fmt.Errorf("cannot create an OCI container from %s source", parts[0]) + } + + if err != nil { + return fmt.Errorf("invalid image source: %w", err) + } + + if b.imgCache != nil { + // Grab the modified source ref from the cache + srcRef, err = oci.ConvertReference(ctx, b.imgCache, srcRef, b.sysCtx) + if err != nil { + return err + } + } + + tmpfsRef, err := ocilayout.ParseReference(tmpDir + ":" + "tmp") + if err != nil { + return err + } + + _, err = copy.Image(ctx, policyCtx, tmpfsRef, srcRef, ©.Options{ + ReportWriter: sylog.Writer(), + SourceCtx: b.sysCtx, + }) + if err != nil { + return err + } + + img, err := srcRef.NewImage(ctx, b.sysCtx) + if err != nil { + return err + } + defer img.Close() + + b.imageSpec, err = img.OCIConfig(ctx) + if err != nil { + return err + } + return nil +} + +func (b *Bundle) extractImage(ctx context.Context, tmpDir string) error { + var mapOptions umocilayer.MapOptions + + loggerLevel := sylog.GetLevel() + // set the apex log level, for umoci + if loggerLevel <= int(sylog.ErrorLevel) { + // silent option + apexlog.SetLevel(apexlog.ErrorLevel) + } else if loggerLevel <= int(sylog.LogLevel) { + // quiet option + apexlog.SetLevel(apexlog.WarnLevel) + } else if loggerLevel < int(sylog.DebugLevel) { + // verbose option(s) or default + apexlog.SetLevel(apexlog.InfoLevel) + } else { + // debug option + apexlog.SetLevel(apexlog.DebugLevel) + } + + // Allow unpacking as non-root + if os.Geteuid() != 0 { + mapOptions.Rootless = true + + uidMap, err := idtools.ParseMapping(fmt.Sprintf("0:%d:1", os.Geteuid())) + if err != nil { + return fmt.Errorf("error parsing uidmap: %s", err) + } + mapOptions.UIDMappings = append(mapOptions.UIDMappings, uidMap) + + gidMap, err := idtools.ParseMapping(fmt.Sprintf("0:%d:1", os.Getegid())) + if err != nil { + return fmt.Errorf("error parsing gidmap: %s", err) + } + mapOptions.GIDMappings = append(mapOptions.GIDMappings, gidMap) + } + + engineExt, err := umoci.OpenLayout(tmpDir) + if err != nil { + return fmt.Errorf("error opening layout: %s", err) + } + + // Obtain the manifest + tmpfsRef, err := ocilayout.ParseReference(tmpDir + ":" + "tmp") + if err != nil { + return err + } + imageSource, err := tmpfsRef.NewImageSource(ctx, b.sysCtx) + if err != nil { + return fmt.Errorf("error creating image source: %s", err) + } + manifestData, mediaType, err := imageSource.GetManifest(ctx, nil) + if err != nil { + return fmt.Errorf("error obtaining manifest source: %s", err) + } + if mediaType != imgspecv1.MediaTypeImageManifest { + return fmt.Errorf("error verifying manifest media type: %s", mediaType) + } + var manifest imgspecv1.Manifest + json.Unmarshal(manifestData, &manifest) + + // UnpackRootfs from umoci v0.4.2 expects a path to a non-existing directory + os.RemoveAll(tools.RootFs(b.bundlePath).Path()) + + // Unpack root filesystem + unpackOptions := umocilayer.UnpackOptions{MapOptions: mapOptions} + err = umocilayer.UnpackRootfs(ctx, engineExt, tools.RootFs(b.bundlePath).Path(), manifest, &unpackOptions) + if err != nil { + return fmt.Errorf("error unpacking rootfs: %s", err) + } + + return nil +} diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go new file mode 100644 index 0000000000..e52c438c79 --- /dev/null +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package native + +import ( + "context" + "io" + "log" + "net/http" + "os" + "os/exec" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/opencontainers/runtime-tools/validate" +) + +const ( + dockerURI = "docker://alpine" + dockerArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-docker-save.tar" + ociArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-oci-archive.tar" + dockerDaemonImage = "alpine:latest" +) + +func setupCache(t *testing.T) *cache.Handle { + dir := t.TempDir() + h, err := cache.New(cache.Config{ParentDir: dir}) + if err != nil { + t.Fatalf("failed to create an image cache handle: %s", err) + } + return h +} + +func getTestTar(url string) (path string, err error) { + dl, err := os.CreateTemp("", "oci-test") + if err != nil { + log.Fatal(err) + } + defer dl.Close() + + r, err := http.Get(url) + if err != nil { + return "", err + } + defer r.Body.Close() + + _, err = io.Copy(dl, r.Body) + if err != nil { + return "", err + } + + return dl.Name(), nil +} + +func validateBundle(t *testing.T, bundlePath string) { + v, err := validate.NewValidatorFromPath(bundlePath, false, "linux") + if err != nil { + t.Errorf("Could not create bundle validator: %v", err) + } + if err := v.CheckAll(); err != nil { + t.Errorf("Bundle not valid: %v", err) + } +} + +func TestFromImageRef(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Prepare docker-archive source + dockerArchive, err := getTestTar(dockerArchiveURI) + if err != nil { + t.Fatalf("Could not download docker archive test file: %v", err) + } + defer os.Remove(dockerArchive) + // Prepare docker-daemon source + hasDocker := false + cmd := exec.Command("docker", "ps") + err = cmd.Run() + if err == nil { + hasDocker = true + cmd = exec.Command("docker", "pull", dockerDaemonImage) + err = cmd.Run() + if err != nil { + t.Fatalf("could not docker pull %s %v", dockerDaemonImage, err) + return + } + } + // Prepare oci-archive source + ociArchive, err := getTestTar(ociArchiveURI) + if err != nil { + t.Fatalf("Could not download oci archive test file: %v", err) + } + defer os.Remove(ociArchive) + // Prepare oci source (oci directory layout) + ociLayout := t.TempDir() + cmd = exec.Command("tar", "-C", ociLayout, "-xf", ociArchive) + err = cmd.Run() + if err != nil { + t.Fatalf("Error extracting oci archive to layout: %v", err) + } + + tests := []struct { + name string + imageRef string + needsDocker bool + }{ + {"docker", dockerURI, false}, + {"docker-archive", "docker-archive:" + dockerArchive, false}, + {"docker-daemon", "docker-daemon:" + dockerDaemonImage, true}, + {"oci-archive", "oci-archive:" + ociArchive, false}, + {"oci", "oci:" + ociLayout, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.needsDocker && !hasDocker { + t.Skipf("docker not available") + } + bundleDir := t.TempDir() + b, err := New( + OptBundlePath(bundleDir), + OptImageRef(tt.imageRef), + OptImgCache(setupCache(t)), + ) + if err != nil { + t.Fatalf("While initializing bundle: %s", err) + } + + if err := b.Create(context.Background(), nil); err != nil { + t.Fatalf("While creating bundle: %s", err) + } + + validateBundle(t, bundleDir) + }) + } +} diff --git a/pkg/ocibundle/sif/bundle_linux.go b/pkg/ocibundle/sif/bundle_linux.go index 5c15a7d137..63e3f8caa6 100644 --- a/pkg/ocibundle/sif/bundle_linux.go +++ b/pkg/ocibundle/sif/bundle_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -10,6 +10,7 @@ package sifbundle import ( + "context" "encoding/json" "fmt" "os" @@ -92,7 +93,7 @@ func (s *sifBundle) writeConfig(img *image.Image, g *generate.Generator) error { } // Create creates an OCI bundle from a SIF image -func (s *sifBundle) Create(ociConfig *specs.Spec) error { +func (s *sifBundle) Create(ctx context.Context, ociConfig *specs.Spec) error { if s.image == "" { return fmt.Errorf("image wasn't set, need one to create bundle") } @@ -156,6 +157,11 @@ func (s *sifBundle) Create(ociConfig *specs.Spec) error { return nil } +// Update will update the OCI config for the OCI bundle, so that it is ready for execution. +func (s *sifBundle) Update(ctx context.Context, ociConfig *specs.Spec) error { + return fmt.Errorf("cannot update config of a SIF OCI bundle: not implemented") +} + // Delete erases OCI bundle create from SIF image func (s *sifBundle) Delete() error { if s.writable { @@ -172,6 +178,15 @@ func (s *sifBundle) Delete() error { return tools.DeleteBundle(s.bundlePath) } +// ImageSpec returns nil for SIF bundles, as they currently do not carry an OCI image spec. +func (s *sifBundle) ImageSpec() (imgSpec *imageSpecs.Image) { + return nil +} + +func (s *sifBundle) Path() string { + return s.bundlePath +} + // FromSif returns a bundle interface to create/delete OCI bundle from SIF image func FromSif(image, bundle string, writable bool) (ocibundle.Bundle, error) { var err error diff --git a/pkg/ocibundle/sif/bundle_linux_test.go b/pkg/ocibundle/sif/bundle_linux_test.go index 381340f3ff..3018d52d2a 100644 --- a/pkg/ocibundle/sif/bundle_linux_test.go +++ b/pkg/ocibundle/sif/bundle_linux_test.go @@ -10,6 +10,7 @@ package sifbundle import ( + "context" "os" "runtime" "testing" @@ -48,7 +49,7 @@ func TestFromSif(t *testing.T) { t.Errorf("unexpected success while opening non existent image") } // create OCI bundle from SIF - if err := bundle.Create(nil); err == nil { + if err := bundle.Create(context.Background(), nil); err == nil { // check if cleanup occurred t.Errorf("unexpected success while creating OCI bundle") } @@ -80,7 +81,7 @@ func TestFromSif(t *testing.T) { g.Config.Linux.Seccomp = nil g.SetProcessArgs([]string{tools.RunScript, "id"}) - if err := bundle.Create(g.Config); err != nil { + if err := bundle.Create(context.Background(), g.Config); err != nil { // check if cleanup occurred t.Fatal(err) } diff --git a/pkg/ocibundle/tools/oci.go b/pkg/ocibundle/tools/oci.go index 5989c4f041..74a7b7e247 100644 --- a/pkg/ocibundle/tools/oci.go +++ b/pkg/ocibundle/tools/oci.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -12,11 +12,14 @@ package tools import ( "fmt" "os" + "os/user" "path/filepath" + "strconv" "syscall" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/internal/pkg/util/passwdfile" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -107,3 +110,18 @@ func DeleteBundle(bundlePath string) error { } return nil } + +// BundleUser returns a user struct for the specified user, from the bundle passwd file. +func BundleUser(bundlePath, user string) (u *user.User, err error) { + passwd := filepath.Join(RootFs(bundlePath).Path(), "etc", "passwd") + if _, err := os.Stat(passwd); err != nil { + return nil, fmt.Errorf("cannot access container passwd file: %w", err) + } + + // We have a numeric container uid + if _, err := strconv.Atoi(user); err == nil { + return passwdfile.LookupUserIDInFile(passwd, user) + } + // We have a container username + return passwdfile.LookupUserInFile(passwd, user) +} diff --git a/pkg/ocibundle/tools/overlay_linux.go b/pkg/ocibundle/tools/overlay_linux.go index e21a40d98e..ffaf715d73 100644 --- a/pkg/ocibundle/tools/overlay_linux.go +++ b/pkg/ocibundle/tools/overlay_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -14,71 +14,118 @@ import ( "os" "path/filepath" "syscall" + + "github.com/apptainer/apptainer/internal/pkg/util/fs/overlay" + "github.com/apptainer/apptainer/pkg/image" + "github.com/apptainer/apptainer/pkg/sylog" ) -// CreateOverlay creates a writable overlay +// CreateOverlay creates a writable overlay using a directory inside the OCI +// bundle. func CreateOverlay(bundlePath string) error { + oldumask := syscall.Umask(0) + defer syscall.Umask(oldumask) + + olDir := filepath.Join(bundlePath, "overlay") + var err error + if err = overlay.EnsureOverlayDir(olDir, true, 0o700); err != nil { + return fmt.Errorf("failed to create %s: %s", olDir, err) + } + // delete overlay directory in case of error + defer func() { + if err != nil { + sylog.Debugf("Encountered error in CreateOverlay; attempting to remove overlay dir %q", olDir) + os.RemoveAll(olDir) + } + }() + + olSet := overlay.Set{WritableOverlay: &overlay.Item{ + SourcePath: olDir, + Type: image.SANDBOX, + Writable: true, + }} + + upDir := filepath.Join(bundlePath, "upper") + if err := os.MkdirAll(upDir, 0o755); err != nil { + return fmt.Errorf("failed to create %s: %s", upDir, err) + } + + return olSet.Mount(filepath.Join(upDir)) +} + +// DeleteOverlay deletes an overlay previously created using a directory inside +// the OCI bundle. +func DeleteOverlay(bundlePath string) error { + olDir := filepath.Join(bundlePath, "overlay") + upDir := filepath.Join(bundlePath, "upper") + + if err := overlay.DetachMount(upDir); err != nil { + return err + } + + return overlay.DetachAndDelete(olDir) +} + +// CreateOverlay creates a writable overlay using tmpfs. +func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { var err error oldumask := syscall.Umask(0) defer syscall.Umask(oldumask) - overlayDir := filepath.Join(bundlePath, "overlay") - if err = os.Mkdir(overlayDir, 0o700); err != nil { - return fmt.Errorf("failed to create %s: %s", overlayDir, err) + olDir := filepath.Join(bundlePath, "overlay") + err = overlay.EnsureOverlayDir(olDir, true, 0o700) + if err != nil { + return "", fmt.Errorf("failed to create %s: %s", olDir, err) } // delete overlay directory in case of error defer func() { if err != nil { - os.RemoveAll(overlayDir) + sylog.Debugf("Encountered error in CreateOverlay; attempting to remove overlay dir %q", olDir) + os.RemoveAll(olDir) } }() - err = syscall.Mount(overlayDir, overlayDir, "", syscall.MS_BIND, "") + options := fmt.Sprintf("mode=1777,size=%dm", sizeMiB) + err = syscall.Mount("tmpfs", olDir, "tmpfs", syscall.MS_NODEV, options) if err != nil { - return fmt.Errorf("failed to bind %s: %s", overlayDir, err) + return "", fmt.Errorf("failed to bind %s: %s", olDir, err) } // best effort to cleanup mount defer func() { if err != nil { - syscall.Unmount(overlayDir, syscall.MNT_DETACH) + sylog.Debugf("Encountered error in CreateOverlayTmpfs; attempting to detach overlay dir %q", olDir) + syscall.Unmount(olDir, syscall.MNT_DETACH) } }() - if err = syscall.Mount("", overlayDir, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("failed to remount %s: %s", overlayDir, err) - } + olSet := overlay.Set{WritableOverlay: &overlay.Item{ + SourcePath: olDir, + Type: image.SANDBOX, + Writable: true, + }} - upperDir := filepath.Join(overlayDir, "upper") - if err = os.Mkdir(upperDir, 0o755); err != nil { - return fmt.Errorf("failed to create %s: %s", upperDir, err) - } - workDir := filepath.Join(overlayDir, "work") - if err = os.Mkdir(workDir, 0o700); err != nil { - return fmt.Errorf("failed to create %s: %s", workDir, err) + err = olSet.Mount(filepath.Join(bundlePath, "upper")) + if err != nil { + return "", err } - rootFsDir := RootFs(bundlePath).Path() - options := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", rootFsDir, upperDir, workDir) - if err = syscall.Mount("overlay", rootFsDir, "overlay", 0, options); err != nil { - return fmt.Errorf("failed to mount %s: %s", overlayDir, err) - } - return nil + return olDir, nil } -// DeleteOverlay deletes overlay -func DeleteOverlay(bundlePath string) error { - overlayDir := filepath.Join(bundlePath, "overlay") - rootFsDir := RootFs(bundlePath).Path() +// DeleteOverlayTmpfs deletes an overlay previously created using tmpfs. +func DeleteOverlayTmpfs(bundlePath, olDir string) error { + upDir := filepath.Join(bundlePath, "upper") - if err := syscall.Unmount(rootFsDir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to unmount %s: %s", rootFsDir, err) - } - if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to unmount %s: %s", overlayDir, err) + if err := overlay.DetachMount(upDir); err != nil { + return err } - if err := os.RemoveAll(overlayDir); err != nil { - return fmt.Errorf("failed to remove %s: %s", overlayDir, err) + + // Because CreateOverlayTmpfs() mounts the tmpfs on olDir, and then + // calls ApplyOverlay(), there needs to be an extra unmount in the this case + if err := overlay.DetachMount(olDir); err != nil { + return err } - return nil + + return overlay.DetachAndDelete(olDir) } diff --git a/pkg/runtime/engine/apptainer/config/config.go b/pkg/runtime/engine/apptainer/config/config.go index 63f5938148..924db912f7 100644 --- a/pkg/runtime/engine/apptainer/config/config.go +++ b/pkg/runtime/engine/apptainer/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" "github.com/apptainer/apptainer/pkg/image" "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/bind" ) // Name is the name of the runtime. @@ -89,7 +90,7 @@ type JSONConfig struct { LibrariesPath []string `json:"librariesPath,omitempty"` FuseMount []FuseMount `json:"fuseMount,omitempty"` ImageList []image.Image `json:"imageList,omitempty"` - BindPath []BindPath `json:"bindpath,omitempty"` + BindPath []bind.Path `json:"bindpath,omitempty"` ApptainerEnv map[string]string `json:"apptainerEnv,omitempty"` UnixSocketPair [2]int `json:"unixSocketPair,omitempty"` OpenFd []int `json:"openFd,omitempty"` @@ -304,12 +305,12 @@ func (e *EngineConfig) GetCustomHome() bool { } // SetBindPath sets the paths to bind into container. -func (e *EngineConfig) SetBindPath(bindpath []BindPath) { +func (e *EngineConfig) SetBindPath(bindpath []bind.Path) { e.JSON.BindPath = bindpath } // GetBindPath retrieves the bind paths. -func (e *EngineConfig) GetBindPath() []BindPath { +func (e *EngineConfig) GetBindPath() []bind.Path { return e.JSON.BindPath } diff --git a/pkg/util/apptainerconf/config.go b/pkg/util/apptainerconf/config.go index bf8623dc26..f9acd171a2 100644 --- a/pkg/util/apptainerconf/config.go +++ b/pkg/util/apptainerconf/config.go @@ -293,10 +293,10 @@ mount slave = {{ if eq .MountSlave true }}yes{{ else }}no{{ end }} # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB). It will -# affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home") and -# it will also affect users of "--writable-tmpfs". +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = {{ .SessiondirMaxSize }} # LIMIT CONTAINER OWNERS: [STRING] diff --git a/pkg/runtime/engine/apptainer/config/bind.go b/pkg/util/bind/bind.go similarity index 86% rename from pkg/runtime/engine/apptainer/config/bind.go rename to pkg/util/bind/bind.go index 24cb738669..d205057a19 100644 --- a/pkg/runtime/engine/apptainer/config/bind.go +++ b/pkg/util/bind/bind.go @@ -1,9 +1,9 @@ -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "fmt" @@ -11,9 +11,9 @@ import ( "strings" ) -// BindOption represents a bind option with its associated +// Option represents a bind option with its associated // value if any. -type BindOption struct { +type Option struct { Value string `json:"value,omitempty"` } @@ -31,17 +31,17 @@ var bindOptions = map[string]bool{ "id": valueOption, } -// BindPath stores a parsed bind path specification. Source and Destination +// Path stores a parsed bind path specification. Source and Destination // paths are required. -type BindPath struct { - Source string `json:"source"` - Destination string `json:"destination"` - Options map[string]*BindOption `json:"options"` +type Path struct { + Source string `json:"source"` + Destination string `json:"destination"` + Options map[string]*Option `json:"options"` } // ImageSrc returns the value of the option image-src for a BindPath, or an // empty string if the option wasn't set. -func (b *BindPath) ImageSrc() string { +func (b *Path) ImageSrc() string { if b.Options != nil && b.Options["image-src"] != nil { src := b.Options["image-src"].Value if src == "" { @@ -54,7 +54,7 @@ func (b *BindPath) ImageSrc() string { // ID returns the value of the option id for a BindPath, or an empty string if // the option wasn't set. -func (b *BindPath) ID() string { +func (b *Path) ID() string { if b.Options != nil && b.Options["id"] != nil { return b.Options["id"].Value } @@ -62,7 +62,7 @@ func (b *BindPath) ID() string { } // Readonly returns true if the ro option was set for a BindPath. -func (b *BindPath) Readonly() bool { +func (b *Path) Readonly() bool { return b.Options != nil && b.Options["ro"] != nil } @@ -70,8 +70,8 @@ func (b *BindPath) Readonly() bool { // more (comma separated) bind paths in src[:dst[:options]] format, and // returns all encountered bind paths as a slice. Options may be simple // flags, e.g. 'rw', or take a value, e.g. 'id=2'. -func ParseBindPath(paths []string) ([]BindPath, error) { - var binds []BindPath +func ParseBindPath(paths []string) ([]Path, error) { + var binds []Path // there is a better regular expression to handle // that directly without all the logic below ... @@ -216,8 +216,8 @@ func splitBy(str string, sep byte) []string { // newBindPath returns BindPath record based on the provided bind // string argument and ensures that the options are valid. -func newBindPath(bind string) (BindPath, error) { - var bp BindPath +func newBindPath(bind string) (Path, error) { + var bp Path splitted := splitBy(bind, ':') @@ -233,17 +233,17 @@ func newBindPath(bind string) (BindPath, error) { } if len(splitted) > 2 { - bp.Options = make(map[string]*BindOption) + bp.Options = make(map[string]*Option) for _, value := range strings.Split(splitted[2], ",") { valid := false for optName, isFlag := range bindOptions { if isFlag && optName == value { - bp.Options[optName] = &BindOption{} + bp.Options[optName] = &Option{} valid = true break } else if strings.HasPrefix(value, optName+"=") { - bp.Options[optName] = &BindOption{Value: value[len(optName+"="):]} + bp.Options[optName] = &Option{Value: value[len(optName+"="):]} valid = true break } diff --git a/pkg/runtime/engine/apptainer/config/bind_test.go b/pkg/util/bind/bind_test.go similarity index 83% rename from pkg/runtime/engine/apptainer/config/bind_test.go rename to pkg/util/bind/bind_test.go index 6431886b41..07e576b404 100644 --- a/pkg/runtime/engine/apptainer/config/bind_test.go +++ b/pkg/util/bind/bind_test.go @@ -1,9 +1,9 @@ -// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "reflect" @@ -14,13 +14,13 @@ func TestParseBindPath(t *testing.T) { tests := []struct { name string bindpaths []string - want []BindPath + want []Path wantErr bool }{ { name: "srcOnly", bindpaths: []string{"/opt"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", @@ -30,7 +30,7 @@ func TestParseBindPath(t *testing.T) { { name: "srcOnlyMultiple", bindpaths: []string{"/opt,/tmp"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", @@ -44,7 +44,7 @@ func TestParseBindPath(t *testing.T) { { name: "srcDst", bindpaths: []string{"/opt:/other"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", @@ -54,7 +54,7 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstMultiple", bindpaths: []string{"/opt:/other,/tmp:/other2,"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", @@ -68,11 +68,11 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstRO", bindpaths: []string{"/opt:/other:ro"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -81,18 +81,18 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstROMultiple", bindpaths: []string{"/opt:/other:ro,/tmp:/other2:ro"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, { Source: "/tmp", Destination: "/other2", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -103,11 +103,11 @@ func TestParseBindPath(t *testing.T) { // parsing multiple simple options. name: "srcDstRORW", bindpaths: []string{"/opt:/other:ro,rw"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, "rw": {}, }, @@ -121,11 +121,11 @@ func TestParseBindPath(t *testing.T) { // delimiting an additional option, vs an additional bind. name: "srcDstRORWMultiple", bindpaths: []string{"/opt:/other:ro,rw,/tmp:/other2:ro,rw"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, "rw": {}, }, @@ -133,7 +133,7 @@ func TestParseBindPath(t *testing.T) { { Source: "/tmp", Destination: "/other2", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, "rw": {}, }, @@ -143,11 +143,11 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstImageSrc", bindpaths: []string{"test.sif:/other:image-src=/opt"}, - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {"/opt"}, }, }, @@ -157,17 +157,17 @@ func TestParseBindPath(t *testing.T) { // Can't use image-src without a value name: "srcDstImageSrcNoVal", bindpaths: []string{"test.sif:/other:image-src"}, - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "srcDstId", bindpaths: []string{"test.sif:/other:image-src=/opt,id=2"}, - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {"/opt"}, "id": {"2"}, }, @@ -177,13 +177,13 @@ func TestParseBindPath(t *testing.T) { { name: "invalidOption", bindpaths: []string{"/opt:/other:invalid"}, - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "invalidSpec", bindpaths: []string{"/opt:/other:rw:invalid"}, - want: []BindPath{}, + want: []Path{}, wantErr: true, }, } diff --git a/pkg/runtime/engine/apptainer/config/mount.go b/pkg/util/bind/mount.go similarity index 68% rename from pkg/runtime/engine/apptainer/config/mount.go rename to pkg/util/bind/mount.go index d4e2b35f78..6312bf3e73 100644 --- a/pkg/runtime/engine/apptainer/config/mount.go +++ b/pkg/util/bind/mount.go @@ -3,7 +3,7 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "encoding/csv" @@ -29,17 +29,17 @@ import ( // // We only support type=bind at present, so assume this if type is missing and // error for other types. -func ParseMountString(mount string) (bindPaths []BindPath, err error) { +func ParseMountString(mount string) (bindPaths []Path, err error) { r := strings.NewReader(mount) c := csv.NewReader(r) records, err := c.ReadAll() if err != nil { - return []BindPath{}, fmt.Errorf("error parsing mount: %v", err) + return []Path{}, fmt.Errorf("error parsing mount: %v", err) } for _, r := range records { - bp := BindPath{ - Options: map[string]*BindOption{}, + bp := Path{ + Options: map[string]*Option{}, } for _, f := range r { @@ -54,41 +54,41 @@ func ParseMountString(mount string) (bindPaths []BindPath, err error) { // TODO - Eventually support volume and tmpfs? Requires structural changes to engine mount functionality. case "type": if val != "bind" { - return []BindPath{}, fmt.Errorf("unsupported mount type %q, only 'bind' is supported", val) + return []Path{}, fmt.Errorf("unsupported mount type %q, only 'bind' is supported", val) } case "source", "src": if val == "" { - return []BindPath{}, fmt.Errorf("mount source cannot be empty") + return []Path{}, fmt.Errorf("mount source cannot be empty") } bp.Source = val case "destination", "dst", "target": if val == "" { - return []BindPath{}, fmt.Errorf("mount destination cannot be empty") + return []Path{}, fmt.Errorf("mount destination cannot be empty") } bp.Destination = val case "ro", "readonly": - bp.Options["ro"] = &BindOption{} + bp.Options["ro"] = &Option{} // Apptainer only - directory inside an image file source to mount from case "image-src": if val == "" { - return []BindPath{}, fmt.Errorf("img-src cannot be empty") + return []Path{}, fmt.Errorf("img-src cannot be empty") } - bp.Options["image-src"] = &BindOption{Value: val} + bp.Options["image-src"] = &Option{Value: val} // Apptainer only - id of the descriptor in a SIF image source to mount from case "id": if val == "" { - return []BindPath{}, fmt.Errorf("id cannot be empty") + return []Path{}, fmt.Errorf("id cannot be empty") } - bp.Options["id"] = &BindOption{Value: val} + bp.Options["id"] = &Option{Value: val} case "bind-propagation": - return []BindPath{}, fmt.Errorf("bind-propagation not supported for individual mounts, check apptainer.conf for global setting") + return []Path{}, fmt.Errorf("bind-propagation not supported for individual mounts, check apptainer.conf for global setting") default: - return []BindPath{}, fmt.Errorf("invalid key %q in mount specification", key) + return []Path{}, fmt.Errorf("invalid key %q in mount specification", key) } } if bp.Source == "" || bp.Destination == "" { - return []BindPath{}, fmt.Errorf("mounts must specify a source and a destination") + return []Path{}, fmt.Errorf("mounts must specify a source and a destination") } bindPaths = append(bindPaths, bp) } diff --git a/pkg/runtime/engine/apptainer/config/mount_test.go b/pkg/util/bind/mount_test.go similarity index 79% rename from pkg/runtime/engine/apptainer/config/mount_test.go rename to pkg/util/bind/mount_test.go index 46820cb732..169c0850db 100644 --- a/pkg/runtime/engine/apptainer/config/mount_test.go +++ b/pkg/util/bind/mount_test.go @@ -1,9 +1,9 @@ -// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "reflect" @@ -14,53 +14,53 @@ func TestParseMountString(t *testing.T) { tests := []struct { name string mountString string - want []BindPath + want []Path wantErr bool }{ { name: "sourceOnly", mountString: "type=bind,source=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "destinationOnly", mountString: "type=bind,destination=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "emptySource", mountString: "type=bind,source=,destination=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "emptyDestination", mountString: "type=bind,source=/opt,destination=", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "invalidType", mountString: "type=potato,source=/opt,destination=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "invalidField", mountString: "type=bind,source=/opt,destination=/opt,color=turquoise", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "simple", mountString: "type=bind,source=/opt,destination=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -68,11 +68,11 @@ func TestParseMountString(t *testing.T) { { name: "simpleSrc", mountString: "type=bind,src=/opt,destination=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -80,11 +80,11 @@ func TestParseMountString(t *testing.T) { { name: "simpleDst", mountString: "type=bind,source=/opt,dst=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -92,11 +92,11 @@ func TestParseMountString(t *testing.T) { { name: "simpleTarget", mountString: "type=bind,source=/opt,target=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -104,11 +104,11 @@ func TestParseMountString(t *testing.T) { { name: "noType", mountString: "source=/opt,destination=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -116,11 +116,11 @@ func TestParseMountString(t *testing.T) { { name: "ro", mountString: "type=bind,source=/opt,destination=/opt,ro", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -130,11 +130,11 @@ func TestParseMountString(t *testing.T) { { name: "readonly", mountString: "type=bind,source=/opt,destination=/opt,readonly", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -144,11 +144,11 @@ func TestParseMountString(t *testing.T) { { name: "imagesrc", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt", - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {Value: "/opt"}, }, }, @@ -158,23 +158,23 @@ func TestParseMountString(t *testing.T) { { name: "imagesrcNoValue", mountString: "type=bind,source=test.sif,destination=/opt,image-src", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "imagesrcEmpty", mountString: "type=bind,source=test.sif,destination=/opt,image-src=", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "id", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt,id=2", - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {Value: "/opt"}, "id": {Value: "2"}, }, @@ -185,29 +185,29 @@ func TestParseMountString(t *testing.T) { { name: "idNoValue", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt,id", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "idEmpty", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt,id=", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "bindpropagation", mountString: "type=bind,source=/opt,destination=/opt,bind-propagation=shared", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "csvEscaped", mountString: `type=bind,"source=/comma,dir","destination=/quote""dir"`, - want: []BindPath{ + want: []Path{ { Source: "/comma,dir", Destination: "/quote\"dir", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -215,16 +215,16 @@ func TestParseMountString(t *testing.T) { { name: "multiple", mountString: "type=bind,source=/opt,destination=/opt\ntype=bind,source=/srv,destination=/srv", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, { Source: "/srv", Destination: "/srv", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, diff --git a/pkg/util/slice/slice.go b/pkg/util/slice/slice.go index c56b4dcfd5..46e0ebba2e 100644 --- a/pkg/util/slice/slice.go +++ b/pkg/util/slice/slice.go @@ -9,6 +9,8 @@ package slice +import "github.com/samber/lo" + // ContainsString returns true if string slice s contains match func ContainsString(s []string, match string) bool { for _, a := range s { @@ -40,3 +42,17 @@ func ContainsInt(s []int, match int) bool { } return false } + +// Subtract removes items in slice b from slice a, returning the result. +// Implemented using a map for greater efficiency than lo.Difference / lo.Without, when operating on large slices. +func Subtract[T comparable](a []T, b []T) []T { + subtractionMap := lo.FromEntries(lo.Map(a, func(item T, _ int) lo.Entry[T, bool] { + return lo.Entry[T, bool]{Key: item, Value: true} + })) + subtractionMap = lo.OmitByKeys(subtractionMap, b) + + return lo.Filter(a, func(x T, _ int) bool { + _, ok := subtractionMap[x] + return ok + }) +} diff --git a/pkg/util/slice/slice_test.go b/pkg/util/slice/slice_test.go index cee3b42e55..fc5e7418f6 100644 --- a/pkg/util/slice/slice_test.go +++ b/pkg/util/slice/slice_test.go @@ -9,7 +9,13 @@ package slice -import "testing" +import ( + "fmt" + "reflect" + "testing" + + "github.com/samber/lo" +) func TestContainsString(t *testing.T) { type args struct { @@ -190,3 +196,95 @@ func TestContainsInt(t *testing.T) { }) } } + +func TestSubtract(t *testing.T) { + type args[T any] struct { + a []T + b []T + want []T + } + intTests := []struct { + name string + args args[int] + }{ + { + name: "Identical", + args: args[int]{ + a: []int{3, 9, 5, 7, 2, 1, 0, 4}, + b: []int{3, 9, 5, 7, 2, 1, 0, 4}, + want: []int{}, + }, + }, + { + name: "EmptyA", + args: args[int]{ + a: []int{}, + b: []int{3, 9, 5, 7, 2, 1, 0, 4}, + want: []int{}, + }, + }, + { + name: "EmptyB", + args: args[int]{ + a: []int{3, 9, 5, 7, 2, 1, 0, 4}, + b: []int{}, + want: []int{3, 9, 5, 7, 2, 1, 0, 4}, + }, + }, + { + name: "EmptyBoth", + args: args[int]{ + a: []int{}, + b: []int{}, + want: []int{}, + }, + }, + { + name: "AsupersetofB", + args: args[int]{ + a: []int{3, 9, 5, 7, 2, 1, 0, 4}, + b: []int{3, 9, 7, 0, 4}, + want: []int{5, 2, 1}, + }, + }, + { + name: "AsubsetofB", + args: args[int]{ + a: []int{5, 2, 1}, + b: []int{5, 7, 2, 1, 0, 4}, + want: []int{}, + }, + }, + { + name: "Intersection", + args: args[int]{ + a: []int{3, 5, 2, 0}, + b: []int{3, 9, 7, 2, 4}, + want: []int{5, 0}, + }, + }, + } + + convertor := func(x int, index int) string { + return fmt.Sprintf("Have an int whose value is %#v, why don't you", x) + } + + for _, tt := range intTests { + t.Run("Int"+tt.name, func(t *testing.T) { + if got := Subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.args.want) { + t.Errorf("Subtract(%#v, %#v) = %#v, want %#v", tt.args.a, tt.args.b, got, tt.args.want) + } + }) + + strArgs := args[string]{ + a: lo.Map(tt.args.a, convertor), + b: lo.Map(tt.args.b, convertor), + want: lo.Map(tt.args.want, convertor), + } + t.Run("String"+tt.name, func(t *testing.T) { + if got := Subtract(strArgs.a, strArgs.b); !reflect.DeepEqual(got, strArgs.want) { + t.Errorf("Subtract(%#v, %#v) = %#v, want %#v", strArgs.a, strArgs.b, got, strArgs.want) + } + }) + } +} diff --git a/scripts/ci-deb-build-test b/scripts/ci-deb-build-test index 6380922827..b8294e6cfd 100755 --- a/scripts/ci-deb-build-test +++ b/scripts/ci-deb-build-test @@ -30,7 +30,9 @@ apt-get install -y \ libssl-dev \ python2 \ uuid-dev \ - golang-go + golang-go \ + conmon \ + crun # for squashfuse_ll build apt-get install -y autoconf automake libtool pkg-config libfuse-dev zlib1g-dev diff --git a/test/cdi/cditemplate.json.tpl b/test/cdi/cditemplate.json.tpl new file mode 100644 index 0000000000..59dd541438 --- /dev/null +++ b/test/cdi/cditemplate.json.tpl @@ -0,0 +1,18 @@ +{ + "cdiVersion": "0.5.0", + "kind": "apptainertesting.sylabs.io/device", + + "devices": [ + { + "name": "TesterDevice", + "containerEdits": { + "deviceNodes": {{tojson .DeviceNodes}}, + "mounts": {{tojson .Mounts}} + } + } + ], + + "containerEdits": { + "env": {{tojson .Env}} + } +} diff --git a/test/cdi/kmsg.json b/test/cdi/kmsg.json new file mode 100644 index 0000000000..78f2035bb7 --- /dev/null +++ b/test/cdi/kmsg.json @@ -0,0 +1,51 @@ +{ + "cdiVersion": "0.5.0", + "kind": "apptainertesting.sylabs.io/device", + + "devices": [ + { + "name": "kmsgDevice", + "containerEdits": { + "deviceNodes": [ + { + "hostPath": "/dev/kmsg", + "path": "/dev/kmsg", + "permissions": "rw", + "uid": 1000, + "gid": 1000 + } + ], + "mounts": [ + { + "containerPath": "/tmpmountforkmsg", + "options": [ + "rw" + ], + "hostPath": "/tmp" + } + ] + } + }, + { + "name": "tmpmountDevice17", + "containerEdits": { + "mounts": [ + { + "containerPath": "/tmpmount17", + "options": [ + "r" + ], + "hostPath": "/tmp" + } + ] + } + } + ], + + "containerEdits": { + "env": [ + "FOO=VALID_SPEC", + "BAR=BARVALUE1" + ] + } +} diff --git a/test/cdi/tmpmount.json b/test/cdi/tmpmount.json new file mode 100644 index 0000000000..77501d5628 --- /dev/null +++ b/test/cdi/tmpmount.json @@ -0,0 +1,45 @@ +{ + "cdiVersion": "0.5.0", + "kind": "apptainertesting.sylabs.io/device", + + "devices": [ + { + "name": "tmpmountDevice1", + "containerEdits": { + "mounts": [ + { + "containerPath": "/tmpmount13", + "options": [ + "rw" + ], + "hostPath": "/tmp" + }, + { + "containerPath": "/tmpmount3", + "options": [ + "rbind", + "nosuid", + "nodev" + ], + "hostPath": "/tmp" + }, + { + "containerPath": "/tmpmount1", + "options": [ + "ro" + ], + "hostPath": "/tmp" + } + ] + } + } + ], + + "containerEdits": { + "env": [ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN" + ] + } +} diff --git a/test/images/extfs-for-overlay.img b/test/images/extfs-for-overlay.img new file mode 100644 index 0000000000..82003e2b72 Binary files /dev/null and b/test/images/extfs-for-overlay.img differ diff --git a/test/images/squashfs-for-overlay.img b/test/images/squashfs-for-overlay.img new file mode 100644 index 0000000000..443f1ae978 Binary files /dev/null and b/test/images/squashfs-for-overlay.img differ