eval-update smoke test (#1114)

* existing updates

* updated evasion scenarios

* update

* dapricot update

* so2sat update

* poisoning

* scenario updates

* remove base

* typedef hint for JSON-like config dict

* add jupyter text

* typehints and docstrings

* avoid name error if attack_type is preloaded

* unbound local errors

* calls via super have implied self

* self reference removed

* torchvision is back-versioned

* typo metrics for metric

* align torchvision version with pytorch version

as prescribed by https://pypi.org/project/torchvision/

* black19.10b0 and flake8 compliant

* update workflow

* forgot to push latest commit

* name changes

* updated names

* simplify

* simplification

* update ART api usage

Co-authored-by: matt wartell <matt.wartell@twosixlabs.com>

armory/__init__.py

@@ -10,6 +10,10 @@
 # Semantic Version
 __version__ = "0.14.0"
 
+# typedef for a widely used JSON-like configuration specification
+from typing import Dict, Any
+
+Config = Dict[str, Any]
 
 # Submodule imports
 try:
@@ -21,11 +25,12 @@
     from armory import docker
     from armory import eval
     from armory import paths
-    from armory import scenarios
     from armory import utils
     from armory import webapi
 except ImportError as e:
     module = e.name
     print(f"ERROR: cannot import '{module}' module")
     print("    Please run: $ pip install -r requirements.txt")
     raise
+
+END_SENTINEL = "Scenario has finished running cleanly"

armory/docker/management.py

@@ -6,7 +6,8 @@
 
 import docker
 
-from armory import paths, scenarios
+import armory
+from armory import paths
 
 
 logger = logging.getLogger(__name__)
@@ -93,7 +94,7 @@ def exec_cmd(self, cmd: str, user="", expect_sentinel=True) -> int:
                 if not inner_output:
                     continue
                 print(inner_output)
-                if inner_output == scenarios.END_SENTINEL:
+                if inner_output == armory.END_SENTINEL:
                     sentinel_found = True
 
         # if we're not running a config (eg armory exec or launch)

armory/eval/evaluator.py

@@ -215,7 +215,15 @@ def run(
             )
             try:
                 if jupyter:
-                    self._run_jupyter(runner, ports)
+                    self._run_jupyter(
+                        runner,
+                        ports,
+                        check_run=check_run,
+                        num_eval_batches=num_eval_batches,
+                        skip_benign=skip_benign,
+                        skip_attack=skip_attack,
+                        skip_misclassified=skip_misclassified,
+                    )
                 elif interactive:
                     self._run_interactive_bash(
                         runner,
@@ -296,7 +304,7 @@ def _run_config(
             kwargs = {"user": self.get_id()}
             python = "python"
 
-        cmd = f"{python} -m armory.scenarios.base {b64_config}{options}"
+        cmd = f"{python} -m armory.scenarios.main {b64_config}{options} --base64"
         return runner.exec_cmd(cmd, **kwargs)
 
     def _run_command(self, runner: ArmoryInstance, command: str) -> int:
@@ -330,14 +338,13 @@ def _run_interactive_bash(
         user_group_id = self.get_id()
         lines = [
             "Container ready for interactive use.",
-            bold(
-                "*** In a new terminal, run the following to attach to the container:"
-            ),
+            bold("# In a new terminal, run the following to attach to the container:"),
             bold(
                 red(
-                    f"    docker exec -it -u {user_group_id} {runner.docker_container.short_id} bash"
+                    f"docker exec -it -u {user_group_id} {runner.docker_container.short_id} bash"
                 )
             ),
+            "",
         ]
         if self.config.get("scenario"):
             options = self._build_options(
@@ -348,6 +355,14 @@ def _run_interactive_bash(
                 skip_misclassified=skip_misclassified,
                 validate_config=validate_config,
             )
+            init_options = self._constructor_options(
+                check_run=check_run,
+                num_eval_batches=num_eval_batches,
+                skip_benign=skip_benign,
+                skip_attack=skip_attack,
+                skip_misclassified=skip_misclassified,
+            )
+
             tmp_dir = os.path.join(self.host_paths.tmp_dir, self.config["eval_id"])
             os.makedirs(tmp_dir)
             self.tmp_config = os.path.join(tmp_dir, "interactive-config.json")
@@ -361,35 +376,82 @@ def _run_interactive_bash(
 
             lines.extend(
                 [
-                    bold("*** To run your scenario in the container:"),
+                    bold("# To run your scenario in the container:"),
+                    bold(
+                        red(
+                            f"python -m armory.scenarios.main {docker_config_path}{options}"
+                        )
+                    ),
+                    "",
+                    bold("# To run your scenario interactively:"),
                     bold(
                         red(
-                            f"    python -m armory.scenarios.base {docker_config_path}{options} --load-config-from-file"
+                            "python\n"
+                            "from armory import scenarios\n"
+                            f's = scenarios.get("{docker_config_path}"{init_options}).load()\n'
+                            "s.evaluate()"
                         )
                     ),
-                    bold("*** To gracefully shut down container, press: Ctrl-C"),
+                    "",
+                    bold("# To gracefully shut down container, press: Ctrl-C"),
                     "",
                 ]
             )
         logger.info("\n".join(lines))
         while True:
             time.sleep(1)
 
-    def _run_jupyter(self, runner: ArmoryInstance, ports: dict) -> None:
+    def _run_jupyter(
+        self,
+        runner: ArmoryInstance,
+        ports: dict,
+        check_run=False,
+        num_eval_batches=None,
+        skip_benign=None,
+        skip_attack=None,
+        skip_misclassified=None,
+    ) -> None:
         if not self.root:
             logger.warning("Running Jupyter Lab as root inside the container.")
 
         user_group_id = self.get_id()
         port = list(ports.keys())[0]
+        tmp_dir = os.path.join(self.host_paths.tmp_dir, self.config["eval_id"])
+        os.makedirs(tmp_dir)
+        self.tmp_config = os.path.join(tmp_dir, "interactive-config.json")
+        docker_config_path = os.path.join(
+            paths.runtime_paths().tmp_dir,
+            self.config["eval_id"],
+            "interactive-config.json",
+        )
+        with open(self.tmp_config, "w") as f:
+            f.write(json.dumps(self.config, sort_keys=True, indent=4) + "\n")
+        init_options = self._constructor_options(
+            check_run=check_run,
+            num_eval_batches=num_eval_batches,
+            skip_benign=skip_benign,
+            skip_attack=skip_attack,
+            skip_misclassified=skip_misclassified,
+        )
         lines = [
             "About to launch jupyter.",
-            bold("*** To connect on the command line as well, in a new terminal, run:"),
+            bold("# To connect on the command line as well, in a new terminal, run:"),
             bold(
                 red(
-                    f"    docker exec -it -u {user_group_id} {runner.docker_container.short_id} bash"
+                    f"docker exec -it -u {user_group_id} {runner.docker_container.short_id} bash"
                 )
             ),
-            bold("*** To gracefully shut down container, press: Ctrl-C"),
+            "",
+            bold("# To run, inside of a notebook:"),
+            bold(
+                red(
+                    "from armory import scenarios\n"
+                    f's = scenarios.get("{docker_config_path}"{init_options}).load()\n'
+                    "s.evaluate()"
+                )
+            ),
+            "",
+            bold("# To gracefully shut down container, press: Ctrl-C"),
             "",
             "Jupyter notebook log:",
         ]
@@ -427,3 +489,21 @@ def _build_options(
         if validate_config:
             options += " --validate-config"
         return options
+
+    def _constructor_options(
+        self,
+        check_run=False,
+        num_eval_batches=None,
+        skip_benign=None,
+        skip_attack=None,
+        skip_misclassified=None,
+    ):
+        kwargs = dict(
+            check_run=check_run,
+            num_eval_batches=num_eval_batches,
+            skip_benign=skip_benign,
+            skip_attack=skip_attack,
+            skip_misclassified=skip_misclassified,
+        )
+        options = "".join(f", {str(k)}={str(v)}" for k, v in kwargs.items() if v)
+        return options

armory/scenarios/__init__.py

@@ -2,4 +2,4 @@
 Evaluation scenarios for novel defenses
 """
 
-from armory.scenarios.common import END_SENTINEL
+from armory.scenarios.main import get