diff --git a/apps/microtvm/reference-vm/.gitignore b/apps/microtvm/reference-vm/.gitignore index d918f5e13cc5..187e6d9f34da 100644 --- a/apps/microtvm/reference-vm/.gitignore +++ b/apps/microtvm/reference-vm/.gitignore @@ -1 +1,2 @@ -/release-test \ No newline at end of file +/release-test +/.vagrant diff --git a/apps/microtvm/reference-vm/README.md b/apps/microtvm/reference-vm/README.md index a5bb63574ce3..3d419cd36463 100644 --- a/apps/microtvm/reference-vm/README.md +++ b/apps/microtvm/reference-vm/README.md @@ -29,9 +29,10 @@ For more information on how to use them, see the ## microTVM Developer Information Each RTOS or platform (like Zephyr, Ardunio, etc) that integrates with microTVM -can check-in a Reference VM in this directory to help the community collaborate. -You should use the tools provided here to ensure a uniform release process -across all platforms. Typically, releases need to be created by TVM committers. +can check-in installation scripts in the Reference VM in this directory to help +the community collaborate. You should use the tools provided here to ensure a +uniform release process across all platforms. Typically, releases need to be +created by TVM committers. Generally speaking, it's expected that any integrated platform with a regression test checked-in to the tvm repository should also define a reference VM. If you @@ -39,16 +40,13 @@ want to integrate a new platform, please raise a discussion on [the forum](https://discuss.tvm.ai). -## Reference VMs Organization +## Reference VM Organization -Reference VMs are organized in this directory as follows: +The Reference VM is organized in this directory as follows: ``` . +-- base-box-tool.py - Reference VM build, test, and release tool. -+-- PLATFORM/ - One or more dirs related to the supported platform(s), - like zephyr/ and arduino/. The dir names are the same to - be passed as arguments to base-box-tool.py as PLATFORM. +-- Vagrantfile - Vagrantfile that end-users will invoke. Should be based | off a base box which contains dependencies other than the | TVM python dependencies. @@ -64,12 +62,12 @@ Reference VMs are organized in this directory as follows: 1. **Build** the base box for a given platform: ```bash -$ ./base-box-tool.py [--provider=PROVIDER] build PLATFORM +$ ./base-box-tool.py [--provider=PROVIDER] build ``` For example: ```bash -$ ./base-box-tool.py --provider virtualbox build zephyr +$ ./base-box-tool.py --provider virtualbox build ``` 2. **Run** release tests for each platform: @@ -90,7 +88,7 @@ $ ./base-box-tool.py --provider virtualbox build zephyr This command does the following for the specified provider: - * Copies all files inside `PLATFORM/` dir except `.vagrant` and `base-box` to + * Copies all files inside this dir except `.vagrant` and `base-box` to `release-test/`. This is done to avoid reusing any VM the developer may have started; @@ -108,7 +106,12 @@ $ ./base-box-tool.py --provider virtualbox build zephyr 4. If release tests pass, **release** the box: ```bash -$ ./base-box-tool.py [--provider=PROVIDER] release --release-version=RELEASE_VER --platform-version=PLATFORM_VER PLATFORM +$ ./base-box-tool.py [--provider=PROVIDER] release --release-version=RELEASE_VER ``` For that step be sure you've logged in to Vagrant Cloud using the `vagrant` tool. + +## Versioning +We use semantic versioning as it is recommended by [Vagrant](https://www.vagrantup.com/docs/boxes/versioning). We use `X.Y.Z` version where we maintain the same major version `X` it has minor changes and newer version is still compatible with older versions and we increase minor version `Y`. However, We increase the major version `X` when new RVM is not compatible with older onces. Updates to the Zephyr SDK or Arduino board SDKs are considered major changes and require incrementing major version `X`. In this versioning, `Z` is barely used but we kept it since Vagrant requires this format. + +**Note**: We will release all microTVM RVM boxes under [microtvm](https://app.vagrantup.com/tlcpack/boxes/microtvm) and use box versioning in Vagrant file. Previous versions like `microtvm-zephyr`, `microtvm-arduino`, `microtvm-zephyr-2.5`, etc. are deprecated and will be removed in the future. diff --git a/apps/microtvm/reference-vm/arduino/Vagrantfile b/apps/microtvm/reference-vm/Vagrantfile similarity index 87% rename from apps/microtvm/reference-vm/arduino/Vagrantfile rename to apps/microtvm/reference-vm/Vagrantfile index ab746c17ee2b..00465a8b8848 100644 --- a/apps/microtvm/reference-vm/arduino/Vagrantfile +++ b/apps/microtvm/reference-vm/Vagrantfile @@ -16,8 +16,8 @@ # under the License. Vagrant.configure("2") do |config| - config.vm.box = "tlcpack/microtvm-arduino" - config.vm.box_version = "2.0.0" + config.vm.box = "tlcpack/microtvm" + config.vm.box_version = "1.0.0" if ENV.has_key?("TVM_RVM_NUM_CORES") num_cores = ENV["TVM_RVM_NUM_CORES"] @@ -31,7 +31,7 @@ Vagrant.configure("2") do |config| ram_bytes = 2048 end - tvm_home = "../../../.." + tvm_home = "../../.." dirs_to_mount = [Pathname.new(Pathname.new(tvm_home).expand_path())] if ENV.has_key?("TVM_PROJECT_DIR") then dirs_to_mount.append(ENV["TVM_PROJECT_DIR"]) @@ -47,10 +47,15 @@ Vagrant.configure("2") do |config| end end - config.vm.provision "shell", path: "provision_setup.sh", env: {"TVM_HOME": dirs_to_mount[0]}, privileged: false + config.vm.provision "shell", + path: "provision_setup.sh", + env: {"TVM_HOME": dirs_to_mount[0], + "TVM_CI_NUM_CORES": num_cores + }, + privileged: false # Enable USB Controller on VirtualBox - vm_name = "microtvm-arduino-#{Time.now.tv_sec}" + vm_name = "microtvm-#{Time.now.tv_sec}" config.vm.provider "virtualbox" do |vb, overrides| vb.name = vm_name vb.cpus = num_cores diff --git a/apps/microtvm/reference-vm/arduino/.gitignore b/apps/microtvm/reference-vm/arduino/.gitignore deleted file mode 100644 index dace7081e3f2..000000000000 --- a/apps/microtvm/reference-vm/arduino/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/.vagrant diff --git a/apps/microtvm/reference-vm/arduino/README.md b/apps/microtvm/reference-vm/arduino/README.md deleted file mode 100644 index 530da71a58f3..000000000000 --- a/apps/microtvm/reference-vm/arduino/README.md +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - -# microTVM Arduino Reference Virtual Machine - -This directory contains setup files for Arduino virtual machine used for testing -microTVM platforms that are supported by [Arduino](https://www.arduino.cc/). - -## VM Information for Developers -Arduino VM is published under [tlcpack](https://app.vagrantup.com/tlcpack). -Here is a list of different release versions and their tools. - -We use semantic versioning as it is recommended by [Vagrant](https://www.vagrantup.com/docs/boxes/versioning). We use `X.Y.Z` version where we maintain the same major version `X` it has minor changes and newer version is still compatible with older versions and we increase minor version `Y`. However, We increase the major version `X` when new RVM is not compatible with older onces. Changing any Arduino board SDKs is considered a major change and requires increasing `X`. - -## Supported Arduino Boards -This RVM has been tested and is known to work with these boards: -- Adafruit Metro M4 -- Adafruit Pybadge -- Arduino Due -- Arduino Nano 33 BLE -- Arduino Portenta H7 -- Feather S2 -- Raspberry Pi Pico -- Sony Spresense -- Wio Terminal - -However, the RVM *should* work with any Arduino with sufficient memory, provided -its core is installed in `base-box/base_box_provision.sh`. - -Note that this RVM does not work with the Teensy boards, even though they are -supported by microTVM. This is because arduino-cli does not support Teensy -boards (https://github.com/arduino/arduino-cli/issues/700)/). diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh deleted file mode 100644 index 8ce9a5a0fa28..000000000000 --- a/apps/microtvm/reference-vm/arduino/base-box/base_box_setup.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -e -set -x - -skip_zeroing_disk=0 -if [ -e "$HOME/skip_zeroing_disk" ]; then - echo "NOTE: will not zero disk at the end due to VMWare Fusion bug" - echo "See: https://communities.vmware.com/t5/VMware-Fusion-Discussions/VMWare-Fusion-Pro-11-15-6-16696540-causes-macOS-crash-during/m-p/2284011#M139190" - skip_zeroing_disk=1 -fi - -# Install common configs -~/base_box_setup_common.sh -rm -f ~/base_box_setup_common.sh - -# Poetry -sed -i "/^# If not running interactively,/ i source \$HOME/.poetry/env" ~/.bashrc -sed -i "/^# If not running interactively,/ i\\ " ~/.bashrc - -# TODO do we need this? -echo 'export PATH=$HOME/vagrant/bin:"$PATH"' >> ~/.profile -source ~/.profile -echo PATH=$PATH - -# Clean box for packaging as a base box -sudo apt-get clean -if [ $skip_zeroing_disk -eq 0 ]; then - echo "Zeroing disk..." - EMPTY_FILE="$HOME/EMPTY" - dd if=/dev/zero "of=${EMPTY_FILE}" bs=1M || /bin/true - if [ ! -e "${EMPTY_FILE}" ]; then - echo "failed to zero empty sectors on disk" - exit 2 - fi - rm -f "${EMPTY_FILE}" -else - echo "NOTE: skipping zeroing disk due to command-line argument." -fi diff --git a/apps/microtvm/reference-vm/arduino/provision_setup.sh b/apps/microtvm/reference-vm/arduino/provision_setup.sh deleted file mode 100644 index 1d54db17fae5..000000000000 --- a/apps/microtvm/reference-vm/arduino/provision_setup.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -ex - -# TVM -# NOTE: TVM is presumed to be mounted already by Vagrantfile. -cd "${TVM_HOME}" - -platform="arduino" -apps/microtvm/reference-vm/rebuild-tvm.sh ${platform} - -# Build poetry -cd apps/microtvm/reference-vm/arduino - -poetry env use 3.7 - -# importers -poetry install -E importer-onnx -poetry install -E importer-tflite -poetry install -E importer-mxnet - -poetry install - -echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm-${platform}" >>~/.profile -echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm/arduino && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile -echo "source \$VENV_PATH/bin/activate" >>~/.profile diff --git a/apps/microtvm/reference-vm/base-box-tool.py b/apps/microtvm/reference-vm/base-box-tool.py index 4a1b5aea3fec..325b9bc0c4c9 100755 --- a/apps/microtvm/reference-vm/base-box-tool.py +++ b/apps/microtvm/reference-vm/base-box-tool.py @@ -18,7 +18,6 @@ import argparse -from ast import arg import copy import json import logging @@ -34,7 +33,7 @@ _LOG = logging.getLogger(__name__) -THIS_DIR = os.path.realpath(os.path.dirname(__file__) or ".") +THIS_DIR = pathlib.Path(os.path.realpath(os.path.dirname(__file__))) # List of vagrant providers supported by this tool ALL_PROVIDERS = ( @@ -52,31 +51,26 @@ # Extra scripts required to execute on provisioning # in [platform]/base-box/base_box_provision.sh -COMMON_SCRIPTS = [ - "apps/microtvm/reference-vm/base_box_setup_common.sh", +EXTRA_SCRIPTS = [ + "apps/microtvm/reference-vm/base-box/base_box_setup_common.sh", "docker/install/ubuntu_install_core.sh", "docker/install/ubuntu_install_python.sh", "docker/utils/apt-install-and-clear.sh", "docker/install/ubuntu1804_install_llvm.sh", + # Zephyr + "docker/install/ubuntu_init_zephyr_project.sh", + "docker/install/ubuntu_install_zephyr_sdk.sh", + "docker/install/ubuntu_install_cmsis.sh", ] -EXTRA_SCRIPTS = { - "arduino": [], - "zephyr": [ - "docker/install/ubuntu_init_zephyr_project.sh", - "docker/install/ubuntu_install_zephyr_sdk.sh", - "docker/install/ubuntu_install_cmsis.sh", - ], -} - PACKER_FILE_NAME = "packer.json" # List of identifying strings for microTVM boards for testing. -with open(pathlib.Path(THIS_DIR) / ".." / "zephyr" / "template_project" / "boards.json") as f: +with open(THIS_DIR / ".." / "zephyr" / "template_project" / "boards.json") as f: zephyr_boards = json.load(f) -with open(pathlib.Path(THIS_DIR) / ".." / "arduino" / "template_project" / "boards.json") as f: +with open(THIS_DIR / ".." / "arduino" / "template_project" / "boards.json") as f: arduino_boards = json.load(f) ALL_MICROTVM_BOARDS = { @@ -232,7 +226,7 @@ def attach_vmware(uuid, vid_hex=None, pid_hex=None, serial=None): } -def generate_packer_config(platform, file_path, providers): +def generate_packer_config(file_path, providers): builders = [] provisioners = [] for provider_name in providers: @@ -253,7 +247,7 @@ def generate_packer_config(platform, file_path, providers): ["git", "rev-parse", "--show-toplevel"], encoding="utf-8" ).strip() - scripts_to_copy = COMMON_SCRIPTS + EXTRA_SCRIPTS[platform] + scripts_to_copy = EXTRA_SCRIPTS for script in scripts_to_copy: script_path = os.path.join(repo_root, script) filename = os.path.basename(script_path) @@ -285,11 +279,9 @@ def generate_packer_config(platform, file_path, providers): def build_command(args): - this_dir = pathlib.Path(THIS_DIR) - base_box_dir = this_dir / args.platform / "base-box" + base_box_dir = THIS_DIR / "base-box" generate_packer_config( - args.platform, os.path.join(base_box_dir, PACKER_FILE_NAME), args.provider or ALL_PROVIDERS, ) @@ -313,9 +305,7 @@ def build_command(args): if box_package_exists: sys.exit("One or more box packages exist (see list above). To rebuild use '--force'") - subprocess.check_call( - packer_args, cwd=os.path.join(THIS_DIR, args.platform, "base-box"), env=env - ) + subprocess.check_call(packer_args, cwd=THIS_DIR / "base-box", env=env) REQUIRED_TEST_CONFIG_KEYS = { @@ -325,10 +315,10 @@ def build_command(args): VM_BOX_RE = re.compile(r'(.*\.vm\.box) = "(.*)"') - +VM_TVM_HOME_RE = re.compile(r'(.*tvm_home) = "(.*)"') # Paths, relative to the platform box directory, which will not be copied to release-test dir. -SKIP_COPY_PATHS = [".vagrant", "base-box"] +SKIP_COPY_PATHS = [".vagrant", "base-box", "scripts"] def do_build_release_test_vm( @@ -365,6 +355,12 @@ def do_build_release_test_vm( if "config.vm.box_version" in line: continue m = VM_BOX_RE.match(line) + tvm_home_m = VM_TVM_HOME_RE.match(line) + + if tvm_home_m: + # Adjust tvm home for testing step + f.write(f'{tvm_home_m.group(1)} = "../../../.."\n') + continue if not m: f.write(line) continue @@ -391,7 +387,7 @@ def do_build_release_test_vm( return True -def do_run_release_test(release_test_dir, platform, provider_name, test_config, test_device_serial): +def do_run_release_test(release_test_dir, provider_name, test_config, test_device_serial): with open( os.path.join(release_test_dir, ".vagrant", "machines", "default", provider_name, "id") ) as f: @@ -405,7 +401,7 @@ def do_run_release_test(release_test_dir, platform, provider_name, test_config, pid_hex=test_config["pid_hex"], serial=test_device_serial, ) - tvm_home = os.path.realpath(os.path.join(THIS_DIR, "..", "..", "..")) + tvm_home = os.path.realpath(THIS_DIR / ".." / ".." / "..") def _quote_cmd(cmd): return " ".join(shlex.quote(a) for a in cmd) @@ -415,7 +411,7 @@ def _quote_cmd(cmd): + " && " + _quote_cmd( [ - f"apps/microtvm/reference-vm/{platform}/base-box/base_box_test.sh", + f"apps/microtvm/reference-vm/base-box/base_box_test.sh", test_config["microtvm_board"], ] ) @@ -424,9 +420,9 @@ def _quote_cmd(cmd): def test_command(args): - user_box_dir = pathlib.Path(THIS_DIR) / args.platform + user_box_dir = THIS_DIR base_box_dir = user_box_dir / "base-box" - boards_file = pathlib.Path(THIS_DIR) / ".." / args.platform / "template_project" / "boards.json" + boards_file = THIS_DIR / ".." / args.platform / "template_project" / "boards.json" with open(boards_file) as f: test_config = json.load(f) @@ -444,7 +440,7 @@ def test_command(args): providers = args.provider - release_test_dir = os.path.join(THIS_DIR, f"release-test-{args.platform}") + release_test_dir = THIS_DIR / f"release-test" if args.skip_build or args.skip_destroy: assert ( @@ -460,7 +456,6 @@ def test_command(args): ) do_run_release_test( release_test_dir, - args.platform, provider_name, microtvm_test_config, args.test_device_serial, @@ -492,7 +487,7 @@ def release_command(args): if args.release_full_name: vm_name = args.release_full_name else: - vm_name = f"tlcpack/microtvm-{args.platform}" + vm_name = "tlcpack/microtvm" if not args.skip_creating_release_version: subprocess.check_call( @@ -518,12 +513,7 @@ def release_command(args): vm_name, args.release_version, provider_name, - os.path.join( - THIS_DIR, - args.platform, - "base-box", - f"output-packer-{provider_name}/package.box", - ), + str(THIS_DIR / "base-box" / f"output-packer-{provider_name}/package.box"), ] ) @@ -550,7 +540,6 @@ def parse_args(): # Options for build subcommand parser_build = subparsers.add_parser("build", help="Build a base box.") parser_build.set_defaults(func=build_command) - parser_build.add_argument("platform", help=platform_help_str, choices=ALL_PLATFORMS) parser_build.add_argument( "--debug-packer", action="store_true", @@ -606,7 +595,6 @@ def parse_args(): # Options for release subcommand parser_release = subparsers.add_parser("release", help="Release base box to cloud.") parser_release.set_defaults(func=release_command) - parser_release.add_argument("platform", help=platform_help_str, choices=ALL_PLATFORMS) parser_release.add_argument( "--release-version", required=True, @@ -634,10 +622,6 @@ def parse_args(): def main(): args = parse_args() - - if os.path.sep in args.platform or not os.path.isdir(os.path.join(THIS_DIR, args.platform)): - sys.exit(f" must be a sub-direcotry of {THIS_DIR}; got {args.platform}") - args.func(args) diff --git a/apps/microtvm/reference-vm/arduino/base-box/.gitignore b/apps/microtvm/reference-vm/base-box/.gitignore similarity index 100% rename from apps/microtvm/reference-vm/arduino/base-box/.gitignore rename to apps/microtvm/reference-vm/base-box/.gitignore diff --git a/apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template b/apps/microtvm/reference-vm/base-box/Vagrantfile.packer-template similarity index 100% rename from apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template rename to apps/microtvm/reference-vm/base-box/Vagrantfile.packer-template diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/base-box/base_box_provision.sh old mode 100644 new mode 100755 similarity index 94% rename from apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh rename to apps/microtvm/reference-vm/base-box/base_box_provision.sh index 4d845d7fed0e..175e4787eb90 --- a/apps/microtvm/reference-vm/arduino/base-box/base_box_provision.sh +++ b/apps/microtvm/reference-vm/base-box/base_box_provision.sh @@ -20,12 +20,23 @@ # virtual machine similar to CI QEMU setup. # -set -e set -x source ~/.profile +# Init Zephyr +cd ~ +~/ubuntu_init_zephyr_project.sh ~/zephyr + +# Install CMSIS +cd ~ +~/ubuntu_install_cmsis.sh ~/cmsis + +# Cleanup +rm -f ubuntu_init_zephyr_project.sh ubuntu_install_cmsis.sh + # Init Arduino +source ~/.profile cd ~ sudo apt-get install -y ca-certificates @@ -68,6 +79,3 @@ arduino-cli core install SPRESENSE:spresense@2.5.0 --additional-urls $SPRESENSE_ # The below sed command avoids the bug, and will be removed when no longer needed. PORTENTA_H7_BUGFIX_PATH=~/.arduino15/packages/arduino/hardware/mbed_portenta/3.1.1/cores/arduino/api/Common.h sed -i '3 i #include ' $PORTENTA_H7_BUGFIX_PATH - -# Cleanup -rm -f *.sh diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh b/apps/microtvm/reference-vm/base-box/base_box_setup.sh old mode 100644 new mode 100755 similarity index 100% rename from apps/microtvm/reference-vm/zephyr/base-box/base_box_setup.sh rename to apps/microtvm/reference-vm/base-box/base_box_setup.sh diff --git a/apps/microtvm/reference-vm/base_box_setup_common.sh b/apps/microtvm/reference-vm/base-box/base_box_setup_common.sh similarity index 100% rename from apps/microtvm/reference-vm/base_box_setup_common.sh rename to apps/microtvm/reference-vm/base-box/base_box_setup_common.sh diff --git a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh b/apps/microtvm/reference-vm/base-box/base_box_test.sh similarity index 60% rename from apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh rename to apps/microtvm/reference-vm/base-box/base_box_test.sh index 5c3d96dfc7df..a8a55a0f40ae 100755 --- a/apps/microtvm/reference-vm/arduino/base-box/base_box_test.sh +++ b/apps/microtvm/reference-vm/base-box/base_box_test.sh @@ -16,25 +16,27 @@ # specific language governing permissions and limitations # under the License. # -# Usage: base_box_test.sh -# Execute microTVM Arduino tests. -# -set -e set -x if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " + echo "Usage: base_box_test.sh " exit -1 fi -board=$1 +platform=$1 +board=$2 -pytest tests/micro/arduino/test_arduino_workflow.py --arduino-board=${board} +if [ "${platform}" == "zephyr" ]; then + pytest tests/micro/zephyr --zephyr-board=${board} +fi -if [ $board == "nano33ble" ]; then - # https://github.com/apache/tvm/issues/8730 - echo "NOTE: skipped test_arduino_rpc_server.py on $board -- known failure" -else - pytest tests/micro/arduino/test_arduino_rpc_server.py --arduino-board=${board} +if [ "${platform}" == "arduino" ]; then + pytest tests/micro/arduino/test_arduino_workflow.py --arduino-board=${board} + if [ $board == "nano33ble" ]; then + # https://github.com/apache/tvm/issues/8730 + echo "NOTE: skipped test_arduino_rpc_server.py on $board -- known failure" + else + pytest tests/micro/arduino/test_arduino_rpc_server.py --arduino-board=${board} + fi fi diff --git a/apps/microtvm/reference-vm/zephyr/provision_setup.sh b/apps/microtvm/reference-vm/provision_setup.sh old mode 100644 new mode 100755 similarity index 78% rename from apps/microtvm/reference-vm/zephyr/provision_setup.sh rename to apps/microtvm/reference-vm/provision_setup.sh index 785055a69658..f6237a82cd8b --- a/apps/microtvm/reference-vm/zephyr/provision_setup.sh +++ b/apps/microtvm/reference-vm/provision_setup.sh @@ -22,11 +22,10 @@ set -ex # NOTE: TVM is presumed to be mounted already by Vagrantfile. cd "${TVM_HOME}" -platform="zephyr" -apps/microtvm/reference-vm/rebuild-tvm.sh ${platform} +apps/microtvm/reference-vm/rebuild_tvm.sh # Build poetry -cd apps/microtvm/reference-vm/zephyr +cd apps/microtvm/reference-vm poetry env use 3.7 @@ -38,8 +37,8 @@ poetry install -E importer-mxnet poetry install poetry run pip3 install -r ${ZEPHYR_BASE}/scripts/requirements.txt -echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm-${platform}" >>~/.profile -echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm/zephyr && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile +echo "export TVM_LIBRARY_PATH=\"$TVM_HOME\"/build-microtvm" >>~/.profile +echo "VENV_PATH=\$((cd \"$TVM_HOME\"/apps/microtvm/reference-vm && poetry env list --full-path) | sed -E 's/^(.*)[[:space:]]\(Activated\)\$/\1/g')" >>~/.profile echo "source \$VENV_PATH/bin/activate" >>~/.profile echo "export PATH=\"\${PATH}:\${HOME}/zephyr-sdk/sysroots/x86_64-pokysdk-linux/usr/bin\"" >>~/.profile echo "export CMSIS_PATH=\"\${HOME}/cmsis\"" >>~/.profile diff --git a/apps/microtvm/reference-vm/rebuild-tvm.sh b/apps/microtvm/reference-vm/rebuild_tvm.sh similarity index 84% rename from apps/microtvm/reference-vm/rebuild-tvm.sh rename to apps/microtvm/reference-vm/rebuild_tvm.sh index ae58cb004c9e..6fdf4fd917f4 100755 --- a/apps/microtvm/reference-vm/rebuild-tvm.sh +++ b/apps/microtvm/reference-vm/rebuild_tvm.sh @@ -16,19 +16,14 @@ # specific language governing permissions and limitations # under the License. # -# "Usage ./apps/microtvm/reference-vm/rebuild-tvm.sh " -# set -e -if [ "$#" -lt 1 -o "$1" == "--help" ]; then - echo "Usage ./apps/microtvm/reference-vm/rebuild-tvm.sh " +if [ "$1" == "--help" ]; then + echo "Usage ./apps/microtvm/reference-vm/rebuild_tvm.sh" exit -1 fi -platform=$1 -shift 1 - # Get number of cores for build if [ -n "${TVM_CI_NUM_CORES}" ]; then num_cores=${TVM_CI_NUM_CORES} @@ -39,7 +34,7 @@ fi cd "$(dirname $0)" cd "$(git rev-parse --show-toplevel)" -BUILD_DIR="build-microtvm-${platform}" +BUILD_DIR="build-microtvm" if [ ! -e "${BUILD_DIR}" ]; then mkdir "${BUILD_DIR}" diff --git a/apps/microtvm/reference-vm/scripts/reference_vm_build.sh b/apps/microtvm/reference-vm/scripts/reference_vm_build.sh index bac31a26cf78..bfbd8aaa26d4 100755 --- a/apps/microtvm/reference-vm/scripts/reference_vm_build.sh +++ b/apps/microtvm/reference-vm/scripts/reference_vm_build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,19 +16,14 @@ # specific language governing permissions and limitations # under the License. # -# Usage: apps/microtvm/reference-vm/scripts/reference_vm_build.sh -# -if [ "$#" -lt 1 -o "$1" == "--help" -o "$1" == "-h" ]; then - echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_build.sh " +if [ "$1" == "--help" -o "$1" == "-h" ]; then + echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_build.sh" exit -1 fi -PLATFORM=$1 -shift - cd "$(dirname "$0")" source "./utils.sh" || exit 2 cd ${RVM_BASE_PATH} -${BASE_BOX_TOOL} --provider=virtualbox build ${PLATFORM} +${BASE_BOX_TOOL} --provider=virtualbox build diff --git a/apps/microtvm/reference-vm/scripts/reference_vm_release.sh b/apps/microtvm/reference-vm/scripts/reference_vm_release.sh index 8719e2c05a9f..beb271bd9e75 100755 --- a/apps/microtvm/reference-vm/scripts/reference_vm_release.sh +++ b/apps/microtvm/reference-vm/scripts/reference_vm_release.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,17 +16,12 @@ # specific language governing permissions and limitations # under the License. # -# Usage: apps/microtvm/reference-vm/scripts/reference_vm_release.sh -# -if [ "$#" -lt 3 -o "$1" == "--help" -o "$1" == "-h" ]; then - echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_release.sh " +if [ "$#" -lt 2 -o "$1" == "--help" -o "$1" == "-h" ]; then + echo "Usage: apps/microtvm/reference-vm/scripts/reference_vm_release.sh " exit -1 fi -PLATFORM=$1 -shift - RELEASE_NAME=$1 shift @@ -37,7 +32,7 @@ cd "$(dirname "$0")" source "./utils.sh" || exit 2 cd ${RVM_BASE_PATH} -${BASE_BOX_TOOL} --provider=virtualbox release ${PLATFORM} \ +${BASE_BOX_TOOL} --provider=virtualbox release \ --release-full-name=${RELEASE_NAME} \ --release-version=${RELEASE_VERSION} \ --skip-creating-release-version diff --git a/apps/microtvm/reference-vm/zephyr/.gitignore b/apps/microtvm/reference-vm/zephyr/.gitignore deleted file mode 100644 index dace7081e3f2..000000000000 --- a/apps/microtvm/reference-vm/zephyr/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/.vagrant diff --git a/apps/microtvm/reference-vm/zephyr/README.md b/apps/microtvm/reference-vm/zephyr/README.md deleted file mode 100644 index c5a1654c3ed3..000000000000 --- a/apps/microtvm/reference-vm/zephyr/README.md +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - -# microTVM Zephyr Reference Virtual Machine - -This directory contains setup files for Zephyr virtual machine used for testing microTVM platforms -that are supported by [Zephyr Project](https://zephyrproject.org/). - -## VM Information for Developers -Zephyr VM is published under [tlcpack](https://app.vagrantup.com/tlcpack). -Here is a list of different release versions and their tools. - -**Note**: We will release all microTVM RVM boxes under [microtvm-zephyr](https://app.vagrantup.com/tlcpack/boxes/microtvm-zephyr) and use box versioning in Vagrant file. Previous versions like `microtvm-zephyr-2.5`, `microtvm-zephyr-2.4` are not continued and will be removed in future. - -## Versioning -We use semantic versioning as it is recommended by [Vagrant](https://www.vagrantup.com/docs/boxes/versioning). We use `X.Y.Z` version where we maintain the same major version `X` it has minor changes and newer version is still compatible with older versions and we increase minor version `Y`. However, We increase the major version `X` when new RVM is not compatible with older onces. Updating Zephyr SDK is considered a major change and it requires incrementing major version `X`. diff --git a/apps/microtvm/reference-vm/zephyr/Vagrantfile b/apps/microtvm/reference-vm/zephyr/Vagrantfile deleted file mode 100644 index fb02f41d17d8..000000000000 --- a/apps/microtvm/reference-vm/zephyr/Vagrantfile +++ /dev/null @@ -1,95 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -Vagrant.configure("2") do |config| - config.vm.box = "tlcpack/microtvm-zephyr" - config.vm.box_version = "2.0.0" - - if ENV.has_key?("TVM_RVM_NUM_CORES") - num_cores = ENV["TVM_RVM_NUM_CORES"] - else - num_cores = 2 - end - - if ENV.has_key?("TVM_RVM_RAM_BYTES") - ram_bytes = ENV["TVM_RVM_RAM_BYTES"] - else - ram_bytes = 2048 - end - - tvm_home = "../../../.." - dirs_to_mount = [Pathname.new(Pathname.new(tvm_home).expand_path())] - if ENV.has_key?("TVM_PROJECT_DIR") then - dirs_to_mount.append(ENV["TVM_PROJECT_DIR"]) - puts "NOTE: also configuring project dir: %s" % [dirs_to_mount[-1]] - end - - git_file = Pathname.new(tvm_home + "/.git") - if git_file.ftype() == "file" then - gitdir_match = Regexp.new('^gitdir: (?.*/.git).*\n$', Regexp::MULTILINE).match(git_file.read()) - if !gitdir_match.nil? then - dirs_to_mount.append(Pathname.new(tvm_home).realpath.join(gitdir_match.named_captures["gitdir"])) - puts "NOTE: also configuring git-worktree gitdir: %s" % [dirs_to_mount[-1]] - end - end - - config.vm.provision "shell", - path: "provision_setup.sh", - env: {"TVM_HOME": dirs_to_mount[0], - "TVM_CI_NUM_CORES": num_cores - }, - privileged: false - - # Enable USB Controller on VirtualBox - vm_name = "microtvm-#{Time.now.tv_sec}" - config.vm.provider "virtualbox" do |vb, overrides| - vb.name = vm_name - vb.cpus = num_cores - vb.memory = ram_bytes - vb.customize ["modifyvm", :id, "--usb", "on"] - vb.customize ["modifyvm", :id, "--usbehci", "on"] - vb.customize ["modifyvm", :id, "--usbxhci", "on"] - vb.customize [ "guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 10000] - dirs_to_mount.each do |d| - overrides.vm.synced_folder d.to_s, d.to_s - end - end - - config.vm.provider "parallels" do |prl, overrides| - prl.name = vm_name - prl.cpus = num_cores - prl.memory = ram_bytes - prl.update_guest_tools = true - prl.customize ["set", :id, "--support-usb30", "on"] - dirs_to_mount.each do |d| - overrides.vm.synced_folder d.to_s, d.to_s, mount_options: ["share", "nosuid", "host_inodes"] - end - end - - config.vm.provider "vmware_desktop" do |vm, overrides| - vm.cpus = num_cores - vm.memory = ram_bytes - vm.vmx["usb_xhci.present"] = "TRUE" - vm.vmx["usb.present"] = "TRUE" - vm.vmx["ehci.present"] = "TRUE" - dirs_to_mount.each do |d| - overrides.vm.synced_folder d.to_s, d.to_s - end - vm.gui = true - end - -end diff --git a/apps/microtvm/reference-vm/zephyr/base-box/.gitignore b/apps/microtvm/reference-vm/zephyr/base-box/.gitignore deleted file mode 100644 index e4406c4f61e2..000000000000 --- a/apps/microtvm/reference-vm/zephyr/base-box/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*.box -.vagrant -/output-packer-* -/packer.json diff --git a/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template b/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template deleted file mode 100644 index b43596bb83c1..000000000000 --- a/apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -Vagrant.configure("2") do |config| - # From hashicorp default template: - # https://github.com/hashicorp/packer/blob/master/builder/vagrant/step_create_vagrantfile.go#L23-L37 - - config.vm.define "source" do |source| - source.vm.box = "{{.SourceBox}}" - config.ssh.insert_key = {{.InsertKey}} - end - - config.vm.define "output" do |output| - output.vm.box = "{{.BoxName}}" - output.vm.box_url = "file://package.box" - config.ssh.insert_key = {{.InsertKey}} - end - - {{ if ne .SyncedFolder "" -}} - config.vm.synced_folder "{{.SyncedFolder}}", "/vagrant" - {{- else -}} - config.vm.synced_folder ".", "/vagrant", disabled: true - {{- end}} - - - {{ if eq .BoxName "microtvm-base-vmware_desktop" -}} - config.vm.provision "shell", inline: "touch ~/skip_zeroing_disk", privileged: false - {{- end}} - - # NOTE: base_box_setup.sh resides in the parent directory (../) because this template is expanded into a - # sub-directory of base-box (output-packer-*). - config.vm.provision "shell", path: "../base_box_setup.sh", privileged: false -end diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh deleted file mode 100644 index 2c55312f3657..000000000000 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_provision.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Using this script we can reuse docker/install scripts to configure the reference -# virtual machine similar to CI QEMU setup. -# - -set -e -set -x - -source ~/.profile - -# Init Zephyr -cd ~ -~/ubuntu_init_zephyr_project.sh ~/zephyr - -# Install CMSIS -cd ~ -~/ubuntu_install_cmsis.sh ~/cmsis - -# Cleanup -rm -f ubuntu_init_zephyr_project.sh ubuntu_install_cmsis.sh diff --git a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh b/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh deleted file mode 100755 index 49f86a6ef9dd..000000000000 --- a/apps/microtvm/reference-vm/zephyr/base-box/base_box_test.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -e -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Usage: base_box_test.sh -# Execute microTVM Zephyr tests. -# - -set -e -set -x - -if [ "$#" -lt 1 ]; then - echo "Usage: base_box_test.sh " - exit -1 -fi - -board=$1 - -pytest tests/micro/zephyr --zephyr-board=${board} diff --git a/tests/lint/check_file_type.py b/tests/lint/check_file_type.py index 37b64433b23e..099ba3c3fa5b 100644 --- a/tests/lint/check_file_type.py +++ b/tests/lint/check_file_type.py @@ -149,10 +149,8 @@ "apps/microtvm/zephyr/template_project/qemu-hack/qemu-system-riscv64", # microTVM Virtual Machines "apps/microtvm/poetry.lock", - "apps/microtvm/reference-vm/arduino/Vagrantfile", - "apps/microtvm/reference-vm/arduino/base-box/Vagrantfile.packer-template", - "apps/microtvm/reference-vm/zephyr/Vagrantfile", - "apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template", + "apps/microtvm/reference-vm/Vagrantfile", + "apps/microtvm/reference-vm/base-box/Vagrantfile.packer-template", # Hexagon "src/runtime/hexagon/rpc/android_bash.sh.template", }