Make linux-image.sh and linux-runner more robust

docker/aarch64-unknown-linux-gnu/Dockerfile

@@ -16,22 +16,16 @@ RUN apt-get install -y --no-install-recommends \
     g++-aarch64-linux-gnu \
     libc6-dev-arm64-cross \
     netcat-openbsd \
-    ipxe-qemu
+    vde2
 
 COPY openssl.sh /
 RUN bash /openssl.sh linux-aarch64 aarch64-linux-gnu-
 
 COPY qemu.sh /
 RUN bash /qemu.sh aarch64 linux softmmu
 
-ENV MIRROR=http://ftp.debian.org/debian/
 COPY linux-image.sh /
-RUN bash /linux-image.sh \
-        aarch64 \
-        gnu \
-        $MIRROR/dists/stretch/main/installer-arm64/current/images/netboot/debian-installer/arm64/linux \
-        $MIRROR/dists/stretch/main/installer-arm64/current/images/netboot/debian-installer/arm64/initrd.gz \
-        $MIRROR/pool/main/l/linux/linux-image-4.9.0-4-arm64_4.9.51-1_arm64.deb
+RUN bash /linux-image.sh aarch64
 
 COPY linux-runner /
 

docker/linux-image.sh

@@ -2,91 +2,115 @@ set -ex
 
 main() {
     local arch=$1 \
-          abi=$2 \
-          url_kernel=$3 \
-          url_initrd=$4 \
-          url_modules=$5
+          kernel=
+
+    case $arch in
+        aarch64)
+            arch=arm64
+            kernel=4.9.0-4-arm64
+            ;;
+        *)
+            echo "Invalid arch: $arch"
+            exit 1
+            ;;
+    esac
 
     local dependencies=(
-        curl
         cpio
+        debian-archive-keyring
     )
 
-    apt-get update
     local purge_list=()
+    apt-get update
     for dep in ${dependencies[@]}; do
         if ! dpkg -L $dep; then
             apt-get install --no-install-recommends -y $dep
             purge_list+=( $dep )
         fi
     done
 
-    mkdir -m 777 /qemu
+    # Download packages
+    mv /etc/apt/sources.list /etc/apt/sources.list.bak
+    echo "deb http://http.debian.net/debian/ stretch main contrib non-free" > \
+        /etc/apt/sources.list
+
+    dpkg --add-architecture $arch
+    apt-get update
+
+    mkdir -p -m 777 /qemu/$arch
+    cd /qemu/$arch
+    apt-get -t stretch -d --no-install-recommends download \
+        busybox:$arch \
+        libc6:$arch \
+        libgcc1:$arch \
+        libssl1*:$arch \
+        libstdc++6:$arch \
+        linux-image-$kernel:$arch \
+        zlib1g:$arch
     cd /qemu
 
-    curl -L $url_kernel -o vmlinuz
-    curl -L $url_initrd -o initrd.gz
-    curl -L $url_modules -o modules.deb
-
-    # Extract initrd
-    mkdir init
-    cd init
-    gunzip -c ../initrd.gz | cpio -id
-    cd -
-
-    # Remove some unecessary modules
-    rm -rf init/lib/modules/*/kernel/drivers/net/wireless/
-    rm -rf init/lib/modules/*/kernel/drivers/net/ethernet/
-    rm -rf init/lib/modules/*/kernel/drivers/net/usb/
-    rm -rf init/lib/modules/*/kernel/drivers/usb/
-    rm -rf init/lib/modules/*/kernel/drivers/gpu/
-    rm -rf init/lib/modules/*/kernel/drivers/mmc/
-    rm -rf init/lib/modules/*/kernel/drivers/staging/
-    rm -rf init/lib/modules/*/kernel/net/wireless/
-
-    # Copy 9p modules
-    dpkg -x modules.deb modules
-    cp ./modules/lib/modules/*/kernel/fs/fscache/fscache.ko \
-       ./modules/lib/modules/*/kernel/net/9p/9pnet.ko \
-       ./modules/lib/modules/*/kernel/net/9p/9pnet_virtio.ko \
-       ./modules/lib/modules/*/kernel/fs/9p/9p.ko \
-       /qemu/init/lib/modules/
-    rm -rf modules
-
-    # Copy libgcc and libstdc++
-    cp /usr/$arch-linux-$abi/lib/libgcc_s.so.1 \
-       /usr/$arch-linux-$abi/lib/libstdc++.so.6 \
-       /qemu/init/usr/lib/
-
-    # Alternative init
-    cat << EOF > /qemu/init/init-alt
-#!/bin/sh
-
-ip addr add 10.0.2.15/24 dev enp0s1
-ip link set enp0s1 up
-ip route add default via 10.0.2.2 dev enp0s1
-
-insmod /lib/modules/fscache.ko
-insmod /lib/modules/9pnet.ko
-insmod /lib/modules/9pnet_virtio.ko
-insmod /lib/modules/9p.ko
+    # Install packages
+    root=root-$arch
+    mkdir -p $root/{bin,etc,root,sys,dev,proc,sbin,usr/{bin,sbin}}
+    for deb in $arch/*deb; do
+        dpkg -x $deb $root/
+    done
+
+    # kernel
+    cp $root/boot/vmlinu* kernel
+
+    # initrd
+    mkdir -p $root/modules
+    cp \
+        $root/lib/modules/*/kernel/drivers/virtio/* \
+        $root/lib/modules/*/kernel/fs/9p/9p.ko \
+        $root/lib/modules/*/kernel/fs/fscache/fscache.ko \
+        $root/lib/modules/*/kernel/net/9p/9pnet.ko \
+        $root/lib/modules/*/kernel/net/9p/9pnet_virtio.ko \
+        $root/modules || true # some file may not exist
+    rm -rf $root/boot
+    rm -rf $root/lib/modules
+
+    cat << 'EOF' > $root/init
+#!/bin/busybox sh
+
+set -e
+
+/bin/busybox --install
+
+mount -t devtmpfs devtmpfs /dev
+mount -t proc none /proc
+mount -t sysfs none /sys
+
+# some archs does not have virtio modules
+insmod /modules/virtio.ko || true
+insmod /modules/virtio_ring.ko || true
+insmod /modules/virtio_mmio.ko || true
+insmod /modules/virtio_pci.ko || true
+insmod /modules/fscache.ko
+insmod /modules/9pnet.ko
+insmod /modules/9pnet_virtio.ko || true
+insmod /modules/9p.ko
 
 mkdir /target
-mount -t 9p -o trans=virtio target /target -oversion=9p2000.L
+mount -t 9p -o trans=virtio target /target -oversion=9p2000.L || true
 
-exec /bin/sh
-EOF
+echo "emulator is ready! $(cut -d' ' -f1 /proc/uptime)"
 
-    chmod +x /qemu/init/init-alt
+exec sh
+EOF
 
-    # Create the new initrd
-    cd init
-    find . | cpio --create --format='newc' --quiet | gzip > ../initrd.gz
-    cd ../
-    rm -rf init
+    chmod +x $root/init
+    cd $root && find . | cpio --create --format='newc' --quiet | gzip > ../initrd.gz
 
     # Clean up
+    rm -rf $root $arch
+    mv -f /etc/apt/sources.list.bak /etc/apt/sources.list
+    # can fail if arch is used (amd64 and/or i386)
+    dpkg --remove-architecture $arch || true
+    apt-get update
     apt-get purge --auto-remove -y ${purge_list[@]}
+    ls -lh /qemu
 }
 
 main "${@}"

docker/linux-runner

@@ -6,75 +6,96 @@ arch=$1
 prog=$2
 
 case "$CROSS_RUNNER" in
-  native)
-    exec $prog
-    ;;
-  qemu-user)
-    exec qemu-$arch $prog
-    ;;
-  qemu-system)
-    true
-    ;;
-  *)
-    echo "Invalid CROSS_RUNNER: $CROSS_RUNNER";
-    echo "This is a bug, please report it at https://github.com/japaric/cross"
-    exit 1
-    ;;
+    native)
+        exec $prog
+        ;;
+    qemu-user)
+        exec qemu-$arch $prog
+        ;;
+    qemu-system)
+        true
+        ;;
+    *)
+        echo "Invalid CROSS_RUNNER: $CROSS_RUNNER";
+        echo "This is a bug, please report it at https://github.com/japaric/cross"
+        exit 1
+        ;;
 esac
 
-if [ "$arch" = "arm" ]; then
-  opt="-machine virt"
-fi
+n=$(nproc)
+memory=1G
+driver9p="virtio-9p-pci"
 
-if [ "$arch" = "aarch64" ]; then
-  opt="-machine virt -cpu cortex-a57"
-fi
+case "$arch" in
+    aarch64)
+        opt="-machine virt -cpu cortex-a57"
+        ;;
+esac
 
 (
-  flock -n 200 || exit 0
+    flock -n 200 || exit 0
+
+    echo Booting QEMU virtual machine with $n cpus...
 
-  qemu-system-$arch \
-    -m 1024 \
-    -smp 4 \
-    -nographic \
-    $opt \
-    -kernel /qemu/vmlinuz \
-    -initrd /qemu/initrd.gz \
-    -serial tcp:127.0.0.1:4444,server,nowait \
-    -virtfs local,path=/target,security_model=mapped,mount_tag=target \
-    -append init=/init-alt > /dev/null &
+    dpipe qemu-system-$arch \
+        -m $memory \
+        -smp $n \
+        -nographic \
+        -monitor none \
+        -net none \
+        -kernel /qemu/kernel \
+        -initrd /qemu/initrd.gz \
+        $opt \
+        -fsdev local,id=fs0,path=/target,security_model=mapped \
+        -device $driver9p,fsdev=fs0,mount_tag=target \
+    = nc -l -k 127.0.0.1 4444 &
 
-  sleep 5
+    # wait nc server to start
+    sleep 3
 
-  nc -w 5 127.0.0.1 4444 > /dev/null
+    # wait the ready message
+    nc 127.0.0.1 4444 | while read line; do
+        line=$(echo "$line" | tr -d '\r')
+        if [[ "$line" == "emulator is ready!"* ]]; then
+            echo "Booted in ${line##* } seconds"
+            break
+        fi
+    done
 
 ) 200>/tmp/qemu.lock
 
-# use a random prefix so we can differenciate the exit code tag from the
+# use a random prefix so we can differentiate the exit code tag from the
 # program output
 begin_tag="oach8aobiePheew1kahl-start"
 end_tag="oach8aobiePheew1kahl-end"
 
 begin=no
 
+rm -f /tmp/xxx
+mkfifo /tmp/xxx
+nc -q 10000 127.0.0.1 4444 << EOF > /tmp/xxx &
+echo $begin_tag; $prog; echo $end_tag \$?
+EOF
+nc_pid=$!
+
 # execute the program and then print the exit code
-echo "echo $begin_tag; $prog; echo $end_tag \$?" | nc -q -1 127.0.0.1 4444 | while read LINE; do
-  LINE=$(echo "$LINE" | tr -d '\r')
-  if [[ "$LINE" == "oach8aobiePheew1kahl-start" ]]; then
-    begin=yes
-    continue
-  fi
-  if [[ $begin == "no" ]]; then
-    continue
-  fi
-  if [[ "$LINE" == $end_tag* ]]; then
-    # kill nc
-    for pid in $(ps | grep nc); do
-      kill $pid || true
-      break
-    done
-    exit ${LINE##* }
-  else
-    echo $LINE
-  fi
-done
+while read line; do
+    line=$(echo "$line" | tr -d '\r')
+    if [[ "$line" == $begin_tag ]]; then
+        begin=yes
+        continue
+    fi
+    if [[ $begin == no ]]; then
+        continue
+    fi
+    if [[ "$line" == $end_tag* ]]; then
+        kill $nc_pid
+        exit ${line##* }
+    else
+        echo $line
+    fi
+done < /tmp/xxx
+
+echo linux-runner crashed!
+
+exit 1

docker/qemu.sh

@@ -120,7 +120,8 @@ EOF
          true
          ;;
       *)
-         echo "Invalid option $softmmu=$softmmu"
+         echo "Invalid softmmu option: $softmmu"
+         exit 1
          ;;
    esac
 
@@ -137,7 +138,6 @@ EOF
     # HACK the binfmt_misc interpreter we'll use expects the QEMU binary to be
     # in /usr/bin. Create an appropriate symlink
     ln -s /usr/local/bin/qemu-$arch /usr/bin/qemu-$arch-static
-    ln -s /usr/local/bin/qemu-system-$arch /usr/bin/qemu-system-$arch || true
 
     # Clean up
     apt-get purge --auto-remove -y ${purge_list[@]}