From 962b690ac3d7a7eb477c73c9f6b7b19bf3d0607f Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Wed, 29 Oct 2014 12:35:01 -0500 Subject: [PATCH 01/11] Move gaia specific stuff to b2g build --- testing/docker/builder/build-b2g-desktop.sh | 6 ++++++ testing/docker/builder/build-setup.sh | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/docker/builder/build-b2g-desktop.sh b/testing/docker/builder/build-b2g-desktop.sh index a56f5d0795400..53bbaca770c31 100755 --- a/testing/docker/builder/build-b2g-desktop.sh +++ b/testing/docker/builder/build-b2g-desktop.sh @@ -14,6 +14,12 @@ cd $gecko_dir hg pull -r $REVISION $REPOSITORY; hg update $REVISION; +### Clone gaia +if [ ! -d "$gaia_dir" ]; then + create_parent_dir $gaia_dir + hg clone https://hg.mozilla.org/integration/gaia-central/ $gaia_dir +fi + ### Pull and update gaia cd $gaia_dir GAIA_REV=$(get_gaia_revision.js) diff --git a/testing/docker/builder/build-setup.sh b/testing/docker/builder/build-setup.sh index 726a1142a5ebd..2615037ac8c69 100755 --- a/testing/docker/builder/build-setup.sh +++ b/testing/docker/builder/build-setup.sh @@ -22,9 +22,3 @@ mkdir -p /home/worker/.mozbuild/ # Create object-folder exists mkdir -p /home/worker/object-folder/ - -### Clone gaia in too -if [ ! -d "$gaia_dir" ]; then - create_parent_dir $gaia_dir - hg clone https://hg.mozilla.org/integration/gaia-central/ $gaia_dir -fi From 39861805af6554128e16f57191ba001d861bda67 Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Thu, 30 Oct 2014 16:07:42 -0500 Subject: [PATCH 02/11] Added linux64-mulet build task --- testing/docker/builder/Dockerfile | 1 + testing/docker/builder/build-firefox.sh | 43 +++++++++++++++++++++++++ testing/taskcluster/job_flags.yml | 1 + 3 files changed, 45 insertions(+) create mode 100755 testing/docker/builder/build-firefox.sh diff --git a/testing/docker/builder/Dockerfile b/testing/docker/builder/Dockerfile index 059c977febfee..30f12c5a61eef 100644 --- a/testing/docker/builder/Dockerfile +++ b/testing/docker/builder/Dockerfile @@ -8,6 +8,7 @@ RUN mkdir -p /home/worker/bin ADD build.sh /home/worker/bin/build.sh ADD build-b2g-desktop.sh /home/worker/bin/build-b2g-desktop.sh ADD get-objdir.py /home/worker/bin/get-objdir.py +ADD build-firefox.sh /home/worker/bin/build-firefox.sh ADD get_gaia_repo.js /home/worker/bin/get_gaia_repo.js ADD get_gaia_revision.js /home/worker/bin/get_gaia_revision.js ADD build-setup.sh /home/worker/bin/build-setup.sh diff --git a/testing/docker/builder/build-firefox.sh b/testing/docker/builder/build-firefox.sh new file mode 100755 index 0000000000000..0b8b41bc5ce4f --- /dev/null +++ b/testing/docker/builder/build-firefox.sh @@ -0,0 +1,43 @@ +#!/bin/bash -live + +################################### build_firefox.sh ################################### + +. build-setup.sh + +### Check that require variables are defined +test $REPOSITORY # Should be an hg repository url to pull from +test $REVISION # Should be an hg revision to pull down +test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder + + +### Pull and update mozilla-central +cd $gecko_dir +hg pull -r $REVISION $REPOSITORY; +hg update $REVISION; + +./mach build; + +### Make package +cd /home/worker/object-folder; +make package package-tests; + +### Extract artifacts +# Navigate to dist/ folder +cd /home/worker/object-folder/dist; + +ls -lah /home/worker/object-folder/dist/ + + +# Target names are cached so make sure we discard them first if found. +rm -f target.linux-x86_64.tar.bz2 target.linux-x86_64.json target.tests.zip + +# Artifacts folder is outside of the cache. +mkdir -p /home/worker/artifacts/ + +# Discard version numbers from packaged files, they just make it hard to write +# the right filename in the task payload where artifacts are declared +mv *.linux-x86_64.tar.bz2 /home/worker/artifacts/target.linux-x86_64.tar.bz2 +mv *.linux-x86_64.json /home/worker/artifacts/target.linux-x86_64.json +mv *.tests.zip /home/worker/artifacts/target.tests.zip + +################################### build.sh ################################### diff --git a/testing/taskcluster/job_flags.yml b/testing/taskcluster/job_flags.yml index 47f644122aba0..2ee2ab014b73e 100644 --- a/testing/taskcluster/job_flags.yml +++ b/testing/taskcluster/job_flags.yml @@ -12,6 +12,7 @@ flags: - linux64_gecko # b2g desktop linux 64 bit - macosx64_gecko # b2g desktop osx 64 bit - win32_gecko # b2g desktop win 32 bit + - linux64-mulet # Firefox desktop - b2g gecko linux 64 bit tests: - reftest From a9a35319f5e180225b221afbb735c76579d59511 Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Sun, 2 Nov 2014 11:13:33 -0600 Subject: [PATCH 03/11] Clean up dockerfile and move things at once instead of individually --- testing/docker/builder/Dockerfile | 12 ++-- testing/docker/builder/build-b2g-desktop.sh | 64 --------------------- testing/docker/builder/build-firefox.sh | 43 -------------- testing/docker/builder/build-setup.sh | 24 -------- testing/docker/builder/build.sh | 35 ----------- testing/docker/builder/get-objdir.py | 22 ------- testing/docker/builder/get_gaia_repo.js | 4 -- testing/docker/builder/get_gaia_revision.js | 4 -- 8 files changed, 4 insertions(+), 204 deletions(-) delete mode 100755 testing/docker/builder/build-b2g-desktop.sh delete mode 100755 testing/docker/builder/build-firefox.sh delete mode 100755 testing/docker/builder/build-setup.sh delete mode 100755 testing/docker/builder/build.sh delete mode 100755 testing/docker/builder/get-objdir.py delete mode 100755 testing/docker/builder/get_gaia_repo.js delete mode 100755 testing/docker/builder/get_gaia_revision.js diff --git a/testing/docker/builder/Dockerfile b/testing/docker/builder/Dockerfile index 30f12c5a61eef..e15bf8d91705d 100644 --- a/testing/docker/builder/Dockerfile +++ b/testing/docker/builder/Dockerfile @@ -2,16 +2,12 @@ FROM quay.io/mozilla/base:0.0.2 MAINTAINER Jonas Finnemann Jensen ENV PATH $PATH:/home/worker/bin/ +ENV TOOLTOOL_CACHE /home/worker/tools/tooltool_cache # Add utilities and configuration -RUN mkdir -p /home/worker/bin -ADD build.sh /home/worker/bin/build.sh -ADD build-b2g-desktop.sh /home/worker/bin/build-b2g-desktop.sh -ADD get-objdir.py /home/worker/bin/get-objdir.py -ADD build-firefox.sh /home/worker/bin/build-firefox.sh -ADD get_gaia_repo.js /home/worker/bin/get_gaia_repo.js -ADD get_gaia_revision.js /home/worker/bin/get_gaia_revision.js -ADD build-setup.sh /home/worker/bin/build-setup.sh +RUN mkdir /home/worker/bin +RUN mkdir /home/worker/tools +ADD bin /home/worker/bin RUN chown -R worker:worker /home/worker/ USER worker diff --git a/testing/docker/builder/build-b2g-desktop.sh b/testing/docker/builder/build-b2g-desktop.sh deleted file mode 100755 index 53bbaca770c31..0000000000000 --- a/testing/docker/builder/build-b2g-desktop.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -live - -################################### build.sh ################################### - -. build-setup.sh - -### Check that require variables are defined -test $REPOSITORY # Should be an hg repository url to pull from -test $REVISION # Should be an hg revision to pull down -test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder - -### Pull and update mozilla-central -cd $gecko_dir -hg pull -r $REVISION $REPOSITORY; -hg update $REVISION; - -### Clone gaia -if [ ! -d "$gaia_dir" ]; then - create_parent_dir $gaia_dir - hg clone https://hg.mozilla.org/integration/gaia-central/ $gaia_dir -fi - -### Pull and update gaia -cd $gaia_dir -GAIA_REV=$(get_gaia_revision.js) -GAIA_REPO="https://hg.mozilla.org$(get_gaia_repo.js)" -hg pull -r $GAIA_REV $GAIA_REPO; -hg update $GAIA_REV; - -cd $gecko_dir - -# Nightly mozconfig expects gaia repo be inside mozilla-central tree -if [ ! -d "gaia" ]; then - ln -s ../../gaia/source gaia -fi - -export MOZ_OBJDIR=$(get-objdir.py $gecko_dir) - -./mach build; - -### Make package -cd $MOZ_OBJDIR -make package package-tests; - -### Extract artifacts -# Navigate to dist/ folder -cd $MOZ_OBJDIR/dist - -ls -lah $MOZ_OBJDIR/dist/ - - -# Target names are cached so make sure we discard them first if found. -rm -f target.linux-x86_64.tar.bz2 target.linux-x86_64.json target.tests.zip - -# Artifacts folder is outside of the cache. -mkdir -p /home/worker/artifacts/ - -# Discard version numbers from packaged files, they just make it hard to write -# the right filename in the task payload where artifacts are declared -mv *.linux-x86_64.tar.bz2 /home/worker/artifacts/target.linux-x86_64.tar.bz2 -mv *.linux-x86_64.json /home/worker/artifacts/target.linux-x86_64.json -mv *.tests.zip /home/worker/artifacts/target.tests.zip - -################################### build.sh ################################### diff --git a/testing/docker/builder/build-firefox.sh b/testing/docker/builder/build-firefox.sh deleted file mode 100755 index 0b8b41bc5ce4f..0000000000000 --- a/testing/docker/builder/build-firefox.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -live - -################################### build_firefox.sh ################################### - -. build-setup.sh - -### Check that require variables are defined -test $REPOSITORY # Should be an hg repository url to pull from -test $REVISION # Should be an hg revision to pull down -test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder - - -### Pull and update mozilla-central -cd $gecko_dir -hg pull -r $REVISION $REPOSITORY; -hg update $REVISION; - -./mach build; - -### Make package -cd /home/worker/object-folder; -make package package-tests; - -### Extract artifacts -# Navigate to dist/ folder -cd /home/worker/object-folder/dist; - -ls -lah /home/worker/object-folder/dist/ - - -# Target names are cached so make sure we discard them first if found. -rm -f target.linux-x86_64.tar.bz2 target.linux-x86_64.json target.tests.zip - -# Artifacts folder is outside of the cache. -mkdir -p /home/worker/artifacts/ - -# Discard version numbers from packaged files, they just make it hard to write -# the right filename in the task payload where artifacts are declared -mv *.linux-x86_64.tar.bz2 /home/worker/artifacts/target.linux-x86_64.tar.bz2 -mv *.linux-x86_64.json /home/worker/artifacts/target.linux-x86_64.json -mv *.tests.zip /home/worker/artifacts/target.tests.zip - -################################### build.sh ################################### diff --git a/testing/docker/builder/build-setup.sh b/testing/docker/builder/build-setup.sh deleted file mode 100755 index 2615037ac8c69..0000000000000 --- a/testing/docker/builder/build-setup.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -vex - -gecko_dir=/home/worker/mozilla-central/source -gaia_dir=/home/worker/gaia/source - -create_parent_dir() { - parent_dir=$(dirname $1) - if [ ! -d "$parent_dir" ]; then - mkdir -p "$parent_dir" - fi -} - -### Firefox Build Setup -# Clone mozilla-central -if [ ! -d "$gecko_dir" ]; then - create_parent_dir $gecko_dir - hg clone https://hg.mozilla.org/mozilla-central/ $gecko_dir -fi - -# Create .mozbuild so mach doesn't complain about this -mkdir -p /home/worker/.mozbuild/ - -# Create object-folder exists -mkdir -p /home/worker/object-folder/ diff --git a/testing/docker/builder/build.sh b/testing/docker/builder/build.sh deleted file mode 100755 index 137be0aece883..0000000000000 --- a/testing/docker/builder/build.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -live - -################################### build.sh ################################### - -build-setup.sh - -### Check that we are running as worker -test `whoami` == 'worker'; - -### Check that require variables are defined -test $REPOSITORY # Should be an hg repository url to pull from -test $REVISION # Should be an hg revision to pull down -test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder - -### Pull, Update and Build -cd /home/worker/mozilla-central; -hg pull -r $REVISION $REPOSITORY; -hg update $REVISION; -./mach build; - -### Make package -cd /home/worker/object-folder; -make package package-tests; - -### Extract artifacts -# Navigate to dist/ folder -cd /home/worker/object-folder/dist; -# Discard version numbers from packaged files, they just make it hard to write -# the right filename in the task payload where artifacts are declared -mv *.linux-x86_64.tar.bz2 target.linux-x86_64.tar.bz2 -mv *.linux-x86_64.json target.linux-x86_64.json -mv *.tests.zip target.tests.zip - - -################################### build.sh ################################### diff --git a/testing/docker/builder/get-objdir.py b/testing/docker/builder/get-objdir.py deleted file mode 100755 index a2744978fb68a..0000000000000 --- a/testing/docker/builder/get-objdir.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python - -from __future__ import print_function -import sys -import os.path - -DEFAULT_OBJDIR = "/home/worker/object-folder" - -gecko_dir = sys.argv[1] - -base_path = os.path.join(gecko_dir, 'python') -sys.path.append(os.path.join(base_path, 'mozbuild')) -sys.path.append(os.path.join(base_path, 'mach')) -sys.path.append(os.path.join(gecko_dir, 'testing', 'mozbase', 'mozprocess')) - -from mozbuild.mozconfig import MozconfigLoader - -loader = MozconfigLoader(gecko_dir) -result = loader.read_mozconfig() - -topobjdir = result['topobjdir'] -print(topobjdir if topobjdir else DEFAULT_OBJDIR) diff --git a/testing/docker/builder/get_gaia_repo.js b/testing/docker/builder/get_gaia_repo.js deleted file mode 100755 index caa97f1231b22..0000000000000 --- a/testing/docker/builder/get_gaia_repo.js +++ /dev/null @@ -1,4 +0,0 @@ -#! /usr/bin/env node -var fs = require('fs'); -var data = fs.readFileSync('/home/worker/mozilla-central/source/b2g/config/gaia.json'); -console.log(JSON.parse(data).repo_path); diff --git a/testing/docker/builder/get_gaia_revision.js b/testing/docker/builder/get_gaia_revision.js deleted file mode 100755 index 000038f9a791e..0000000000000 --- a/testing/docker/builder/get_gaia_revision.js +++ /dev/null @@ -1,4 +0,0 @@ -#! /usr/bin/env node -var fs = require('fs'); -var data = fs.readFileSync('/home/worker/mozilla-central/source/b2g/config/gaia.json'); -console.log(JSON.parse(data).revision); From 2ff17b301a9ab04667651c835917077ce4b6f6f5 Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Sun, 2 Nov 2014 11:14:00 -0600 Subject: [PATCH 04/11] Move build scripts --- .../docker/builder/bin/build-b2g-desktop.sh | 64 +++++++++++++++++++ testing/docker/builder/bin/build-firefox.sh | 43 +++++++++++++ testing/docker/builder/bin/build-setup.sh | 24 +++++++ testing/docker/builder/bin/build.sh | 35 ++++++++++ testing/docker/builder/bin/get-objdir.py | 22 +++++++ testing/docker/builder/bin/get_gaia_repo.js | 4 ++ .../docker/builder/bin/get_gaia_revision.js | 4 ++ 7 files changed, 196 insertions(+) create mode 100755 testing/docker/builder/bin/build-b2g-desktop.sh create mode 100755 testing/docker/builder/bin/build-firefox.sh create mode 100755 testing/docker/builder/bin/build-setup.sh create mode 100755 testing/docker/builder/bin/build.sh create mode 100755 testing/docker/builder/bin/get-objdir.py create mode 100755 testing/docker/builder/bin/get_gaia_repo.js create mode 100755 testing/docker/builder/bin/get_gaia_revision.js diff --git a/testing/docker/builder/bin/build-b2g-desktop.sh b/testing/docker/builder/bin/build-b2g-desktop.sh new file mode 100755 index 0000000000000..53bbaca770c31 --- /dev/null +++ b/testing/docker/builder/bin/build-b2g-desktop.sh @@ -0,0 +1,64 @@ +#!/bin/bash -live + +################################### build.sh ################################### + +. build-setup.sh + +### Check that require variables are defined +test $REPOSITORY # Should be an hg repository url to pull from +test $REVISION # Should be an hg revision to pull down +test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder + +### Pull and update mozilla-central +cd $gecko_dir +hg pull -r $REVISION $REPOSITORY; +hg update $REVISION; + +### Clone gaia +if [ ! -d "$gaia_dir" ]; then + create_parent_dir $gaia_dir + hg clone https://hg.mozilla.org/integration/gaia-central/ $gaia_dir +fi + +### Pull and update gaia +cd $gaia_dir +GAIA_REV=$(get_gaia_revision.js) +GAIA_REPO="https://hg.mozilla.org$(get_gaia_repo.js)" +hg pull -r $GAIA_REV $GAIA_REPO; +hg update $GAIA_REV; + +cd $gecko_dir + +# Nightly mozconfig expects gaia repo be inside mozilla-central tree +if [ ! -d "gaia" ]; then + ln -s ../../gaia/source gaia +fi + +export MOZ_OBJDIR=$(get-objdir.py $gecko_dir) + +./mach build; + +### Make package +cd $MOZ_OBJDIR +make package package-tests; + +### Extract artifacts +# Navigate to dist/ folder +cd $MOZ_OBJDIR/dist + +ls -lah $MOZ_OBJDIR/dist/ + + +# Target names are cached so make sure we discard them first if found. +rm -f target.linux-x86_64.tar.bz2 target.linux-x86_64.json target.tests.zip + +# Artifacts folder is outside of the cache. +mkdir -p /home/worker/artifacts/ + +# Discard version numbers from packaged files, they just make it hard to write +# the right filename in the task payload where artifacts are declared +mv *.linux-x86_64.tar.bz2 /home/worker/artifacts/target.linux-x86_64.tar.bz2 +mv *.linux-x86_64.json /home/worker/artifacts/target.linux-x86_64.json +mv *.tests.zip /home/worker/artifacts/target.tests.zip + +################################### build.sh ################################### diff --git a/testing/docker/builder/bin/build-firefox.sh b/testing/docker/builder/bin/build-firefox.sh new file mode 100755 index 0000000000000..0b8b41bc5ce4f --- /dev/null +++ b/testing/docker/builder/bin/build-firefox.sh @@ -0,0 +1,43 @@ +#!/bin/bash -live + +################################### build_firefox.sh ################################### + +. build-setup.sh + +### Check that require variables are defined +test $REPOSITORY # Should be an hg repository url to pull from +test $REVISION # Should be an hg revision to pull down +test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder + + +### Pull and update mozilla-central +cd $gecko_dir +hg pull -r $REVISION $REPOSITORY; +hg update $REVISION; + +./mach build; + +### Make package +cd /home/worker/object-folder; +make package package-tests; + +### Extract artifacts +# Navigate to dist/ folder +cd /home/worker/object-folder/dist; + +ls -lah /home/worker/object-folder/dist/ + + +# Target names are cached so make sure we discard them first if found. +rm -f target.linux-x86_64.tar.bz2 target.linux-x86_64.json target.tests.zip + +# Artifacts folder is outside of the cache. +mkdir -p /home/worker/artifacts/ + +# Discard version numbers from packaged files, they just make it hard to write +# the right filename in the task payload where artifacts are declared +mv *.linux-x86_64.tar.bz2 /home/worker/artifacts/target.linux-x86_64.tar.bz2 +mv *.linux-x86_64.json /home/worker/artifacts/target.linux-x86_64.json +mv *.tests.zip /home/worker/artifacts/target.tests.zip + +################################### build.sh ################################### diff --git a/testing/docker/builder/bin/build-setup.sh b/testing/docker/builder/bin/build-setup.sh new file mode 100755 index 0000000000000..2615037ac8c69 --- /dev/null +++ b/testing/docker/builder/bin/build-setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash -vex + +gecko_dir=/home/worker/mozilla-central/source +gaia_dir=/home/worker/gaia/source + +create_parent_dir() { + parent_dir=$(dirname $1) + if [ ! -d "$parent_dir" ]; then + mkdir -p "$parent_dir" + fi +} + +### Firefox Build Setup +# Clone mozilla-central +if [ ! -d "$gecko_dir" ]; then + create_parent_dir $gecko_dir + hg clone https://hg.mozilla.org/mozilla-central/ $gecko_dir +fi + +# Create .mozbuild so mach doesn't complain about this +mkdir -p /home/worker/.mozbuild/ + +# Create object-folder exists +mkdir -p /home/worker/object-folder/ diff --git a/testing/docker/builder/bin/build.sh b/testing/docker/builder/bin/build.sh new file mode 100755 index 0000000000000..137be0aece883 --- /dev/null +++ b/testing/docker/builder/bin/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash -live + +################################### build.sh ################################### + +build-setup.sh + +### Check that we are running as worker +test `whoami` == 'worker'; + +### Check that require variables are defined +test $REPOSITORY # Should be an hg repository url to pull from +test $REVISION # Should be an hg revision to pull down +test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder + +### Pull, Update and Build +cd /home/worker/mozilla-central; +hg pull -r $REVISION $REPOSITORY; +hg update $REVISION; +./mach build; + +### Make package +cd /home/worker/object-folder; +make package package-tests; + +### Extract artifacts +# Navigate to dist/ folder +cd /home/worker/object-folder/dist; +# Discard version numbers from packaged files, they just make it hard to write +# the right filename in the task payload where artifacts are declared +mv *.linux-x86_64.tar.bz2 target.linux-x86_64.tar.bz2 +mv *.linux-x86_64.json target.linux-x86_64.json +mv *.tests.zip target.tests.zip + + +################################### build.sh ################################### diff --git a/testing/docker/builder/bin/get-objdir.py b/testing/docker/builder/bin/get-objdir.py new file mode 100755 index 0000000000000..a2744978fb68a --- /dev/null +++ b/testing/docker/builder/bin/get-objdir.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python + +from __future__ import print_function +import sys +import os.path + +DEFAULT_OBJDIR = "/home/worker/object-folder" + +gecko_dir = sys.argv[1] + +base_path = os.path.join(gecko_dir, 'python') +sys.path.append(os.path.join(base_path, 'mozbuild')) +sys.path.append(os.path.join(base_path, 'mach')) +sys.path.append(os.path.join(gecko_dir, 'testing', 'mozbase', 'mozprocess')) + +from mozbuild.mozconfig import MozconfigLoader + +loader = MozconfigLoader(gecko_dir) +result = loader.read_mozconfig() + +topobjdir = result['topobjdir'] +print(topobjdir if topobjdir else DEFAULT_OBJDIR) diff --git a/testing/docker/builder/bin/get_gaia_repo.js b/testing/docker/builder/bin/get_gaia_repo.js new file mode 100755 index 0000000000000..caa97f1231b22 --- /dev/null +++ b/testing/docker/builder/bin/get_gaia_repo.js @@ -0,0 +1,4 @@ +#! /usr/bin/env node +var fs = require('fs'); +var data = fs.readFileSync('/home/worker/mozilla-central/source/b2g/config/gaia.json'); +console.log(JSON.parse(data).repo_path); diff --git a/testing/docker/builder/bin/get_gaia_revision.js b/testing/docker/builder/bin/get_gaia_revision.js new file mode 100755 index 0000000000000..000038f9a791e --- /dev/null +++ b/testing/docker/builder/bin/get_gaia_revision.js @@ -0,0 +1,4 @@ +#! /usr/bin/env node +var fs = require('fs'); +var data = fs.readFileSync('/home/worker/mozilla-central/source/b2g/config/gaia.json'); +console.log(JSON.parse(data).revision); From 2885193762189def19f6d8dc7d5ef383e7ffe437 Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Sun, 2 Nov 2014 22:32:50 -0600 Subject: [PATCH 05/11] Pull tooltool dependencies --- testing/docker/builder/Dockerfile | 6 +++--- testing/docker/builder/bin/build-firefox.sh | 10 +++++++++- testing/docker/builder/system-setup.sh | 16 ++++++++++++++++ 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100755 testing/docker/builder/system-setup.sh diff --git a/testing/docker/builder/Dockerfile b/testing/docker/builder/Dockerfile index e15bf8d91705d..8db1fc6bc00bc 100644 --- a/testing/docker/builder/Dockerfile +++ b/testing/docker/builder/Dockerfile @@ -5,10 +5,10 @@ ENV PATH $PATH:/home/worker/bin/ ENV TOOLTOOL_CACHE /home/worker/tools/tooltool_cache # Add utilities and configuration -RUN mkdir /home/worker/bin -RUN mkdir /home/worker/tools +ADD system-setup.sh /tmp/system-setup.sh ADD bin /home/worker/bin -RUN chown -R worker:worker /home/worker/ +RUN /tmp/system-setup.sh +RUN chown -R worker:worker /home/worker/* USER worker diff --git a/testing/docker/builder/bin/build-firefox.sh b/testing/docker/builder/bin/build-firefox.sh index 0b8b41bc5ce4f..16608879a19b9 100755 --- a/testing/docker/builder/bin/build-firefox.sh +++ b/testing/docker/builder/bin/build-firefox.sh @@ -9,12 +9,20 @@ test $REPOSITORY # Should be an hg repository url to pull from test $REVISION # Should be an hg revision to pull down test $MOZCONFIG # Should be a mozconfig file from mozconfig/ folder - ### Pull and update mozilla-central cd $gecko_dir hg pull -r $REVISION $REPOSITORY; hg update $REVISION; +### Retrieve latest releng manifest for tooltool +tooltool=/home/worker/tools/tooltool.py +manifest=browser/config/tooltool-manifests/linux64/releng.manifest +tooltool_url=http://tooltool.pub.build.mozilla.org/temp-sm-stuff + +python $tooltool --url $tooltool_url --overwrite -m $manifest fetch -c $TOOLTOOL_CACHE +chmod +x setup.sh +./setup.sh + ./mach build; ### Make package diff --git a/testing/docker/builder/system-setup.sh b/testing/docker/builder/system-setup.sh new file mode 100755 index 0000000000000..cd07f82e5a038 --- /dev/null +++ b/testing/docker/builder/system-setup.sh @@ -0,0 +1,16 @@ +#!/bin/bash -ve + +############################### system-setup.sh ############################### + +home="/home/worker" + +mkdir -p $home/bin +mkdir -p $home/tools +mkdir -p $home/tools/tooltool_cache + +wget -O $home/tools/tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py + +# Remove the setup.sh setup, we don't really need this script anymore, deleting +# it keeps the image as clean as possible. +rm $0; echo "Deleted $0"; + From 6e88a447303386828fa6e2707bf666a85f61a171 Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Mon, 3 Nov 2014 11:56:13 -0600 Subject: [PATCH 06/11] Add MOZ_OBJDIR support to build file --- testing/docker/builder/bin/build-firefox.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/testing/docker/builder/bin/build-firefox.sh b/testing/docker/builder/bin/build-firefox.sh index 16608879a19b9..aeae4320c2ff4 100755 --- a/testing/docker/builder/bin/build-firefox.sh +++ b/testing/docker/builder/bin/build-firefox.sh @@ -14,7 +14,7 @@ cd $gecko_dir hg pull -r $REVISION $REPOSITORY; hg update $REVISION; -### Retrieve latest releng manifest for tooltool +### Retrieve and install latest tooltool manifest tooltool=/home/worker/tools/tooltool.py manifest=browser/config/tooltool-manifests/linux64/releng.manifest tooltool_url=http://tooltool.pub.build.mozilla.org/temp-sm-stuff @@ -23,17 +23,19 @@ python $tooltool --url $tooltool_url --overwrite -m $manifest fetch -c $TOOLTOOL chmod +x setup.sh ./setup.sh +export MOZ_OBJDIR=$(get-objdir.py $gecko_dir) + ./mach build; ### Make package -cd /home/worker/object-folder; +cd $MOZ_OBJDIR; make package package-tests; ### Extract artifacts # Navigate to dist/ folder -cd /home/worker/object-folder/dist; +cd $MOZ_OBJDIR/dist; -ls -lah /home/worker/object-folder/dist/ +ls -lah $MOZ_OBJDIR/dist/ # Target names are cached so make sure we discard them first if found. From ea4c4dc2841aaa5580385b17fb6fc6a2ecb607be Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Mon, 3 Nov 2014 14:47:34 -0600 Subject: [PATCH 07/11] Add linux64-mulet build task and update job tasks yml --- testing/docker/builder/Dockerfile | 2 +- ...{build-firefox.sh => build-mulet-linux.sh} | 2 +- testing/taskcluster/job_flags.yml | 6 +++ .../taskcluster/tasks/builds/mulet_linux.yml | 52 +++++++++++++++++++ 4 files changed, 60 insertions(+), 2 deletions(-) rename testing/docker/builder/bin/{build-firefox.sh => build-mulet-linux.sh} (94%) create mode 100644 testing/taskcluster/tasks/builds/mulet_linux.yml diff --git a/testing/docker/builder/Dockerfile b/testing/docker/builder/Dockerfile index 8db1fc6bc00bc..f4a73c9cd6f2a 100644 --- a/testing/docker/builder/Dockerfile +++ b/testing/docker/builder/Dockerfile @@ -2,7 +2,7 @@ FROM quay.io/mozilla/base:0.0.2 MAINTAINER Jonas Finnemann Jensen ENV PATH $PATH:/home/worker/bin/ -ENV TOOLTOOL_CACHE /home/worker/tools/tooltool_cache +ENV TOOLTOOL_CACHE /home/worker/tools/tooltool-cache # Add utilities and configuration ADD system-setup.sh /tmp/system-setup.sh diff --git a/testing/docker/builder/bin/build-firefox.sh b/testing/docker/builder/bin/build-mulet-linux.sh similarity index 94% rename from testing/docker/builder/bin/build-firefox.sh rename to testing/docker/builder/bin/build-mulet-linux.sh index aeae4320c2ff4..0b9e5249ee37f 100755 --- a/testing/docker/builder/bin/build-firefox.sh +++ b/testing/docker/builder/bin/build-mulet-linux.sh @@ -1,6 +1,6 @@ #!/bin/bash -live -################################### build_firefox.sh ################################### +################################### build-mulet-linux.sh ################################### . build-setup.sh diff --git a/testing/taskcluster/job_flags.yml b/testing/taskcluster/job_flags.yml index 2ee2ab014b73e..3e512c7bd02f2 100644 --- a/testing/taskcluster/job_flags.yml +++ b/testing/taskcluster/job_flags.yml @@ -56,6 +56,11 @@ builds: types: opt: tasks/builds/b2g_desktop.yml debug: tasks/builds/b2g_desktop_debug.yml + linux64-mulet: + platforms: + - Mulet Linux + types: + opt: tasks/builds/mulet_linux.yml # Test section covers the -u options in the try flags tests: @@ -75,5 +80,6 @@ tests: allowed_build_tasks: - tasks/builds/b2g_desktop.yml - tasks/builds/b2g_desktop_debug.yml + - tasks/builds/mulet_linux.yml task: tasks/tests/b2g_mochitest.yml chunks: 5 diff --git a/testing/taskcluster/tasks/builds/mulet_linux.yml b/testing/taskcluster/tasks/builds/mulet_linux.yml new file mode 100644 index 0000000000000..a027d7e96b070 --- /dev/null +++ b/testing/taskcluster/tasks/builds/mulet_linux.yml @@ -0,0 +1,52 @@ +taskId: {{build_slugid}} +task: + created: '{{now}}' + deadline: '{{#from_now}}24 hours{{/from_now}}' + metadata: + source: http://todo.com/soon + owner: {{owner}} + name: Mulet Linux x64 Opt + description: Mulet Linux x64 Opt + + workerType: b2gbuild + provisionerId: aws-provisioner + + scopes: + - 'docker-worker:cache:sources-mozilla-central' + - 'docker-worker:cache:build-mulet-linux-objects' + - 'docker-worker:cache:tooltool-cache' + + payload: + cache: + sources-mozilla-central: '/home/worker/mozilla-central' + build-mulet-linux-objects: '/home/worker/object-folder' + tooltool-cache: '/home/worker/tools/tooltool-cache' + + env: + MOZCONFIG: 'b2g/dev/config/mozconfigs/linux64/mulet' + # revision/project params defined originally here https://github.com/taskcluster/taskcluster-try/blob/master/try/instantiate.js + REVISION: '{{revision}}' + REPOSITORY: '{{repository}}' + + image: '{{#docker_image}}builder{{/docker_image}}' + maxRunTime: 3600 + + command: + - build-mulet-linux.sh + + artifacts: + 'public/build': + type: directory + path: '/home/worker/artifacts/' + expires: '{{#from_now}}1 year{{/from_now}}' + + extra: + # Rather then enforcing particular conventions we require that all build + # tasks provide the "build" extra field to specify where the build and tests + # files are located. + locations: + build: 'public/build/target.linux-x86_64.tar.bz2' + tests: 'public/build/target.tests.zip' + + treeherder: + symbol: B From 3be347319e251c9e6b24565fdabfee6bf3c8b34c Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Mon, 3 Nov 2014 16:10:42 -0600 Subject: [PATCH 08/11] Fixed command paths --- testing/taskcluster/tasks/builds/b2g_desktop.yml | 2 +- testing/taskcluster/tasks/builds/b2g_desktop_debug.yml | 2 +- testing/taskcluster/tasks/builds/mulet_linux.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/taskcluster/tasks/builds/b2g_desktop.yml b/testing/taskcluster/tasks/builds/b2g_desktop.yml index 915e4d814e89c..6993f2aa84bcc 100644 --- a/testing/taskcluster/tasks/builds/b2g_desktop.yml +++ b/testing/taskcluster/tasks/builds/b2g_desktop.yml @@ -32,7 +32,7 @@ task: maxRunTime: 3600 command: - - build-b2g-desktop.sh + - bin/build-b2g-desktop.sh artifacts: 'public/build': diff --git a/testing/taskcluster/tasks/builds/b2g_desktop_debug.yml b/testing/taskcluster/tasks/builds/b2g_desktop_debug.yml index edca6059422c3..dd57905dac0ec 100644 --- a/testing/taskcluster/tasks/builds/b2g_desktop_debug.yml +++ b/testing/taskcluster/tasks/builds/b2g_desktop_debug.yml @@ -32,7 +32,7 @@ task: maxRunTime: 3600 command: - - build-b2g-desktop.sh + - bin/build-b2g-desktop.sh artifacts: 'public/build': diff --git a/testing/taskcluster/tasks/builds/mulet_linux.yml b/testing/taskcluster/tasks/builds/mulet_linux.yml index a027d7e96b070..d7e6a20580149 100644 --- a/testing/taskcluster/tasks/builds/mulet_linux.yml +++ b/testing/taskcluster/tasks/builds/mulet_linux.yml @@ -32,7 +32,7 @@ task: maxRunTime: 3600 command: - - build-mulet-linux.sh + - bin/build-mulet-linux.sh artifacts: 'public/build': From 82ea3991ab23dba17a58ae6af79a334c5f860d3a Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Mon, 3 Nov 2014 20:30:02 -0600 Subject: [PATCH 09/11] Run as worker --- testing/docker/builder/Dockerfile | 5 +++-- testing/docker/builder/VERSION | 2 +- testing/docker/builder/run-as-worker.sh | 6 ++++++ 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100755 testing/docker/builder/run-as-worker.sh diff --git a/testing/docker/builder/Dockerfile b/testing/docker/builder/Dockerfile index f4a73c9cd6f2a..d6e3b0736b282 100644 --- a/testing/docker/builder/Dockerfile +++ b/testing/docker/builder/Dockerfile @@ -6,11 +6,12 @@ ENV TOOLTOOL_CACHE /home/worker/tools/tooltool-cache # Add utilities and configuration ADD system-setup.sh /tmp/system-setup.sh +ADD run-as-worker.sh /usr/local/bin/run-as-worker.sh ADD bin /home/worker/bin RUN /tmp/system-setup.sh RUN chown -R worker:worker /home/worker/* -USER worker - # Set a default command useful for debugging CMD ["/bin/bash", "--login"] + +ENTRYPOINT ["run-as-worker.sh"] diff --git a/testing/docker/builder/VERSION b/testing/docker/builder/VERSION index 9789c4ccb0c8a..e3b86dd9cc103 100644 --- a/testing/docker/builder/VERSION +++ b/testing/docker/builder/VERSION @@ -1 +1 @@ -0.0.14 +0.0.16 diff --git a/testing/docker/builder/run-as-worker.sh b/testing/docker/builder/run-as-worker.sh new file mode 100755 index 0000000000000..4cc27cacf77ba --- /dev/null +++ b/testing/docker/builder/run-as-worker.sh @@ -0,0 +1,6 @@ +#!/bin/bash -e + +# Take care of any volume mounts created by a user other than worker +chown -R worker:worker /home/worker/ + +sudo -E -u worker "$@" From 7f37b80f593f336517fa44403cce6a7028c8f536 Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Tue, 4 Nov 2014 12:04:02 -0600 Subject: [PATCH 10/11] User root user for now for builds until volume mount permissions are resolved --- testing/docker/builder/Dockerfile | 7 ++++--- testing/docker/builder/run-as-worker.sh | 6 ------ testing/docker/builder/system-setup.sh | 4 +++- 3 files changed, 7 insertions(+), 10 deletions(-) delete mode 100755 testing/docker/builder/run-as-worker.sh diff --git a/testing/docker/builder/Dockerfile b/testing/docker/builder/Dockerfile index d6e3b0736b282..a9067f36a34ba 100644 --- a/testing/docker/builder/Dockerfile +++ b/testing/docker/builder/Dockerfile @@ -6,12 +6,13 @@ ENV TOOLTOOL_CACHE /home/worker/tools/tooltool-cache # Add utilities and configuration ADD system-setup.sh /tmp/system-setup.sh -ADD run-as-worker.sh /usr/local/bin/run-as-worker.sh ADD bin /home/worker/bin RUN /tmp/system-setup.sh -RUN chown -R worker:worker /home/worker/* + +# TODO enable worker +# TODO volume mount permissions will be an issue +# USER worker # Set a default command useful for debugging CMD ["/bin/bash", "--login"] -ENTRYPOINT ["run-as-worker.sh"] diff --git a/testing/docker/builder/run-as-worker.sh b/testing/docker/builder/run-as-worker.sh deleted file mode 100755 index 4cc27cacf77ba..0000000000000 --- a/testing/docker/builder/run-as-worker.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -e - -# Take care of any volume mounts created by a user other than worker -chown -R worker:worker /home/worker/ - -sudo -E -u worker "$@" diff --git a/testing/docker/builder/system-setup.sh b/testing/docker/builder/system-setup.sh index cd07f82e5a038..819ff0e7c8a8a 100755 --- a/testing/docker/builder/system-setup.sh +++ b/testing/docker/builder/system-setup.sh @@ -6,10 +6,12 @@ home="/home/worker" mkdir -p $home/bin mkdir -p $home/tools -mkdir -p $home/tools/tooltool_cache +mkdir -p $home/tools/tooltool-cache wget -O $home/tools/tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py +chown -R worker:worker /home/worker/* /home/worker/.* + # Remove the setup.sh setup, we don't really need this script anymore, deleting # it keeps the image as clean as possible. rm $0; echo "Deleted $0"; From 07a77e424341b6ed80bb27f5c8fac15b70babfca Mon Sep 17 00:00:00 2001 From: Greg Arndt Date: Tue, 4 Nov 2014 14:44:01 -0600 Subject: [PATCH 11/11] Add tooltool manifest for b2g builds --- testing/docker/builder/bin/build-b2g-desktop.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/testing/docker/builder/bin/build-b2g-desktop.sh b/testing/docker/builder/bin/build-b2g-desktop.sh index 53bbaca770c31..40009ff1b81b8 100755 --- a/testing/docker/builder/bin/build-b2g-desktop.sh +++ b/testing/docker/builder/bin/build-b2g-desktop.sh @@ -14,6 +14,15 @@ cd $gecko_dir hg pull -r $REVISION $REPOSITORY; hg update $REVISION; +### Retrieve and install latest tooltool manifest +tooltool=/home/worker/tools/tooltool.py +manifest=b2g/config/tooltool-manifests/linux64/releng.manifest +tooltool_url=http://tooltool.pub.build.mozilla.org/temp-sm-stuff + +python $tooltool --url $tooltool_url --overwrite -m $manifest fetch -c $TOOLTOOL_CACHE +chmod +x setup.sh +./setup.sh + ### Clone gaia if [ ! -d "$gaia_dir" ]; then create_parent_dir $gaia_dir