Skip to content

Commit

Permalink
[2.1 backport] Port the release script from master
Browse files Browse the repository at this point in the history
This would allow us to use the release script as-is from the tag instead of from master. This would help reproducibility
  • Loading branch information
kit-ty-kate committed Jul 9, 2024
1 parent dfbe6a0 commit 11b42f4
Show file tree
Hide file tree
Showing 5 changed files with 222 additions and 98 deletions.
36 changes: 20 additions & 16 deletions release/Dockerfile.in
Original file line number Diff line number Diff line change
@@ -1,29 +1,33 @@
FROM multiarch/debian-debootstrap:%TARGET_TAG%
FROM multiarch/alpine:%TARGET_TAG%
# May need configuration on the host:
# docker run --rm --privileged multiarch/qemu-user-static:register --reset
LABEL Description="opam release builds" Vendor="OCamlPro" Version="1.0"

RUN apt-get update && apt-get install bzip2 g++ make patch wget libltdl-dev --yes && apt-get clean --yes
RUN useradd -U --create-home opam
RUN apk add gcc g++ make coreutils openssl

ADD https://caml.inria.fr/pub/distrib/ocaml-4.07/ocaml-4.07.1.tar.gz /root/
RUN addgroup -S opam && adduser -S opam -G opam -s /bin/sh

ADD https://github.com/ocaml/ocaml/archive/refs/tags/%OCAMLV%.tar.gz /root/

WORKDIR /root
RUN tar xzf ocaml-4.07.1.tar.gz
WORKDIR ocaml-4.07.1
RUN tar xzf %OCAMLV%.tar.gz
WORKDIR ocaml-%OCAMLV%
RUN sed -i 's/gnueabi/*eabi/' configure
RUN sed -i 's/musl/musl*/' configure
RUN sed -i -e 's/NGROUPS_MAX/65536/' otherlibs/unix/getgroups.c
RUN ./configure %CONF% -prefix /usr/local
RUN make world opt.opt
RUN make install
RUN rm -rf /root/ocaml-4.07.1 /root/ocaml-4.07.1.tar.gz
RUN make "-j$(nproc)" && make install && rm -rf /root/ocaml-%OCAMLV% /root/ocaml-%OCAMLV%.tar.gz

RUN apk add patch

ENV PATH /usr/local/bin:/usr/bin:/bin
USER opam
VOLUME /src
WORKDIR /home/opam/
CMD tar xzf /src/opam-full-${VERSION}.tar.gz && \
cd opam-full-${VERSION} && \
echo "(${LINKING})" > src/client/linking.sexp && \
CMD tar xz >&2 && \
cd opam-full-${VERSION} >&2 && \
./configure --with-mccs && \
make lib-ext opam && \
strip opam && \
cp opam /src/opam-${VERSION}-${TARGET}
make lib-ext && \
echo "(${LINKING})" > src/client/linking.sexp && \
make opam >&2 && \
strip opam >&2 && \
cat opam
134 changes: 83 additions & 51 deletions release/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,71 +3,80 @@ VERSION = $(shell git describe $(TAG))
OPAM_VERSION = $(subst -,~,$(VERSION))
GIT_URL = ..

FULL_ARCHIVE_URL = https://github.com/ocaml/opam/releases/download/$(VERSION)/opam-full-$(VERSION).tar.gz

TARGETS = x86_64-linux i686-linux armhf-linux arm64-linux
# todo: x86_64-darwin

OCAMLV = 4.10.2
OCAMLV = 4.14.2
# currently hardcoded in Dockerfile.in
OCAML_URL = https://caml.inria.fr/pub/distrib/ocaml-$(basename $(OCAMLV))/ocaml-$(OCAMLV).tar.gz
OCAML_URL = https://github.com/ocaml/ocaml/archive/refs/tags/$(OCAMLV).tar.gz

HOST_OS = $(shell uname -s | tr A-Z a-z | sed 's/darwin/macos/')
HOST_OS = $(shell uname -s | tr A-Z a-z | sed -e 's/_.*//' -e 's/darwin/macos/')
HOST = $(shell uname -m | sed 's/amd64/x86_64/')-$(HOST_OS)
OUTDIR = out/$(TAG)

# The equivalent of "which <cmd>". Taken from the GNU Make documentation
pathsearch = $(firstword $(wildcard $(addsuffix /$(1),$(subst :, ,$(PATH)))))

x86_64-linux: $(OUTDIR)/opam-$(VERSION)-x86_64-linux
i686-linux: $(OUTDIR)/opam-$(VERSION)-i686-linux
armhf-linux: $(OUTDIR)/opam-$(VERSION)-armhf-linux
arm64-linux: $(OUTDIR)/opam-$(VERSION)-arm64-linux
ppc64le-linux: $(OUTDIR)/opam-$(VERSION)-ppc64le-linux
s390x-linux: $(OUTDIR)/opam-$(VERSION)-s390x-linux

all: $(patsubst %,out/opam-$(VERSION)-%,$(TARGETS))

out/opam-full-$(VERSION).tar.gz:
mkdir -p out
cd out && curl -OfL $(FULL_ARCHIVE_URL) || { \
git clone $(GIT_URL) -b $(TAG) --depth 1 opam-full-$(VERSION); \
sed 's/^AC_INIT(opam,.*)/AC_INIT(opam,$(OPAM_VERSION))/' opam-full-$(VERSION)/configure.ac > \
opam-full-$(VERSION)/configure.ac.tmp; \
mv opam-full-$(VERSION)/configure.ac.tmp \
opam-full-$(VERSION)/configure.ac; \
$(MAKE) -C opam-full-$(VERSION) configure download-ext; \
tar cz --exclude-vcs opam-full-$(VERSION) -f $(notdir $@); \
rm -rf opam-full-$(VERSION); \
}
$(OUTDIR)/opam-full-$(VERSION).tar.gz:
mkdir -p "$(OUTDIR)"
git clone $(GIT_URL) -b $(TAG) "$(OUTDIR)/opam-full-$(VERSION)"
$(MAKE) -C "$(OUTDIR)/opam-full-$(VERSION)" OCAML=$(call pathsearch,ocaml) download-ext
cd "$(OUTDIR)" && tar cz -f $(notdir $@) --exclude .git opam-full-$(VERSION)
rm -rf "$(OUTDIR)/opam-full-$(VERSION)"

build/Dockerfile.x86_64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/amd64-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86_64-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.i686-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/i386-jessie/g' $^ | sed 's/%CONF%/-host i686-linux/g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/x86-v3.13/g' -e 's/%CONF%/-build i586-alpine-linux-musl/g' $^ >$@

# Need to lie about gnueabihf instead of musleabihf, because of a ./configure bug
build/Dockerfile.armhf-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/armhf-jessie/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/armv7-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.arm64-linux: Dockerfile.in
mkdir -p build && sed 's/%TARGET_TAG%/arm64-stretch/g' $^ | sed 's/%CONF%//g' >$@
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/arm64-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.ppc64le-linux: Dockerfile.in
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/ppc64le-v3.13/g' -e 's/%CONF%//g' $^ >$@
build/Dockerfile.s390x-linux: Dockerfile.in
mkdir -p build && sed -e "s/%OCAMLV%/$(OCAMLV)/g" -e 's/%TARGET_TAG%/s390x-v3.13/g' -e 's/%CONF%//g' $^ >$@


build/%.image: build/Dockerfile.%
docker build -t opam-build-$* -f $^ build
touch $@

# Actually, this is for debian 8 jessie, and varies wildly
# Actually, this is for alpine 3.13, and varies
CLINKING_linux = \
-Wl,-Bstatic \
-lunix -lmccs_stubs -lmccs_glpk_stubs \
-lstdc++ \
-Wl,-Bdynamic \
-static-libgcc
-static-libgcc \
-static
# -Wl,-Bdynamic

CLINKING_macos = \
-lunix -lmccs_stubs -lmccs_glpk_stubs \
-lstdc++

CLINKING_openbsd = $(CLINKING_macos)
CLINKING_freebsd = $(CLINKING_macos)

LINKING = (-noautolink $(patsubst %,-cclib %,$(CLINKING_$(1))))

EXPORTS_openbsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \

EXPORTS_freebsd = \
CPATH=/usr/local/include: \
LIBRARY_PATH=/usr/local/lib: \

%: opam-$(VERSION)-%

opam-$(VERSION)-%: out/opam-$(VERSION)-%
opam-$(VERSION)-%: $(OUTDIR)/opam-$(VERSION)-%
ln -sf $^ $@

# host: opam-$(VERSION)-$(HOST)
Expand All @@ -76,40 +85,44 @@ opam-$(VERSION)-%: out/opam-$(VERSION)-%
build/$(HOST).env:
mkdir -p build/$(HOST)
cd build/$(HOST) && curl -OL $(OCAML_URL)
cd build/$(HOST) && tar xzf ocaml-$(OCAMLV).tar.gz
cd build/$(HOST) && tar xzf $(OCAMLV).tar.gz
cd build/$(HOST)/ocaml-$(OCAMLV) && \
./configure -prefix $(shell pwd)/build/$(HOST) && \
$(MAKE) world opt.opt && \
./configure --prefix "$(shell pwd)/build/$(HOST)" \
--disable-debug-runtime --disable-debugger --disable-instrumented-runtime \
--disable-ocamldoc --disable-stdlib-manpages --disable-ocamltest && \
$(MAKE) -j$(JOBS) && \
$(MAKE) install
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/ocaml-$(OCAMLV).tar.gz
rm -rf build/$(HOST)/ocaml-$(OCAMLV) build/$(HOST)/$(OCAMLV).tar.gz
touch $@

# Actually builds out/opam-$(VERSION)-$(HOST), but we don't want to override the
# Actually builds $(OUTDIR)/opam-$(VERSION)-$(HOST), but we don't want to override the
# rule that goes through a container
host: out/opam-full-$(VERSION).tar.gz build/$(HOST).env
host: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/$(HOST).env
rm -rf build/opam-full-$(VERSION)
cd build && tar xzf ../$<
( export \
PATH=$(shell pwd)/build/$(HOST)/bin:$$PATH \
MAKE=$(MAKE) \
$(EXPORTS_$(HOST_OS)); \
cd build/opam-full-$(VERSION) && \
./configure && \
./configure --with-mccs && \
$(MAKE) lib-ext && \
echo "$(call LINKING,$(HOST_OS))" >src/client/linking.sexp && \
$(MAKE) lib-ext DUNE_ARGS="--root=`pwd`"; \
$(MAKE) opam DUNE_ARGS="--root=`pwd`"; \
$(MAKE) opam; \
)
cp build/opam-full-$(VERSION)/opam out/opam-$(VERSION)-$(HOST)
strip out/opam-$(VERSION)-$(HOST)
cp build/opam-full-$(VERSION)/opam $(OUTDIR)/opam-$(VERSION)-$(HOST)
strip $(OUTDIR)/opam-$(VERSION)-$(HOST)
$(OUTDIR)/opam-$(VERSION)-$(HOST) --version
rm -rf build/opam-full-$(VERSION)

# Containerised builds
out/opam-$(VERSION)-%-linux: build/%-linux.image out/opam-full-$(VERSION).tar.gz
docker run --rm -v `pwd`/out:/src \
$(OUTDIR)/opam-$(VERSION)-%-linux: $(OUTDIR)/opam-full-$(VERSION).tar.gz build/%-linux.image
docker run --rm -i \
-e "VERSION=$(VERSION)" \
-e "TARGET=$*-linux" \
-e "LINKING=$(call LINKING,$(HOST_OS))" \
opam-build-$*-linux
-e "LINKING=$(call LINKING,linux)" \
opam-build-$*-linux \
<$< >$@

clean:
rm -rf build
Expand All @@ -119,9 +132,28 @@ distclean: clean

REMOTE_DIR = /tmp/opam-release
REMOTE_MAKE = make
remote: out/opam-full-$(VERSION).tar.gz
ssh "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/out"
scp Makefile "$(REMOTE):$(REMOTE_DIR)/"
scp "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
ssh "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
scp "$(REMOTE):$(REMOTE_DIR)/out/opam-$(VERSION)*" out/
REMOTE_SHELL = /bin/sh
SSH = ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
SCP = scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
remote: $(OUTDIR)/opam-full-$(VERSION).tar.gz
$(SSH) "$(REMOTE)" "mkdir -p $(REMOTE_DIR)/$(OUTDIR)"
$(SCP) Makefile "$(REMOTE):$(REMOTE_DIR)/"
$(SCP) "$^" "$(REMOTE):$(REMOTE_DIR)/$^"
$(SSH) "$(REMOTE)" 'sh -c "cd $(REMOTE_DIR) && ulimit -s 8192 && $(REMOTE_MAKE) host TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)"'
$(SCP) "$(REMOTE):$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"

SSH_USER = root
ULIMIT = ulimit -s 8192 &&
qemu: $(OUTDIR)/opam-full-$(VERSION).tar.gz
$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"mkdir -p $(REMOTE_DIR)/$(OUTDIR)\""
$(SCP) -P "$(QEMU_PORT)" Makefile "$(SSH_USER)@localhost:$(REMOTE_DIR)/"
$(SCP) -P "$(QEMU_PORT)" "$^" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$^"
$(SSH) -p "$(QEMU_PORT)" $(SSH_USER)@localhost "$(REMOTE_SHELL) -c \"cd $(REMOTE_DIR) && $(ULIMIT) $(REMOTE_MAKE) host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV)\""
$(SCP) -P "$(QEMU_PORT)" "$(SSH_USER)@localhost:$(REMOTE_DIR)/$(OUTDIR)/opam-$(VERSION)*" "$(OUTDIR)/"

macos-local: $(OUTDIR)/opam-full-$(VERSION).tar.gz
LOCAL_RELEASE_DIR=$(shell mktemp -d); \
mkdir -p "$${LOCAL_RELEASE_DIR}/${OUTDIR}" && \
cp Makefile "$^" "$${LOCAL_RELEASE_DIR}" && \
cd "$${LOCAL_RELEASE_DIR}" && ulimit -s 8192 && arch -arch $(MACOS_ARCH) make host JOBS=$(JOBS) TAG=$(TAG) VERSION=$(VERSION) OCAMLV=$(OCAMLV) && \
cp $${LOCAL_RELEASE_DIR}/$(OUTDIR)/opam-$(VERSION)* "$(GIT_URL)/release/$(OUTDIR)/"
43 changes: 34 additions & 9 deletions release/readme.md
Original file line number Diff line number Diff line change
@@ -1,29 +1,54 @@
# Steps to follow for each release

## Finalise opam code for release
* update version in opam files, configure.ac
* update version in all the opam files and in configure.ac
* run `make configure` to regenerate `./configure` [checked by github actions]
* update copyright headers
* run `make tests`, `opam-rt` [checked by github actions & appveyor]
* update the CHANGE file: take `master_changes.md` content to fil it
* if you're releasing the first final release of a new branch (e.g. 2.2.0): make sure `root_version` in OpamFile.ml is set to the final release number (e.g. for 2.2.0, root_version should be 2.2). Make sure that opamFormatUpgrade.ml also contains an upgrade function from the previous version (that function will most likely be empty)
* run `make tests`, `opam-rt` [checked by github actions]
* update the CHANGE file: take `master_changes.md` content to fill it

## Github release

[ once bump version & changes PRs merged ]
* tag the release (git tag -a 2.2.0; git push origin 2.2.0)
* /!\ Once the tag pushed, it can be updated only in case of severe issue
* tag the release (git tag -am 2.2.0 2.2.0; git push origin 2.2.0)
* /!\ Once the tag pushed, it can be updated [different commit] only in case of severe issue
* create a release (or prerelease if intermediate release) draft on github based on your tag (https://github.com/ocaml/opam/releases/new)
* generate opam artifacts, using `shell/release.sh`, it requires to have Docker install with several remotes, the different arches
* add releases notes (content of `master_changes.md`) in the release
* upload signature of artefacts
* fetch locally the tag
* launch docker using the Docker GUI macOS app
* generate opam artifacts, using `release/release.sh <tag>` from a macOS/arm64 machine, it requires to have Docker and QEMU installed (see below device requirements)
* generate the signatures using `release/sign.sh <tag>`
* add releases notes (content of `master_changes.md`) in the release draft
* upload everything from `release/out/<tag>`
* finalise the release (publish)

## Publish the release

* add hashes in `install.sh` (and check signatures)
* publish opam packages in opam-repository
* publish opam packages in opam-repository (and add `flags: avoid-version` and `available: opam-version >= "2.1.0"` to each packages if this is not a stable version)
* update versions (and messages, if necessary) in https://github.com/ocaml/opam-repository/blob/master/repo

## Announce!

* a blog entry in opam.ocaml.org
* a announcement in discuss.ocaml.org
* copy the blog entry from opam.ocaml.org for https://github.com/ocaml/ocaml.org/tree/main/data/changelog/opam
* announce the release on the OCaml Discord server


## After release

* Bump the version with a `~dev` at the end (e.g. `2.2.0~alpha~dev`)
* Check if reftests needs an update
* Bring the changes to the changelog (CHANGES) from the branch of the release to the `master` branch

### On a release candidate
* create a branch to a `x.y` for rc's and the final release

---

## Device requirements
* Mac M1 or above with Rosetta2
* brew dependencies: git, gpg, qemu>=8.1.0, docker>=24.0.0, md5sha1sum
* opam repo with the tag fetched
* Have the secret key available
Loading

0 comments on commit 11b42f4

Please sign in to comment.