Skip to content

Commit

Permalink
fix packing
Browse files Browse the repository at this point in the history
  • Loading branch information
pmp-p committed Oct 25, 2023
1 parent f97fcca commit b215e2b
Show file tree
Hide file tree
Showing 4 changed files with 430 additions and 59 deletions.
143 changes: 143 additions & 0 deletions python-wasi-sdk.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
#!/bin/bash
reset

# TODO: check how dbg tools work with default settings
# https://developer.chrome.com/blog/wasm-debugging-2020/


. /etc/lsb-release
DISTRIB="${DISTRIB_ID}-${DISTRIB_RELEASE}"

export SDKROOT=/opt/python-wasm-sdk

export CIVER=${CIVER:-$DISTRIB}
export CI=true

if echo $0|grep -q python-wasm-sdk
then
echo " * adding emsdk to wasm-sdk"
emsdk=true
else
emsdk=false
fi

echo " * adding wasi-sdk to wasm-sdk"
wasisdk=true


sudo mkdir -p ${SDKROOT}
sudo chmod 777 ${SDKROOT}

ORIGIN=$(pwd)

# 3.12 3.11 3.10

BUILDS=${BUILDS:-3.12 3.11}

for PYBUILD in $BUILDS
do
cd "$ORIGIN"

if [ -f ${SDKROOT}/dev ]
then
echo "${SDKROOT}/dev found : using build cache"
else
echo "doing a clean build"
rm -rf ${SDKROOT}/* ${SDKROOT}/.??*
fi

cp -Rf * ${SDKROOT}/

if cd ${SDKROOT}/
then
mkdir -p build/pycache
export PYTHONDONTWRITEBYTECODE=1

# make install cpython will force bytecode generation
export PYTHONPYCACHEPREFIX="$(realpath build/pycache)"

. ${CONFIG:-config}

cd ${SDKROOT}

if [ -f $HPY ]
then
echo " re-using host python HPY=$HPY"

else
cd ${SDKROOT}
. scripts/cpython-fetch.sh

cd ${SDKROOT}
. support/__EMSCRIPTEN__.sh

. scripts/cpython-build-host.sh 2>&1 >/dev/null

. scripts/cpython-build-host-deps.sh >/dev/null

fi

cd ${SDKROOT}

if $wasisdk
then
echo WASI SDK TODO
> ${SDKROOT}/python3-wasi
> ${SDKROOT}/wasm32-wasi-shell.sh

chmod +x ${SDKROOT}/python3-wasi ${SDKROOT}/wasm32-wasi-shell.sh

mkdir -p src build ${SDKROOT}/devices/wasi ${SDKROOT}/prebuilt/wasisdk

fi

if $emsdk
then
# use ./ or emsdk will pollute env
./scripts/emsdk-fetch.sh

echo " ------------ building cpython wasm ${PYBUILD} ${CIVER} ----------------" 1>&2

if ./scripts/cpython-build-emsdk.sh > /dev/null
then
echo " ---------- building cpython wasm plus ${PYBUILD} ${CIVER} -----------" 1>&2
if ./scripts/cpython-build-emsdk-deps.sh > /dev/null
then

echo " --------- adding some usefull pkg ${PYBUILD} ${CIVER} ---------" 1>&2
./scripts/cpython-build-emsdk-prebuilt.sh


echo "
==========================================================
stripping emsdk ${PYBUILD} ${CIVER}
==========================================================
" 1>&2
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports*
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports/sdl2/SDL-*
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports-builds
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/tests

else
echo " cpython-build-emsdk-deps failed" 1>&2
exit 124
fi
else
echo " cpython-build-emsdk failed" 1>&2
exit 128
fi

fi

. ${SDKROOT}/scripts/pack-sdk.sh

else
echo "cd failed" 1>&2
exit 137
fi
done

exit 0

123 changes: 64 additions & 59 deletions python-wasm-sdk.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,17 @@ export SDKROOT=/opt/python-wasm-sdk
export CIVER=${CIVER:-$DISTRIB}
export CI=true

if echo $0|grep -q python-wasm-sdk
then
echo " * adding emsdk to wasm-sdk"
emsdk=true
else
emsdk=false
fi

echo " * adding wasi-sdk to wasm-sdk"
wasisdk=true


sudo mkdir -p ${SDKROOT}
sudo chmod 777 ${SDKROOT}
Expand All @@ -29,9 +40,10 @@ do

if [ -f ${SDKROOT}/dev ]
then
rm ${SDKROOT}/emsdk/.complete
echo "${SDKROOT}/dev found : using build cache"
else
rm -rf ${SDKROOT}/*
echo "doing a clean build"
rm -rf ${SDKROOT}/* ${SDKROOT}/.??*
fi

cp -Rf * ${SDKROOT}/
Expand All @@ -47,90 +59,83 @@ do
. ${CONFIG:-config}

cd ${SDKROOT}
. scripts/cpython-fetch.sh

cd ${SDKROOT}
. support/__EMSCRIPTEN__.sh
if [ -f $HPY ]
then
echo " re-using host python HPY=$HPY"

else
cd ${SDKROOT}
. scripts/cpython-fetch.sh

cd ${SDKROOT}
. support/__EMSCRIPTEN__.sh

. scripts/cpython-build-host.sh 2>&1 >/dev/null
. scripts/cpython-build-host.sh 2>&1 >/dev/null

. scripts/cpython-build-host-deps.sh >/dev/null
. scripts/cpython-build-host-deps.sh >/dev/null

# cool down machine a bit
[ -f /lowend ] && sleep 45
fi

cd ${SDKROOT}
# use ./ or emsdk will pollute env
./scripts/emsdk-fetch.sh

echo " ------------ building cpython wasm ${PYBUILD} ${CIVER} ----------------" 1>&2
if $wasisdk
then
echo WASI SDK TODO
> ${SDKROOT}/python3-wasi
> ${SDKROOT}/wasm32-wasi-shell.sh

chmod +x ${SDKROOT}/python3-wasi ${SDKROOT}/wasm32-wasi-shell.sh

if ./scripts/cpython-build-emsdk.sh > /dev/null
mkdir -p src build ${SDKROOT}/devices/wasi ${SDKROOT}/prebuilt/wasisdk

fi

if $emsdk
then
echo " ---------- building cpython wasm plus ${PYBUILD} ${CIVER} -----------" 1>&2
if ./scripts/cpython-build-emsdk-deps.sh > /dev/null
then
[ -f /lowend ] && sleep 45
# use ./ or emsdk will pollute env
./scripts/emsdk-fetch.sh

echo " ------------ building cpython wasm ${PYBUILD} ${CIVER} ----------------" 1>&2

if ./scripts/cpython-build-emsdk.sh > /dev/null
then
echo " ---------- building cpython wasm plus ${PYBUILD} ${CIVER} -----------" 1>&2
if ./scripts/cpython-build-emsdk-deps.sh > /dev/null
then

echo " --------- adding some usefull pkg ${PYBUILD} ${CIVER} ---------" 1>&2
./scripts/cpython-build-emsdk-prebuilt.sh
echo " --------- adding some usefull pkg ${PYBUILD} ${CIVER} ---------" 1>&2
./scripts/cpython-build-emsdk-prebuilt.sh


echo "
echo "
==========================================================
stripping emsdk ${PYBUILD} ${CIVER}
==========================================================
" 1>&2
==========================================================
stripping emsdk ${PYBUILD} ${CIVER}
==========================================================
" 1>&2
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports*
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports/sdl2/SDL-*
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/cache/ports-builds
rm -rf ${SDKROOT}/emsdk/upstream/emscripten/tests



echo "making tarball" 1>&2

cd /
mkdir -p /tmp/sdk
tar -cpPR \
${SDKROOT}/config \
${SDKROOT}/python3-wasm \
${SDKROOT}/wasm32-*-shell.sh \
${SDKROOT}/emsdk \
${SDKROOT}/devices/* \
${SDKROOT}/prebuilt/* \
> /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar
lz4 -c --favor-decSpeed --best /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar \
> /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar.lz4

#if [ -f /lowend ]
if true
then
# bzip2 is energy waste for local build
rm /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar
else
# bzip2 will remove original
[ -f /lowend ] || bzip2 -9 /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar
fi
else
echo " cpython-build-emsdk-deps failed" 1>&2
exit 124
fi
else
echo " cpython-build-emsdk-deps failed" 1>&2
exit 111
echo " cpython-build-emsdk failed" 1>&2
exit 128
fi
else
echo " cpython-build-emsdk failed" 1>&2
exit 115

fi

echo "done" 1>&2
. ${SDKROOT}/scripts/pack-sdk.sh

# cool down
[ -f /lowend ] && sleep 60
else
echo "cd failed" 1>&2
exit 124
exit 137
fi
done

Expand Down
18 changes: 18 additions & 0 deletions scripts/pack-sdk.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
echo "making tarball" 1>&2

cd /
mkdir -p /tmp/sdk
tar -cpPR \
${SDKROOT}/config \
${SDKROOT}/python3-was? \
${SDKROOT}/wasm32-*-shell.sh \
${SDKROOT}/*sdk \
${SDKROOT}/devices/* \
${SDKROOT}/prebuilt/* \
> /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar
lz4 -c --favor-decSpeed --best /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar \
> /tmp/sdk/python${PYBUILD}-wasm-sdk-${CIVER}.tar.lz4


echo "done" 1>&2

Loading

0 comments on commit b215e2b

Please sign in to comment.