Rewrite repo-build and package-build in Python

(See #211.) * Replace `scripts/repo-build` (Bash script) with `scripts/repo_build.py` * Replace `scripts/package-build` (Bash script) with `scripts/package_build.py` * Create `toltec` Python support library (in `scripts/toltec`) * Remove superseded `opkg` Python library (in `scripts/opkg`) * Add Python linting and formatting checks * Update Makefile to work with the new scripts * Update workflow files to work with the new scripts * Update koreader to not use the `$recipedir` variable (not documented, should not be used)
toltec-dev · Jan 15, 2021 · 4c44d06 · 4c44d06
1 parent cb362f5
commit 4c44d06
Show file tree

Hide file tree

Showing 25 changed files with 1,771 additions and 1,489 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -2,11 +2,9 @@ name: Setup Toltec dependencies
 runs:
     using: "composite"
     steps:
-        - name: Install Ubuntu packages
+        - name: Install missing packages
           shell: bash
-          run: |
-            sudo apt-get update -yq
-            sudo apt-get install -yq bsdtar tree
+          run: sudo apt-get install bsdtar
         - name: Install shfmt and Shellcheck
           shell: bash
           run: |
@@ -35,3 +33,8 @@ runs:
             chmod a+x shellcheck
             sudo chown root:root shellcheck
             sudo mv shellcheck "$install_dir"
+        - name: Install Python dependencies
+          shell: bash
+          run: |
+            python -m pip install --upgrade pip
+            pip install -r requirements.txt
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -8,7 +8,11 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
-            - name: Setup Toltec environment
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.x'
+            - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Check formatting
               run: make format
@@ -21,10 +25,14 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
-            - name: Setup Toltec environment
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.x'
+            - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages
-              run: remote_repo='https://toltec-dev.org/${{ github.base_ref }}' make repo-new
+              run: make repo-new FLAGS='--remote-repo https://toltec-dev.org/${{ github.base_ref }}'
             - name: Save the build output
               uses: actions/upload-artifact@v2
               with:

diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml
@@ -10,11 +10,14 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.x'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages
-              run: |
-                remote_repo="https://toltec-dev.org/stable" make repo
+              run: make repo FLAGS="--remote-repo https://toltec-dev.org/stable"
             - name: Sync packages with the remote repository
               uses: ./.github/actions/sync-repository
               with:

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -10,11 +10,14 @@ jobs:
         steps:
             - name: Checkout the Git repository
               uses: actions/checkout@v2
+            - name: Setup Python
+              uses: actions/setup-python@v2
+              with:
+                python-version: '3.x'
             - name: Setup Toltec dependencies
               uses: ./.github/actions/setup
             - name: Build packages
-              run: |
-                remote_repo="https://toltec-dev.org/testing" make repo
+              run: make repo FLAGS="--remote-repo https://toltec-dev.org/testing"
             - name: Sync packages with the remote repository
               uses: ./.github/actions/sync-repository
               with:

diff --git a/.mypy.ini b/.mypy.ini
@@ -0,0 +1,3 @@
+[mypy]
+[mypy-docker.*]
+ignore_missing_imports = True
diff --git a/.pylintrc b/.pylintrc
@@ -0,0 +1,7 @@
+[MESSAGES CONTROL]
+# See <https://github.com/PyCQA/pylint/issues/3882>
+disable=unsubscriptable-object
+
+[SIMILARITIES]
+# Do not consider duplicate imports in several files as duplicated code
+ignore-imports=yes
diff --git a/Makefile b/Makefile
@@ -40,35 +40,43 @@ help:
 	@echo "$$USAGE"
 
 repo:
-	./scripts/repo-build package build/package build/repo "$$remote_repo"
+	./scripts/repo_build.py $(FLAGS)
 
 repo-local:
-	./scripts/repo-build -l package build/package build/repo "$$remote_repo"
+	./scripts/repo_build.py --local
 
 repo-new:
-	./scripts/repo-build -n package build/package build/repo "$$remote_repo"
+	./scripts/repo_build.py --no-fetch $(FLAGS)
 
 repo-check:
 	./scripts/repo-check build/repo
 
 $(RECIPES): %:
-	./scripts/package-build package/"${@}" build/package/"${@}"
+	./scripts/package_build.py $(FLAGS) "$(@)"
 
-$(RECIPES_PUSH): %:
-	ssh root@"${HOST}" mkdir -p .cache/opkg
-	scp build/package/"$(@:%-push=%)"/*/*.ipk root@"${HOST}":.cache/opkg
+push: %:
+	ssh root@"$(HOST)" rm -rf ~/.cache/toltec
+	scp -r build/repo root@"$(HOST)":~/.cache/toltec
 
 format:
-	@echo "==> Checking the formatting of shell scripts"
+	@echo "==> Checking Bash formatting"
 	shfmt -d .
+	@echo "==> Checking Python formatting"
+	black --line-length 80 --check --diff scripts
 
 format-fix:
-	@echo "==> Fixing the formatting of shell scripts"
+	@echo "==> Fixing Bash formatting"
 	shfmt -l -w .
+	@echo "==> Fixing Python formatting"
+	black --line-length 80 scripts
 
 lint:
-	@echo "==> Linting shell scripts"
+	@echo "==> Linting Bash scripts"
 	shellcheck $$(shfmt -f .)
+	@echo "==> Typechecking Python files"
+	MYPYPATH=scripts mypy scripts
+	@echo "==> Linting Python files"
+	PYTHONPATH=: pylint scripts
 	@echo "==> Verifying that the bootstrap checksum is correct"
 	./scripts/bootstrap/checksum-check
 

diff --git a/package/koreader/package b/package/koreader/package
@@ -18,12 +18,14 @@ source=(
     koreader.draft
     KOReader.oxide
     rm2-support.patch
+    koreader
 )
 sha256sums=(
     38db8f3895472828d7c1d63fc54db426135a8dbf8425633e299a1a9abfdb69b2
     SKIP
     SKIP
     SKIP
+    SKIP
 )
 
 prepare() {
@@ -42,5 +44,5 @@ package() {
     install -D -m 644 -t "$pkgdir"/opt/etc/draft/ "$srcdir"/koreader.draft
     install -D -m 644 -t "$pkgdir"/opt/usr/share/applications/ "$srcdir"/KOReader.oxide
     install -D -m 644 -t "$pkgdir"/opt/etc/draft/icons/ "$srcdir"/resources/koreader.png
-    install -D -m 755 -t "$pkgdir"/opt/bin/ "$recipedir"/koreader
+    install -D -m 755 -t "$pkgdir"/opt/bin/ "$srcdir"/koreader
 }
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,25 @@
+appdirs==1.4.4
+astroid==2.4.2
+black==20.8b1
+certifi==2020.12.5
+chardet==4.0.0
+click==7.1.2
+docker==4.4.1
+idna==2.10
+isort==5.7.0
+lazy-object-proxy==1.4.3
+mccabe==0.6.1
+mypy==0.790
+mypy-extensions==0.4.3
+pathspec==0.8.1
+pylint==2.6.0
+python-dateutil==2.8.1
+regex==2020.11.13
+requests==2.25.1
+six==1.15.0
+toml==0.10.2
+typed-ast==1.4.2
+typing-extensions==3.7.4.3
+urllib3==1.26.2
+websocket-client==0.57.0
+wrapt==1.12.1
diff --git a/scripts/opkg/arfile.py b/scripts/opkg/arfile.py