ip-tools · amotl · Nov 29, 2022 · Apr 13, 2022 · Apr 14, 2022 · Apr 14, 2022
diff --git a/.coveragerc b/.coveragerc
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -5,29 +5,65 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+
+  # Allow job to be triggered manually.
   workflow_dispatch:
 
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
 jobs:
 
   tests-backend:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ['ubuntu-latest', 'macos-latest']
+        os: [
+          'ubuntu-20.04',
+
+          # Don't use macOS for now, it is currently unstable, otherwise slow. -- 2022-04-19, amo
+          # 'macos-latest',
+        ]
         python-version: ['2.7']
 
     defaults:
       run:
         shell: bash
 
+    env:
+      OPS_API_CONSUMER_KEY: ${{ secrets.OPS_API_CONSUMER_KEY }}
+      OPS_API_CONSUMER_SECRET: ${{ secrets.OPS_API_CONSUMER_SECRET }}
+      IFICLAIMS_API_URI: ${{ secrets.IFICLAIMS_API_URI }}
+      IFICLAIMS_API_USERNAME: ${{ secrets.IFICLAIMS_API_USERNAME }}
+      IFICLAIMS_API_PASSWORD: ${{ secrets.IFICLAIMS_API_PASSWORD }}
+
     name: Python ${{ matrix.python-version }} on OS ${{ matrix.os }}
     steps:
 
     - name: Acquire sources
       uses: actions/checkout@v3
 
+    - name: Install dependencies for Linux
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt install --yes pdftk poppler-utils
+
+    - name: Install dependencies for macOS
+      if: runner.os == 'macOS'
+      run: |
+        brew install poppler
+
+    - name: Configure policy for ImageMagick
+      # https://stackoverflow.com/q/52998331
+      # https://stackoverflow.com/a/57721936
+      if: runner.os == 'Linux'
+      run: |
+        sudo rm /etc/ImageMagick-6/policy.xml
+
     - name: Setup Python
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
         architecture: x64
@@ -40,7 +76,7 @@ jobs:
         make setup-test
 
     - name: Run tests
-      run: make test
+      run: make test-coverage options="--app-cache-backend=memory"
 
     - name: Upload coverage results to Codecov
       uses: codecov/codecov-action@v3
@@ -51,11 +87,42 @@ jobs:
         name: codecov-umbrella
         fail_ci_if_error: false
 
+  tests-docs:
+    runs-on: "ubuntu-20.04"
+
+    defaults:
+      run:
+        shell: bash
+
+    name: Build documentation
+    steps:
+
+    - name: Acquire sources
+      uses: actions/checkout@v3
+
+    - name: Setup Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '2.7'
+        architecture: 'x64'
+        cache: 'pip'
+        cache-dependency-path: 'requirements-docs.txt'
+
+    - name: Invoke Sphinx
+      run: |
+        pip install virtualenv
+        make docs-html
+
   tests-ui:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ['ubuntu-latest', 'macos-latest']
+        os: [
+          'ubuntu-20.04',
+
+          # Don't use macOS for now, it is currently unstable, otherwise slow. -- 2022-04-19, amo
+          # 'macos-latest',
+        ]
         node-version: ['11']
 
     defaults:

diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,4 @@ coverage.xml
 o-*.min.js
 o-*.min.map
 yarn-error.log
+patzilla-uspto-cache.sqlite
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -15,8 +15,43 @@ Development
 - [ui] EPO/OPS: Support CPCI classifications
 - [ui] Fix dependencies: Upgrade Moment.js and Async
 - [ci] Add CI/GHA configuration
-- [ci] Add macOS to test matrix
 - [ui] Update a few JavaScript dependencies
+- [qa] Switch test framework from ``nose`` to ``pytest``
+- [qa] Improve Python test harness
+- [mw] Fix getting PDF documents and drawings from USPTO. Thanks, @aghster!
+- [qa] Add software test infrastructure for data acquisition subsystems
+- [qa] Add tests covering the acquisition of USPTO PDF documents and drawings
+- [mw] Refactor the bootloader subsystem
+- [qa] Add tests for EPO OPS data source
+- [ui] Fix depa.tech data source logo
+- [ui] Improve loading behaviour for vendor data source logos
+- [mw] EPO/OPS: Update list of full text authorities. Thanks, @epo and @aghster!
+- [mw] Fix JSON Web Token adapter for generating shareable links
+- [mw] Improve reading lists of document numbers from text files
+- [mw] Fix acquisition of claims and description for DE and US documents
+- [mw] Improve gracefulness when decoding document numbers
+- [mw] Improve parameter validation in authentication handler
+- [mw] Honour ``fulltext_enabled`` setting of data source
+- [mw] Disable acquisition of PDF documents from DPMA/DEPATISconnect
+- [mw] EPO/OPS: Unlock inquiry of image information for documents without kind code
+- [mw] EPO/Espacenet: Repair access layer. Thanks, @aghster!
+- [ui] EPO/Espacenet: Repair "external sources" links
+- [mw] EPO/Espacenet: Acquire "abstract" text
+- [mw] EPO/Espacenet: Make access layer production ready
+- [mw] EPO/Espacenet: Acquire "abstract" text from OPS API
+- [ui] Adjust JavaScript dependencies for ``bootbox``
+- [infra] Don't use designated port 9999 to avoid confusion
+- [infra] Rework the Docker setup
+- [doc] Various documentation updates across the board
+- [doc] Fix building the documentation
+- [mw] Fix logging. Thanks, @think-nice-things!
+- [cli] Start using the ``Click`` command line parser
+- [mw] Improve "minimal" application framework bootstrapping
+- [mw] Add CLI interface for basic operations on EPO/OPS
+- [mw] Fix access to DPMAregister due to upstream changes
+- [mw] Add CLI interface for search operation on IFI CLAIMS
+- [mw] Fix access to USPTO documents using USPTO PPUBS, PatFT and AppFT are deprecated
+- [mw] Generalize image format conversion
 
 
 2019-11-01 0.169.3
@@ -189,7 +224,7 @@ Development
 - [ui] Add ApplicationProfile subsystem
 - [ui] Improve Stack subsystem data model
 - [ui] Improve jQuery3 compatibility
-- [ui] Improve INID code labels, refactoring
+- [ui] Improve INID code labels, refactoring
 - [ui] Add display flavour with verbose INID code labels
 - [ui] Add a few Material Design Components
 - [ui] Add Stack opener button
@@ -262,7 +297,7 @@ Development
 - [ui] Fix occasional cropped display of drawings
 - [ui] Prepare drawings loader for fetching new items when navigating to previous item
 - [ui] Reorder comfort form fields again
-- [ui] Nail Javascript module dependencies to exact versions
+- [ui] Nail Javascript module dependencies to exact versions
 - [mw] Fix error reporting and logging issues
 - [ui] Hide previous/next drawing navigation buttons when displaying a single drawing
 - [ui] Prevent out of bounds navigation to the previous drawing if there's just one item in the collection
@@ -2362,15 +2397,12 @@ Bug fixes and minor updates
 2013-10-24 0.0.7
 ================
 
-feature:
 - backpropagate current basket entries into checkbox state
 - display "inventor" attribute
 - add portfolio demo frameset
 - add ship-mode=single-bibdata
 - fix: be more graceful if applicants or inventors are missing from data
-- renamed ingress query parameters "ship_*" to "ship-*"
-
-tech:
+- renamed ingress query parameters ``ship_*`` to ``ship-*``
 - route refactoring
 - ui refactoring: more responsive through "twitter bootstrap responsive css"
 

diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst
@@ -8,31 +8,32 @@ the people who contributed to this software:
 * Alexander Wagner <https://ip7.tech/>
 * Andreas Büchler <https://www.coperion.com/>
 * Andreas Motl <andreas.motl@elmyra.de>
-* Andreas Sykora <http://www.q-ip.de/sykora-lang.php>
+* Andreas Sykora <http://q-ip.de/de/menschen.html#Syko_Content_Btn>
 * Andrii Danyleiko <https://github.com/DefteZ>
 * Arne Krüger <https://depa.tech/>
 * Benjamin Walter <b.walter@europatent.net>
-* David Santamauro <https://www.ificlaims.com/>
+* David Santamauro <https://www.ificlaims.com/about-team.htm>
 * Erik Zscherpe <http://blumpatentrecherchen.de/>
-* Ernst Fischer <http://www.pateris.de/dr-ernst-fischer/>
-* Felix Berkemeier <http://www.patente.de/en/dr-felix-berkemeier.html>
+* Ernst Fischer <https://www.pateris.de/dr-ernst-fischer/>
+* Felix Berkemeier <https://www.patente.de/en/dr-felix-berkemeier.html>
 * Frank Heider <https://github.com/frankheider>
 * Gaby Strauß <http://www.patentberichterstatter-muenchen.de/>
 * Johannes Steimel <https://www.festool.com/>
+* Johannes Zellner <johannes@zellner.org>
 * Klaus Hager <http://www.patentberichterstatter-muenchen.de/>
 * Lars Witte <https://www.patentfamily.com/>
 * Luca Martinuzzi <http://www.patentberichterstatter-muenchen.de/>
 * Marc Haus <https://depa.tech/>
-* Martin Bock <http://www.serviva.com/>
+* Martin Bock <https://serviva.com/>
 * Martin Glatzer <https://www.guentner.com/>
 * Michael Behl <m.behl@europatent.net>
 * Papoteur <https://github.com/papoteur-mga>
-* Raphael Jung <http://www.serviva.com/>
-* Robert Frenzel <http://www.serviva.com/>
+* Raphael Jung <https://serviva.com/>
+* Robert Frenzel <https://serviva.com/>
 * Siegfried Greiner <http://greiner-mv.de/>
 * Steffen Köhler <s.koehler@europatent.net>
-* Uwe Spadaccini <http://www.serviva.com/>
+* Uwe Spadaccini <https://www.infovida.de/ueber-infovida.html>
 * Wilhelm von Krüchten <http://www.patentberichterstatter-muenchen.de/>
-* Wolf-Rüdiger von Gartzen <gartzen@europatent.net>
+* Wolf-Rüdiger von Gartzen <info@innovationship.de>
 
 Thanks a bunch!
diff --git a/Makefile b/Makefile
@@ -5,9 +5,24 @@ $(eval venvpath     := .venv2)
 $(eval pip          := $(venvpath)/bin/pip)
 $(eval twine        := $(venvpath)/bin/twine)
 $(eval python       := $(venvpath)/bin/python)
+$(eval pserve       := $(venvpath)/bin/pserve)
+$(eval pytest       := $(venvpath)/bin/pytest)
 $(eval bumpversion  := $(venvpath)/bin/bumpversion)
 $(eval fab          := $(venvpath)/bin/fab)
 
+$(eval venv3path     := .venv)
+$(eval yarn          := $(venv3path)/bin/yarn)
+$(eval npx           := $(venv3path)/bin/npx)
+
+
+setup: setup-py
+
+jswatch:
+	npx yarn watch
+
+pywatch:
+	HUPPER_DEFAULT_MONITOR=hupper.watchdog.WatchdogFileMonitor $(pserve) --reload patzilla/config/development-local.ini
+
 
 js:
 	# url cleaner
@@ -28,6 +43,7 @@ js-release: js
 	@echo ------------------------------------------
 	npx yarn release
 
+
 sdist:
 	$(python) setup.py sdist
 
@@ -51,9 +67,11 @@ upload-pypi:
 setup-virtualenv:
 	@test -e $(python) || virtualenv --python=python2 $(venvpath)
 
-setup-test: setup-virtualenv
+setup-py: setup-virtualenv
 	$(pip) install --editable=.[test]
 
+setup-test: setup-py
+
 setup-deployment:
 	$(pip) install --requirement requirements-deploy.txt
 
@@ -81,39 +99,31 @@ install-nginx-auth:
 	fab upload_nginx_auth
 
 test:
-	@$(python) runtests.py      \
-		$(options)              \
-		--all-modules           \
-		--traverse-namespace    \
-		--with-doctest          \
-		--doctest-tests         \
-		--doctest-extension=rst \
-		--doctest-options=doctestencoding=utf-8,+ELLIPSIS,+NORMALIZE_WHITESPACE,+REPORT_UDIFF \
-		--exclude-dir=patzilla/navigator/static \
-		--exclude-dir=patzilla/navigator/templates \
-		--exclude-dir=patzilla/util/database \
-		--exclude-dir=patzilla/util/web/uwsgi \
-		--exclude-dir=patzilla/access/sip \
-		--ignore-files=setup.py \
-		--ignore-files=fabfile.py \
-		--nocapture \
-		--nologcapture \
-		--verbose
-
-# +REPORT_ONLY_FIRST_FAILURE
-
-
-test-cover:
-	$(MAKE) test options="--with-coverage"
-
-nginx_path=/Users/amo/dev/celeraone/sources/c1-ocb-integrator/rem_rp/parts/openresty
-nginx-start:
-	@$(nginx_path)/nginx/sbin/nginx -p $(nginx_path)/nginx -c `pwd`/nginx-auth/etc/nginx.conf -g "daemon off; error_log /dev/stdout info;"
-
-mongodb-start:
+	@$(pytest) \
+		$(options) \
+		patzilla tests
+
+test-coverage:
+	@$(MAKE) test options="$(options) --cov"
+
+# --all-modules
+# --traverse-namespace
+# --doctest-options=doctestencoding=utf-8,+ELLIPSIS,+NORMALIZE_WHITESPACE,+REPORT_UDIFF
+#                                   +REPORT_ONLY_FIRST_FAILURE
+# --nocapture
+# --nologcapture
+
+nginx:
+	nginx -c `pwd`/nginx-auth/etc/nginx.conf -g "daemon off; error_log /dev/stdout info;"
+
+nginx-start: nginx
+
+mongodb:
 	mkdir -p ./var/lib/mongodb
 	mongod --dbpath=./var/lib/mongodb
 
+mongodb-start: mongodb
+
 mongodb-sip-export:
 	mkdir -p var/tmp/mongodb
 	mongoexport --db patzilla_development --collection sip_country > var/tmp/mongodb/sip_country.mongodb
@@ -132,22 +142,29 @@ sloccount:
 genlicenses:
 	$(pip) install third-party-license-file-generator
 	$(pip) freeze > /tmp/requirements.txt
-	$(python) -m third_party_license_file_generator --requirements-path /tmp/requirements.txt --python-path $(python) --output-file licenses-backend.txt
+	$(python) -m third_party_license_file_generator \
+	    --requirements-path /tmp/requirements.txt --python-path $(python) \
+	    --permit-gpl --permit-commercial \
+	    --output-file licenses-backend.txt
 	npx yarn licenses generate-disclaimer > licenses-frontend.txt
 
 clear-cache:
 	mongo beaker --eval 'db.dropDatabase();'
 
 docs-virtualenv:
 	$(eval venvpath := ".venv_sphinx")
-	@test -e $(venvpath)/bin/python || `command -v virtualenv` --python=`command -v python` --no-site-packages $(venvpath)
+	@test -e $(venvpath)/bin/python || `command -v virtualenv` --python=`command -v python2` $(venvpath)
 	@$(venvpath)/bin/pip --quiet install --requirement requirements-docs.txt
 
 docs-html: docs-virtualenv
 	$(eval venvpath := ".venv_sphinx")
 	touch docs/index.rst
 	export SPHINXBUILD="`pwd`/$(venvpath)/bin/sphinx-build"; cd docs; make html
 
+docs-linkcheck: docs-virtualenv
+	$(eval venvpath := ".venv_sphinx")
+	export SPHINXBUILD="`pwd`/$(venvpath)/bin/sphinx-build"; cd docs; make linkcheck
+
 pdf-EP666666:
 	/usr/local/bin/wkhtmltopdf \
 		--no-stop-slow-scripts --debug-javascript \