Merge branch 'develop'

.gitlab-ci.yml

@@ -2,41 +2,74 @@ image: python:3.7
 
 before_script:
   - python --version
-  - pip install -e .[dev] --extra-index-url https://download.pytorch.org/whl/cu113
-  - pwd
-  - ls -l
-  - python -c "import sys;print(sys.path)"
+  - pip install -e ".[dev]"
+
+# jobs extending .scheduled_only only run in scheduled pipelines not on every commit
+.scheduled_only:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+
+# Checking codestyle
 
 .codestyle:
   stage: test
   script:
     - flake8 --version
     - mypy --version
+    - black --version
     - flake8
     - mypy --config-file mypy.ini
+    - black . --check --verbose --diff --color
 
 codestyle:
-  extends: .codestyle
+  extends:
+    - .codestyle
 
-codestyle:3.8:
-  extends: .codestyle
-  image: python:3.8
+codestyle:3.10:
+  extends:
+    - .codestyle
+    - .scheduled_only
+  image: python:3.10
 
-.test:
-  stage: test
+# Running tests
+
+test_and_coverage:
+  script:
+    - coverage run -m pytest
+    - coverage report
+    - coverage xml
+  coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
+  artifacts:
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+
+test:3.10:
+  extends:
+    - .scheduled_only
+  image: python:3.10
   script:
-    - pytest --version
     - python -m pytest .
 
-test:
-  extends: .test
+test:torch_backwards_compatibility:
+  script:
+    - pip install torch==1.9.0 torchvision==0.10.0
+    - python -m pytest .
 
-test:3.8:
-  extends: .test
-  image: python:3.8
+# Documentation
 
-test-build-doc:
+test_build_doc:
   stage: test
   script:
     - apt-get update && apt-get install -y pandoc
     - sphinx-build -b html docs/source/ docs/build/ -a
+
+test_doc_completeness:
+  extends: .scheduled_only
+  stage: test
+  allow_failure: true
+  script:
+    - flake8 --version
+    # explicitly select Docstring errors and ignore to overwrite config in setup.cfg
+    - flake8 --select=D1 --ignore=E501

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        language_version: python3.8

.readthedocs.yaml

@@ -0,0 +1,30 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-20.04
+  tools:
+    python: "3.9"
+    # You can also specify other tool versions:
+    # nodejs: "16"
+    # rust: "1.55"
+    # golang: "1.17"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+
+# If using Sphinx, optionally build your docs in additional formats such as PDF
+# formats:
+#    - pdf
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+    - requirements: requirements.txt
+    - requirements: requirements-dev.txt

AUTHORS

@@ -0,0 +1,10 @@
+The development of BITorch was started by Joseph Bethge and PD Dr. Haojin Yang.
+
+The current maintainers can be contacted at: fb10-xnor@hpi.de
+
+The following people have contributed code to BITorch (in alphabetical order):
+
+    Christopher Aust
+    Joseph Bethge
+    Paul Mattes
+    Haojing Yang

CHANGELOG.md

@@ -4,6 +4,44 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [0.3.0] - 2023/01/13
+
+### Added
+
+- new models:
+  - [MeliusNet](bitorch/models/meliusnet.py)
+  - [BinaryDenseNet](bitorch/models/densenet.py)
+  - [QuickNet](bitorch/models/quicknet.py)
+- simple example script for MNIST
+- support for integration of bitorch's inference engine for the following layers
+  - QLinear
+  - QConv
+- a quantized DLRM version, derived from [this](https://github.com/facebookresearch/dlrm) implementation
+- example code for training the quantized DLRM model
+- new quantization function: [Progressive Sign](bitorch/quantizations/progressive_sign.py)
+- new features in PyTorch Lightning example:
+  - training with Knowledge Distillation
+  - improved logging
+  - callback to update Progressive Sign module
+- option to integrate custom models, datasets, quantization functions
+- a quantization scheduler which lets you change quantization methods during training
+- a padding layer
+
+### Changed
+
+- requirements changed:
+  - code now depends on torch 1.12.x and torchvision 0.13.x
+  - requirements for examples are now stored at their respective folders
+  - optional requirements now install everything needed to run all examples
+- code is now formatted with the black code formatter
+- using PyTorch's implementation of RAdam
+- renamed the `bitwidth` attribute of quantization functions to `bit_width`
+- moved the image datasets out of the bitorch core package into the image classification example
+
+### Fixed
+
+- fix error from updated protobuf package
+
 ## [0.2.0] - 2022/05/19
 
 ### Added

README.md

@@ -9,7 +9,10 @@ Our current roadmap contains:
 - Extending the model zoo with pre-trained models of state-of-the-art approaches
 - Adding examples for advanced training methods with multiple stages, knowledge distillation, etc.
 
-All changes are tracked in the [changelog](CHANGELOG.md).
+All changes are tracked in the [changelog](https://github.com/hpi-xnor/bitorch/blob/main/CHANGELOG.md).
+
+Please refer to [our wiki](https://bitorch.readthedocs.io/en/latest/) for a comprehensive introduction into
+the library or use the introduction notebook in `examples/notebooks`.
 
 ## Installation
 
@@ -18,11 +21,11 @@ Currently, the only supported installation is pip (a conda package is planned in
 
 ### Pip
 
-If you wish to use a *specific version* of PyTorch for compatibility with certain devices or CUDA versions,
+If you wish to use a _specific version_ of PyTorch for compatibility with certain devices or CUDA versions,
 we advise on installing the corresponding versions of `pytorch` and `torchvision` first (or afterwards),
 please consult [pytorch's getting started guide](https://pytorch.org/get-started/locally/).
 
-Afterwards simply run:
+Otherwise, simply run:
 ```bash
 pip install bitorch
 ```
@@ -32,9 +35,7 @@ Note, that you can also request a specific PyTorch version directly, e.g. for CU
 pip install bitorch --extra-index-url https://download.pytorch.org/whl/cu113
 ```
 
-To use advanced logging capabilities with [tensorboardX](https://github.com/lanpa/tensorboardX),
-install the optional dependencies as well:
-
+If you want to run the examples install the optional dependencies as well:
 ```bash
 pip install "bitorch[opt]"
 ```
@@ -45,13 +46,8 @@ The package can also be installed locally for editing and development.
 First, clone the [repository](https://github.com/hpi-xnor/bitorch), then run:
 
 ```bash
-pip install -e .
-```
-
-To activate advanced logging with Tensorboard and model summary, install the optional dependencies as well:
-
-```bash
-pip install -e ".[opt]"
+pip install -e .         # without optional dependencies
+pip install -e ".[opt]"  # with optional dependencies
 ```
 
 ### Dali Preprocessing
@@ -61,31 +57,60 @@ e.g. with CUDA 11.x, (currently only supported for imagenet)
 you need to install the `nvidia-dali-cuda110` package by running the following command:
 
 ```
- pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda110
+pip install --extra-index-url https://developer.download.nvidia.com/compute/redist --upgrade nvidia-dali-cuda110
 ```
 
-### Code formatting and typing
+## Development
 
-Install the _dev_ requirements for (local) development:
+Install the package and _dev_ requirements locally for development:
 
 ```bash
 pip install -e ".[dev]"
 ```
 
-New code should be compatible with Python 3.X versions and be compliant with PEP8. To check the codebase, please run
+### Tests
+
+The tests can be run with [pytest](https://docs.pytest.org/):
+
+```bash
+pytest
+```
+
+### Code formatting and typing
 
+For conveniently checking whether your code suites the required style (more details below), run
+```bash
+./check-codestyle.sh
+```
+
+New code should be compatible with Python 3.X versions and be compliant with PEP8. To check the codebase, please run
 ```bash
 flake8
 ```
 
 The codebase has type annotations, please make sure to add type hints if required. We use `mypy` for type checking:
-
 ```bash
 mypy --config-file mypy.ini
 ```
 
-Finally, the tests can be run with:
+For code formatting we use `black`:
+```bash
+black . --check --verbose --diff --color  # check what changes the formatter would do
+black .  # apply the formatter
+```
+
+In order to automatically apply the code formatting with every commit, you can also install pre-commit
+and use the pre-commit hook:
+```bash
+pre-commit install
+```
 
+### Documentation
+
+We use [Google's Python Docstring Format](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+to document our code.
+
+Documentation can be generated with
 ```bash
-pytest
+sphinx-build -b html docs/source/ docs/build/ -a
 ```

bitorch/__init__.py

@@ -1,11 +1,18 @@
+"""
+BITorch is a library currently under development to simplify building quantized and binary neural networks with PyTorch.
+It contains implementation of the required layers, different quantization functions and examples.
+"""
 import os
 from argparse import ArgumentParser, Namespace
 from importlib import import_module
 from pathlib import Path
 from typing import List
 
 from .config import Config
+from .runtime_mode import RuntimeMode, runtime_mode_type, change_mode, pause_wrapping  # noqa: F401
+from .layers import convert  # noqa: F401
 
+mode: RuntimeMode = RuntimeMode.DEFAULT
 
 configs_by_name = {}
 
@@ -50,7 +57,7 @@ def config_from_name(name: str) -> Config:
 
 
 def config_names() -> List:
-    """getter for list of config names for argparse
+    """Get the list of config names for argparse.
 
     Returns:
         List: the config names
@@ -59,20 +66,29 @@ def config_names() -> List:
 
 
 def add_config_args(parser: ArgumentParser) -> None:
-    """adds all config arguments
+    """Adds all arguments from all registered configs.
 
     Args:
         parser (ArgumentParser): parser to add the arguments to
     """
-    for config in configs_by_name.values():
-        config.add_config_arguments(parser)
+    for config_ in configs_by_name.values():
+        config_.add_config_arguments(parser)
 
 
 def apply_args_to_configuration(args: Namespace) -> None:
-    """applys the cli configurations to the config objects.
+    """Applies the cli configurations to the config objects.
 
     Args:
         args (Namespace): the cli configurations
     """
-    for config in configs_by_name.values():
-        config.apply_args_to_configuration(args)
+    for config_ in configs_by_name.values():
+        config_.apply_args_to_configuration(args)
+
+
+def register_custom_config(custom_config: Config) -> None:
+    """Register a custom (external) config in bitorch.
+
+    Args:
+        custom_config: the custom config which should be added to bitorch
+    """
+    configs_by_name[custom_config.name] = custom_config