[misc] several important updates

.env.local

@@ -10,6 +10,7 @@ DISABLE_VERSION_CHECK=
 FORCE_CHECK_IMPORTS=
 LLAMAFACTORY_VERBOSITY=
 USE_MODELSCOPE_HUB=
+USE_OPENMIND_HUB=
 RECORD_VRAM=
 # torchrun
 FORCE_TORCHRUN=

.github/workflows/publish.yml

@@ -26,15 +26,15 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.8"
-      
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           python -m pip install build
-      
+
       - name: Build package
         run: |
           python -m build
-      
+
       - name: Publish package
         uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/tests.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-          - "3.8"
+          - "3.8"  # TODO: remove py38 in next transformers release
           - "3.9"
           - "3.10"
           - "3.11"
@@ -54,7 +54,6 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          python -m pip install git+https://github.com/huggingface/transformers.git
           python -m pip install ".[torch,dev]"
 
       - name: Check quality

.gitignore

@@ -162,6 +162,7 @@ cython_debug/
 # custom .gitignore
 ms_cache/
 hf_cache/
+om_cache/
 cache/
 config/
 saves/

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: check-ast
+    -   id: check-added-large-files
+        args: ['--maxkb=25000']
+    -   id: check-merge-conflict
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+    -   id: no-commit-to-branch
+        args: ['--branch', 'master']
+
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.17.0
+    hooks:
+    -   id: pyupgrade
+        args: [--py38-plus]
+
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+    -   id: ruff
+        args: [--fix]
+    -   id: ruff-format

Makefile

@@ -1,7 +1,14 @@
-.PHONY: quality style test
+.PHONY: build commit quality style test
 
 check_dirs := scripts src tests setup.py
 
+build:
+	pip install build && python -m build
+
+commit:
+	pre-commit install
+	pre-commit run --all-files
+
 quality:
 	ruff check $(check_dirs)
 	ruff format --check $(check_dirs)
@@ -11,4 +18,4 @@ style:
 	ruff format $(check_dirs)
 
 test:
-	CUDA_VISIBLE_DEVICES= pytest tests/
+	CUDA_VISIBLE_DEVICES= WANDB_DISABLED=true pytest tests/