Merge pull request #302 from HiSPARC/testing-updates

Improve lint tests

.github/dependabot.yml

@@ -9,3 +9,8 @@ updates:
     directory: '/provisioning/roles/publicdb/files'
     schedule:
       interval: 'weekly'
+
+  - package-ecosystem: 'docker'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

.github/workflows/tests.yml

@@ -30,7 +30,7 @@ jobs:
       - run: make devinstall
       - run: make coveragetests
 
-  flake:
+  lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -40,7 +40,9 @@ jobs:
           cache: pip
           cache-dependency-path: 'requirements-dev.txt'
       - run: pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt
-      - run: make flaketest
+      - run: make linttest
+        env:
+          RUFF_OUTPUT_FORMAT: github
 
   ansible-lint:
     runs-on: ubuntu-latest
@@ -62,7 +64,7 @@ jobs:
 
   docs:
     if: github.ref == 'refs/heads/master'
-    needs: [tests, flake, ansible-lint, hadolint]
+    needs: [tests, lint, ansible-lint, hadolint]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

Makefile

@@ -1,23 +1,29 @@
-.PHONY: devinstall test unittests coveragetests flaketest doctest ansibletest
-
+.PHONY: devinstall
 devinstall:
 	pip install --upgrade --upgrade-strategy eager -r requirements-dev.txt
 	conda install --quiet --yes --channel conda-forge --file provisioning/roles/publicdb/files/requirements-conda.txt
 	pip install -r provisioning/roles/publicdb/files/requirements-pip.txt
 
-test: coveragetests flaketest doctest ansibletest
+.PHONY: test
+test: coveragetests linttest doctest ansibletest
 
+.PHONY: unittests
 unittests:
 	coverage run ./manage.py test $(tests)
 
+.PHONY: coveragetests
 coveragetests: unittests
 	coverage report
 
-flaketest:
-	flake8
+.PHONY: linttest
+linttest:
+	ruff check .
+	typos .
 
+.PHONY: doctest
 doctest:
 	PYTHONPATH=$(CURDIR):$(PYTHONPATH) sphinx-build -anW doc doc/_build/html
 
+.PHONY: ansibletest
 ansibletest:
 	ansible-lint -p provisioning/playbook.yml || true

manage.py

@@ -2,8 +2,8 @@
 import os
 import sys
 
-if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "publicdb.settings")
+if __name__ == '__main__':
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'publicdb.settings')
 
     from django.core.management import execute_from_command_line
 

publicdb/analysissessions/models.py

@@ -26,35 +26,35 @@ class AnalysisSession(models.Model):
     starts = models.DateTimeField()
     ends = models.DateTimeField()
 
-    def in_progress(self):
-        return self.starts <= datetime.datetime.now() < self.ends
+    class Meta:
+        verbose_name = 'Analysis session'
+        verbose_name_plural = 'Analysis sessions'
 
-    in_progress.boolean = True
+    def __str__(self):
+        return self.title
 
     def save(self, *args, **kwargs):
         self.hash = hashlib.md5(self.slug.encode('utf-8')).hexdigest()
         super().save(*args, **kwargs)
         Student(session=self, name='Test student').save()
 
-    def __str__(self):
-        return self.title
+    def in_progress(self):
+        return self.starts <= datetime.datetime.now() < self.ends
 
-    class Meta:
-        verbose_name = 'Analysis session'
-        verbose_name_plural = 'Analysis sessions'
+    in_progress.boolean = True
 
 
 class Student(models.Model):
     session = models.ForeignKey(AnalysisSession, models.CASCADE, related_name='students')
     name = models.CharField(max_length=255)
 
-    def __str__(self):
-        return f'{self.session} - {self.name}'
-
     class Meta:
         verbose_name = 'Student'
         verbose_name_plural = 'Students'
 
+    def __str__(self):
+        return f'{self.session} - {self.name}'
+
 
 class AnalyzedCoincidence(models.Model):
     session = models.ForeignKey(AnalysisSession, models.CASCADE, related_name='analyzed_coincidences')
@@ -68,14 +68,14 @@ class AnalyzedCoincidence(models.Model):
     phi = models.FloatField(null=True, blank=True)
     error_estimate = models.FloatField(null=True, blank=True)
 
-    def __str__(self):
-        return f"{self.coincidence} - {self.student}"
-
     class Meta:
         verbose_name = 'Analyzed coincidence'
         verbose_name_plural = 'Analyzed coincidences'
         ordering = ['coincidence']
 
+    def __str__(self):
+        return f'{self.coincidence} - {self.student}'
+
 
 class SessionRequest(models.Model):
     first_name = models.CharField(max_length=255)
@@ -100,7 +100,7 @@ class Meta:
 
     @property
     def name(self):
-        return f"{self.first_name} {self.sur_name}"
+        return f'{self.first_name} {self.sur_name}'
 
     def create_session(self):
         self.session_pending = False
@@ -157,7 +157,7 @@ def find_coincidence(self, date, session):
             all_coincidences = cq.any(stations)
             coincidences = cq.events_from_stations(all_coincidences, stations, n=3)
             for coincidence in coincidences:
-                # Todo: Double check for multiple events from same station,
+                # TODO: Double check for multiple events from same station,
                 self.save_coincidence(coincidence, session)
                 number_of_coincidences += 1
 
@@ -212,14 +212,14 @@ def generate_url(self):
     def sendmail_request(self):
         subject = 'HiSPARC analysis session request'
         message = textwrap.dedent(
-            f'''\
+            f"""\
             Hello {self.name},
 
             Please click on this link to confirm your request for an analysis session with jSparc:
             https://data.hisparc.nl/analysis-session/request/{self.url}/
 
             Greetings,
-            The HiSPARC Team'''
+            The HiSPARC Team""",
         )
         sender = 'Beheer HiSPARC <bhrhispa@nikhef.nl>'
         send_mail(subject, message, sender, [self.email], fail_silently=False)
@@ -229,7 +229,7 @@ def sendmail_request(self):
     def sendmail_created(self):
         subject = 'HiSPARC analysis session created'
         message = textwrap.dedent(
-            f'''\
+            f"""\
             Hello {self.name},
 
             Your analysis session for jSparc has been created.
@@ -244,15 +244,15 @@ def sendmail_created(self):
             https://data.hisparc.nl/analysis-session/{slugify(self.sid)}/data/
 
             Greetings,
-            The HiSPARC Team'''
+            The HiSPARC Team""",
         )
         sender = 'Beheer HiSPARC <bhrhispa@nikhef.nl>'
         send_mail(subject, message, sender, [self.email], fail_silently=False)
 
     def sendmail_created_less(self):
         subject = 'HiSPARC analysis session created with less events'
         message = textwrap.dedent(
-            f'''\
+            f"""\
             Hello {self.name},
 
             Your analysis session for jSparc has been created.
@@ -269,23 +269,23 @@ def sendmail_created_less(self):
             https://data.hisparc.nl/analysis-session/{slugify(self.sid)}/data/
 
             Greetings,
-            The HiSPARC Team'''
+            The HiSPARC Team""",
         )
         sender = 'Beheer HiSPARC <bhrhispa@nikhef.nl>'
         send_mail(subject, message, sender, [self.email], fail_silently=False)
 
     def sendmail_zero(self):
         subject = 'HiSPARC analysis session creation failed'
         message = textwrap.dedent(
-            f'''\
+            f"""\
             Hello {self.name},
 
             Your analysis session for jSparc could not be created.
             Perhaps there was no data for the date and/or stations you selected.
             Please try selecting a different cluster or date.
 
             Greetings,
-            The HiSPARC Team'''
+            The HiSPARC Team""",
         )
         sender = 'Beheer HiSPARC <bhrhispa@nikhef.nl>'
         send_mail(subject, message, sender, [self.email], fail_silently=False)

publicdb/analysissessions/urls.py

@@ -4,10 +4,10 @@
 
 app_name = 'sessions'
 urlpatterns = [
-    path('<slug:slug>/data/', views.data_display, name="data_display"),
-    path('request/', views.request_form, name="request"),
-    path('request/validate/', views.validate_request_form, name="validate"),
-    path('request/<slug:url>/', views.confirm_request, name="confirm"),
-    path('get_coincidence/', views.get_coincidence, name="get_coincidence"),
-    path('result/', views.result, name="result"),
+    path('<slug:slug>/data/', views.data_display, name='data_display'),
+    path('request/', views.request_form, name='request'),
+    path('request/validate/', views.validate_request_form, name='validate'),
+    path('request/<slug:url>/', views.confirm_request, name='confirm'),
+    path('get_coincidence/', views.get_coincidence, name='get_coincidence'),
+    path('result/', views.result, name='result'),
 ]

publicdb/analysissessions/views.py

@@ -45,7 +45,7 @@ def get_coincidence(request):
         raise
     else:
         if not session.in_progress():
-            return error_json(404, "The requested session has not started yet or is already expired.")
+            return error_json(404, 'The requested session has not started yet or is already expired.')
 
     if not student_name:
         student = Student.objects.get(session=session, name='Test student')
@@ -61,7 +61,7 @@ def get_coincidence(request):
             analyzed_coincidence.student = student
             analyzed_coincidence.save()
         except IndexError:
-            return error_json(404, "No unanalysed coincidences available, request a new session.")
+            return error_json(404, 'No unanalysed coincidences available, request a new session.')
 
     events = get_events(analyzed_coincidence)
     response = data_json(analyzed_coincidence, events)
@@ -82,41 +82,41 @@ def get_events(analyzed_coincidence):
             continue
 
         timestamp = datetime_to_gps(datetime.combine(event.date, event.time))
-        event_dict = dict(
-            timestamp=timestamp,
-            nanoseconds=event.nanoseconds,
-            number=event.station.number,
-            latitude=config.gps_latitude,
-            longitude=config.gps_longitude,
-            altitude=config.gps_altitude,
-            status='on',
-            detectors=len(event.traces),
-            traces=event.traces,
-            pulseheights=event.pulseheights,
-            integrals=event.integrals,
-            mips=[ph / 200.0 if ph > 0 else ph for ph in event.pulseheights],
-        )
+        event_dict = {
+            'timestamp': timestamp,
+            'nanoseconds': event.nanoseconds,
+            'number': event.station.number,
+            'latitude': config.gps_latitude,
+            'longitude': config.gps_longitude,
+            'altitude': config.gps_altitude,
+            'status': 'on',
+            'detectors': len(event.traces),
+            'traces': event.traces,
+            'pulseheights': event.pulseheights,
+            'integrals': event.integrals,
+            'mips': [ph / 200.0 if ph > 0 else ph for ph in event.pulseheights],
+        }
         events.append(event_dict)
     return events
 
 
 def data_json(coincidence, events):
     """Construct json with data for jSparc to display"""
     timestamp = datetime_to_gps(datetime.combine(coincidence.coincidence.date, coincidence.coincidence.time))
-    data = dict(
-        pk=coincidence.pk,
-        timestamp=timestamp,
-        nanoseconds=coincidence.coincidence.nanoseconds,
-        events=events,
-    )
+    data = {
+        'pk': coincidence.pk,
+        'timestamp': timestamp,
+        'nanoseconds': coincidence.coincidence.nanoseconds,
+        'events': events,
+    }
     response = HttpResponse(json.dumps(data), content_type='application/json')
     response['Access-Control-Allow-Origin'] = '*'
     return response
 
 
 def error_json(error_code, message):
     """Construct error response json for jSparc requests"""
-    data = dict(message=message, code=error_code)
+    data = {'message': message, 'code': error_code}
     response = HttpResponse(json.dumps(data), status=error_code, content_type='application/json')
     response['Access-Control-Allow-Origin'] = '*'
     return response
@@ -143,7 +143,7 @@ def top_lijst(slug):
                     'wgh_error': wgh_error,
                     'min_error': min_error,
                     'num_events': num_events,
-                }
+                },
             )
 
     return sorted(scores, key=operator.itemgetter('wgh_error'))
@@ -192,17 +192,17 @@ def result(request):
         rank = [x['name'] for x in ranking].index(student_name) + 1
     except ValueError:
         rank = None
-    msg = "OK [result stored]"
-    response = HttpResponse(json.dumps(dict(msg=msg, rank=rank)), content_type='application/json')
+    msg = 'OK [result stored]'
+    response = HttpResponse(json.dumps({'msg': msg, 'rank': rank}), content_type='application/json')
     response['Access-Control-Allow-Origin'] = '*'
     return response
 
 
 def test_result():
     """Generate random ranking for test sessions"""
-    msg = "Test session, result not stored"
+    msg = 'Test session, result not stored'
     rank = randint(1, 10)
-    response = HttpResponse(json.dumps(dict(msg=msg, rank=rank)), content_type='application/json')
+    response = HttpResponse(json.dumps({'msg': msg, 'rank': rank}), content_type='application/json')
     response['Access-Control-Allow-Origin'] = '*'
     return response