From e6145d8ebe620037c8cd83a818d260f5b1e04763 Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Sun, 1 May 2022 23:01:49 +0100 Subject: [PATCH 1/7] Add some useful make commands --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 3fe042ec..8f4f4858 100644 --- a/Makefile +++ b/Makefile @@ -51,3 +51,12 @@ clean-test: clean-dev-server: docker-compose rm -sfv docker volume rm -f core_postgres + +dev-logs: + docker-compose logs -f backend + +dev-sql: + docker-compose exec database psql -U postgres postgres + +dev-fake-bulk-data: + docker-compose exec backend python -m scripts.fake generate --teams 10000 --users 2 --categories 10 --challenges 100 --solves 10000 From 9676a65f6108d56ac9fa3c5ac8e5f5f9d38f5658 Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Sun, 1 May 2022 23:07:11 +0100 Subject: [PATCH 2/7] Fix handling reverse proxies in IP logs --- .../0010_fix_client_ip_addresses.py | 28 +++++++++++++++++++ src/member/models.py | 4 +-- 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 src/member/migrations/0010_fix_client_ip_addresses.py diff --git a/src/member/migrations/0010_fix_client_ip_addresses.py b/src/member/migrations/0010_fix_client_ip_addresses.py new file mode 100644 index 00000000..84b099e4 --- /dev/null +++ b/src/member/migrations/0010_fix_client_ip_addresses.py @@ -0,0 +1,28 @@ +# Generated by Django 4.0.2 on 2022-05-01 19:51 + +from django.db import migrations, models + +def fix_ip_addresses(apps, schema_editor): + UserIP = apps.get_model('member', 'userip') + db_alias = schema_editor.connection.alias + for userip in UserIP.objects.using(db_alias).all(): + userip.ip = userip.ip.split(",")[0] + userip.save() + +class Migration(migrations.Migration): + dependencies = [ + ('member', '0009_alter_member_options_alter_member_username_and_more'), + ] + + operations = [ + migrations.RunPython( + fix_ip_addresses, + # Marked as elidable as all new IP addresses will be in the correct format + elidable=True, + ), + migrations.AlterField( + model_name='userip', + name='ip', + field=models.GenericIPAddressField(), + ), + ] diff --git a/src/member/models.py b/src/member/models.py index 07ed9e7f..2ea3ad00 100644 --- a/src/member/models.py +++ b/src/member/models.py @@ -81,7 +81,7 @@ def should_deny_admin(self): class UserIP(ExportModelOperationsMixin("user_ip"), models.Model): user = models.ForeignKey(get_user_model(), on_delete=SET_NULL, null=True) - ip = models.CharField(max_length=255) + ip = models.GenericIPAddressField() seen = models.IntegerField(default=1) last_seen = models.DateTimeField(default=timezone.now) user_agent = models.CharField(max_length=255) @@ -90,7 +90,7 @@ class UserIP(ExportModelOperationsMixin("user_ip"), models.Model): def hook(request): if not request.user.is_authenticated: return - ip = request.headers.get("x-forwarded-for", "0.0.0.0") + ip = request.headers.get("x-forwarded-for", request.META.get("REMOTE_ADDR", "0.0.0.0")).split(",")[0] user_agent = request.headers.get("user-agent", "???")[:255] qs = UserIP.objects.filter(user=request.user, ip=ip) if qs.exists(): From 160e8cef9cb9dc0edc7cdbef6a91c8be5928f2a7 Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Mon, 2 May 2022 00:11:58 +0100 Subject: [PATCH 3/7] Another useful command --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 8f4f4858..a5f5fd5d 100644 --- a/Makefile +++ b/Makefile @@ -60,3 +60,6 @@ dev-sql: dev-fake-bulk-data: docker-compose exec backend python -m scripts.fake generate --teams 10000 --users 2 --categories 10 --challenges 100 --solves 10000 + +dev-shell: + docker-compose exec backend ./src/manage.py shell From 75f23fe54a8c4b0745d0ba62ab903d7162791fa3 Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Mon, 2 May 2022 00:12:19 +0100 Subject: [PATCH 4/7] Allow querying user's IPs --- src/member/migrations/0010_fix_client_ip_addresses.py | 7 +++++++ src/member/models.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/member/migrations/0010_fix_client_ip_addresses.py b/src/member/migrations/0010_fix_client_ip_addresses.py index 84b099e4..82b42364 100644 --- a/src/member/migrations/0010_fix_client_ip_addresses.py +++ b/src/member/migrations/0010_fix_client_ip_addresses.py @@ -1,6 +1,7 @@ # Generated by Django 4.0.2 on 2022-05-01 19:51 from django.db import migrations, models +from django.conf import settings def fix_ip_addresses(apps, schema_editor): UserIP = apps.get_model('member', 'userip') @@ -25,4 +26,10 @@ class Migration(migrations.Migration): name='ip', field=models.GenericIPAddressField(), ), + migrations.AlterField( + model_name='userip', + name='user', + field=models.ForeignKey(null=True, on_delete=models.deletion.SET_NULL, + related_name='ips', to=settings.AUTH_USER_MODEL), + ), ] diff --git a/src/member/models.py b/src/member/models.py index 2ea3ad00..408ac652 100644 --- a/src/member/models.py +++ b/src/member/models.py @@ -80,7 +80,7 @@ def should_deny_admin(self): class UserIP(ExportModelOperationsMixin("user_ip"), models.Model): - user = models.ForeignKey(get_user_model(), on_delete=SET_NULL, null=True) + user = models.ForeignKey(get_user_model(), on_delete=SET_NULL, null=True, related_name="ips") ip = models.GenericIPAddressField() seen = models.IntegerField(default=1) last_seen = models.DateTimeField(default=timezone.now) From 274bed19200b369aed6b673cd6a65a99595b4139 Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Mon, 2 May 2022 00:53:38 +0100 Subject: [PATCH 5/7] Add a command to group users by IP --- src/ractf/management/commands/group_ips.py | 49 ++++++++++++++++++++++ src/ractf/tests.py | 33 ++++++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 src/ractf/management/commands/group_ips.py diff --git a/src/ractf/management/commands/group_ips.py b/src/ractf/management/commands/group_ips.py new file mode 100644 index 00000000..3af0dcc2 --- /dev/null +++ b/src/ractf/management/commands/group_ips.py @@ -0,0 +1,49 @@ +from pprint import pprint +from django.core.management import BaseCommand + +from member.models import UserIP +from json import dumps + + +class Command(BaseCommand): + help = "Group users by source IP address to try and spot cheating." + + def add_arguments(self, parser): + parser.add_argument( + '--multiple', + action='store_true', + help='Show only IP addresses with multiple users', + ) + + parser.add_argument( + '--json', + action='store_true', + help='Output JSON', + ) + + def handle(self, *args, **options) -> None: + self.stderr.write(self.style.WARNING("Due to use of CGNAT, source IP addresses may be unreliable. Proceed with caution.")) + + ips = UserIP.objects.all() + + grouped = {} + + for ip in ips: + if ip.ip not in grouped: + grouped[ip.ip] = [] + if ip.user.username not in grouped[ip.ip]: + grouped[ip.ip].append(ip.user.username) + + if options["multiple"]: + multiple_grouped = {} + + for group in grouped.items(): + if len(group[1]) > 1: + multiple_grouped |= {group[0]: group[1]} + + grouped = multiple_grouped + + if options["json"]: + self.stdout.write(dumps(grouped)) + else: + pprint(grouped) diff --git a/src/ractf/tests.py b/src/ractf/tests.py index a39b155a..26070d0b 100644 --- a/src/ractf/tests.py +++ b/src/ractf/tests.py @@ -1 +1,32 @@ -# Create your tests here. +from io import StringIO +from django.test import TestCase +from django.core.management import call_command +from member.models import UserIP +from django.contrib.auth import get_user_model + +class GroupIpsTest(TestCase): + def setUp(self): + one = get_user_model()(username="one", email="one@one.one") + one.save() + + + two = get_user_model()(username="two", email="two@two.two") + two.save() + + three = get_user_model()(username="three", email="three@three.three") + three.save() + + UserIP.objects.create(user=one, ip="1.1.1.1", user_agent="Django Tests") + UserIP.objects.create(user=two, ip="1.1.1.1", user_agent="Django Tests") + UserIP.objects.create(user=three, ip="2.2.2.2", user_agent="Django Tests") + + def test_group_ips(self): + out = StringIO() + call_command("group_ips", '--json', stdout=out) + self.assertIn('1.1.1.1', out.getvalue()) + + def test_group_ips_multiple(self): + out = StringIO() + call_command("group_ips", '--json', '--multiple', stdout=out) + self.assertIn('1.1.1.1', out.getvalue()) + self.assertNotIn('2.2.2.2', out.getvalue()) From 4c80f2e0b0d01269735e8b3e92bb8857f4d655eb Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Mon, 2 May 2022 02:28:52 +0100 Subject: [PATCH 6/7] Add more tests to appease CodeCov --- .../0010_fix_client_ip_addresses.py | 3 ++- src/ractf/management/commands/group_ips.py | 6 +++--- src/ractf/tests.py | 21 ++++++++++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/member/migrations/0010_fix_client_ip_addresses.py b/src/member/migrations/0010_fix_client_ip_addresses.py index 82b42364..9a8e8d43 100644 --- a/src/member/migrations/0010_fix_client_ip_addresses.py +++ b/src/member/migrations/0010_fix_client_ip_addresses.py @@ -1,7 +1,8 @@ # Generated by Django 4.0.2 on 2022-05-01 19:51 -from django.db import migrations, models from django.conf import settings +from django.db import migrations, models + def fix_ip_addresses(apps, schema_editor): UserIP = apps.get_model('member', 'userip') diff --git a/src/ractf/management/commands/group_ips.py b/src/ractf/management/commands/group_ips.py index 3af0dcc2..26b13bfd 100644 --- a/src/ractf/management/commands/group_ips.py +++ b/src/ractf/management/commands/group_ips.py @@ -1,8 +1,8 @@ -from pprint import pprint +from json import dumps + from django.core.management import BaseCommand from member.models import UserIP -from json import dumps class Command(BaseCommand): @@ -46,4 +46,4 @@ def handle(self, *args, **options) -> None: if options["json"]: self.stdout.write(dumps(grouped)) else: - pprint(grouped) + self.stdout.write(str(grouped)) diff --git a/src/ractf/tests.py b/src/ractf/tests.py index 26070d0b..28972c7c 100644 --- a/src/ractf/tests.py +++ b/src/ractf/tests.py @@ -1,8 +1,11 @@ from io import StringIO -from django.test import TestCase + +from django.contrib.auth import get_user_model from django.core.management import call_command +from django.test import TestCase + from member.models import UserIP -from django.contrib.auth import get_user_model + class GroupIpsTest(TestCase): def setUp(self): @@ -22,10 +25,22 @@ def setUp(self): def test_group_ips(self): out = StringIO() - call_command("group_ips", '--json', stdout=out) + call_command("group_ips", stdout=out) self.assertIn('1.1.1.1', out.getvalue()) def test_group_ips_multiple(self): + out = StringIO() + call_command("group_ips", '--multiple', stdout=out) + self.assertIn('1.1.1.1', out.getvalue()) + self.assertNotIn('2.2.2.2', out.getvalue()) + + + def test_group_ips_json(self): + out = StringIO() + call_command("group_ips", '--json', stdout=out) + self.assertIn('1.1.1.1', out.getvalue()) + + def test_group_ips_multiple_json(self): out = StringIO() call_command("group_ips", '--json', '--multiple', stdout=out) self.assertIn('1.1.1.1', out.getvalue()) From da1b0e050f80050f4c809eb4148124a3bbd81c20 Mon Sep 17 00:00:00 2001 From: thebeanogamer Date: Mon, 2 May 2022 02:34:33 +0100 Subject: [PATCH 7/7] Run Black --- src/ractf/management/commands/group_ips.py | 16 +++++++++------- src/ractf/tests.py | 20 +++++++++----------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/ractf/management/commands/group_ips.py b/src/ractf/management/commands/group_ips.py index 26b13bfd..ec495639 100644 --- a/src/ractf/management/commands/group_ips.py +++ b/src/ractf/management/commands/group_ips.py @@ -10,19 +10,21 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--multiple', - action='store_true', - help='Show only IP addresses with multiple users', + "--multiple", + action="store_true", + help="Show only IP addresses with multiple users", ) parser.add_argument( - '--json', - action='store_true', - help='Output JSON', + "--json", + action="store_true", + help="Output JSON", ) def handle(self, *args, **options) -> None: - self.stderr.write(self.style.WARNING("Due to use of CGNAT, source IP addresses may be unreliable. Proceed with caution.")) + self.stderr.write( + self.style.WARNING("Due to use of CGNAT, source IP addresses may be unreliable. Proceed with caution.") + ) ips = UserIP.objects.all() diff --git a/src/ractf/tests.py b/src/ractf/tests.py index 28972c7c..b8b27b65 100644 --- a/src/ractf/tests.py +++ b/src/ractf/tests.py @@ -12,7 +12,6 @@ def setUp(self): one = get_user_model()(username="one", email="one@one.one") one.save() - two = get_user_model()(username="two", email="two@two.two") two.save() @@ -26,22 +25,21 @@ def setUp(self): def test_group_ips(self): out = StringIO() call_command("group_ips", stdout=out) - self.assertIn('1.1.1.1', out.getvalue()) + self.assertIn("1.1.1.1", out.getvalue()) def test_group_ips_multiple(self): out = StringIO() - call_command("group_ips", '--multiple', stdout=out) - self.assertIn('1.1.1.1', out.getvalue()) - self.assertNotIn('2.2.2.2', out.getvalue()) - + call_command("group_ips", "--multiple", stdout=out) + self.assertIn("1.1.1.1", out.getvalue()) + self.assertNotIn("2.2.2.2", out.getvalue()) def test_group_ips_json(self): out = StringIO() - call_command("group_ips", '--json', stdout=out) - self.assertIn('1.1.1.1', out.getvalue()) + call_command("group_ips", "--json", stdout=out) + self.assertIn("1.1.1.1", out.getvalue()) def test_group_ips_multiple_json(self): out = StringIO() - call_command("group_ips", '--json', '--multiple', stdout=out) - self.assertIn('1.1.1.1', out.getvalue()) - self.assertNotIn('2.2.2.2', out.getvalue()) + call_command("group_ips", "--json", "--multiple", stdout=out) + self.assertIn("1.1.1.1", out.getvalue()) + self.assertNotIn("2.2.2.2", out.getvalue())