Skip to content

Commit

Permalink
[CHANGE] Refactor shard assignment from service to Project #147
Browse files Browse the repository at this point in the history
There are cases when a single service contains enough projects that it
outgrows a single shard. Since service.name is unique, we could not use
the same service.name on multiple shards.

Refactoring Shard assignment to a Project gives us more flexibility for
managing our Projects, and changes Service to a grouping of related rules
and notifications

Service.shard is moved to Project.shard
Service block groups Projects by Shard
Service list gets pagination. It's expected that in the default case,
users will subscribe to their projects updates, and therefore see them
on the home page
  • Loading branch information
kfdm authored May 29, 2019
2 parents a333420 + 4bf1383 commit 2107ef6
Show file tree
Hide file tree
Showing 38 changed files with 663 additions and 483 deletions.
15 changes: 8 additions & 7 deletions promgen/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,16 @@ class ShardAdmin(admin.ModelAdmin):

@admin.register(models.Service)
class ServiceAdmin(admin.ModelAdmin):
list_display = ('name', 'shard', 'owner')
list_filter = ('shard', ('owner', admin.RelatedOnlyFieldListFilter))
list_display = ('name', 'owner')
list_filter = (('owner', admin.RelatedOnlyFieldListFilter),)
list_select_related = ('owner',)


@admin.register(models.Project)
class ProjectAdmin(admin.ModelAdmin):
list_display = ('name', 'service', 'farm', 'owner')
list_select_related = ('service', 'farm', 'service__shard')
list_filter = (('owner', admin.RelatedOnlyFieldListFilter),)
list_display = ('name', 'shard', 'service', 'farm', 'owner')
list_select_related = ('service', 'farm', 'shard', 'owner')
list_filter = ('shard', ('owner', admin.RelatedOnlyFieldListFilter),)


class SenderForm(forms.ModelForm):
Expand All @@ -61,7 +62,7 @@ class SenderAdmin(admin.ModelAdmin):

@admin.register(models.Farm)
class FarmAdmin(admin.ModelAdmin):
list_display = ('name',)
list_display = ('name', 'source')
list_filter = ('source',)


Expand All @@ -81,7 +82,7 @@ class DefaultExporterAdmin(admin.ModelAdmin):
@admin.register(models.URL)
class URLAdmin(admin.ModelAdmin):
list_display = ('url', 'project')
list_select_related = ('project', 'project__service', 'project__service__shard')
list_select_related = ('project', 'project__service', 'project__shard')


class RuleLabelInline(admin.TabularInline):
Expand Down
3 changes: 1 addition & 2 deletions promgen/filters.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ class ShardFilter(django_filters.rest_framework.FilterSet):

class ServiceFilter(django_filters.rest_framework.FilterSet):
name = django_filters.CharFilter(field_name="name", lookup_expr="contains")
shard = django_filters.CharFilter(field_name="shard__name", lookup_expr="contains")


class ProjectFilter(django_filters.rest_framework.FilterSet):
Expand All @@ -16,7 +15,7 @@ class ProjectFilter(django_filters.rest_framework.FilterSet):
field_name="service__name", lookup_expr="contains"
)
shard = django_filters.CharFilter(
field_name="service__shard__name", lookup_expr="contains"
field_name="shard__name", lookup_expr="contains"
)


Expand Down
12 changes: 0 additions & 12 deletions promgen/forms.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,18 +97,6 @@ class Meta:
exclude = []


class ProjectRegister(forms.ModelForm):
class Meta:
model = models.Project
exclude = ['service', 'farm']


class ProjectUpdate(forms.ModelForm):
class Meta:
model = models.Project
exclude = ['farm']


class URLForm(forms.ModelForm):
class Meta:
model = models.URL
Expand Down
39 changes: 39 additions & 0 deletions promgen/migrations/0008_shard_service.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Generated by Django 2.1.8 on 2019-04-22 07:15

from django.db import migrations, models
import django.db.models.deletion


def project_to_shard(apps, schema_editor):
Project = apps.get_model("promgen", "Project")
for project in Project.objects.all():
project.shard_id = project.service.shard_id
project.save()


def project_to_service(apps, schema_editor):
Project = apps.get_model("promgen", "Project")
for project in Project.objects.all():
project.service.shard_id = project.shard_id
project.service.save()


class Migration(migrations.Migration):

dependencies = [("promgen", "0007_message_filter")]

operations = [
migrations.AddField(
model_name="project",
name="shard",
field=models.ForeignKey(
default=1,
on_delete=django.db.models.deletion.CASCADE,
to="promgen.Shard",
),
preserve_default=False,
),
migrations.RunPython(project_to_shard, project_to_service),
migrations.AlterModelOptions(name="service", options={"ordering": ["name"]}),
migrations.RemoveField(model_name="service", name="shard"),
]
32 changes: 13 additions & 19 deletions promgen/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,26 +179,27 @@ def get_absolute_url(self):
return reverse('shard-detail', kwargs={'pk': self.pk})

def __str__(self):
return self.name
if self.enabled:
return self.name
return self.name + ' (disabled)'


class Service(models.Model):
name = models.CharField(max_length=128, unique=True)
notifiers = GenericRelation(Sender)
rule_set = GenericRelation('Rule')
shard = models.ForeignKey('Shard', on_delete=models.CASCADE)
description = models.TextField(blank=True)

owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, default=None)

notifiers = GenericRelation(Sender)
rule_set = GenericRelation('Rule')

class Meta:
ordering = ['shard', 'name']
ordering = ['name']

def get_absolute_url(self):
return reverse('service-detail', kwargs={'pk': self.pk})

def __str__(self):
return '{} » {}'.format(self.shard.name, self.name)
return self.name

@classmethod
def default(cls, service_name='Default', shard_name='Default'):
Expand All @@ -216,25 +217,18 @@ def default(cls, service_name='Default', shard_name='Default'):
logger.info('Created default service')
return service

@property
def check_notifiers(self):
if self.notifiers.count() > 0:
return True
for project in self.project_set.all():
if project.notifiers.count() == 0:
return False
return True


class Project(models.Model):
name = models.CharField(max_length=128, unique=True)
description = models.TextField(blank=True)
owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, default=None)

service = models.ForeignKey('Service', on_delete=models.CASCADE)
shard = models.ForeignKey('Shard', on_delete=models.CASCADE)
farm = models.ForeignKey('Farm', blank=True, null=True, on_delete=models.SET_NULL)

notifiers = GenericRelation(Sender)
rule_set = GenericRelation('Rule')
description = models.TextField(blank=True)

owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, default=None)

class Meta:
ordering = ['name']
Expand Down
11 changes: 6 additions & 5 deletions promgen/prometheus.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,12 @@ def render_urls():
for url in models.URL.objects.prefetch_related(
'project__farm__host_set',
'project__farm',
'project__service__shard',
'project__service',
'project__service',
'project__shard',
'project'):
urls[(
url.project.name, url.project.service.name, url.project.service.shard.name,
url.project.name, url.project.service.name, url.project.shard.name,
)].append(url.url)

data = [{'labels': {'project': k[0], 'service': k[1], '__shard': k[2]}, 'targets': v} for k, v in urls.items()]
Expand All @@ -144,8 +145,8 @@ def render_config(service=None, project=None):
prefetch_related(
'project__farm__host_set',
'project__farm',
'project__service__shard',
'project__service',
'project__shard',
'project',
):
if not exporter.project.farm:
Expand All @@ -158,7 +159,7 @@ def render_config(service=None, project=None):
continue

labels = {
'__shard': exporter.project.service.shard.name,
'__shard': exporter.project.shard.name,
'service': exporter.project.service.name,
'project': exporter.project.name,
'farm': exporter.project.farm.name,
Expand Down Expand Up @@ -361,7 +362,6 @@ def import_config(config, replace_shard=None):

service, created = models.Service.objects.get_or_create(
name=entry['labels']['service'],
defaults={'shard': shard}
)
if created:
logger.debug('Created service %s', service)
Expand All @@ -382,6 +382,7 @@ def import_config(config, replace_shard=None):
project, created = models.Project.objects.get_or_create(
name=entry['labels']['project'],
service=service,
shard=shard,
defaults={'farm': farm}
)
if created:
Expand Down
4 changes: 2 additions & 2 deletions promgen/rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def format(self, rules=None, name='promgen'):


class ServiceViewSet(SharedViewSet, viewsets.ModelViewSet):
queryset = models.Service.objects.prefetch_related('shard')
queryset = models.Service.objects.all()
filterset_class = filters.ServiceFilter
serializer_class = serializers.ServiceSerializer
lookup_value_regex = '[^/]+'
Expand Down Expand Up @@ -71,7 +71,7 @@ def notifiers(self, request, name):

class ProjectViewSet(SharedViewSet, viewsets.ModelViewSet):
queryset = models.Project.objects.prefetch_related(
'service', 'service__shard', 'farm'
'service', 'shard', 'farm'
)
filterset_class = filters.ProjectFilter
serializer_class = serializers.ProjectSerializer
Expand Down
22 changes: 22 additions & 0 deletions promgen/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,3 +268,25 @@ def check_user_subscription(sender, instance, created, request):
messages.success(request, "Subscribed using %s" % (instance.owner.email))
else:
messages.warning(request, "No email configured")


@receiver(post_save, sender=models.Service)
def add_default_service_subscription(instance, created, **kwargs):
if created and instance.owner:
sender, new_notifier = models.Sender.objects.get_or_create(
obj=instance,
sender="promgen.notification.user",
value=instance.owner.username,
)
print(sender)


@receiver(post_save, sender=models.Project)
def add_default_project_subscription(instance, created, **kwargs):
if created and instance.owner:
sender, new_notifier = models.Sender.objects.get_or_create(
obj=instance,
sender="promgen.notification.user",
value=instance.owner.username,
)
print(sender)
102 changes: 52 additions & 50 deletions promgen/templates/promgen/farm_list.html
Original file line number Diff line number Diff line change
@@ -1,66 +1,68 @@
{% extends "base.html" %}
{% load i18n %}
{% load promgen %}

{% block content %}

<div class="page-header">
<h1>Farms</h1>
</div>

<ol class="breadcrumb">
<li><a href="{% url 'service-list' %}">Home</a></li>
<li class="active">All Farms</li>
</ol>
{% breadcrumb label="Farms" %}

{% include "promgen/pagination.html" %}

<div class="panel panel-default">
<table class="table table-bordered table-condensed">
<thead>
<tr>
<th>Farm</th>
<th>Source</th>
<th>Hosts</th>
<th>Project</th>
<th>&nbsp;</th>
</tr>
</thead>
{% for farm in farm_list %}
<table class="table table-bordered table-condensed">
<thead>
<tr>
<td><a href="{% url 'farm-detail' farm.id %}">{{ farm.name }}</a></td>
<td>
{% if farm.driver.remote %}
<span class="glyphicon glyphicon-cloud-download" aria-hidden="true"></span>
{% endif %}
{{ farm.source }}
</td>
<td>
<ul>
{% for host in farm.host_set.all %}
<li><a href="{% url 'host-detail' host.name %}">{{ host.name }}</a></li>
{% endfor %}
</ul>
</td>
<td>
<ul>
<th>Farm</th>
<th>Source</th>
<th>Hosts</th>
<th>Project</th>
<th>&nbsp;</th>
</tr>
</thead>
{% for farm in farm_list %}
<tr>
<td><a href="{% url 'farm-detail' farm.id %}">{{ farm.name }}</a></td>
<td>
{% if farm.driver.remote %}
<span class="glyphicon glyphicon-cloud-download" aria-hidden="true"></span>
{% endif %}
{{ farm.source }}
</td>
<td>
<ul>
{% for host in farm.host_set.all %}
<li><a href="{% url 'host-detail' host.name %}">{{ host.name }}</a></li>
{% endfor %}
</ul>
</td>
<td>
<ul>
{% for project in farm.project_set.all %}
<li><a href="{% url 'project-detail' project.id %}">{{ project.name }}</a></li>
<li><a href="{% url 'project-detail' project.id %}">{{ project.name }}</a></li>
{% endfor %}
</ul>
</td>
<td>
{% if farm.driver.remote %}
<form method="post" action="{% url 'farm-convert' farm.id %}" onsubmit="return confirm('Convert this farm to local?')" style="display: inline">
{% csrf_token %}
<button class="btn btn-warning">{% trans "Convert to Local Farm" %}</button>
</form>
{% endif %}
<form method="post" action="{% url 'farm-delete' farm.id %}" onsubmit="return confirm('Delete this farm?')" style="display: inline">
{% csrf_token %}
<button class="btn btn-danger">{% trans "Delete" %}</button>
</form>
</td>
</tr>
{% endfor %}
</table>

</ul>
</td>
<td>
{% if farm.driver.remote %}
<form method="post" action="{% url 'farm-convert' farm.id %}" onsubmit="return confirm('Convert this farm to local?')" style="display: inline">
{% csrf_token %}
<button class="btn btn-warning">{% trans "Convert to Local Farm" %}</button>
</form>
{% endif %}
<form method="post" action="{% url 'farm-delete' farm.id %}" onsubmit="return confirm('Delete this farm?')" style="display: inline">
{% csrf_token %}
<button class="btn btn-danger">{% trans "Delete" %}</button>
</form>
</td>
</tr>
{% endfor %}
</table>
</div>

{% include "promgen/pagination.html" %}

{% endblock %}
Loading

0 comments on commit 2107ef6

Please sign in to comment.