Search analytics

readthedocs/projects/constants.py

@@ -355,3 +355,19 @@
 )
 
 GITHUB_PR_PULL_PATTERN = 'pull/{id}/head:external-{id}'
+
+SEARCH_ANALYTICS_PARAMS = {
+    'period': (
+        ('recent', 'Recent Queries'),
+        ('last-24-hrs', 'Top Queries of Last 24 Today'),
+        ('last-48-hrs', 'Top Queries of Yesterday'),
+        ('last-7-days', 'Top Queries of Last 7 Days'),
+        ('last-1-month', 'Top Queries of Last 1 Month'),
+    ),
+    'size': (
+        (5, 5),
+        (10, 10),
+        (50, 50),
+        (100, 100),
+    )
+}

readthedocs/projects/forms.py

@@ -19,6 +19,7 @@
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.integrations.models import Integration
 from readthedocs.oauth.models import RemoteRepository
+from readthedocs.projects.constants import SEARCH_ANALYTICS_PARAMS
 from readthedocs.projects.exceptions import ProjectSpamError
 from readthedocs.projects.models import (
     Domain,
@@ -806,3 +807,30 @@ def clean_name(self):
                 _('Only letters, numbers and underscore are allowed'),
             )
         return name
+
+
+class SearchAnalyticsForm(forms.Form):
+    version = forms.ChoiceField(required=False)
+    period = forms.ChoiceField(
+        choices=SEARCH_ANALYTICS_PARAMS['period'],
+        required=False,
+        help_text=_('Choose the time-period for the results.')
+    )
+    size = forms.ChoiceField(
+        choices=SEARCH_ANALYTICS_PARAMS['size'],
+        required=False
+    )
+
+    def __init__(self, *args, **kwargs):
+        self.project = kwargs.pop('project', None)
+        super().__init__(*args, **kwargs)
+
+        versions = (
+            self.project.versions(manager=INTERNAL)
+            .filter(project=self.project, active=True)
+        )
+        sorted_versions = sort_version_aware(versions)
+
+        self.fields['version'].choices = [
+            (version.slug, version.verbose_name) for version in sorted_versions
+        ]

readthedocs/projects/urls/private.py

@@ -105,6 +105,10 @@
         r'^(?P<project_slug>[-\w]+)/advertising/$',
         ProjectAdvertisingUpdate.as_view(), name='projects_advertising',
     ),
+    url(
+        r'^(?P<project_slug>[-\w]+)/search-analytics/$',
+        private.search_analytics_view, name='projects_search_analytics',
+    ),
 ]
 
 domain_urls = [

readthedocs/projects/views/private.py

@@ -16,6 +16,7 @@
 from django.middleware.csrf import get_token
 from django.shortcuts import get_object_or_404, render
 from django.urls import reverse
+from django.utils import timezone
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import ListView, TemplateView, View
@@ -42,6 +43,7 @@
     ProjectExtraForm,
     ProjectRelationshipForm,
     RedirectForm,
+    SearchAnalyticsForm,
     TranslationForm,
     UpdateProjectForm,
     UserForm,
@@ -58,6 +60,7 @@
 from readthedocs.projects.notifications import EmailConfirmNotification
 from readthedocs.projects.views.base import ProjectAdminMixin, ProjectSpamMixin
 from readthedocs.projects.views.mixins import ProjectImportMixin
+from readthedocs.search.models import SearchQuery
 
 from ..tasks import retry_domain_verification
 
@@ -899,3 +902,84 @@ class EnvironmentVariableDelete(EnvironmentVariableMixin, DeleteView):
     # This removes the delete confirmation
     def get(self, request, *args, **kwargs):
         return self.http_method_not_allowed(request, *args, **kwargs)
+
+
+@login_required
+def search_analytics_view(request, project_slug):
+    project = get_object_or_404(
+        Project.objects.for_admin_user(request.user),
+        slug=project_slug,
+    )
+
+    version_slug = request.GET.get('version', project.default_version)
+    period = request.GET.get('period', 'recent')
+    size = request.GET.get('size', 5)
+
+    try:
+        size = int(size)
+    except ValueError:
+        size = 5
+
+    data = {
+        'version': version_slug,
+        'period': period,
+        'size': size,
+    }
+
+    form = SearchAnalyticsForm(data=data, project=project)
+
+    if version_slug:
+        version_qs = Version.objects.filter(project=project, slug=version_slug)
+
+        if version_qs.exists():
+            version = version_qs.first()
+            search_queries = SearchQuery.objects.filter(
+                project=project,
+                version=version,
+            )
+
+            now = timezone.now()
+
+            if period == 'recent':
+                queries = search_queries.order_by('-modified')
+
+            elif period == 'last-24-hrs':
+                last_24_hrs = now - timezone.timedelta(days=1)
+                queries = search_queries.filter(
+                    modified__gte=last_24_hrs,
+                    modified__lte=now,
+                ).order_by('-count')
+
+            elif period == 'last-48-hrs':
+                last_48_hrs = now - timezone.timedelta(days=2)
+                queries = search_queries.filter(
+                    modified__gte=last_48_hrs,
+                    modified__lte=now,
+                ).order_by('-count')
+
+            elif period == 'last-7-days':
+                last_7_days = now - timezone.timedelta(days=7)
+                queries = search_queries.filter(
+                    modified__gte=last_7_days,
+                    modified__lte=now,
+                ).order_by('-count')
+
+            elif period == 'last-1-month':
+                last_30_days = now - timezone.timedelta(days=30)
+                queries = search_queries.filter(
+                    modified__gte=last_30_days,
+                    modified__lte=now,
+                ).order_by('-count')
+
+            else:
+                queries = []
+    else:
+        queries = []
+
+    queries = queries[:size]
+
+    return render(
+        request,
+        'projects/projects_search_analytics.html',
+        {'form': form, 'project': project, 'queries': queries},
+    )

readthedocs/search/admin.py

@@ -0,0 +1,16 @@
+"""SearchQuery Admin classes."""
+
+from django.contrib import admin
+
+from .models import SearchQuery
+
+
+class SearchQueryAdmin(admin.ModelAdmin):
+    list_filter = ('created',)
+    list_display = ('__str__', 'count', 'created', 'modified')
+    search_fields = ('project__slug', 'version__slug', 'query')
+    readonly_fields = ('created', 'modified')
+    list_select_related = ('project', 'version', 'version__project')
+
+
+admin.site.register(SearchQuery, SearchQueryAdmin)

readthedocs/search/api.py

@@ -152,3 +152,29 @@ def get_all_projects_url(self):
         for project in all_projects:
             projects_url[project.slug] = project.get_docs_url(version_slug=version_slug)
         return projects_url
+
+    def list(self, request, *args, **kwargs):
+        """Overriding ``list`` method to record query in database."""
+
+        response = super().list(request, *args, **kwargs)
+
+        project_slug = self.request.query_params.get('project', None)
+        version_slug = self.request.query_params.get('version', None)
+        query = self.request.query_params.get('q', '')
+        total_results = response.data.get('count', 0)
+
+        try:
+            utils.record_search_query(
+                project_slug,
+                version_slug,
+                query,
+                total_results,
+            )
+        except Exception:
+            log.exception('[%s] [%s] [Query: %s] Error recording search query in database.' % (
+                project_slug,
+                version_slug,
+                query,
+            ))
+
+        return response

readthedocs/search/migrations/0001_initial.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.22 on 2019-07-31 14:25
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('projects', '0044_auto_20190703_1300'),
+        ('builds', '0009_added_external_version_type'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SearchQuery',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('query', models.CharField(max_length=4092, verbose_name='Query')),
+                ('count', models.PositiveIntegerField(default=1, verbose_name='No. of times this query was searched')),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='search_queries', to='projects.Project')),
+                ('version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='search_queries', to='builds.Version', verbose_name='Version')),
+            ],
+            options={
+                'verbose_name': 'Search query',
+                'verbose_name_plural': 'Search queries',
+            },
+        ),
+    ]

readthedocs/search/models.py

@@ -0,0 +1,43 @@
+"""Search Queries."""
+
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+
+from django_extensions.db.models import TimeStampedModel
+
+from readthedocs.builds.models import Version
+from readthedocs.projects.models import Project
+from readthedocs.projects.querysets import RelatedProjectQuerySet
+
+
+class SearchQuery(TimeStampedModel):
+
+    """Information about the search queries."""
+
+    project = models.ForeignKey(
+        Project,
+        related_name='search_queries',
+        on_delete=models.CASCADE,
+    )
+    version = models.ForeignKey(
+        Version,
+        verbose_name=_('Version'),
+        related_name='search_queries',
+        on_delete=models.CASCADE,
+    )
+    query = models.CharField(
+        _('Query'),
+        max_length=4092,
+    )
+    count = models.PositiveIntegerField(
+        _('No. of times this query was searched'),
+        default=1,
+    )
+    objects = RelatedProjectQuerySet.as_manager()
+
+    class Meta:
+        verbose_name = 'Search query'
+        verbose_name_plural = 'Search queries'
+
+    def __str__(self):
+        return f'[{self.project.slug}:{self.version.slug}]: {self.query}'

readthedocs/search/tasks.py

@@ -1,9 +1,11 @@
 import logging
 
 from django.apps import apps
+from django.utils import timezone
 from django_elasticsearch_dsl.registries import registry
 
 from readthedocs.worker import app
+from .models import SearchQuery
 from .utils import _get_index, _get_document
 
 log = logging.getLogger(__name__)
@@ -118,3 +120,15 @@ def index_missing_objects(app_label, model_name, document_class, index_generatio
     log.info("Indexed %s missing objects from model: %s'", queryset.count(), model.__name__)
 
     # TODO: Figure out how to remove the objects from ES index that has been deleted
+
+
+@app.task(queue='web')
+def delete_old_search_queries_from_db():
+    last_3_months = timezone.timedelta(days=90)
+    search_queries_qs = SearchQuery.objects.filter(
+        created__lte=last_3_months
+    )
+
+    if search_queries_qs.exists():
+        log.info('Deleting search queries for last 3 months. Total: %s', search_queries_qs.count())
+        search_queries_qs.delete()

readthedocs/search/utils.py

@@ -10,6 +10,7 @@
 from readthedocs.builds.models import Version
 from readthedocs.projects.models import HTMLFile, Project
 from readthedocs.search.documents import PageDocument
+from readthedocs.search.models import SearchQuery
 
 
 log = logging.getLogger(__name__)
@@ -168,3 +169,38 @@ def _get_sorted_results(results, source_key='_source'):
     ]
 
     return sorted_results
+
+
+def record_search_query(project_slug, version_slug, query, total_results):
+    """Record search query in database."""
+    if not project_slug or not version_slug or not query or not total_results:
+        return
+
+    project_qs = Project.objects.filter(slug=project_slug)
+
+    if not project_qs.exists():
+        return
+
+    project = project_qs.first()
+    version_qs = Version.objects.filter(project=project, slug=version_slug)
+
+    if not version_qs.exists():
+        return
+
+    version = version_qs.first()
+    search_query_qs = SearchQuery.objects.filter(
+        project=project,
+        version=version,
+        query=query,
+    )
+
+    if search_query_qs.exists():
+        search_query_obj = search_query_qs.first()
+        search_query_obj.count += 1
+        search_query_obj.save()
+    else:
+        SearchQuery.objects.create(
+            project=project,
+            version=version,
+            query=query,
+        )

readthedocs/settings/base.py

@@ -326,6 +326,16 @@ def USE_PROMOS(self):  # noqa
             'schedule': crontab(minute=0, hour='*/3'),
             'options': {'queue': 'web'},
         },
+        'every-three-month-delete-old-search-queries': {
+            'task': 'readthedocs.search.tasks.delete_old_search_queries_from_db',
+            'schedule': crontab(
+                minute=0,
+                hour=0,
+                day_of_month=1,
+                month_of_year='*/4'
+            ),
+            'options': {'queue': 'web'},
+        }
     }
     MULTIPLE_APP_SERVERS = [CELERY_DEFAULT_QUEUE]
     MULTIPLE_BUILD_SERVERS = [CELERY_DEFAULT_QUEUE]