diff --git a/lumisections/admin.py b/lumisections/admin.py new file mode 100644 index 0000000..ac2b267 --- /dev/null +++ b/lumisections/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from .models import Lumisection + +# Register your models here. +admin.site.register(Lumisection) diff --git a/lumisections/apps.py b/lumisections/apps.py new file mode 100644 index 0000000..fcdc9d6 --- /dev/null +++ b/lumisections/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class LumisectionsConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'lumisections' diff --git a/lumisections/management/commands/extract_lumisections.py b/lumisections/management/commands/extract_lumisections.py new file mode 100644 index 0000000..4ed46ed --- /dev/null +++ b/lumisections/management/commands/extract_lumisections.py @@ -0,0 +1,39 @@ +from django.core.management.base import BaseCommand + +from runs.models import Run +from lumisections.models import Lumisection + +# https://betterprogramming.pub/3-techniques-for-importing-large-csv-files-into-a-django-app-2b6e5e47dba0 +import pandas as pd + +class Command(BaseCommand): + help = "Extracts lumisections from files" + + def add_arguments(self, parser): + parser.add_argument("file_path", type=str) + + def handle(self, *args, **options): + file_path = options["file_path"] + + df = pd.read_csv(file_path) + + lumisections = [] + + for index, row in df.iterrows(): + run_number = row["fromrun"] + lumi_number = row["fromlumi"] + #print(run_number, lumi_number) + + run, _ = Run.objects.get_or_create(run_number=run_number) + + lumisection = Lumisection( + run_number = run, + ls_number = lumi_number, + ) + + lumisections.append(lumisection) + + Lumisection.objects.bulk_create(lumisections) + print(f'lumisections successfully added!') + + diff --git a/lumisections/models.py b/lumisections/models.py new file mode 100644 index 0000000..75d82ef --- /dev/null +++ b/lumisections/models.py @@ -0,0 +1,11 @@ +from django.db import models +from runs.models import Run + +# Create your models here. +class Lumisection(models.Model): + ls_number = models.IntegerField() + run_number = models.ForeignKey(Run, on_delete=models.CASCADE) + date = models.DateTimeField(auto_now_add=True) + + def __str__(self): + return f"run {self.run_number.run_number} / lumisection {self.ls_number}" diff --git a/lumisections/views.py b/lumisections/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/lumisections/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/mlp/settings.py b/mlp/settings.py index 9f6741c..203267f 100644 --- a/mlp/settings.py +++ b/mlp/settings.py @@ -10,13 +10,11 @@ https://docs.djangoproject.com/en/3.2/ref/settings/ """ -import os - +from pathlib import Path from decouple import config # Build paths inside the project like this: BASE_DIR / 'subdir'. -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - +BASE_DIR = Path(__file__).resolve().parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ @@ -53,6 +51,7 @@ 'runs.apps.RunsConfig', 'run_histos.apps.RunHistosConfig', + 'lumisections', ] MIDDLEWARE = [ @@ -71,7 +70,7 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, "templates")], + 'DIRS': [BASE_DIR / 'templates'], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -132,8 +131,13 @@ # https://docs.djangoproject.com/en/3.2/howto/static-files/ STATIC_URL = '/static/' -#STATICFILES_DIRS = (os.path.join(BASE_DIR, 'home/static'),) -STATIC_ROOT = os.path.join(BASE_DIR, 'static') + +STATICFILES_DIRS = [ BASE_DIR / 'static_project' ] + +STATIC_ROOT = BASE_DIR / 'static_cdn/static_root' + +MEDIA_URL = '/media/' +MEDIA_ROOT = BASE_DIR / 'static_cdn/media_root' # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field diff --git a/mlp/urls.py b/mlp/urls.py index 50ba8ee..1261efe 100644 --- a/mlp/urls.py +++ b/mlp/urls.py @@ -19,5 +19,7 @@ urlpatterns = [ path("", include("home.urls")), path("datasets/", include("datasets.urls")), + path("runs/", include("runs.urls")), + path("run_histos/", include("run_histos.urls")), path('admin/', admin.site.urls), ] diff --git a/notebook.ipynb b/notebook.ipynb index ce0d3f9..edca606 100644 --- a/notebook.ipynb +++ b/notebook.ipynb @@ -28,6 +28,34 @@ { "cell_type": "code", "execution_count": 3, + "id": "0162a7cc", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import altair as alt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "253b0e45", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "786ff643", + "metadata": {}, + "source": [ + "## Checking apps" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "id": "64de1c6d", "metadata": {}, "outputs": [], @@ -37,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "b056afe4", "metadata": {}, "outputs": [ @@ -45,11 +73,11 @@ "data": { "text/plain": [ "{'id': 1,\n", - " 'run_number': 315257,\n", - " 'date': datetime.datetime(2021, 10, 22, 12, 35, 30, 551814, tzinfo=)}" + " 'run_number': 315258,\n", + " 'date': datetime.datetime(2021, 10, 26, 11, 37, 39, 211844, tzinfo=)}" ] }, - "execution_count": 4, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -60,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "a897f790", "metadata": {}, "outputs": [], @@ -70,7 +98,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "bd9589f5", "metadata": {}, "outputs": [ @@ -79,18 +107,18 @@ "text/plain": [ "{'id': 1,\n", " 'run_id': 1,\n", - " 'date': datetime.datetime(2021, 10, 22, 12, 35, 52, 542313, tzinfo=),\n", - " 'primary_dataset': 'ZeroBias',\n", + " 'date': datetime.datetime(2021, 10, 26, 11, 37, 48, 578496, tzinfo=),\n", + " 'primary_dataset': 'EGamma',\n", " 'path': 'TrackParameters/generalTracks/GeneralProperties',\n", " 'title': 'AbsDistanceOfClosestApproachToBS_GenTk',\n", - " 'entries': 34430387,\n", - " 'mean': 0.4408986607028722,\n", - " 'rms': 2.7660848356477787,\n", - " 'skewness': 3.1844432366039466,\n", - " 'kurtosis': 35.49796721670949}" + " 'entries': 786498,\n", + " 'mean': 0.4792563974038401,\n", + " 'rms': 2.9386499655611944,\n", + " 'skewness': 2.917119692221068,\n", + " 'kurtosis': 29.826709167508028}" ] }, - "execution_count": 6, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -106,6 +134,716 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "markdown", + "id": "2f876830", + "metadata": {}, + "source": [ + "## Converting to dataframes" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "76e78f5f", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(run_histos)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3a0284f1", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idrun_iddateprimary_datasetpathtitleentriesmeanrmsskewnesskurtosis
0112021-10-26 11:37:48.578496+00:00EGammaTrackParameters/generalTracks/GeneralPropertiesAbsDistanceOfClosestApproachToBS_GenTk7864980.4792562.9386502.91712029.826709
1212021-10-26 11:37:48.578595+00:00EGammaTrackParameters/generalTracks/GeneralPropertiesChi2Prob_GenTk7864980.4924660.322329-0.069946-1.345512
2312021-10-26 11:37:48.578635+00:00EGammaTrackParameters/generalTracks/GeneralPropertiesChi2_GenTk78649819.08340268.6988310.043568-2.917017
3412021-10-26 11:37:48.578670+00:00EGammaTrackParameters/generalTracks/GeneralPropertiesChi2oNDFVsEta_ImpactPoint_GenTk786288-0.1061331.5892740.272494-0.112022
4512021-10-26 11:37:48.578703+00:00EGammaTrackParameters/generalTracks/GeneralPropertiesChi2oNDFVsNHits_ImpactPoint_GenTk78628811.3311336.68478537.555632171.534894
\n", + "
" + ], + "text/plain": [ + " id run_id date primary_dataset \\\n", + "0 1 1 2021-10-26 11:37:48.578496+00:00 EGamma \n", + "1 2 1 2021-10-26 11:37:48.578595+00:00 EGamma \n", + "2 3 1 2021-10-26 11:37:48.578635+00:00 EGamma \n", + "3 4 1 2021-10-26 11:37:48.578670+00:00 EGamma \n", + "4 5 1 2021-10-26 11:37:48.578703+00:00 EGamma \n", + "\n", + " path \\\n", + "0 TrackParameters/generalTracks/GeneralProperties \n", + "1 TrackParameters/generalTracks/GeneralProperties \n", + "2 TrackParameters/generalTracks/GeneralProperties \n", + "3 TrackParameters/generalTracks/GeneralProperties \n", + "4 TrackParameters/generalTracks/GeneralProperties \n", + "\n", + " title entries mean rms \\\n", + "0 AbsDistanceOfClosestApproachToBS_GenTk 786498 0.479256 2.938650 \n", + "1 Chi2Prob_GenTk 786498 0.492466 0.322329 \n", + "2 Chi2_GenTk 786498 19.083402 68.698831 \n", + "3 Chi2oNDFVsEta_ImpactPoint_GenTk 786288 -0.106133 1.589274 \n", + "4 Chi2oNDFVsNHits_ImpactPoint_GenTk 786288 11.331133 6.684785 \n", + "\n", + " skewness kurtosis \n", + "0 2.917120 29.826709 \n", + "1 -0.069946 -1.345512 \n", + "2 0.043568 -2.917017 \n", + "3 0.272494 -0.112022 \n", + "4 37.555632 171.534894 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31f284d1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "1ba4f204", + "metadata": {}, + "source": [ + "## Filtering" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "89c1c27d", + "metadata": {}, + "outputs": [], + "source": [ + "select_title = ['Chi2Prob_GenTk', 'Chi2_GenTk']" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "20786fd3", + "metadata": {}, + "outputs": [], + "source": [ + "df_filtered = df[df['title'].isin(select_title)][['primary_dataset', 'run_id', 'title', 'mean']]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f21db4be", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
primary_datasetrun_idtitlemean
1EGamma1Chi2Prob_GenTk0.492466
2EGamma1Chi2_GenTk19.083402
84EGamma2Chi2Prob_GenTk0.486288
85EGamma2Chi2_GenTk19.146579
167EGamma3Chi2Prob_GenTk0.499791
...............
17182ZeroBias49Chi2_GenTk19.138309
17277ZeroBias50Chi2Prob_GenTk0.505047
17278ZeroBias50Chi2_GenTk19.139480
17373ZeroBias51Chi2Prob_GenTk0.516412
17374ZeroBias51Chi2_GenTk19.086274
\n", + "

404 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " primary_dataset run_id title mean\n", + "1 EGamma 1 Chi2Prob_GenTk 0.492466\n", + "2 EGamma 1 Chi2_GenTk 19.083402\n", + "84 EGamma 2 Chi2Prob_GenTk 0.486288\n", + "85 EGamma 2 Chi2_GenTk 19.146579\n", + "167 EGamma 3 Chi2Prob_GenTk 0.499791\n", + "... ... ... ... ...\n", + "17182 ZeroBias 49 Chi2_GenTk 19.138309\n", + "17277 ZeroBias 50 Chi2Prob_GenTk 0.505047\n", + "17278 ZeroBias 50 Chi2_GenTk 19.139480\n", + "17373 ZeroBias 51 Chi2Prob_GenTk 0.516412\n", + "17374 ZeroBias 51 Chi2_GenTk 19.086274\n", + "\n", + "[404 rows x 4 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_filtered" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a890359c", + "metadata": {}, + "outputs": [], + "source": [ + "df_filtered = df_filtered.pivot(index=['primary_dataset', 'run_id'], columns='title', values='mean').reset_index()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "5b78479c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titleprimary_datasetrun_idChi2Prob_GenTkChi2_GenTk
0EGamma10.49246619.083402
1EGamma20.48628819.146579
2EGamma30.49979119.027085
3EGamma40.48909519.103232
4EGamma50.48394719.190387
...............
197ZeroBias510.51641219.086274
198ZeroBias520.49762318.862586
199ZeroBias530.50235718.838694
200ZeroBias540.47968419.094168
201ZeroBias550.48712818.958929
\n", + "

202 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + "title primary_dataset run_id Chi2Prob_GenTk Chi2_GenTk\n", + "0 EGamma 1 0.492466 19.083402\n", + "1 EGamma 2 0.486288 19.146579\n", + "2 EGamma 3 0.499791 19.027085\n", + "3 EGamma 4 0.489095 19.103232\n", + "4 EGamma 5 0.483947 19.190387\n", + ".. ... ... ... ...\n", + "197 ZeroBias 51 0.516412 19.086274\n", + "198 ZeroBias 52 0.497623 18.862586\n", + "199 ZeroBias 53 0.502357 18.838694\n", + "200 ZeroBias 54 0.479684 19.094168\n", + "201 ZeroBias 55 0.487128 18.958929\n", + "\n", + "[202 rows x 4 columns]" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_filtered" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "19e79e5e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['EGamma', 'JetHT', 'SingleMuon', 'ZeroBias'], dtype=object)" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_filtered.primary_dataset.unique()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e70de59", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "dad6535a", + "metadata": {}, + "source": [ + "## Testing cross-filtering with altair" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "e2ce2510", + "metadata": {}, + "outputs": [], + "source": [ + "source = df_filtered" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "bb329f3d", + "metadata": {}, + "outputs": [], + "source": [ + "brush_EGamma = alt.selection(type='interval')\n", + "brush_JetHT = alt.selection(type='interval')\n", + "brush_ZeroBias = alt.selection(type='interval')" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "b5a40a03", + "metadata": {}, + "outputs": [], + "source": [ + "EGamma = alt.Chart(source[source['primary_dataset']=='EGamma']).mark_point(clip=True).encode(\n", + " alt.X('Chi2Prob_GenTk:Q',\n", + " scale=alt.Scale(domain=(0.4, 0.6))\n", + " ),\n", + " alt.Y('Chi2_GenTk:Q',\n", + " scale=alt.Scale(domain=(16, 22))\n", + " ),\n", + " color=alt.condition(brush_EGamma|brush_JetHT|brush_ZeroBias, 'primary_dataset', alt.value('lightgray'))\n", + ").add_selection(\n", + " brush_EGamma\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "28351c10", + "metadata": {}, + "outputs": [], + "source": [ + "JetHT = alt.Chart(source[source['primary_dataset']=='JetHT']).mark_point(clip=True).encode(\n", + " alt.X('Chi2Prob_GenTk:Q',\n", + " scale=alt.Scale(domain=(0.4, 0.6))\n", + " ),\n", + " alt.Y('Chi2_GenTk:Q',\n", + " scale=alt.Scale(domain=(16, 22))\n", + " ),\n", + " color=alt.condition(brush_EGamma|brush_JetHT|brush_ZeroBias, 'primary_dataset', alt.value('lightgray'))\n", + ").add_selection(\n", + " brush_JetHT\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "bc80295e", + "metadata": {}, + "outputs": [], + "source": [ + "ZeroBias = alt.Chart(source[source['primary_dataset']=='ZeroBias']).mark_point(clip=True).encode(\n", + " alt.X('Chi2Prob_GenTk:Q',\n", + " scale=alt.Scale(domain=(0.4, 0.6))\n", + " ),\n", + " alt.Y('Chi2_GenTk:Q',\n", + " scale=alt.Scale(domain=(16, 22))\n", + " ),\n", + " color=alt.condition(brush_EGamma|brush_JetHT|brush_ZeroBias, 'primary_dataset', alt.value('lightgray'))\n", + ").add_selection(\n", + " brush_ZeroBias\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "7f3a3588", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + "" + ], + "text/plain": [ + "alt.VConcatChart(...)" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "EGamma & JetHT & ZeroBias" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b74199fb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a96dbb5", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/run_histos/management/commands/extract_run_histos.py b/run_histos/management/commands/extract_run_histos.py index 8e59e8f..a271c2c 100644 --- a/run_histos/management/commands/extract_run_histos.py +++ b/run_histos/management/commands/extract_run_histos.py @@ -14,18 +14,23 @@ def add_arguments(self, parser): def handle(self, *args, **options): file_path = options["file_path"] - + split_file_path = file_path.replace('.csv', "").split('/')[-1].split('_') + print(split_file_path) + # opening per run file from ML4DQM df = pd.read_csv(file_path) # extracting and creating run number (ForeignKey of histo1DRun) - run_number = df.run.unique()[0] - print(type(run_number)) + dataset = split_file_path[0] + run_number = split_file_path[1] + workspace = 'TrackParameters/generalTracks/GeneralProperties' + tag = 'generalTracks' + run, _ = Run.objects.get_or_create(run_number=run_number) print(f'run {run_number} successfully added!') # extracting set of histograms corresponding to the run - dataset = 'ZeroBias' + dataset = split_file_path[0] workspace = 'TrackParameters/generalTracks/GeneralProperties' tag = 'generalTracks' diff --git a/run_histos/templates/run_histos/main.html b/run_histos/templates/run_histos/main.html new file mode 100644 index 0000000..05910f1 --- /dev/null +++ b/run_histos/templates/run_histos/main.html @@ -0,0 +1,71 @@ +{% extends "base.html" %} + +{% block title %} Run Histos {% endblock title %} + +{% block content %} + +
+ {% if error_message %} + {{ error_message }} + {% endif %} + +

Choosing Dataset / Variable / Plot type

+ +
+ {% csrf_token %} +
+ +
+ + +
----
+
+
ZeroBias
+
SingleMuon
+
EGamma
+
JetHT
+
+
+
+
+ +
+ + +
----
+
+
Chi2Prob_GenTk
+
Chi2_GenTk
+
+
+
+
+ +
+ + +
----
+
+
Histogram
+
Time Serie
+
+
+
+ +
+ +
+ +
+ +

Plot

+ +
+ + + +{% endblock content %} diff --git a/run_histos/urls.py b/run_histos/urls.py new file mode 100644 index 0000000..8f9c1be --- /dev/null +++ b/run_histos/urls.py @@ -0,0 +1,9 @@ +from django.urls import path +from .views import run_histos_view, chart_view_altair + +app_name = 'run_histos' + +urlpatterns = [ + path('', run_histos_view, name='main-runhistos-view'), + path('altair/', chart_view_altair, name='altair-view') +] diff --git a/run_histos/views.py b/run_histos/views.py index 91ea44a..b1c27df 100644 --- a/run_histos/views.py +++ b/run_histos/views.py @@ -1,3 +1,78 @@ from django.shortcuts import render +from django.http import JsonResponse + +from runs.models import Run +from .models import RunHisto + +import pandas as pd +import altair as alt # Create your views here. + +def run_histos_view(request): + + error_message = None + dataset = None + variable = None + plot_type = None + df = None + chart = {} + + # objects.all().values() provides a dictionary while objects.all().values_list() provides a tuple + runs_df = pd.DataFrame(Run.objects.all().values()) + runhistos_df = pd.DataFrame(RunHisto.objects.all().values()) + + if runhistos_df.shape[0] > 0: + df = pd.merge(runs_df, runhistos_df, left_on='id', right_on='run_id').drop(['id_x', 'id_y', 'run_id', 'date_x', 'date_y'], axis=1) + + if request.method == 'POST': + dataset = request.POST['dataset'] + variable = request.POST['variable'] + plot_type = request.POST['plot_type'] + print(f"dataset: {dataset} / variable: {variable} / plot_type: {plot_type}") + + #df = df.query('primary_dataset.str.lower()=="zerobias" & title.str.lower()=="chi2prob_gentk"') + df = df.query('primary_dataset.str.lower()==@dataset & title.str.lower()==@variable') + + if plot_type == 'histogram': + chart = alt.Chart(df).mark_bar().encode( + alt.X("mean", bin=True), + y='count()', + ).to_json(indent=None) + elif plot_type == 'time serie': + chart = alt.Chart(df).mark_circle(size=60).encode( + alt.X('run_number', + scale=alt.Scale(domain=(315000, 316000)) # shouldn't be hardcoded + ), + y='mean', + tooltip=['run_number', 'mean'] + ).to_json(indent=None) + else: + print("No chart type was selected.") + + else: + error_message = "No runhistos in the database" + + context = { + 'error_message': error_message, + 'df': df, + 'chart' : chart, + } + + return render(request, 'run_histos/main.html', context) + +def chart_view_altair(request): + + chart = {} + + runhistos_df = pd.DataFrame(RunHisto.objects.all().values()).head(100) + + if runhistos_df.shape[0] > 0: + chart_obj = alt.Chart(runhistos_df).mark_bar().encode( + x='mean', + ).to_json(indent=None) + + else: + print("No runshistos in the database") + + return JsonResponse(chart_obj) diff --git a/runs/models.py b/runs/models.py index b8302ea..0ddfcc3 100644 --- a/runs/models.py +++ b/runs/models.py @@ -6,4 +6,4 @@ class Run(models.Model): date = models.DateTimeField(auto_now_add=True) def __str__(self): - return f"run {run_number}" + return f"run {self.run_number}" diff --git a/runs/templates/runs/main.html b/runs/templates/runs/main.html index 08f28dd..e253505 100644 --- a/runs/templates/runs/main.html +++ b/runs/templates/runs/main.html @@ -1,11 +1,15 @@ -{% extends 'base.html' %} +{% extends "base.html" %} {% block title %} Runs {% endblock title %} {% block content %} -

Welcome to the run view

-{{runs}} +{% if error_message %} +{{ error_message }} +{% endif %} -{% endblock content %} +{% if runs %} +{{ runs | safe }} +{% endif %} +{% endblock content %} diff --git a/runs/templates/runs/run.html b/runs/templates/runs/run.html new file mode 100644 index 0000000..380acae --- /dev/null +++ b/runs/templates/runs/run.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} + +{% block title %} Run view {% endblock title %} + +{% block content %} + +Specific view of run number ... + +{% endblock content %} diff --git a/runs/urls.py b/runs/urls.py index 779c8be..8080340 100644 --- a/runs/urls.py +++ b/runs/urls.py @@ -1,8 +1,9 @@ from django.urls import path -from .views import chart_select_view +from .views import runs_view, run_view app_name = 'runs' urlpatterns = [ - path('', chart_select_view, name='main-runs-view') + path('', runs_view, name='main-runs-view'), + path('run/', run_view, name='main-run-view') ] diff --git a/runs/views.py b/runs/views.py index 80ce6b4..a1ce08c 100644 --- a/runs/views.py +++ b/runs/views.py @@ -5,10 +5,27 @@ # Create your views here. -def chart_select_view(request): +def runs_view(request): + + error_message = None + df = None + # objects.all().values() provides a dictionary while objects.all().values_list() provides a tuple runs_df = pd.DataFrame(Run.objects.all().values()) + + if runs_df.shape[0] > 0: + df = runs_df.drop(['id'], axis=1) + + else: + error_message = "No runs in the database" + context = { - 'runs': runs_df, + 'error_message': error_message, + 'runs': df, } return render(request, 'runs/main.html', context) + + +def run_view(request): + return render(request, 'runs/run.html') + diff --git a/scripts/project_notebook.sh b/scripts/project_notebook.sh new file mode 100644 index 0000000..480d4ef --- /dev/null +++ b/scripts/project_notebook.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd .. + +./manage.py shell_plus --notebook diff --git a/scripts/project_visualization.sh b/scripts/project_visualization.sh new file mode 100644 index 0000000..f9e11d0 --- /dev/null +++ b/scripts/project_visualization.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +cd .. + +./manage.py graph_models -a -g -o ad_project.png +./manage.py graph_models -a -I Run,RunHisto,Lumisection -o ad_project_classes.png diff --git a/scripts/step1_extract_runs.sh b/scripts/step1_extract_runs.sh index b4ff3a7..f6f6c2f 100644 --- a/scripts/step1_extract_runs.sh +++ b/scripts/step1_extract_runs.sh @@ -3,7 +3,7 @@ cd .. # testing one file -python manage.py extract_runs /eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/ZeroBias_315257_UL2018.csv +./manage.py extract_runs /eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/ZeroBias_315257_UL2018.csv FILES="/eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/ZeroBia*" @@ -12,5 +12,5 @@ for f in $FILES do echo "Processing $f file..." - #python manage.py extract_runs f + #./manage.py extract_runs $f done diff --git a/scripts/step2_extract_run_histos.sh b/scripts/step2_extract_run_histos.sh index c802749..18a461c 100644 --- a/scripts/step2_extract_run_histos.sh +++ b/scripts/step2_extract_run_histos.sh @@ -3,14 +3,14 @@ cd .. # testing one file -python manage.py extract_run_histos /eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/ZeroBias_315257_UL2018.csv +#./manage.py extract_run_histos /eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/ZeroBias_315257_UL2018.csv -FILES="/eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/ZeroBia*" +FILES="/eos/project/c/cmsml4dc/ML_2020/PerRun_UL2018_Data/*_315*" # extracting from all files for f in $FILES do echo "Processing $f file..." - #python manage.py extract_run_histos f + ./manage.py extract_run_histos $f done diff --git a/scripts/step3_extract_lumisections.sh b/scripts/step3_extract_lumisections.sh new file mode 100644 index 0000000..24e0432 --- /dev/null +++ b/scripts/step3_extract_lumisections.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cd .. + +# testing one file +./manage.py extract_lumisections /eos/project/c/cmsml4dc/ML_2020/UL2018_Data/DF2018A_1D_Complete/ZeroBias_2018A_DataFrame_1D_1.csv + +FILES="/eos/project/c/cmsml4dc/ML_2020/UL2018_Data/DF2018A_1D_Complete/*" + +# extracting from all files +for f in $FILES + +do + echo "Processing $f file..." + #./manage.py extract_lumisections $f +done diff --git a/static_project/main.js b/static_project/main.js new file mode 100644 index 0000000..1fc70d7 --- /dev/null +++ b/static_project/main.js @@ -0,0 +1,6 @@ +$(document).ready(function(){ + $('.ui.dropdown') + .dropdown() +; +}) + diff --git a/static_project/style.css b/static_project/style.css new file mode 100644 index 0000000..e69de29 diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..5d16cdb --- /dev/null +++ b/templates/base.html @@ -0,0 +1,40 @@ +{% load static %} + + + + + + + + + + + + + + + + + + + + + ML Playground - {% block title %}{% endblock title %} + + + + + + {% include 'navbar.html' %} + +
+ {% block content %} + {% endblock content %} +
+ + + + + + + diff --git a/templates/navbar.html b/templates/navbar.html new file mode 100644 index 0000000..218633a --- /dev/null +++ b/templates/navbar.html @@ -0,0 +1,2 @@ +

Navbar

+