From 7f4933ec28fb3b066e291b274f907ac8667ffb99 Mon Sep 17 00:00:00 2001 From: "S. Pothier" Date: Tue, 27 Aug 2024 09:08:26 -0700 Subject: [PATCH] Tickets/dm 45892 (#5) * WIP * WIP * WIP * WIP * WIP --- README.md | 2 + .../README.md | 0 .../logrep_proto_1.ipynb | 193 ++++++++++++++---- .../logrep_proto_1.yaml | 8 - .../requirements.txt | 0 .../scaffolding.org | 0 times-square.yaml | 2 +- 7 files changed, 157 insertions(+), 48 deletions(-) rename {notebooks_for_times_square => notebooks_tsqr}/README.md (100%) rename {notebooks_for_times_square => notebooks_tsqr}/logrep_proto_1.ipynb (66%) rename {notebooks_for_times_square => notebooks_tsqr}/logrep_proto_1.yaml (61%) rename {notebooks_for_times_square => notebooks_tsqr}/requirements.txt (100%) rename {notebooks_for_times_square => notebooks_tsqr}/scaffolding.org (100%) diff --git a/README.md b/README.md index 85d6661..626e655 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Project-wide Logging and Reporting + ## Set up For local development, create a virtual environment and install the required packages: @@ -15,5 +16,6 @@ pip install -r requirements.txt This repository uses pre-commit hooks to ensure code is formatted correctly and that notebooks are cleared of outputs before committing. To set up the pre-commit hooks, run the following command in the root directory of the repository: ```bash + pre-commit install ``` diff --git a/notebooks_for_times_square/README.md b/notebooks_tsqr/README.md similarity index 100% rename from notebooks_for_times_square/README.md rename to notebooks_tsqr/README.md diff --git a/notebooks_for_times_square/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb similarity index 66% rename from notebooks_for_times_square/logrep_proto_1.ipynb rename to notebooks_tsqr/logrep_proto_1.ipynb index ff93343..0bd42a0 100644 --- a/notebooks_for_times_square/logrep_proto_1.ipynb +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -15,10 +15,23 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "markdown", "id": "1", "metadata": {}, + "source": [ + "\n", + "## Imports and General Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, "outputs": [], "source": [ "# Only use packages available in the Rubin Science Platform\n", @@ -34,8 +47,12 @@ { "cell_type": "code", "execution_count": null, - "id": "2", - "metadata": {}, + "id": "3", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, "outputs": [], "source": [ "log = log_name\n", @@ -58,14 +75,30 @@ " tucson = 'https://tucson-teststand.lsst.codes',\n", ")\n", "server = envs[env]\n", - "server" + "service = f'{server}/{log}'\n", + "service" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, + "source": [ + "\n", + "## Setup Source" ] }, { "cell_type": "code", "execution_count": null, - "id": "3", - "metadata": {}, + "id": "5", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, "outputs": [], "source": [ "recs = None\n", @@ -79,15 +112,40 @@ " limit=limit,\n", " )\n", "qstr = urlencode(qparams)\n", - "url = f'{server}/{log}/messages?{qstr}'\n", - "print(f'\\nAttempt to get logs from {url=}')\n", + "url = f'{service}/messages?{qstr}'\n", + "\n", "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", " 'date_invalidated', 'date_begin', 'date_end',\n", " 'time_lost', # float\n", " #'systems','subsystems','cscs', # values are lists, special handling\n", - " ])\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "\n", + "## Get Records" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO Often fails on first request. Find out why!\n", + "try:\n", + " response = requests.get(url, timeout=timeout)\n", + "except:\n", + " pass \n", + " \n", "try:\n", + " print(f'Attempt to get logs from {url=}')\n", " response = requests.get(url, timeout=timeout)\n", " response.raise_for_status()\n", " recs = response.json()\n", @@ -102,69 +160,107 @@ " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", "numf = len(flds) if ok else 0\n", "numr = len(recs) if ok else 0\n", - "print(f'{numr=} {numf=}')" + "print(f'Retrieved {numr} records, each with {numf=} fields.')" ] }, { - "cell_type": "code", - "execution_count": null, - "id": "4", + "cell_type": "markdown", + "id": "8", "metadata": {}, - "outputs": [], "source": [ - "flds" + "\n", + "## Tables of (mostly raw) results" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### Fields names provided in records from log." ] }, { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "10", "metadata": {}, "outputs": [], "source": [ - "facets" + "pd.DataFrame(flds, columns=['Field Name'])" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Facets from log records.\n", + "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." ] }, { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "12", "metadata": {}, "outputs": [], "source": [ - "recs[0]" + "pd.DataFrame.from_dict(facets, orient='index')" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Table of selected log record fields.\n", + "Table can be retrieved as CSV file for local use." ] }, { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "14", "metadata": {}, "outputs": [], "source": [ "cols = ['date_added', 'time_lost']\n", "df = pd.DataFrame(recs)[cols]\n", + "\n", + "# Allow download of CSV version of DataFrame\n", + "csvfile = 'tl.csv'\n", + "df.to_csv(csvfile)\n", + "myfile = FileLink(csvfile)\n", + "print('Table available as CSV file: ')\n", + "display(myfile)\n", "df" ] }, { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "15", "metadata": {}, "outputs": [], "source": [ - "# Allow download of CSV version of DataFrame\n", - "csvfile = 'tl.csv'\n", - "df.to_csv(csvfile)\n", - "myfile = FileLink(csvfile)\n", - "display(myfile)" + "df = pd.DataFrame(recs)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "\n", + "## Plots from log" ] }, { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -175,38 +271,57 @@ ] }, { - "cell_type": "code", - "execution_count": null, - "id": "10", + "cell_type": "markdown", + "id": "18", "metadata": {}, - "outputs": [], - "source": [] + "source": [ + "\n", + "## Raw Content Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "### Example of one record" + ] }, { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "20", "metadata": {}, "outputs": [], "source": [ - "recs[:4]" + "rec = recs[0]\n", + "rec" ] }, { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "21", "metadata": {}, "outputs": [], "source": [ - "df = pd.DataFrame(recs)\n", - "df" + "msg = rec[\"message_text\"]\n", + "print(msg)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "\n", + "## Stakeholder Elicitation" ] }, { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "23", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks_for_times_square/logrep_proto_1.yaml b/notebooks_tsqr/logrep_proto_1.yaml similarity index 61% rename from notebooks_for_times_square/logrep_proto_1.yaml rename to notebooks_tsqr/logrep_proto_1.yaml index a7e82ac..3c1d085 100644 --- a/notebooks_for_times_square/logrep_proto_1.yaml +++ b/notebooks_tsqr/logrep_proto_1.yaml @@ -8,14 +8,6 @@ tags: - reporting - prototype parameters: - env: - type: string - description: "tucson | usdf_dev | summit" - default: "tucson" - log: - type: string - description: "The log name: narrativelog | exposurelog" - default: narrativelog record_limit: type: integer description: Max number of records to output diff --git a/notebooks_for_times_square/requirements.txt b/notebooks_tsqr/requirements.txt similarity index 100% rename from notebooks_for_times_square/requirements.txt rename to notebooks_tsqr/requirements.txt diff --git a/notebooks_for_times_square/scaffolding.org b/notebooks_tsqr/scaffolding.org similarity index 100% rename from notebooks_for_times_square/scaffolding.org rename to notebooks_tsqr/scaffolding.org diff --git a/times-square.yaml b/times-square.yaml index 9aafe7e..3e6e1d5 100644 --- a/times-square.yaml +++ b/times-square.yaml @@ -1,4 +1,4 @@ enabled: true -root: notebooks_for_times_square +root: notebooks_tsqr description: > Times Square for project-wide Logging and Reporting.