Tickets/dm 45892 (#5)

* WIP * WIP * WIP * WIP * WIP
lsst-ts · Aug 27, 2024 · 7f4933e · 7f4933e
1 parent 198dd93
commit 7f4933e
Show file tree

Hide file tree

Showing 7 changed files with 157 additions and 48 deletions.
diff --git a/README.md b/README.md
@@ -2,6 +2,7 @@
 
 Project-wide Logging and Reporting
 
+
 ## Set up
 
 For local development, create a virtual environment and install the required packages:
@@ -15,5 +16,6 @@ pip install -r requirements.txt
 This repository uses pre-commit hooks to ensure code is formatted correctly and that notebooks are cleared of outputs before committing. To set up the pre-commit hooks, run the following command in the root directory of the repository:
 
 ```bash
+
 pre-commit install
 ```
diff --git a/notebooks_for_times_square/README.md → notebooks_tsqr/README.md b/notebooks_for_times_square/README.md → notebooks_tsqr/README.md
diff --git a/...oks_for_times_square/logrep_proto_1.ipynb → notebooks_tsqr/logrep_proto_1.ipynb b/...oks_for_times_square/logrep_proto_1.ipynb → notebooks_tsqr/logrep_proto_1.ipynb
@@ -15,10 +15,23 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "id": "1",
    "metadata": {},
+   "source": [
+    "<a class=\"anchor\" id=\"imports\"></a>\n",
+    "## Imports and General Setup"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "# Only use packages available in the Rubin Science Platform\n",
@@ -34,8 +47,12 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "2",
-   "metadata": {},
+   "id": "3",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "log = log_name\n",
@@ -58,14 +75,30 @@
     "    tucson = 'https://tucson-teststand.lsst.codes',\n",
     ")\n",
     "server = envs[env]\n",
-    "server"
+    "service = f'{server}/{log}'\n",
+    "service"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4",
+   "metadata": {
+    "jp-MarkdownHeadingCollapsed": true
+   },
+   "source": [
+    "<a class=\"anchor\" id=\"setup_source\"></a>\n",
+    "## Setup Source"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "3",
-   "metadata": {},
+   "id": "5",
+   "metadata": {
+    "jupyter": {
+     "source_hidden": true
+    }
+   },
    "outputs": [],
    "source": [
     "recs = None\n",
@@ -79,15 +112,40 @@
     "               limit=limit,\n",
     "              )\n",
     "qstr = urlencode(qparams)\n",
-    "url = f'{server}/{log}/messages?{qstr}'\n",
-    "print(f'\\nAttempt to get logs from {url=}')\n",
+    "url = f'{service}/messages?{qstr}'\n",
+    "\n",
     "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n",
     "                     'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n",
     "                     'date_invalidated', 'date_begin', 'date_end',\n",
     "                     'time_lost', # float\n",
     "                     #'systems','subsystems','cscs',  # values are lists, special handling\n",
-    "                    ])\n",
+    "                    ])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "<a class=\"anchor\" id=\"get_records\"></a>\n",
+    "## Get Records"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# TODO Often fails on first request.  Find out why!\n",
+    "try:\n",
+    "    response = requests.get(url, timeout=timeout)\n",
+    "except:\n",
+    "    pass \n",
+    "    \n",
     "try:\n",
+    "    print(f'Attempt to get logs from {url=}')\n",
     "    response = requests.get(url, timeout=timeout)\n",
     "    response.raise_for_status()\n",
     "    recs = response.json()\n",
@@ -102,69 +160,107 @@
     "    print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n",
     "numf = len(flds) if ok else 0\n",
     "numr = len(recs) if ok else 0\n",
-    "print(f'{numr=} {numf=}')"
+    "print(f'Retrieved {numr} records, each with {numf=} fields.')"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "4",
+   "cell_type": "markdown",
+   "id": "8",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "flds"
+    "<a class=\"anchor\" id=\"table\"></a>\n",
+    "## Tables of (mostly raw) results"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9",
+   "metadata": {},
+   "source": [
+    "### Fields names provided in records from log."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "5",
+   "id": "10",
    "metadata": {},
    "outputs": [],
    "source": [
-    "facets"
+    "pd.DataFrame(flds, columns=['Field Name'])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11",
+   "metadata": {},
+   "source": [
+    "### Facets from log records.\n",
+    "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "6",
+   "id": "12",
    "metadata": {},
    "outputs": [],
    "source": [
-    "recs[0]"
+    "pd.DataFrame.from_dict(facets, orient='index')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "13",
+   "metadata": {},
+   "source": [
+    "### Table of selected log record fields.\n",
+    "Table can be retrieved as CSV file for local use."
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7",
+   "id": "14",
    "metadata": {},
    "outputs": [],
    "source": [
     "cols = ['date_added', 'time_lost']\n",
     "df = pd.DataFrame(recs)[cols]\n",
+    "\n",
+    "# Allow download of CSV version of DataFrame\n",
+    "csvfile = 'tl.csv'\n",
+    "df.to_csv(csvfile)\n",
+    "myfile = FileLink(csvfile)\n",
+    "print('Table available as CSV file: ')\n",
+    "display(myfile)\n",
     "df"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "8",
+   "id": "15",
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Allow download of CSV version of DataFrame\n",
-    "csvfile = 'tl.csv'\n",
-    "df.to_csv(csvfile)\n",
-    "myfile = FileLink(csvfile)\n",
-    "display(myfile)"
+    "df = pd.DataFrame(recs)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "16",
+   "metadata": {},
+   "source": [
+    "<a class=\"anchor\" id=\"plot\"></a>\n",
+    "## Plots from log"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "9",
+   "id": "17",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -175,38 +271,57 @@
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "10",
+   "cell_type": "markdown",
+   "id": "18",
    "metadata": {},
-   "outputs": [],
-   "source": []
+   "source": [
+    "<a class=\"anchor\" id=\"raw_analysis\"></a>\n",
+    "## Raw Content Analysis"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "19",
+   "metadata": {},
+   "source": [
+    "### Example of one record"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "11",
+   "id": "20",
    "metadata": {},
    "outputs": [],
    "source": [
-    "recs[:4]"
+    "rec = recs[0]\n",
+    "rec"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "12",
+   "id": "21",
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = pd.DataFrame(recs)\n",
-    "df"
+    "msg = rec[\"message_text\"]\n",
+    "print(msg)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "22",
+   "metadata": {},
+   "source": [
+    "<a class=\"anchor\" id=\"elicitation\"></a>\n",
+    "## Stakeholder Elicitation"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "13",
+   "id": "23",
    "metadata": {},
    "outputs": [],
    "source": []

diff --git a/...ooks_for_times_square/logrep_proto_1.yaml → notebooks_tsqr/logrep_proto_1.yaml b/...ooks_for_times_square/logrep_proto_1.yaml → notebooks_tsqr/logrep_proto_1.yaml
@@ -8,14 +8,6 @@ tags:
   - reporting
   - prototype
 parameters:
-  env:
-    type: string
-    description: "tucson | usdf_dev | summit"
-    default: "tucson"
-  log:
-    type: string
-    description: "The log name: narrativelog | exposurelog"
-    default: narrativelog
   record_limit:
     type: integer
     description: Max number of records to output

diff --git a/notebooks_for_times_square/requirements.txt → notebooks_tsqr/requirements.txt b/notebooks_for_times_square/requirements.txt → notebooks_tsqr/requirements.txt
diff --git a/notebooks_for_times_square/scaffolding.org → notebooks_tsqr/scaffolding.org b/notebooks_for_times_square/scaffolding.org → notebooks_tsqr/scaffolding.org
diff --git a/times-square.yaml b/times-square.yaml
@@ -1,4 +1,4 @@
 enabled: true
-root: notebooks_for_times_square
+root: notebooks_tsqr
 description: >
   Times Square for project-wide Logging and Reporting.