feat: display docs generation jobs in admin pages (#8)

src/application.rs

@@ -17,6 +17,7 @@ use crate::model::cargo::{
 };
 use crate::model::config::Configuration;
 use crate::model::deps::DepsAnalysis;
+use crate::model::docs::DocGenJob;
 use crate::model::packages::CrateInfo;
 use crate::model::stats::{DownloadStats, GlobalStats};
 use crate::model::{CrateVersion, JobCrate, RegistryInformation};
@@ -437,6 +438,17 @@ impl Application {
         .await
     }
 
+    /// Gets the documentation jobs
+    pub async fn get_doc_gen_jobs(&self, auth_data: &AuthData) -> Result<Vec<DocGenJob>, ApiError> {
+        let mut connection: sqlx::pool::PoolConnection<Sqlite> = self.service_db_pool.acquire().await?;
+        in_transaction(&mut connection, |transaction| async move {
+            let app = self.with_transaction(transaction);
+            let _principal = app.authenticate(auth_data).await?;
+            Ok(self.service_docs_generator.get_jobs())
+        })
+        .await
+    }
+
     /// Force the re-generation for the documentation of a package
     pub async fn regen_crate_version_doc(&self, auth_data: &AuthData, package: &str, version: &str) -> Result<(), ApiError> {
         let mut connection: sqlx::pool::PoolConnection<Sqlite> = self.service_db_pool.acquire().await?;

src/main.rs

@@ -102,7 +102,8 @@ async fn main_serve_app(application: Arc<Application>, cookie_key: Key) -> Resul
                                 .route("/", get(routes::api_v1_get_global_tokens))
                                 .route("/", put(routes::api_v1_create_global_token))
                                 .route("/:token_id", delete(routes::api_v1_revoke_global_token)),
-                        ),
+                        )
+                        .route("/jobs/docgen", get(routes::api_v1_get_doc_gen_jobs)),
                 )
                 .nest(
                     "/crates",

src/model/docs.rs

@@ -0,0 +1,37 @@
+/*******************************************************************************
+ * Copyright (c) 2024 Cénotélie Opérations SAS (cenotelie.fr)
+ ******************************************************************************/
+
+//! Data types around documentation generation
+
+use chrono::NaiveDateTime;
+use serde_derive::{Deserialize, Serialize};
+
+use super::JobCrate;
+
+/// The state of a documentation generation job
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum DocGenJobState {
+    /// The job is queued
+    Queued,
+    /// The worker is working on this job
+    Working,
+    /// The job is finished and succeeded
+    Success,
+    /// The worker failed to complete this job
+    Failure,
+}
+
+/// A documentation generation job
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DocGenJob {
+    /// The specification for the job
+    pub spec: JobCrate,
+    /// The state of the job
+    pub state: DocGenJobState,
+    /// Timestamp the last time this job was touched
+    #[serde(rename = "lastUpdate")]
+    pub last_update: NaiveDateTime,
+    /// The output log, if any
+    pub output: String,
+}

src/model/mod.rs

@@ -8,6 +8,7 @@ pub mod auth;
 pub mod cargo;
 pub mod config;
 pub mod deps;
+pub mod docs;
 pub mod errors;
 pub mod namegen;
 pub mod osv;

src/routes.rs

@@ -26,6 +26,7 @@ use crate::model::cargo::{
     CrateUploadResult, OwnersChangeQuery, OwnersQueryResult, RegistryUser, SearchResults, YesNoMsgResult, YesNoResult,
 };
 use crate::model::deps::DepsAnalysis;
+use crate::model::docs::DocGenJob;
 use crate::model::packages::CrateInfo;
 use crate::model::stats::{DownloadStats, GlobalStats};
 use crate::model::{AppVersion, CrateVersion, RegistryInformation};
@@ -379,6 +380,11 @@ pub async fn api_v1_revoke_global_token(
     response(state.application.revoke_global_token(&auth_data, token_id).await)
 }
 
+/// Gets the documentation jobs
+pub async fn api_v1_get_doc_gen_jobs(auth_data: AuthData, State(state): State<Arc<AxumState>>) -> ApiResult<Vec<DocGenJob>> {
+    response(state.application.get_doc_gen_jobs(&auth_data).await)
+}
+
 /// Gets the known users
 pub async fn api_v1_get_users(auth_data: AuthData, State(state): State<Arc<AxumState>>) -> ApiResult<Vec<RegistryUser>> {
     response(state.application.get_users(&auth_data).await)

src/services/database/mod.rs

@@ -44,7 +44,7 @@ impl<'c> Database<'c> {
     }
 
     /// Checks that a user is an admin
-    async fn check_is_admin(&self, uid: i64) -> Result<(), ApiError> {
+    pub async fn check_is_admin(&self, uid: i64) -> Result<(), ApiError> {
         let is_admin = self.get_is_admin(uid).await?;
         if is_admin {
             Ok(())

src/services/docs.rs

@@ -6,8 +6,9 @@
 
 use std::path::{Path, PathBuf};
 use std::process::Stdio;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 
+use chrono::Local;
 use flate2::bufread::GzDecoder;
 use futures::StreamExt;
 use log::{error, info};
@@ -18,6 +19,7 @@ use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
 use tokio_stream::wrappers::UnboundedReceiverStream;
 
 use crate::model::config::Configuration;
+use crate::model::docs::{DocGenJob, DocGenJobState};
 use crate::model::JobCrate;
 use crate::services::database::Database;
 use crate::services::storage::Storage;
@@ -27,6 +29,9 @@ use crate::utils::db::in_transaction;
 
 /// Service to generate documentation for a crate
 pub trait DocsGenerator {
+    /// Gets all the jobs
+    fn get_jobs(&self) -> Vec<DocGenJob>;
+
     /// Queues a job for documentation generation
     fn queue(&self, job: JobCrate) -> Result<(), ApiError>;
 }
@@ -42,7 +47,8 @@ pub fn get_docs_generator(
         configuration,
         service_db_pool,
         service_storage,
-        queue: sender,
+        jobs: Arc::new(Mutex::new(Vec::with_capacity(64))),
+        jobs_sender: sender,
     });
     let service2 = service.clone();
     let _handle = tokio::spawn(async move {
@@ -60,34 +66,70 @@ struct DocsGeneratorImpl {
     service_db_pool: Pool<Sqlite>,
     /// The storage layer
     service_storage: Arc<dyn Storage + Send + Sync>,
-    /// The queue of waiting jobs
-    queue: UnboundedSender<JobCrate>,
+    /// The documentation jobs
+    jobs: Arc<Mutex<Vec<DocGenJob>>>,
+    /// Sender to send job signals to the worker
+    jobs_sender: UnboundedSender<usize>,
 }
 
 impl DocsGenerator for DocsGeneratorImpl {
+    /// Gets all the jobs
+    fn get_jobs(&self) -> Vec<DocGenJob> {
+        self.jobs.lock().unwrap().clone()
+    }
+
     /// Queues a job for documentation generation
-    fn queue(&self, job: JobCrate) -> Result<(), ApiError> {
-        self.queue.send(job)?;
+    fn queue(&self, spec: JobCrate) -> Result<(), ApiError> {
+        let now = Local::now().naive_local();
+        let index = {
+            let mut jobs = self.jobs.lock().unwrap();
+            let index = jobs.len();
+            jobs.push(DocGenJob {
+                spec,
+                state: DocGenJobState::Queued,
+                last_update: now,
+                output: String::new(),
+            });
+            index
+        };
+        self.jobs_sender.send(index)?;
         Ok(())
     }
 }
 
 impl DocsGeneratorImpl {
+    /// Update a job
+    fn update_job(&self, job_index: usize, state: DocGenJobState, output: Option<&str>) {
+        let now = Local::now().naive_local();
+        let mut jobs = self.jobs.lock().unwrap();
+        let job = jobs.get_mut(job_index).unwrap();
+        job.state = state;
+        job.last_update = now;
+        if let Some(output) = output {
+            job.output.push_str(output);
+        }
+    }
+
     /// Implementation of the worker
-    async fn worker(&self, receiver: UnboundedReceiver<JobCrate>) {
+    async fn worker(&self, receiver: UnboundedReceiver<usize>) {
         let mut stream = UnboundedReceiverStream::new(receiver);
-        while let Some(job) = stream.next().await {
-            if let Err(e) = self.docs_worker_job(job).await {
+        while let Some(job_index) = stream.next().await {
+            self.update_job(job_index, DocGenJobState::Working, None);
+            if let Err(e) = self.docs_worker_job(job_index).await {
                 error!("{e}");
                 if let Some(backtrace) = &e.backtrace {
                     error!("{backtrace}");
                 }
+                self.update_job(job_index, DocGenJobState::Failure, Some(&e.to_string()));
+            } else {
+                self.update_job(job_index, DocGenJobState::Success, None);
             }
         }
     }
 
     /// Executes a documentation generation job
-    async fn docs_worker_job(&self, job: JobCrate) -> Result<(), ApiError> {
+    async fn docs_worker_job(&self, job_index: usize) -> Result<(), ApiError> {
+        let job = self.jobs.lock().unwrap().get(job_index).unwrap().spec.clone();
         info!("generating doc for {} {}", job.name, job.version);
         let content = self.service_storage.download_crate(&job.name, &job.version).await?;
         let temp_folder = Self::extract_content(&job.name, &job.version, &content)?;

src/webapp/admin-jobs-docgen.html

@@ -0,0 +1,116 @@
+<!DOCTYPE html>
+<html lang="en" class="dark">
+
+<head>
+  <meta charset="UTF-8">
+  <meta name="description" content="">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <link rel="icon" type="image/png" href="/webapp/favicon.png">
+  <title>
+    Cratery -- Documentation generation jobs
+  </title>
+  <script src="https://cdn.tailwindcss.com"></script>
+</head>
+
+<header style="position: sticky; top: 0;">
+  <nav class="bg-white border-gray-200 px-4 lg:px-6 py-2.5 dark:bg-gray-800">
+      <div class="flex flex-wrap justify-between items-center mx-auto max-w-screen-xl">
+          <a href="/webapp/index.html" class="flex items-center">
+              <img src="./logo-white.svg" class="mr-3 h-6 sm:h-9" style="min-width: 200px;" alt="Cratery Logo" />
+          </a>
+          <div class="flex items-center lg:order-2">
+            <a id="link-admin" href="/webapp/admin.html" style="cursor: pointer;" class="text-gray-800 dark:text-white hover:bg-gray-50 focus:ring-4 focus:ring-gray-300 font-medium rounded-lg text-sm px-4 lg:px-5 py-2 lg:py-2.5 mr-2 dark:hover:bg-gray-700 focus:outline-none dark:focus:ring-gray-800">Admin</a>
+            <a id="link-account" href="/webapp/account.html" style="cursor: pointer;" class="text-gray-800 dark:text-white hover:bg-gray-50 focus:ring-4 focus:ring-gray-300 font-medium rounded-lg text-sm px-4 lg:px-5 py-2 lg:py-2.5 mr-2 dark:hover:bg-gray-700 focus:outline-none dark:focus:ring-gray-800">My Account</a>
+            <a onclick="doLogout()" style="cursor: pointer;" class="text-gray-800 dark:text-white hover:bg-gray-50 focus:ring-4 focus:ring-gray-300 font-medium rounded-lg text-sm px-4 lg:px-5 py-2 lg:py-2.5 mr-2 dark:hover:bg-gray-700 focus:outline-none dark:focus:ring-gray-800">Logout</a>
+          </div>
+      </div>
+  </nav>
+</header>
+<body onload="doPageLoad()" class="bg-white dark:bg-gray-800">
+  <section class="bg-white dark:bg-gray-900">
+    <div class="p-2 flex flex-row flex-wrap">
+      <a href="/webapp/admin.html" class="font-medium text-blue-600 dark:text-blue-500 hover:underline">
+        <svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke-width="1.5" stroke="currentColor" class="size-6" style="display: inline-block;">
+          <path stroke-linecap="round" stroke-linejoin="round" d="M10.5 19.5 3 12m0 0 7.5-7.5M3 12h18" />
+        </svg>
+       Back to admin
+      </a>
+    </div>
+    <div class="py-4 lg:py-4 px-4 mx-auto max-w-screen-xxl">
+      <h2 class="mb-4 text-4xl tracking-tight font-extrabold text-center text-gray-900 dark:text-white">Documentation generation jobs</h2>
+      <div class="relative overflow-x-auto space-y-8">
+        <table class="w-full text-sm text-left rtl:text-right text-gray-500 dark:text-gray-400">
+            <thead class="text-xs text-gray-700 uppercase bg-gray-50 dark:bg-gray-700 dark:text-gray-400">
+              <tr>
+                <th scope="col" class="px-6 py-3">
+                    Crate
+                </th>
+                <th scope="col" class="px-6 py-3">
+                    Version
+                </th>
+                <th scope="col" class="px-6 py-3">
+                    Status
+                </th>
+              </tr>
+            </thead>
+            <tbody id="jobs">
+            </tbody>
+        </table>
+      </div>
+    </div>
+  </section>
+</body>
+<footer class="p-4 bg-white md:p-8 lg:p-10 dark:bg-gray-800">
+  <div class="mx-auto max-w-screen-xl text-center">
+      <span class="text-sm text-gray-500 sm:text-center dark:text-gray-400">Version <span id="version"></span>, Copyright © <span id="year"></span> <a href="https://cenotelie.fr/" target="_blank" class="hover:underline">Cénotélie</a>. All Rights Reserved.</span>
+  </div>
+</footer>
+
+<link href="/webapp/index.css" rel="stylesheet" />
+<script src="/webapp/api.js"></script>
+<script src="/webapp/index.js"></script>
+<script>
+    function doPageLoad() {
+      onPageLoad().then((_user) => {
+        apiGetDocGenJobs().then(jobs => {
+          const jobsEl = document.getElementById("jobs");
+          for (const job of jobs) {
+            // jobsEl.appendChild(renderJob(job));
+          }
+        })
+      });
+    }
+
+    function renderJob(job) {
+      const inputLoginEl = document.createElement("input");
+      inputLoginEl.setAttribute("type", "text");
+      inputLoginEl.className = "block p-3 w-full text-sm text-gray-900 bg-gray-50 rounded-lg border border-gray-300 shadow-sm focus:ring-primary-500 focus:border-primary-500 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-primary-500 dark:focus:border-primary-500 dark:shadow-sm-light";
+      inputLoginEl.value = user.login;
+      const inputNameEl = document.createElement("input");
+      inputNameEl.setAttribute("type", "text");
+      inputNameEl.className = "block p-3 w-full text-sm text-gray-900 bg-gray-50 rounded-lg border border-gray-300 shadow-sm focus:ring-primary-500 focus:border-primary-500 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-primary-500 dark:focus:border-primary-500 dark:shadow-sm-light";
+      inputNameEl.value = user.name;
+      const inputRolesEl = document.createElement("input");
+      inputRolesEl.setAttribute("type", "text");
+      inputRolesEl.className = "block p-3 w-full text-sm text-gray-900 bg-gray-50 rounded-lg border border-gray-300 shadow-sm focus:ring-primary-500 focus:border-primary-500 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-primary-500 dark:focus:border-primary-500 dark:shadow-sm-light";
+      inputRolesEl.value = user.roles;
+
+      const row = document.createElement("tr");
+      const cell1 = document.createElement("th");
+      cell1.setAttribute("scope", "row");
+      cell1.className = "px-6 py-4 font-medium text-gray-900 whitespace-nowrap dark:text-white";
+      cell1.appendChild(document.createTextNode(user.email));
+      const cell2 = document.createElement("td");
+      cell2.className = "px-6 py-4";
+      cell2.appendChild(inputLoginEl);
+      const cell3 = document.createElement("td");
+      cell3.className = "px-6 py-4";
+      cell3.appendChild(inputNameEl);
+
+      row.appendChild(cell1);
+      row.appendChild(cell2);
+      row.appendChild(cell3);
+      return row;
+    }
+</script>
+</html>

src/webapp/admin.html

@@ -35,6 +35,9 @@ <h1 class="text-xl font-bold leading-tight tracking-tight text-gray-900 md:text-
     </div>
     <div class="p-6 mb-4 flex flex-row flex-wrap">
       <ul class="max-w-md space-y-1 text-gray-500 list-disc list-inside dark:text-gray-400">
+        <li>
+          <a href="/webapp/admin-jobs-docgen.html" class="font-medium text-blue-600 dark:text-blue-500 hover:underline">See documentation generation jobs</a>
+        </li>
         <li>
           <a href="/webapp/admin-users.html" class="font-medium text-blue-600 dark:text-blue-500 hover:underline">Manage users</a>
         </li>

src/webapp/api.js

@@ -116,6 +116,16 @@ function apiRevokeGlobalToken(token_id) {
   );
 }
 
+function apiGetDocGenJobs() {
+  return fetch("/api/v1/admin/jobs/docgen").then((response) => {
+    if (response.status !== 200) {
+      throw response.text();
+    } else {
+      return response.json();
+    }
+  });
+}
+
 function apiGetUsers() {
   return fetch("/api/v1/admin/users").then((response) => {
     if (response.status !== 200) {

src/webapp/mod.rs

@@ -27,6 +27,9 @@ pub fn get_resources() -> EmbeddedResources {
     add!(resources, "index-outdated.html");
     add!(resources, "account.html");
     add!(resources, "admin.html");
+    add!(resources, "admin-users.html");
+    add!(resources, "admin-tokens.html");
+    add!(resources, "admin-jobs-docgen.html");
     add!(resources, "crate.html");
     add!(resources, "oauthcallback.html");
     // CSS