From 309f6bf5fc81b9dff64e67e87a3f091a51f4de3c Mon Sep 17 00:00:00 2001 From: Florian Dieminger Date: Mon, 14 Oct 2024 21:51:39 +0200 Subject: [PATCH] fix(diff): fix subpages and banners Also add support for --sidebars in diff-test. --- Cargo.lock | 2 + Cargo.toml | 1 + crates/diff-test/Cargo.toml | 2 + crates/diff-test/src/main.rs | 29 ++++++++++-- crates/rari-doc/src/helpers/subpages.rs | 31 ++++++++++++- crates/rari-doc/src/html/rewriter.rs | 46 +++++++++---------- crates/rari-doc/src/html/sidebar.rs | 35 ++++++++++---- crates/rari-doc/src/sidebars/apiref.rs | 2 +- crates/rari-doc/src/templ/templs/banners.rs | 2 +- .../rari-doc/src/templ/templs/listsubpages.rs | 2 + crates/rari-md/Cargo.toml | 2 +- crates/rari-md/src/html.rs | 6 ++- crates/rari-md/src/lib.rs | 6 ++- 13 files changed, 120 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 925a073..7ee4302 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -799,6 +799,7 @@ version = "0.0.12" dependencies = [ "ansi-to-html", "anyhow", + "base64", "clap", "html-minifier", "ignore", @@ -810,6 +811,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "similar", ] diff --git a/Cargo.toml b/Cargo.toml index fcad091..b2b535f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ reqwest = { version = "0.12", default-features = false, features = [ "gzip", ] } indoc = "2" +base64 = "0.22" [dependencies] diff --git a/crates/diff-test/Cargo.toml b/crates/diff-test/Cargo.toml index 86c011a..bcb7841 100644 --- a/crates/diff-test/Cargo.toml +++ b/crates/diff-test/Cargo.toml @@ -14,6 +14,7 @@ itertools.workspace = true regex.workspace = true serde.workspace = true serde_json.workspace = true +base64.workspace = true jsonpath_lib = "0.3.0" prettydiff = "0.7" @@ -21,4 +22,5 @@ html-minifier = "5" ansi-to-html = "0.2" similar = "2" quick-xml = "0.36" +sha2 = "0.10" clap = { version = "4.5.1", features = ["derive"] } diff --git a/crates/diff-test/src/main.rs b/crates/diff-test/src/main.rs index a7d9de4..0307cc2 100644 --- a/crates/diff-test/src/main.rs +++ b/crates/diff-test/src/main.rs @@ -1,14 +1,15 @@ use std::cmp::max; -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs; use std::fs::File; use std::io::Write; use std::path::Path; use std::sync::atomic::AtomicUsize; use std::sync::atomic::Ordering::Relaxed; -use std::sync::LazyLock; +use std::sync::{Arc, LazyLock, RwLock}; use anyhow::{anyhow, Error}; +use base64::prelude::{Engine as _, BASE64_STANDARD_NO_PAD}; use clap::{Args, Parser, Subcommand}; use ignore::types::TypesBuilder; use ignore::WalkBuilder; @@ -18,6 +19,7 @@ use prettydiff::{diff_lines, diff_words}; use rayon::prelude::*; use regex::Regex; use serde_json::Value; +use sha2::{Digest, Sha256}; use xml::fmt_html; mod xml; @@ -147,6 +149,8 @@ struct BuildArgs { value: bool, #[arg(short, long)] verbose: bool, + #[arg(long)] + sidebars: bool, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -175,7 +179,7 @@ const IGNORE: &[&str] = &[ "doc.popularity", "doc.source.github_url", "doc.source.last_commit_url", - "doc.sidebarHTML", + //"doc.sidebarHTML", "doc.sidebarMacro", "doc.hasMathML", "doc.other_translations", @@ -187,12 +191,16 @@ static WS_DIFF: LazyLock = static DATA_FLAW_SRC: LazyLock = LazyLock::new(|| Regex::new(r#" data-flaw-src="[^"]+""#).unwrap()); +static DIFF_MAP: LazyLock>>> = + LazyLock::new(|| Arc::new(RwLock::new(HashMap::new()))); + fn full_diff( lhs: &Value, rhs: &Value, path: &[PathIndex], diff: &mut BTreeMap, fast: bool, + sidebars: bool, ) { if path.len() == 1 { if let PathIndex::Object(s) = &path[0] { @@ -203,7 +211,7 @@ fn full_diff( } if lhs != rhs { let key = make_key(path); - if IGNORE.iter().any(|i| key.starts_with(i)) { + if IGNORE.iter().any(|i| key.starts_with(i)) || key == "doc.sidebarHTML" && !sidebars { return; } match (lhs, rhs) { @@ -218,6 +226,7 @@ fn full_diff( &path, diff, fast, + sidebars, ); } } @@ -233,6 +242,7 @@ fn full_diff( &path, diff, fast, + sidebars, ); } } @@ -259,6 +269,15 @@ fn full_diff( rhs = fmt_html(&html_minifier::minify(rhs_t).unwrap()); } if lhs != rhs { + let mut diff_hash = Sha256::new(); + diff_hash.write_all(lhs.as_bytes()).unwrap(); + diff_hash.write_all(rhs.as_bytes()).unwrap(); + let diff_hash = BASE64_STANDARD_NO_PAD.encode(&diff_hash.finalize()[..]); + if let Some(hash) = (*DIFF_MAP.read().unwrap()).get(&diff_hash) { + diff.insert(key, format!("See {hash}")); + return; + } + (*DIFF_MAP.write().unwrap()).insert(diff_hash, "somewhere else".into()); diff.insert( key, ansi_to_html::convert(&if fast { @@ -309,7 +328,7 @@ fn main() -> Result<(), anyhow::Error> { let left = v; let right = b.get(k).unwrap_or(&Value::Null); let mut diff = BTreeMap::new(); - full_diff(left, right, &[], &mut diff, arg.fast); + full_diff(left, right, &[], &mut diff, arg.fast, arg.sidebars); if !diff.is_empty() { return Some(format!( r#"
  • {k}
    {}
    • "#, diff --git a/crates/rari-doc/src/helpers/subpages.rs b/crates/rari-doc/src/helpers/subpages.rs index d1a53a7..f7d9f58 100644 --- a/crates/rari-doc/src/helpers/subpages.rs +++ b/crates/rari-doc/src/helpers/subpages.rs @@ -8,6 +8,7 @@ use rari_types::fm_types::{FeatureStatus, PageType}; use rari_types::globals::{cache_content, deny_warnings}; use rari_types::locale::Locale; +use super::l10n::l10n_json_data; use super::titles::api_page_title; use crate::error::DocError; use crate::pages::page::{Page, PageLike, PageReader}; @@ -95,6 +96,14 @@ pub fn add_inline_badges(out: &mut String, page: &Page, locale: Locale) -> Resul Ok(()) } +pub fn write_parent_li(out: &mut String, page: &Page, locale: Locale) -> Result<(), DocError> { + let title = l10n_json_data("Template", "overview", locale)?; + write!(out, "
  • {}", page.url(), title)?; + add_inline_badges(out, page, locale)?; + write!(out, "
    • ")?; + Ok(()) +} + pub fn list_sub_pages_reverse_internal( out: &mut String, url: &str, @@ -120,10 +129,14 @@ pub fn list_sub_pages_internal( depth: Option, sorter: Option, page_types: &[PageType], + include_parent: bool, ) -> Result<(), DocError> { let sub_pages = get_sub_pages(url, Some(1), sorter.unwrap_or_default())?; - let depth = depth.map(|i| i.saturating_sub(1)); + if include_parent { + let page = Page::from_url_with_other_locale_and_fallback(url, Some(locale))?; + write_parent_li(out, &page, locale)?; + } for sub_page in sub_pages { if !page_types.is_empty() && !page_types.contains(&sub_page.page_type()) { continue; @@ -134,7 +147,16 @@ pub fn list_sub_pages_internal( } else { write_li_with_badges(out, &sub_page, locale, false)?; out.push_str("
      "); - list_sub_pages_internal(out, sub_page.url(), locale, depth, sorter, page_types)?; + + list_sub_pages_internal( + out, + sub_page.url(), + locale, + depth, + sorter, + page_types, + include_parent, + )?; out.push_str("
    "); out.push_str(""); } @@ -148,6 +170,7 @@ pub fn list_sub_pages_grouped_internal( locale: Locale, sorter: Option, page_types: &[PageType], + include_parent: bool, ) -> Result<(), DocError> { let sub_pages = get_sub_pages(url, None, sorter.unwrap_or_default())?; @@ -171,6 +194,10 @@ pub fn list_sub_pages_grouped_internal( grouped.insert(sub_page.title(), vec![sub_page]); } } + if include_parent { + let page = Page::from_url_with_other_locale_and_fallback(url, Some(locale))?; + write_parent_li(out, &page, locale)?; + } for (prefix, group) in grouped { let keep_group = group.len() > 2; if keep_group { diff --git a/crates/rari-doc/src/html/rewriter.rs b/crates/rari-doc/src/html/rewriter.rs index 393d7e6..ac7977a 100644 --- a/crates/rari-doc/src/html/rewriter.rs +++ b/crates/rari-doc/src/html/rewriter.rs @@ -277,29 +277,21 @@ pub fn post_process_html( el.after("", ContentType::Html); Ok(()) }), - element!("div.notecard.callout > p:first-child", |el| { - el.prepend( - &concat_strs!( - "", - NoteCard::Callout.prefix_for_locale(page.locale()), - "" - ), - ContentType::Html, - ); - Ok(()) - }), - element!("div.notecard.warning > p:first-child", |el| { - el.prepend( - &concat_strs!( - "", - NoteCard::Warning.prefix_for_locale(page.locale()), - "" - ), - ContentType::Html, - ); - Ok(()) - }), - element!("div.notecard.note > p:first-child", |el| { + element!( + "div.notecard.warning[data-add-warning] > p:first-child", + |el| { + el.prepend( + &concat_strs!( + "", + NoteCard::Warning.prefix_for_locale(page.locale()), + "" + ), + ContentType::Html, + ); + Ok(()) + } + ), + element!("div.notecard.note[data-add-note] > p:first-child", |el| { el.prepend( &concat_strs!( "", @@ -330,6 +322,14 @@ pub fn post_process_html( el.remove_attribute("data-sourcepos"); Ok(()) }), + element!("*[data-add-note]", |el| { + el.remove_attribute("data-add-note"); + Ok(()) + }), + element!("*[data-add-warning]", |el| { + el.remove_attribute("data-add-warning"); + Ok(()) + }), ]; if sidebar { element_content_handlers.push(element!("html", |el| { diff --git a/crates/rari-doc/src/html/sidebar.rs b/crates/rari-doc/src/html/sidebar.rs index 382cb54..903ef73 100644 --- a/crates/rari-doc/src/html/sidebar.rs +++ b/crates/rari-doc/src/html/sidebar.rs @@ -224,6 +224,8 @@ pub struct SubPageEntry { pub tags: Vec, #[serde(default)] pub details: Details, + #[serde(default)] + pub include_parent: bool, } #[derive(Serialize, Deserialize, Default, Debug)] @@ -248,8 +250,8 @@ pub enum SidebarEntry { #[derive(Debug, Default)] pub enum MetaChildren { Children(Vec), - ListSubPages(String, Vec), - ListSubPagesGrouped(String, Vec), + ListSubPages(String, Vec, bool), + ListSubPagesGrouped(String, Vec, bool), WebExtApi, #[default] None, @@ -327,12 +329,13 @@ impl From for SidebarMetaEntry { link, title, path, + include_parent, }) => SidebarMetaEntry { section: false, details, code: false, content: SidebarMetaEntryContent::Link { link, title }, - children: MetaChildren::ListSubPages(path, tags), + children: MetaChildren::ListSubPages(path, tags, include_parent), }, SidebarEntry::ListSubPagesGrouped(SubPageEntry { details, @@ -340,12 +343,13 @@ impl From for SidebarMetaEntry { link, title, path, + include_parent, }) => SidebarMetaEntry { section: false, details, code: false, content: SidebarMetaEntryContent::Link { link, title }, - children: MetaChildren::ListSubPagesGrouped(path, tags), + children: MetaChildren::ListSubPagesGrouped(path, tags, include_parent), }, SidebarEntry::Default(BasicEntry { link, @@ -457,11 +461,24 @@ impl SidebarMetaEntry { child.render(out, locale, slug, l10n)?; } } - MetaChildren::ListSubPages(url, page_types) => { - list_sub_pages_internal(out, url, locale, Some(1), None, page_types)? - } - MetaChildren::ListSubPagesGrouped(url, page_types) => { - list_sub_pages_grouped_internal(out, url, locale, None, page_types)? + MetaChildren::ListSubPages(url, page_types, include_parent) => list_sub_pages_internal( + out, + url, + locale, + Some(1), + None, + page_types, + *include_parent, + )?, + MetaChildren::ListSubPagesGrouped(url, page_types, include_parent) => { + list_sub_pages_grouped_internal( + out, + url, + locale, + None, + page_types, + *include_parent, + )? } MetaChildren::WebExtApi => { let children = &helpers::webextapi::children( diff --git a/crates/rari-doc/src/sidebars/apiref.rs b/crates/rari-doc/src/sidebars/apiref.rs index 110b00c..097be24 100644 --- a/crates/rari-doc/src/sidebars/apiref.rs +++ b/crates/rari-doc/src/sidebars/apiref.rs @@ -166,7 +166,7 @@ fn build_interface_list(entries: &mut Vec, interfaces: &[&str] .map(|interface| SidebarMetaEntry { code: true, content: SidebarMetaEntryContent::Link { - title: None, + title: Some(interface.to_string()), link: Some(format!( "/Web/API/{}", interface.replace("()", "").replace('.', "/") diff --git a/crates/rari-doc/src/templ/templs/banners.rs b/crates/rari-doc/src/templ/templs/banners.rs index 89de192..63ae2c8 100644 --- a/crates/rari-doc/src/templ/templs/banners.rs +++ b/crates/rari-doc/src/templ/templs/banners.rs @@ -37,7 +37,7 @@ pub fn available_in_workers(typ: Option) -> Result { .unwrap_or(l10n_json_data("Template", default_typ, env.locale)?); Ok(concat_strs!( - r#"

    "#, + r#"

    "#, copy, "

    " )) diff --git a/crates/rari-doc/src/templ/templs/listsubpages.rs b/crates/rari-doc/src/templ/templs/listsubpages.rs index f3dbce2..acaf0a1 100644 --- a/crates/rari-doc/src/templ/templs/listsubpages.rs +++ b/crates/rari-doc/src/templ/templs/listsubpages.rs @@ -42,6 +42,7 @@ pub fn list_sub_pages( Some(depth), Some(SubPagesSorter::SlugNatural), &[], + false, )?; } out.push_str(if ordered { "" } else { "" }); @@ -74,6 +75,7 @@ pub fn list_sub_pages_grouped( }) .as_deref() .unwrap_or_default(), + false, )?; out.push_str(""); Ok(out) diff --git a/crates/rari-md/Cargo.toml b/crates/rari-md/Cargo.toml index 6954159..5f5c08b 100644 --- a/crates/rari-md/Cargo.toml +++ b/crates/rari-md/Cargo.toml @@ -12,6 +12,6 @@ regex.workspace = true thiserror.workspace = true rari-types.workspace = true itertools.workspace = true +base64.workspace = true -base64 = "0.22" comrak = { version = "0.29", default-features = false } diff --git a/crates/rari-md/src/html.rs b/crates/rari-md/src/html.rs index cfe105c..584c02c 100644 --- a/crates/rari-md/src/html.rs +++ b/crates/rari-md/src/html.rs @@ -526,13 +526,15 @@ impl<'o, 'c: 'o> HtmlFormatter<'o, 'c> { return Ok((false, Flag::Card)); } Some(NoteCard::Note) => { - self.output.write_all(b"
    \n")?; return Ok((false, Flag::Card)); } Some(NoteCard::Warning) => { - self.output.write_all(b"
    \n")?; return Ok((false, Flag::Card)); diff --git a/crates/rari-md/src/lib.rs b/crates/rari-md/src/lib.rs index 3f4d816..568a712 100644 --- a/crates/rari-md/src/lib.rs +++ b/crates/rari-md/src/lib.rs @@ -135,8 +135,10 @@ mod test { #[test] fn li_p() -> Result<(), anyhow::Error> { + let out = m2h("- foo\n- bar\n", Locale::EnUs)?; + assert_eq!(out, "
      \n
    • foo
      • \n
    • bar
      • \n
    \n"); let out = m2h("- foo\n\n- bar\n", Locale::EnUs)?; - assert_eq!(out, "
      \n
    • foo
      • \n
    • bar
      • \n
    \n"); + assert_eq!(out, "
      \n
    • \n

      foo

      \n
      • \n
    • \n

      bar

      \n
      • \n
    \n"); Ok(()) } @@ -162,7 +164,7 @@ mod test { let out = m2h("> **Note:** foobar", Locale::EnUs)?; assert_eq!( out, - "
    \n

    foobar

    \n
    \n" + "
    \n

    foobar

    \n
    \n" ); Ok(()) }