From 7d608459c295e7c4fb6c8281b9310d4f0d7372f7 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Thu, 12 Jan 2023 22:01:23 +0100 Subject: [PATCH 1/3] Post creation from Mastodon (fixes #2590) --- crates/apub/assets/mastodon/objects/page.json | 54 +++++++++++++++++++ crates/apub/src/objects/post.rs | 21 ++++++-- crates/apub/src/protocol/objects/mod.rs | 1 + crates/apub/src/protocol/objects/page.rs | 2 +- 4 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 crates/apub/assets/mastodon/objects/page.json diff --git a/crates/apub/assets/mastodon/objects/page.json b/crates/apub/assets/mastodon/objects/page.json new file mode 100644 index 0000000000..176b403570 --- /dev/null +++ b/crates/apub/assets/mastodon/objects/page.json @@ -0,0 +1,54 @@ +{ + "@context": [ + "https://www.w3.org/ns/activitystreams", + { + "ostatus": "http://ostatus.org#", + "atomUri": "ostatus:atomUri", + "inReplyToAtomUri": "ostatus:inReplyToAtomUri", + "conversation": "ostatus:conversation", + "sensitive": "as:sensitive", + "toot": "http://joinmastodon.org/ns#", + "votersCount": "toot:votersCount" + } + ], + "id": "https://mastodon.madrid/users/felix/statuses/107224289116410645", + "type": "Note", + "summary": null, + "inReplyTo": "https://mamot.fr/users/retiolus/statuses/107224244380204526", + "published": "2021-11-05T11:46:50Z", + "url": "https://mastodon.madrid/@felix/107224289116410645", + "attributedTo": "https://mastodon.madrid/users/felix", + "to": [ + "https://mastodon.madrid/users/felix/followers" + ], + "cc": [ + "https://www.w3.org/ns/activitystreams#Public", + "https://mamot.fr/users/retiolus" + ], + "sensitive": false, + "atomUri": "https://mastodon.madrid/users/felix/statuses/107224289116410645", + "inReplyToAtomUri": "https://mamot.fr/users/retiolus/statuses/107224244380204526", + "conversation": "tag:mamot.fr,2021-11-05:objectId=64635960:objectType=Conversation", + "content": "

@retiolus i have never been disappointed by a thinkpad. if you want to save money, get a model from a few years ago, there isnt a huge difference anyway.

", + "contentMap": { + "en": "

@retiolus i have never been disappointed by a thinkpad. if you want to save money, get a model from a few years ago, there isnt a huge difference anyway.

" + }, + "attachment": [], + "tag": [ + { + "type": "Mention", + "href": "https://mamot.fr/users/retiolus", + "name": "@retiolus@mamot.fr" + } + ], + "replies": { + "id": "https://mastodon.madrid/users/felix/statuses/107224289116410645/replies", + "type": "Collection", + "first": { + "type": "CollectionPage", + "next": "https://mastodon.madrid/users/felix/statuses/107224289116410645/replies?only_other_accounts=true&page=true", + "partOf": "https://mastodon.madrid/users/felix/statuses/107224289116410645/replies", + "items": [] + } + } +} \ No newline at end of file diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index e15e1b2dc5..11dae68e19 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -21,6 +21,7 @@ use activitypub_federation::{ utils::verify_domains_match, }; use activitystreams_kinds::public; +use anyhow::anyhow; use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, @@ -40,7 +41,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::LemmyError, - utils::{check_slurs, convert_datetime, markdown_to_html, remove_slurs}, + utils::{check_slurs_opt, convert_datetime, markdown_to_html, remove_slurs}, }; use std::ops::Deref; use url::Url; @@ -108,7 +109,7 @@ impl ApubObject for ApubPost { attributed_to: AttributedTo::Lemmy(ObjectId::new(creator.actor_id)), to: vec![community.actor_id.clone().into(), public()], cc: vec![], - name: self.name.clone(), + name: Some(self.name.clone()), content: self.body.as_ref().map(|b| markdown_to_html(b)), media_type: Some(MediaTypeMarkdownOrHtml::Html), source: self.body.clone().map(Source::new), @@ -151,7 +152,7 @@ impl ApubObject for ApubPost { verify_person_in_community(&page.creator()?, &community, context, request_counter).await?; let slur_regex = &local_site_opt_to_slur_regex(&local_site_data.local_site); - check_slurs(&page.name, slur_regex)?; + check_slurs_opt(&page.name, slur_regex)?; verify_domains_match(page.creator()?.inner(), page.id.inner())?; verify_is_public(&page.to, &page.cc)?; @@ -169,6 +170,16 @@ impl ApubObject for ApubPost { .dereference(context, local_instance(context).await, request_counter) .await?; let community = page.community(context, request_counter).await?; + let name = page + .name + .clone() + .or_else(|| { + page.content.clone().and_then(|c| { + // take the first line, and limit to first 60 characters + c.lines().next().map(|l| l.chars().take(60).collect()) + }) + }) + .ok_or_else(|| anyhow!("Object must have name or content"))?; let form = if !page.is_mod_action(context).await? { let first_attachment = page.attachment.into_iter().map(Attachment::url).next(); @@ -197,7 +208,7 @@ impl ApubObject for ApubPost { let language_id = LanguageTag::to_language_id_single(page.language, context.pool()).await?; PostInsertForm { - name: page.name.clone(), + name, url: url.map(Into::into), body: body_slurs_removed, creator_id: creator.id, @@ -221,7 +232,7 @@ impl ApubObject for ApubPost { } else { // if is mod action, only update locked/stickied fields, nothing else PostInsertForm::builder() - .name(page.name.clone()) + .name(name) .creator_id(creator.id) .community_id(community.id) .ap_id(Some(page.id.clone().into())) diff --git a/crates/apub/src/protocol/objects/mod.rs b/crates/apub/src/protocol/objects/mod.rs index 5a3b90bf61..2dcf1eed74 100644 --- a/crates/apub/src/protocol/objects/mod.rs +++ b/crates/apub/src/protocol/objects/mod.rs @@ -131,6 +131,7 @@ mod tests { fn test_parse_objects_mastodon() { test_json::("assets/mastodon/objects/person.json").unwrap(); test_json::("assets/mastodon/objects/note.json").unwrap(); + test_json::("assets/mastodon/objects/page.json").unwrap(); } #[test] diff --git a/crates/apub/src/protocol/objects/page.rs b/crates/apub/src/protocol/objects/page.rs index 3aadb20c1a..672e5f74c6 100644 --- a/crates/apub/src/protocol/objects/page.rs +++ b/crates/apub/src/protocol/objects/page.rs @@ -46,7 +46,7 @@ pub struct Page { pub(crate) attributed_to: AttributedTo, #[serde(deserialize_with = "deserialize_one_or_many")] pub(crate) to: Vec, - pub(crate) name: String, + pub(crate) name: Option, #[serde(deserialize_with = "deserialize_one_or_many", default)] pub(crate) cc: Vec, From 2891a42ddf908ee4c4cc0e4db154c62e96a576d6 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Fri, 13 Jan 2023 22:49:12 +0100 Subject: [PATCH 2/3] better logic for page title --- crates/apub/src/objects/post.rs | 15 ++++++++++----- scripts/test.sh | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 11dae68e19..c4b2b8210e 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -46,6 +46,8 @@ use lemmy_utils::{ use std::ops::Deref; use url::Url; +const MAX_TITLE_LENGTH: usize = 100; + #[derive(Clone, Debug)] pub struct ApubPost(pub(crate) Post); @@ -170,16 +172,19 @@ impl ApubObject for ApubPost { .dereference(context, local_instance(context).await, request_counter) .await?; let community = page.community(context, request_counter).await?; - let name = page + let mut name = page .name .clone() .or_else(|| { - page.content.clone().and_then(|c| { - // take the first line, and limit to first 60 characters - c.lines().next().map(|l| l.chars().take(60).collect()) - }) + page + .content + .clone() + .and_then(|c| c.lines().next().map(ToString::to_string)) }) .ok_or_else(|| anyhow!("Object must have name or content"))?; + if name.chars().count() > MAX_TITLE_LENGTH { + name = name.chars().take(MAX_TITLE_LENGTH).collect(); + } let form = if !page.is_mod_action(context).await? { let first_attachment = page.attachment.into_iter().map(Attachment::url).next(); diff --git a/scripts/test.sh b/scripts/test.sh index 44c40ad237..a64d99d426 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -ex +set -e PACKAGE="$1" echo "$PACKAGE" From 92ae3901ff5fb2ad2b7c61089bd33cd3e7c038a1 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Sat, 21 Jan 2023 00:34:59 +0900 Subject: [PATCH 3/3] add deserialize helper --- crates/apub/assets/mastodon/objects/page.json | 1 - crates/apub/src/api/resolve_object.rs | 7 +---- crates/apub/src/objects/post.rs | 1 + crates/apub/src/protocol/objects/page.rs | 29 +++++++++++++++++-- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/apub/assets/mastodon/objects/page.json b/crates/apub/assets/mastodon/objects/page.json index 176b403570..06d9b2215a 100644 --- a/crates/apub/assets/mastodon/objects/page.json +++ b/crates/apub/assets/mastodon/objects/page.json @@ -14,7 +14,6 @@ "id": "https://mastodon.madrid/users/felix/statuses/107224289116410645", "type": "Note", "summary": null, - "inReplyTo": "https://mamot.fr/users/retiolus/statuses/107224244380204526", "published": "2021-11-05T11:46:50Z", "url": "https://mastodon.madrid/@felix/107224289116410645", "attributedTo": "https://mastodon.madrid/users/felix", diff --git a/crates/apub/src/api/resolve_object.rs b/crates/apub/src/api/resolve_object.rs index c179ed5821..dd39218bc1 100644 --- a/crates/apub/src/api/resolve_object.rs +++ b/crates/apub/src/api/resolve_object.rs @@ -46,12 +46,7 @@ async fn convert_response( ) -> Result { use SearchableObjects::*; let removed_or_deleted; - let mut res = ResolveObjectResponse { - comment: None, - post: None, - community: None, - person: None, - }; + let mut res = ResolveObjectResponse::default(); match object { Person(p) => { removed_or_deleted = p.deleted; diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index c4b2b8210e..2ef6401f51 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -124,6 +124,7 @@ impl ApubObject for ApubPost { published: Some(convert_datetime(self.published)), updated: self.updated.map(convert_datetime), audience: Some(ObjectId::new(community.actor_id)), + in_reply_to: None, }; Ok(page) } diff --git a/crates/apub/src/protocol/objects/page.rs b/crates/apub/src/protocol/objects/page.rs index 672e5f74c6..9055b1fcc3 100644 --- a/crates/apub/src/protocol/objects/page.rs +++ b/crates/apub/src/protocol/objects/page.rs @@ -23,7 +23,7 @@ use itertools::Itertools; use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::newtypes::DbUrl; use lemmy_utils::error::LemmyError; -use serde::{Deserialize, Serialize}; +use serde::{de::Error, Deserialize, Deserializer, Serialize}; use serde_with::skip_serializing_none; use url::Url; @@ -46,8 +46,11 @@ pub struct Page { pub(crate) attributed_to: AttributedTo, #[serde(deserialize_with = "deserialize_one_or_many")] pub(crate) to: Vec, - pub(crate) name: Option, + // If there is inReplyTo field this is actually a comment and must not be parsed + #[serde(deserialize_with = "deserialize_not_present", default)] + pub(crate) in_reply_to: Option, + pub(crate) name: Option, #[serde(deserialize_with = "deserialize_one_or_many", default)] pub(crate) cc: Vec, pub(crate) content: Option, @@ -259,3 +262,25 @@ impl InCommunity for Page { } } } + +/// Only allows deserialization if the field is missing or null. If it is present, throws an error. +pub fn deserialize_not_present<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let result: Option = Deserialize::deserialize(deserializer)?; + match result { + None => Ok(None), + Some(_) => Err(D::Error::custom("Post must not have inReplyTo property")), + } +} + +#[cfg(test)] +mod tests { + use crate::protocol::{objects::page::Page, tests::test_parse_lemmy_item}; + + #[test] + fn test_not_parsing_note_as_page() { + assert!(test_parse_lemmy_item::("assets/lemmy/objects/note.json").is_err()); + } +}