refactor

src/datasource_database.rs

@@ -760,7 +760,7 @@ impl SourceDatabase {
         let mut after: String = self.params.after.clone();
         let mut is_before_after_done: bool = false;
         if let Some(max_age) = self.params.max_age {
-            let utc = Utc::now().sub(Duration::hours(max_age));
+            let utc = Utc::now().sub(Duration::try_hours(max_age).unwrap_or_default());
             before = String::new();
             after = utc.format("%Y%m%d%H%M%S").to_string();
         }

src/datasource_labels.rs

@@ -0,0 +1,48 @@
+use crate::datasource::DataSource;
+use crate::pagelist::*;
+use crate::platform::Platform;
+use async_trait::async_trait;
+use mysql_async::from_row;
+use mysql_async::prelude::Queryable;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct SourceLabels {}
+
+#[async_trait]
+impl DataSource for SourceLabels {
+    fn name(&self) -> String {
+        "labels".to_string()
+    }
+
+    fn can_run(&self, platform: &Platform) -> bool {
+        platform.has_param("labels_yes") || platform.has_param("labels_any")
+    }
+
+    async fn run(&mut self, platform: &Platform) -> Result<PageList, String> {
+        let sql = platform.get_label_sql();
+        let mut conn = platform
+            .state()
+            .get_wiki_db_connection("wikidatawiki")
+            .await?;
+        let rows = conn
+            .exec_iter(sql.0.as_str(), mysql_async::Params::Positional(sql.1))
+            .await
+            .map_err(|e| format!("{:?}", e))?
+            .map_and_drop(from_row::<(Vec<u8>,)>)
+            .await
+            .map_err(|e| format!("{:?}", e))?;
+        conn.disconnect().await.map_err(|e| format!("{:?}", e))?;
+        let ret = PageList::new_from_wiki_with_capacity("wikidatawiki", rows.len());
+        rows.iter()
+            .map(|row| String::from_utf8_lossy(&row.0))
+            .filter_map(|item| Platform::entry_from_entity(&item))
+            .for_each(|entry| ret.add_entry(entry).unwrap_or(()));
+        Ok(ret)
+    }
+}
+
+impl SourceLabels {
+    pub fn new() -> Self {
+        Self {}
+    }
+}

src/datasource_manual.rs

@@ -0,0 +1,50 @@
+use crate::datasource::DataSource;
+use crate::pagelist::*;
+use crate::pagelist_entry::PageListEntry;
+use crate::platform::Platform;
+use async_trait::async_trait;
+use wikibase::mediawiki::title::Title;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct SourceManual {}
+
+#[async_trait]
+impl DataSource for SourceManual {
+    fn name(&self) -> String {
+        "manual".to_string()
+    }
+
+    fn can_run(&self, platform: &Platform) -> bool {
+        platform.has_param("manual_list") && platform.has_param("manual_list_wiki")
+    }
+
+    async fn run(&mut self, platform: &Platform) -> Result<PageList, String> {
+        let wiki = platform
+            .get_param("manual_list_wiki")
+            .ok_or_else(|| "Missing parameter \'manual_list_wiki\'".to_string())?;
+        let api = platform.state().get_api_for_wiki(wiki.to_string()).await?;
+        let ret = PageList::new_from_wiki(&wiki);
+        platform
+            .get_param("manual_list")
+            .ok_or_else(|| "Missing parameter \'manual_list\'".to_string())?
+            .split('\n')
+            .filter_map(|line| {
+                let line = line.trim().to_string();
+                if !line.is_empty() {
+                    let title = Title::new_from_full(&line, &api);
+                    let entry = PageListEntry::new(title);
+                    Some(entry)
+                } else {
+                    None
+                }
+            })
+            .for_each(|entry| ret.add_entry(entry).unwrap_or(()));
+        Ok(ret)
+    }
+}
+
+impl SourceManual {
+    pub fn new() -> Self {
+        Self {}
+    }
+}

src/datasource_pagepile.rs

@@ -0,0 +1,71 @@
+use crate::datasource::DataSource;
+use crate::pagelist::*;
+use crate::pagelist_entry::PageListEntry;
+use crate::platform::Platform;
+use async_trait::async_trait;
+use serde_json::value::Value;
+use std::time;
+use wikibase::mediawiki::api::Api;
+use wikibase::mediawiki::title::Title;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct SourcePagePile {}
+
+#[async_trait]
+impl DataSource for SourcePagePile {
+    fn name(&self) -> String {
+        "pagepile".to_string()
+    }
+
+    fn can_run(&self, platform: &Platform) -> bool {
+        platform.has_param("pagepile")
+    }
+
+    async fn run(&mut self, platform: &Platform) -> Result<PageList, String> {
+        let pagepile = platform
+            .get_param("pagepile")
+            .ok_or_else(|| "Missing parameter \'pagepile\'".to_string())?;
+        let timeout = time::Duration::from_secs(240);
+        let builder = reqwest::ClientBuilder::new().timeout(timeout);
+        let api = Api::new_from_builder("https://www.wikidata.org/w/api.php", builder)
+            .await
+            .map_err(|e| e.to_string())?;
+        let params = api.params_into(&[
+            ("id", &pagepile.to_string()),
+            ("action", "get_data"),
+            ("format", "json"),
+            ("doit", "1"),
+        ]);
+        let text = api
+            .query_raw("https://tools.wmflabs.org/pagepile/api.php", &params, "GET")
+            .await
+            .map_err(|e| format!("PagePile: {:?}", e))?;
+        let v: Value =
+            serde_json::from_str(&text).map_err(|e| format!("PagePile JSON: {:?}", e))?;
+        let wiki = v["wiki"]
+            .as_str()
+            .ok_or(format!("PagePile {} does not specify a wiki", &pagepile))?;
+        let api = platform.state().get_api_for_wiki(wiki.to_string()).await?; // Just because we need query_raw
+        let ret = PageList::new_from_wiki(wiki);
+        v["pages"]
+            .as_array()
+            .ok_or(format!(
+                "PagePile {} does not have a 'pages' array",
+                &pagepile
+            ))?
+            .iter()
+            .filter_map(|title| title.as_str())
+            .map(|title| PageListEntry::new(Title::new_from_full(title, &api)))
+            .for_each(|entry| ret.add_entry(entry).unwrap_or(()));
+        if ret.is_empty()? {
+            platform.warn("<span tt=\'warn_pagepile\'></span>".to_string())?;
+        }
+        Ok(ret)
+    }
+}
+
+impl SourcePagePile {
+    pub fn new() -> Self {
+        Self {}
+    }
+}

src/datasource_search.rs

@@ -0,0 +1,87 @@
+use crate::datasource::DataSource;
+use crate::pagelist::*;
+use crate::pagelist_entry::PageListEntry;
+use crate::platform::Platform;
+use async_trait::async_trait;
+use rayon::prelude::*;
+use wikibase::mediawiki::api::Api;
+
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct SourceSearch {}
+
+#[async_trait]
+impl DataSource for SourceSearch {
+    fn name(&self) -> String {
+        "search".to_string()
+    }
+
+    fn can_run(&self, platform: &Platform) -> bool {
+        platform.has_param("search_query")
+            && platform.has_param("search_wiki")
+            && platform.has_param("search_max_results")
+            && !platform.is_param_blank("search_query")
+            && !platform.is_param_blank("search_wiki")
+    }
+
+    async fn run(&mut self, platform: &Platform) -> Result<PageList, String> {
+        let wiki = platform
+            .get_param("search_wiki")
+            .ok_or_else(|| "Missing parameter \'search_wiki\'".to_string())?;
+        let query = platform
+            .get_param("search_query")
+            .ok_or_else(|| "Missing parameter \'search_query\'".to_string())?;
+        let max = match platform
+            .get_param("search_max_results")
+            .ok_or_else(|| "Missing parameter \'search_max_results\'".to_string())?
+            .parse::<usize>()
+        {
+            Ok(max) => max,
+            Err(e) => return Err(format!("{:?}", e)),
+        };
+        let api = platform.state().get_api_for_wiki(wiki.to_string()).await?;
+        let srlimit = if max > 500 { 500 } else { max };
+        let srlimit = format!("{}", srlimit);
+        let namespace_ids = platform
+            .form_parameters()
+            .ns
+            .par_iter()
+            .cloned()
+            .collect::<Vec<usize>>();
+        let namespace_ids = if namespace_ids.is_empty() {
+            "*".to_string()
+        } else {
+            namespace_ids
+                .iter()
+                .map(|i| i.to_string())
+                .collect::<Vec<String>>()
+                .join(",")
+        };
+        let params = api.params_into(&[
+            ("action", "query"),
+            ("list", "search"),
+            ("srlimit", srlimit.as_str()),
+            ("srsearch", query.as_str()),
+            ("srnamespace", namespace_ids.as_str()),
+        ]);
+        let result = match api.get_query_api_json_limit(&params, Some(max)).await {
+            Ok(result) => result,
+            Err(e) => return Err(format!("{:?}", e)),
+        };
+        let titles = Api::result_array_to_titles(&result);
+        let ret = PageList::new_from_wiki(&wiki);
+        titles
+            .iter()
+            .map(|title| PageListEntry::new(title.to_owned()))
+            .for_each(|entry| ret.add_entry(entry).unwrap_or(()));
+        if ret.is_empty()? {
+            platform.warn("<span tt=\'warn_search\'></span>".to_string())?;
+        }
+        Ok(ret)
+    }
+}
+
+impl SourceSearch {
+    pub fn new() -> Self {
+        Self {}
+    }
+}