From 887d8c8a002efc9309e4a73701f0246ad29882bc Mon Sep 17 00:00:00 2001 From: "Victor I. Afolabi" Date: Fri, 11 Dec 2020 19:55:13 -0600 Subject: [PATCH] Minor changes and refactor --- Cargo.lock | 49 +---- Cargo.toml | 6 +- README.md | 80 +++----- src/api.rs | 79 ++++---- src/download.rs | 43 ++-- src/engines.rs | 32 --- src/engines/freemp3cloud.rs | 40 ++-- src/engines/mod.rs | 32 +++ src/engines/mp3red.rs | 35 ++-- src/main.rs | 34 ++-- src/types.rs | 10 +- src/utils.rs | 393 ++++++++++++++++++------------------ 12 files changed, 381 insertions(+), 452 deletions(-) delete mode 100644 src/engines.rs create mode 100644 src/engines/mod.rs diff --git a/Cargo.lock b/Cargo.lock index bf1fa06..75bb375 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -357,20 +357,6 @@ dependencies = [ "serde_urlencoded 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "backtrace" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace-sys 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.81 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "backtrace" version = "0.3.55" @@ -384,15 +370,6 @@ dependencies = [ "rustc-demangle 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "backtrace-sys" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cc 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.81 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "base-x" version = "0.2.8" @@ -786,15 +763,6 @@ dependencies = [ "syn 1.0.54 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "dbghelp-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "derive_more" version = "0.99.11" @@ -924,19 +892,12 @@ dependencies = [ "termcolor 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "error-chain" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "error-chain" version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "backtrace 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)", "version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1568,7 +1529,7 @@ dependencies = [ [[package]] name = "mythra" -version = "0.1.0" +version = "0.1.1" dependencies = [ "actix-cors 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "actix-rt 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1577,7 +1538,7 @@ dependencies = [ "clap 3.0.0-beta.2 (registry+https://github.com/rust-lang/crates.io-index)", "cursive 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "error-chain 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)", "fantoccini 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "indicatif 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3151,9 +3112,7 @@ dependencies = [ "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" "checksum awc 2.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b381e490e7b0cfc37ebc54079b0413d8093ef43d14a4e4747083f7fa47a9e691" -"checksum backtrace 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "346d7644f0b5f9bc73082d3b2236b69a05fd35cce0cfa3724e184e6a5c9e2a2f" "checksum backtrace 0.3.55 (registry+https://github.com/rust-lang/crates.io-index)" = "ef5140344c85b01f9bbb4d4b7288a8aa4b3287ccef913a14bcc78a1063623598" -"checksum backtrace-sys 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "18fbebbe1c9d1f383a9cc7e8ccdb471b91c8d024ee9c2ca5b5346121fe8b4399" "checksum base-x 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" "checksum base64 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" "checksum base64 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" @@ -3198,7 +3157,6 @@ dependencies = [ "checksum darling 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)" = "15d658711a12632b5574c8d5b3fc5d2f0d2f87b9fbf9237ee0f759b88bb6bdec" "checksum darling_core 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)" = "16d3514d243331d8acde56bfcf45d0147aabbda853c2f49dce081ea85f9a7220" "checksum darling_macro 0.10.3 (registry+https://github.com/rust-lang/crates.io-index)" = "44f63c369ef0c17ad17585d31d5f2bf10dece2710bf0766e01db57a6f9849a2e" -"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum derive_more 0.99.11 (registry+https://github.com/rust-lang/crates.io-index)" = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" @@ -3216,7 +3174,6 @@ dependencies = [ "checksum enumset_derive 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "74bef436ac71820c5cf768d7af9ba33121246b09a00e09a55d94ef8095a875ac" "checksum env_logger 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f26ecb66b4bdca6c1409b40fb255eefc2bd4f6d135dab3c3124f80ffa2a9661e" "checksum error-chain 0.12.4 (registry+https://github.com/rust-lang/crates.io-index)" = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -"checksum error-chain 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a8b6b917cc8ec29b15e925bc6faaaea9df5a91967cd0279ed14d11db1beb20db" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" "checksum fantoccini 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "fc4eadcfdf79804c8be3835451fa83d3e41ba0ec10c33355b7670e49fdc40c08" "checksum flate2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" diff --git a/Cargo.toml b/Cargo.toml index 7c02e51..757b1a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,20 +1,20 @@ [package] name = "mythra" -version = "0.1.0" +version = "0.1.1" authors = ["Diretnan Domnan "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -actix-web = { version = "3.1.0", features = ["openssl"] } +actix-web = { version = "3.3.2", features = ["openssl"] } actix-cors = "0.5.3" actix-rt = "1.1.1" async-trait = "0.1.41" clap = { version = "3.0.0-beta.2", features = ["yaml"] } cursive = "0.15.0" env_logger = "0.8.1" -error-chain = "0.4" +error-chain = "0.12" fantoccini = "0.14.2" futures = "0.3.5" indicatif = "0.15.0" diff --git a/README.md b/README.md index 266ffa2..6519b03 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,52 @@ +# Mythra + +

mythra

-
- - - - - - - - - - - - - - - - - -
Web API - - Deploy Status - -
Build and Test - - Test Status - -
Documentation - - Docs Status - -
Releases - - Releases Status - -
-
+ -# Mythra +| Build Type | Status | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Web API | [![Deploy status](https://github.com/deven96/mythra/workflows/Deploy%20to%20Heroku/badge.svg)](https://github.com/deven96/mythra/actions/) | +| Build and Test | [![Test status](https://github.com/deven96/mythra/workflows/Build%20and%20Test/badge.svg)](https://github.com/deven96/mythra/actions/) | +| Documentation | [![Docs status](https://github.com/deven96/mythra/workflows/Deploy%20docs%20to%20Stoplight/badge.svg)](https://bisoncorps.stoplight.io/docs/mythra/reference/Mythra.v1.yaml) | +| Releases | [![Release status](https://github.com/deven96/mythra/workflows/Release%20to%20GitHub/badge.svg)](https://github.com/deven96/mythra/releases) | In my bid to learn rust I am trying to make a music web scraper - - In order to use either the `ncurses` or the `pancurses` backend of a dependent library `cursive`, you will need the ncurses library installed on your system. ## Archlinux -``` +```sh pacman -S ncurses ``` ## Ubuntu -``` -apt-get install libncursesw5-dev +```sh +apt-get install libncursesw5-dev libssl-dev ``` ## Fedora -``` -yum install ncurses-devel +```sh +yum install ncurses-devel openssl-devel ``` ## macOS -``` +```sh brew install ncurses ``` ### Engines - FreeMP3Cloud - +- MP3Red ## Installation + With Rust installed You must have [chromedriver](https://chromedriver.chromium.org/) available on path and running on port 4444 @@ -92,11 +64,13 @@ cargo run ./target/debug/mythra search --query "Justin Timberlake Mirrors" ``` + Or download from Github [Releases](https://github.com/deven96/mythra/releases) ## Example -

mythra example

+![Mythra example](assets/example.gif) + ## Deployment @@ -104,11 +78,11 @@ The deployed API version from `mythra api` is available. Please read the [API do ## License -This project is opened under the [GNU AGPLv3](https://github.com/deven96/mythra/blob/master/LICENSE) which allows very broad use for both academic and commercial purposes. - +This project is opened under the [GNU AGPLv3](./LICENSE) which allows very broad use for both academic and commercial purposes. ## Credits -Library/Resource | Use -------- | ----- -[Stoplight](https://stoplight.io) | Generating API docs -[Fantoccini](https://github.com/jonhoo/fantoccini/) | Scraping javascript sites using chromedriver/geckodriver + +| Library/Resource | Use | +| --------------------------------------------------- | -------------------------------------------------------- | +| [Stoplight](https://stoplight.io) | Generating API docs | +| [Fantoccini](https://github.com/jonhoo/fantoccini/) | Scraping javascript sites using chromedriver/geckodriver | diff --git a/src/api.rs b/src/api.rs index e54496e..4a07c98 100644 --- a/src/api.rs +++ b/src/api.rs @@ -1,49 +1,48 @@ -use actix_web::{http::StatusCode, web, App, HttpServer, HttpResponse}; -//use actix_cors::Cors; -use actix_web::{middleware::Logger}; -use crate::types::MusicRequest; -use crate::engines::mp3red; use crate::engines::freemp3cloud; -use log::{error, debug}; +use crate::engines::mp3red; +use crate::types::MusicRequest; + +use actix_web::{http::StatusCode, web, App, HttpResponse, HttpServer}; +//use actix_cors::Cors; +use actix_web::middleware::Logger; +use log::{debug, error}; async fn search(web::Query(info): web::Query) -> HttpResponse { - debug!("Request for client with engine={} and query={}!", info.engine, info.query); - let query = info.query.clone(); - let engine = info.engine.clone(); - let engine_match = engine.as_str(); - match engine_match { - "mp3red" => { - let e = mp3red::MP3Red{}; - let res = e.search(query).await.ok(); - HttpResponse::Ok().json(res.unwrap()) - }, - "freemp3cloud" => { - let e = freemp3cloud::FreeMP3Cloud{}; - let res = e.search(query).await.ok(); - HttpResponse::Ok().json(res.unwrap()) - }, - _ => { - error!("Engine {} is unsupported", engine_match); - HttpResponse::new(StatusCode::NOT_FOUND) - }, + debug!( + "Request for client with engine={} and query={}!", + info.engine, info.query + ); + let query = info.query.clone(); + let engine = info.engine.clone(); + let engine_match = engine.as_str(); + match engine_match { + "mp3red" => { + let e = mp3red::MP3Red {}; + let res = e.search(query).await.ok(); + HttpResponse::Ok().json(res.unwrap()) + } + "freemp3cloud" => { + let e = freemp3cloud::FreeMP3Cloud {}; + let res = e.search(query).await.ok(); + HttpResponse::Ok().json(res.unwrap()) + } + _ => { + error!("Engine {} is unsupported", engine_match); + HttpResponse::new(StatusCode::NOT_FOUND) } + } } pub async fn api(port: &str) -> std::io::Result<()> { let address: &str = &(format!("0.0.0.0:{}", port))[..]; - HttpServer::new(|| - App::new() - .wrap(Logger::default()) - .wrap(Logger::new("%a %{User-Agent}i")) - .service( - web::resource("/search") - .route( - web::get().to(search) - ) - ) - ) - .bind(address) - .unwrap() - .run() - .await + HttpServer::new(|| { + App::new() + .wrap(Logger::default()) + .wrap(Logger::new("%a %{User-Agent}i")) + .service(web::resource("/search").route(web::get().to(search))) + }) + .bind(address) + .unwrap() + .run() + .await } diff --git a/src/download.rs b/src/download.rs index d1ce2d0..cee41d9 100644 --- a/src/download.rs +++ b/src/download.rs @@ -1,19 +1,18 @@ use cursive::utils::Counter; -use reqwest::{Url}; -use std::{ - fs, - io::{self, copy, Read}, - path::Path, -}; +use reqwest::Url; + +use std::fs; +use std::io::{self, copy, Read}; +use std::path::Path; fn append_frag(text: &mut String, frag: &mut String) { if !frag.is_empty() { - let encoded = frag.chars() + let encoded = frag + .chars() .collect::>() .chunks(2) - .map(|ch| { - u8::from_str_radix(&ch.iter().collect::(), 16).unwrap() - }).collect::>(); + .map(|ch| u8::from_str_radix(&ch.iter().collect::(), 16).unwrap()) + .collect::>(); text.push_str(&std::str::from_utf8(&encoded).unwrap()); frag.clear(); } @@ -59,34 +58,34 @@ pub fn download_size(url: &str) -> Result { .parse::() .unwrap() } else { - return Err(format!( - "Couldn't download URL: {}. Error: {:?}", - url, - resp.status(), - ) - .into()); + return Err( + format!("Couldn't download URL: {}. Error: {:?}", url, resp.status(),).into(), + ); } }; Ok(total_size) } -pub fn download_from_url(counter: Counter, url:String){ +pub fn download_from_url(counter: Counter, url: String) { let parsed_url = Url::parse(&url[..]).unwrap(); let mut request = ureq::get(url.as_str()); - let segment = parsed_url.path_segments() + let segment = parsed_url + .path_segments() .and_then(|segments| { let output = decode_uri(&segments.last().unwrap().to_owned()); Some(output) }) .unwrap_or("tmp.bin".to_owned()); - + let file = Path::new(&segment); - + // if file already exists if file.exists() { let size = file.metadata().unwrap().len() - 1; - request = request.set("Content-Length", &(format!("bytes={}-", size))[..]).build(); + request = request + .set("Content-Length", &(format!("bytes={}-", size))[..]) + .build(); &counter.set(size as usize); } @@ -99,6 +98,6 @@ pub fn download_from_url(counter: Counter, url:String){ .append(true) .open(&file) .unwrap(); - + copy(&mut source, &mut dest).unwrap(); } diff --git a/src/engines.rs b/src/engines.rs deleted file mode 100644 index 29ffeac..0000000 --- a/src/engines.rs +++ /dev/null @@ -1,32 +0,0 @@ -pub mod mp3red; -pub mod freemp3cloud; -use crate::utils::render_select_music; -use crate::types::Music; -use log::error; - -pub async fn search_all(engine:&str, query:&str) -> Result, Box> { - let query = String::from(query); - match engine { - "mp3red" => { - let e = mp3red::MP3Red{}; - e.search(query).await - }, - "freemp3cloud" => { - let e = freemp3cloud::FreeMP3Cloud{}; - e.search(query).await - }, - _ => { - let empty: Vec = vec![]; - error!("Engine is unsupported"); - Ok(empty) - }, - - } -} - -pub async fn cli(engine: &str, query:&str){ - let title: &str = &(format!("Searching {} for {}", - engine, query))[..]; - let results = search_all(engine, query).await.ok().unwrap(); - render_select_music(results, title); -} diff --git a/src/engines/freemp3cloud.rs b/src/engines/freemp3cloud.rs index 52e7640..d37a6fc 100644 --- a/src/engines/freemp3cloud.rs +++ b/src/engines/freemp3cloud.rs @@ -1,25 +1,24 @@ -use std::collections::HashMap; -use crate::types::{Engine, Music}; -use crate::utils::extract_from_el; +use crate::types::{Engine, Music, MythraResult}; use crate::utils::cached_reqwest; -use scraper::{Html, Selector, ElementRef}; +use crate::utils::extract_from_el; + use indicatif::ProgressBar; use log::debug; +use scraper::{ElementRef, Html, Selector}; + +use std::collections::HashMap; pub struct FreeMP3Cloud; -pub static CONFIG:Engine = Engine { +pub static CONFIG: Engine = Engine { name: "FreeMP3Cloud", - base_url:"https://freemp3cloud.com/", + base_url: "https://freemp3cloud.com/", search_url: "https://freemp3cloud.com/", }; impl FreeMP3Cloud { - pub async fn search(&self, _query:String) -> Result, Box> { - let _query:&str = &_query[..]; - let form_params: HashMap<&str, &str> = [("searchSong", _query)] - .iter() - .cloned() - .collect(); + pub async fn search(&self, _query: String) -> MythraResult> { + let _query: &str = &_query[..]; + let form_params: HashMap<&str, &str> = [("searchSong", _query)].iter().cloned().collect(); let bar = ProgressBar::new(100); let full_url: String = CONFIG.search_url.to_owned(); let res = cached_reqwest::js_post(&full_url, ".el-input", &form_params).await; @@ -35,24 +34,25 @@ impl FreeMP3Cloud { _ => (), } // increment progress bar - let inc: u64 = (100/size) as u64; + let inc: u64 = (100 / size) as u64; bar.inc(inc); } bar.finish(); - return Ok(vec) + + Ok(vec) } - pub async fn parse_single_music(&self, ind:usize, element:ElementRef<'_>) -> Option{ + pub async fn parse_single_music(&self, ind: usize, element: ElementRef<'_>) -> Option { let title = extract_from_el(&element, ".s-title", "text"); let artiste = extract_from_el(&element, ".s-artist", "text"); let duration = extract_from_el(&element, ".s-time", "text"); let download_link = extract_from_el(&element, ".play-ctrl", "data-src"); debug!("Retrieving song with title -> {}", title); - - Some(Music{ - index: ind+1, - artiste: Some(artiste), - title, + + Some(Music { + index: ind + 1, + artiste: Some(artiste), + title, download_link, picture_link: None, collection: None, diff --git a/src/engines/mod.rs b/src/engines/mod.rs new file mode 100644 index 0000000..97bdead --- /dev/null +++ b/src/engines/mod.rs @@ -0,0 +1,32 @@ +pub mod freemp3cloud; +pub mod mp3red; + +use crate::types::{Music, MythraResult}; +use crate::utils::render_select_music; + +use log::error; + +pub async fn search_all(engine: &str, query: &str) -> MythraResult> { + let query = String::from(query); + match engine { + "mp3red" => { + let e = mp3red::MP3Red {}; + e.search(query).await + } + "freemp3cloud" => { + let e = freemp3cloud::FreeMP3Cloud {}; + e.search(query).await + } + _ => { + let empty: Vec = vec![]; + error!("Engine is unsupported"); + Ok(empty) + } + } +} + +pub async fn cli(engine: &str, query: &str) { + let title: &str = &(format!("Searching {} for {}", engine, query))[..]; + let results = search_all(engine, query).await.ok().unwrap(); + render_select_music(results, title); +} diff --git a/src/engines/mp3red.rs b/src/engines/mp3red.rs index 5493c91..942cb0f 100644 --- a/src/engines/mp3red.rs +++ b/src/engines/mp3red.rs @@ -1,28 +1,31 @@ -use crate::types::{Engine, Music}; -use crate::utils::extract_from_el; +use crate::types::{Engine, Music, MythraResult}; use crate::utils::cached_reqwest; -use scraper::{Html, Selector, ElementRef}; +use crate::utils::extract_from_el; + use indicatif::ProgressBar; +use scraper::{ElementRef, Html, Selector}; pub struct MP3Red; -pub static CONFIG:Engine = Engine { +pub static CONFIG: Engine = Engine { name: "MP3Red", - base_url:"https://mp3red.best/", + base_url: "https://mp3red.best/", search_url: "https://mp3red.best/mp3/", }; impl MP3Red { - pub async fn search(&self, _query:String) -> Result, Box> { - let _query:&str = &_query.replace(" ", "-")[..]; + pub async fn search(&self, _query: String) -> MythraResult> { + let _query: &str = &_query.replace(" ", "-")[..]; let bar = ProgressBar::new(100); let mut full_url: String = CONFIG.search_url.to_owned(); full_url.push_str(_query); + let res = cached_reqwest::get(&full_url).await; let document = Html::parse_document(res.as_str()); let selector = Selector::parse("div.box-post").unwrap(); let mut vec: Vec = Vec::new(); let elems = document.select(&selector); let size = elems.count(); + for (ind, element) in document.select(&selector).enumerate() { let single_music = self.parse_single_music(ind, element).await; match single_music { @@ -30,30 +33,30 @@ impl MP3Red { _ => (), } // increment progress bar - let inc: u64 = (100/size) as u64; + let inc: u64 = (100 / size) as u64; bar.inc(inc); } bar.finish(); - return Ok(vec) + Ok(vec) } - pub async fn parse_single_music(&self, ind:usize, element:ElementRef<'_>) -> Option{ + pub async fn parse_single_music(&self, ind: usize, element: ElementRef<'_>) -> Option { let picture_link = extract_from_el(&element, "img", "src"); let title = extract_from_el(&element, "img", "alt"); let duration = extract_from_el(&element, ".duration", "text"); let size = extract_from_el(&element, ".file-size", "text"); - let initial_download_link = extract_from_el(&element,".pull-left","href"); + let initial_download_link = extract_from_el(&element, ".pull-left", "href"); let res = cached_reqwest::get(&initial_download_link).await; let document = Html::parse_document(res.as_str()); let dl_selector = Selector::parse(".dl-list").unwrap(); let dl_element = document.select(&dl_selector).next().unwrap(); let download_link = extract_from_el(&dl_element, "[class='btn']", "href"); - - Some(Music{ - index: ind+1, - artiste: None, - title, + + Some(Music { + index: ind + 1, + artiste: None, + title, download_link, picture_link: Some(picture_link), collection: None, diff --git a/src/main.rs b/src/main.rs index fcbcfbe..3c96475 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,10 +1,11 @@ +mod api; +mod download; mod engines; mod types; mod utils; -mod download; -mod api; -use clap::{App, load_yaml}; -use log::{info, error}; + +use clap::{load_yaml, App}; +use log::{error, info}; #[actix_web::main] async fn main() { @@ -14,40 +15,33 @@ async fn main() { Some(("clear-cache", cache_matches)) => { // Clear cache // Start API server on port - let verbosity = cache_matches.value_of("verbose") - .unwrap(); + let verbosity = cache_matches.value_of("verbose").unwrap(); utils::configure_log(verbosity); utils::clear_cache(); - }, + } Some(("search", search_matches)) => { // Search on CLI - let verbosity = search_matches.value_of("verbose") - .unwrap(); + let verbosity = search_matches.value_of("verbose").unwrap(); utils::configure_log(verbosity); - let engine = search_matches.value_of("engine") - .unwrap(); - let query = search_matches.value_of("query") - .unwrap(); + let engine = search_matches.value_of("engine").unwrap(); + let query = search_matches.value_of("query").unwrap(); engines::cli(engine, query).await; - }, + } Some(("api", api_matches)) => { // Start API server on port - let verbosity = api_matches.value_of("verbose") - .unwrap(); + let verbosity = api_matches.value_of("verbose").unwrap(); utils::configure_log(verbosity); - let port = api_matches.value_of("port") - .unwrap(); + let port = api_matches.value_of("port").unwrap(); info!("Running API on {:?}", port); let server = api::api(port).await; match server { Err(_) => info!("Error starting server"), Ok(_) => info!("Exiting..."), } - }, + } _ => error!("Select a valid subcommand"), - } } diff --git a/src/types.rs b/src/types.rs index 17c4e9e..66e0db6 100644 --- a/src/types.rs +++ b/src/types.rs @@ -1,7 +1,10 @@ -use serde::{Serialize, Deserialize}; use scraper::ElementRef; +use serde::{Deserialize, Serialize}; use std::borrow::Borrow; +// Result type. +pub type MythraResult = std::result::Result>; + #[derive(Serialize, Deserialize, Debug)] pub struct Music { // Option<> elements are struct elements @@ -31,10 +34,9 @@ pub struct Engine { pub search_url: &'static str, } - pub trait EngineTraits { - fn search(&self, query:String) -> Result, Box>; - fn parse_single_music(&self, ind:usize, el:ElementRef) -> Result>; + fn search(&self, query: String) -> MythraResult>; + fn parse_single_music(&self, ind: usize, el: ElementRef) -> MythraResult; } // Music request parser for API diff --git a/src/utils.rs b/src/utils.rs index b3c5f03..462b603 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,43 +1,46 @@ -use scraper::{Selector, ElementRef}; -use log::debug; -use crate::types::Music; use crate::download::{download_from_url, download_size}; -use cursive::views::{Dialog, ProgressBar, SelectView}; +use crate::types::{Music, MythraResult}; + use cursive::align::HAlign; +use cursive::view::{Resizable, Scrollable}; +use cursive::views::{Dialog, ProgressBar, SelectView}; use cursive::Cursive; -use cursive::view::{Scrollable, Resizable}; + use env_logger::Builder; -use std::{env}; +use fantoccini; +use log::{debug, info}; +use scraper::{ElementRef, Selector}; + +use std::collections::{hash_map::DefaultHasher, HashMap}; +use std::env; +use std::fs; +use std::hash::{Hash, Hasher}; +use std::io::{Read, Write}; +use std::path::Path; pub static CACHE_NAME: &str = ".mythra-cache"; + // Reduces the stress of repetitive extraction of elements // as the raw scraper library is too verbose -pub fn extract_from_el(element:&ElementRef, selector:&str, attr:&str) -> String { +pub fn extract_from_el(element: &ElementRef, selector: &str, attr: &str) -> String { let selector = Selector::parse(selector).unwrap(); let tag = element.select(&selector).next().unwrap(); match attr { "text" => { - return tag.text().collect::(); - }, - others => - { - let val = String::from( - tag.value().attr(others).unwrap() - ); + return tag.text().collect::(); + } + others => { + let val = String::from(tag.value().attr(others).unwrap()); if others.eq("href") { debug!("Retrieved link {}", val) }; - return val - }, + return val; + } } } - // Removes cache directory -pub fn clear_cache(){ - use std::fs; - use std::path::Path; - use log::info; +pub fn clear_cache() { match env::current_exe() { Ok(current_exe) => { let path: &Path = Path::new(current_exe.to_str().unwrap()); @@ -50,143 +53,161 @@ pub fn clear_cache(){ Err(_) => { info!("Mythra cache does not exist"); } - } + } + Err(_) => (), + } +} - }, - Err(_) => { () } +pub fn render_select_music(songs: Vec, title: &str) { + let mut select = SelectView::new().h_align(HAlign::Center).autojump(); + for song in songs { + let title_copy = &song.title.clone(); + let title = title_copy.as_str(); + select.add_item(title, song); } + let mut siv = cursive::default(); + select.set_on_submit(render_downloading_song); + // Let's add a ResizedView to keep the + // list at a reasonable size (it can scroll anyway). + siv.add_layer(Dialog::around(select.scrollable().fixed_size((50, 10))).title(title)); + siv.run(); +} + +fn render_downloading_song(siv: &mut Cursive, song: &Music) { + // replace previous view + let link_copy = song.download_link.clone(); + let use_link = &link_copy[..]; + let download_size_u64 = download_size(use_link).unwrap(); + let cb = siv.cb_sink().clone(); + siv.add_layer( + Dialog::around( + ProgressBar::new() + .range(0, download_size_u64 as usize) + .with_task(move |counter| { + download_from_url(counter, link_copy.to_owned()); + cb.send(Box::new(completed_download)).unwrap(); + }) + .full_width(), + ) + .button("Minimize", |siv| { + siv.pop_layer(); + }), + ); + siv.set_autorefresh(true); +} + +// complete download callback +fn completed_download(siv: &mut Cursive) { + siv.set_autorefresh(false); + siv.pop_layer(); + siv.add_layer( + Dialog::new() + .title("Download complete") + .button("Return", |siv| { + siv.pop_layer(); + }), + ); +} + +// configure logging +pub fn configure_log(level: &str) { + let logname: &str = "MYTHRA_LOG_FMT"; + // activate debugging for only actix_web and mythra + let logfmt = format!("actix_web={},mythra={}", level, level); + env::set_var(logname, logfmt); + Builder::new().parse_env(logname).init(); } // Wrapper around the reqwest module // Retrieve web pages from cache if they exist // else retrieve from url pub mod cached_reqwest { -use std::env; -use std::path::Path; -use std::collections::{hash_map::DefaultHasher, HashMap}; -use std::hash::{Hash, Hasher}; -use std::fs::{OpenOptions, create_dir_all}; -use std::io::{Read, Write}; -use log::debug; -use fantoccini; -//std::fs::File, String - - pub fn create_or_retrieve(url: String, exe_path: std::path::PathBuf) -> (std::fs::File, String){ - let path: &Path = Path::new(exe_path.to_str().unwrap()); - let parent: &str = path.parent().unwrap().to_str().unwrap(); - // hash url - let mut hasher = DefaultHasher::new(); - url.hash(&mut hasher); - let hashed_url: &str = &(hasher.finish().to_string())[..]; - let full_dir_path = format!("{}/{}", parent, crate::utils::CACHE_NAME); - let full_path = format!("{}/{}",full_dir_path, hashed_url); - // create all parent directories necessary - create_dir_all(full_dir_path).ok().unwrap(); - let mut file = OpenOptions::new() - .write(true).read(true) - .create(true).open(full_path) - .unwrap(); - // read file contents to String - let mut contents = String::new(); - file.read_to_string(&mut contents).unwrap(); -// Ok((file, contents)) - (file, contents) + #[allow(dead_code)] + use super::*; + + pub fn create_or_retrieve( + url: String, + exe_path: std::path::PathBuf, + ) -> (std::fs::File, String) { + let path: &Path = Path::new(exe_path.to_str().unwrap()); + let parent: &str = path.parent().unwrap().to_str().unwrap(); + // hash url + let mut hasher = DefaultHasher::new(); + url.hash(&mut hasher); + let hashed_url: &str = &(hasher.finish().to_string())[..]; + let full_dir_path = format!("{}/{}", parent, crate::utils::CACHE_NAME); + let full_path = format!("{}/{}", full_dir_path, hashed_url); + // create all parent directories necessary + fs::create_dir_all(full_dir_path).ok().unwrap(); + let mut file = fs::OpenOptions::new() + .write(true) + .read(true) + .create(true) + .open(full_path) + .unwrap(); + // read file contents to String + let mut contents = String::new(); + file.read_to_string(&mut contents).unwrap(); + // Ok((file, contents)) + (file, contents) } - pub async fn fantoccini_form(url: &String, form_selector: &str, params: &HashMap<&str, &str>) -> Result> { - let mut results = String::new(); - match env::current_exe() { + pub async fn fantoccini_form( + url: &String, + form_selector: &str, + params: &HashMap<&str, &str>, + ) -> MythraResult { + let results = match env::current_exe() { Ok(exe_path) => { let mut val_map = String::from(""); - for (key, val) in params{ - val_map = val_map + &(format!("{}={}", key,val))[..] + for (key, val) in params { + val_map = val_map + &(format!("{}={}", key, val))[..] } let concat_url = url.to_owned() + &val_map[..]; let (mut file, contents) = create_or_retrieve(concat_url, exe_path); // if file is empty then cache does not exist // then retrieve directly using reqwest if (contents.as_str()).eq("") { - let mut c = fantoccini::Client::new("http://localhost:4444").await.expect("failed to connect to WebDriver"); + let mut c = fantoccini::Client::new("http://localhost:4444") + .await + .expect("failed to connect to WebDriver"); c.goto(url).await.unwrap(); - let mut form = c.form( - fantoccini::Locator::Css(form_selector) - ).await.unwrap(); + let mut form = c + .form(fantoccini::Locator::Css(form_selector)) + .await + .unwrap(); for (key, val) in params { form.set_by_name(key, val).await.unwrap(); - }; + } let mut res_client = form.submit().await.unwrap(); let res = res_client.source().await.unwrap(); file.write_all((res.as_str()).as_bytes())?; c.close().await.unwrap(); debug!("Retrieving {} [POST] data from web (fantoccini)", url); - results = res; + res } else { debug!("Retrieving {} [POST] data from cache (fantoccini)", url); - results = contents; + contents } - }, - Err(e) => { - format!("failed to get current exe path: {}", e); - }, - }; - return Ok(results) - - } - - pub async fn poster(url: &String, params: &HashMap<&str, &str>) -> Result> { - let mut results = String::new(); - match env::current_exe() { - Ok(exe_path) => { - // hash url - let mut val_map = String::from(""); - for (key, val) in params{ - val_map = val_map + &(format!("{}={}", key,val))[..] - } - let concat_url = url.to_owned() + &val_map[..]; - // if file is empty then cache does not exist - // then retrieve directly using reqwest - let (mut file, contents) = create_or_retrieve(concat_url, exe_path); - if (contents.as_str()).eq("") { - let client = reqwest::Client::new(); - let res = client.post(url) - .form(params) - .send() - .await - .unwrap() - .text() - .await - .unwrap(); - file.write_all((res.as_str()).as_bytes())?; - debug!("Retrieving {} [POST] data from web", url); - results = res; - } else { - debug!("Retrieving {} [POST] data from cache", url); - results = contents; - } - }, + } Err(e) => { - format!("failed to get current exe path: {}", e); - }, + return Err(Box::new(e)); + // format!("failed to get current exe path: {}", e); + } }; - return Ok(results) - + Ok(results) } - pub async fn getter(url: &String) -> Result> { - let mut results = String::new(); - match env::current_exe() { + pub async fn getter(url: &String) -> MythraResult { + let mut results = String::new(); + match env::current_exe() { Ok(exe_path) => { let (mut file, contents) = create_or_retrieve(url.to_string(), exe_path); // if file is empty then cache does not exist // then retrieve directly using reqwest if (contents.as_str()).eq("") { - let res = reqwest::get(url) - .await - .unwrap() - .text() - .await - .unwrap(); + let res = reqwest::get(url).await.unwrap().text().await.unwrap(); file.write_all((res.as_str()).as_bytes())?; debug!("Retrieving {} [GET] data from web", url); results = res; @@ -194,12 +215,12 @@ use fantoccini; debug!("Retrieving {} [GET] data from cache", url); results = contents; } - }, + } Err(e) => { format!("failed to get current exe path: {}", e); - }, + } }; - return Ok(results) + Ok(results) } pub async fn get(url: &String) -> String { @@ -207,81 +228,61 @@ use fantoccini; getter(&new_url).await.ok().unwrap() } - pub async fn post(url: &String, params: &HashMap<&str, &str>) -> String { + pub async fn js_post( + url: &String, + form_selector: &str, + params: &HashMap<&str, &str>, + ) -> String { let new_url = url.clone(); - poster(&new_url, params).await.ok().unwrap() - } - - pub async fn js_post(url: &String, form_selector: &str, params: &HashMap<&str, &str>) -> String { - let new_url = url.clone(); - fantoccini_form(&new_url, form_selector, params).await.ok().unwrap() - } -} - -pub fn render_select_music(songs:Vec, title: &str){ - let mut select = SelectView::new() - .h_align(HAlign::Center) - .autojump(); - for song in songs { - let title_copy = &song.title.clone(); - let title = title_copy.as_str(); - select.add_item(title, song); + fantoccini_form(&new_url, form_selector, params) + .await + .ok() + .unwrap() } - let mut siv = cursive::default(); - select.set_on_submit(render_downloading_song); - // Let's add a ResizedView to keep the - // list at a reasonable size (it can scroll anyway). - siv.add_layer( - Dialog::around(select.scrollable().fixed_size((50, 10))) - .title(title), - ); - siv.run(); -} - -fn render_downloading_song(siv: &mut Cursive, song: &Music){ -// replace previous view - let link_copy = song.download_link.clone(); - let use_link = &link_copy[..]; - let download_size_u64 = download_size(use_link).unwrap(); - let cb = siv.cb_sink().clone(); - siv.add_layer(Dialog::around( - ProgressBar::new() - .range(0, download_size_u64 as usize) - .with_task(move |counter| { - download_from_url(counter, link_copy.to_owned()); - cb.send(Box::new(completed_download)).unwrap(); - }) - .full_width(), - ) - .button("Minimize", |siv| { - siv.pop_layer(); - }) - ); - siv.set_autorefresh(true); -} -// complete download callback -fn completed_download(siv: &mut Cursive) { - siv.set_autorefresh(false); - siv.pop_layer(); - siv.add_layer( - Dialog::new() - .title("Download complete") - .button("Return", |siv| { - siv.pop_layer(); - }), - ); + /* + * pub async fn poster(url: &String, params: &HashMap<&str, &str>) -> MythraResult { + * let results = match env::current_exe() { + * Ok(exe_path) => { + * // hash url + * let mut val_map = String::from(""); + * for (key, val) in params { + * val_map = val_map + &(format!("{}={}", key, val))[..] + * } + * let concat_url = url.to_owned() + &val_map[..]; + * // if file is empty then cache does not exist + * // then retrieve directly using reqwest + * let (mut file, contents) = create_or_retrieve(concat_url, exe_path); + * if (contents.as_str()).eq("") { + * let client = reqwest::Client::new(); + * let res = client + * .post(url) + * .form(params) + * .send() + * .await + * .unwrap() + * .text() + * .await + * .unwrap(); + * file.write_all((res.as_str()).as_bytes())?; + * debug!("Retrieving {} [POST] data from web", url); + * res + * } else { + * debug!("Retrieving {} [POST] data from cache", url); + * contents + * } + * } + * Err(e) => { + * return Err(Box::new(e)); + * // format!("failed to get current exe path: {}", e); + * } + * }; + * return Ok(results); + * } + * + * pub async fn post(url: &String, params: &HashMap<&str, &str>) -> String { + * let new_url = url.clone(); + * poster(&new_url, params).await.ok().unwrap() + * } + */ } - - -// configure logging -pub fn configure_log(level: &str){ - let logname: &str = "MYTHRA_LOG_FMT" ; - // activate debugging for only actix_web and mythra - let logfmt = format!("actix_web={},mythra={}", level, level); - env::set_var(logname, logfmt); - Builder::new() - .parse_env(logname) - .init(); -} -