27.07.2024

Mithronn · Mithronn · commit 9ca056751dde · 2024-07-27T20:45:59.000+03:00
* Added age restricted videos support
diff --git a/Cargo.toml b/Cargo.toml
@@ -26,34 +26,34 @@ reqwest = { version = "0.12.5", features = [
     "cookies",
     "gzip",
 ], default-features = false }
-scraper = "0.19.0"
-serde = "1.0.202"
-serde_json = "1.0.117"
+scraper = "0.19.1"
+serde = "1.0.204"
+serde_json = "1.0.120"
 serde_qs = "0.13.0"
-regex = "1.10.3"
-url = "2.5.0"
+regex = "1.10.5"
+url = "2.5.2"
 urlencoding = "2.1.3"
-thiserror = "1.0.60"
-derive_more = "0.99.17"
+thiserror = "1.0.63"
+derive_more = "0.99.18"
 derivative = "2.2.0"
 once_cell = "1.19.0"
-tokio = { version = "1.37.0", default-features = false, features = ["sync"] }
+tokio = { version = "1.39.2", default-features = false, features = ["sync"] }
 rand = "0.8.5"
 reqwest-middleware = { version = "0.3.2", features = ["json"] }
 reqwest-retry = "0.6.0"
 m3u8-rs = "6.0.0"
-async-trait = "0.1.80"
+async-trait = "0.1.81"
 aes = "0.8.4"
 cbc = { version = "0.1.2", features = ["std"] }
 hex = "0.4.3"
 boa_engine = "0.17.3"
 mime = "0.3.17"
-bytes = "1.6.0"
+bytes = "1.6.1"
 flame = { version = "0.2.2", optional = true }
 flamer = { version = "0.5.0", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1.37.0", features = ["full"] }
+tokio = { version = "1.39.2", features = ["full"] }
 
 [features]
 default = ["search", "live", "default-tls"]
diff --git a/src/blocking/search/youtube.rs b/src/blocking/search/youtube.rs
@@ -98,6 +98,7 @@ impl Playlist {
     /// Get next chunk of videos from playlist and return fetched [`Video`] array.
     /// - If limit is [`None`] it will be [`u64::MAX`]
     /// - If [`Playlist`] is coming from [`SearchResult`] this function always return empty [`Vec<Video>`]!
+    ///
     /// to use this function with [`SearchResult`] follow example
     ///
     /// # Example
@@ -131,6 +132,7 @@ impl Playlist {
     /// Try to fetch all playlist videos and return [`Playlist`].
     /// - If limit is [`None`] it will be [`u64::MAX`]
     /// - If [`Playlist`] is coming from [`SearchResult`] this function always return [`Playlist`] with empty [`Vec<Video>`]!
+    ///
     /// to use this function with [`SearchResult`] follow example
     ///
     /// # Example
diff --git a/src/info.rs b/src/info.rs
@@ -1,5 +1,6 @@
+use once_cell::sync::Lazy;
 use reqwest::{
-    header::{HeaderMap, HeaderValue, COOKIE},
+    header::{HeaderMap, HeaderName, HeaderValue, COOKIE},
     Client,
 };
 use reqwest_middleware::{ClientBuilder, ClientWithMiddleware};
@@ -17,11 +18,12 @@ use crate::{
     constants::BASE_URL,
     info_extras::{get_media, get_related_videos},
     stream::{NonLiveStream, NonLiveStreamOptions, Stream},
-    structs::{PlayerResponse, VideoError, VideoInfo, VideoOptions},
+    structs::{PlayerResponse, VideoError, VideoInfo, VideoOptions, YTConfig},
     utils::{
         between, choose_format, clean_video_details, get_functions, get_html, get_html5player,
-        get_random_v6_ip, get_video_id, is_not_yet_broadcasted, is_play_error, is_private_video,
-        is_rental, parse_live_video_formats, parse_video_formats, sort_formats,
+        get_random_v6_ip, get_video_id, get_ytconfig, is_age_restricted_from_html,
+        is_not_yet_broadcasted, is_play_error, is_private_video, is_rental,
+        parse_live_video_formats, parse_video_formats, sort_formats,
     },
 };
 
@@ -120,7 +122,7 @@ impl Video {
 
         let response = get_html(client, url_parsed.as_str(), None).await?;
 
-        let (player_response, initial_response): (PlayerResponse, serde_json::Value) = {
+        let (mut player_response, initial_response): (PlayerResponse, serde_json::Value) = {
             let document = Html::parse_document(&response);
             let scripts_selector = Selector::parse("script").unwrap();
             let player_response_string = document
@@ -158,14 +160,23 @@ impl Video {
             return Err(VideoError::VideoNotFound);
         }
 
-        if is_private_video(&player_response) {
+        let is_age_restricted = is_age_restricted_from_html(&player_response, &response);
+
+        if is_private_video(&player_response) && !is_age_restricted {
             return Err(VideoError::VideoIsPrivate);
         }
 
-        if player_response.streaming_data.is_none()
-            || is_rental(&player_response)
-            || is_not_yet_broadcasted(&player_response)
-        {
+        if is_age_restricted {
+            let embed_ytconfig = self.get_embeded_ytconfig(&response).await?;
+
+            let player_response_new =
+                serde_json::from_str::<PlayerResponse>(&embed_ytconfig).unwrap();
+
+            player_response.streaming_data = player_response_new.streaming_data;
+            player_response.storyboards = player_response_new.storyboards;
+        }
+
+        if is_rental(&player_response) || is_not_yet_broadcasted(&player_response) {
             return Err(VideoError::VideoSourceNotFound);
         }
 
@@ -456,6 +467,68 @@ impl Video {
     pub(crate) fn get_options(&self) -> VideoOptions {
         self.options.clone()
     }
+
+    #[cfg_attr(feature = "performance_analysis", flamer::flame)]
+    async fn get_embeded_ytconfig(&self, html: &str) -> Result<String, VideoError> {
+        let ytcfg = get_ytconfig(html)?;
+
+        // This client can access age restricted videos (unless the uploader has disabled the 'allow embedding' option)
+        // See: https://github.com/yt-dlp/yt-dlp/blob/28d485714fef88937c82635438afba5db81f9089/yt_dlp/extractor/youtube.py#L231
+        let query = serde_json::json!({
+            "context": {
+                "client": {
+                    "clientName": "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
+                    "clientVersion": "2.0",
+                    "hl": "en",
+                    "clientScreen": "EMBED",
+                },
+                "thirdParty": {
+                    "embedUrl": "https://google.com",
+                },
+            },
+            "playbackContext": {
+                "contentPlaybackContext": {
+                    "signatureTimestamp": ytcfg.sts.unwrap_or(0),
+                    "html5Preference": "HTML5_PREF_WANTS",
+                },
+            },
+            "videoId": self.get_video_id(),
+        });
+
+        static CONFIGS: Lazy<(HeaderMap, &str)> = Lazy::new(|| {
+            use std::str::FromStr;
+
+            (HeaderMap::from_iter([
+            (HeaderName::from_str("content-type").unwrap(), HeaderValue::from_str("application/json").unwrap()),
+            (HeaderName::from_str("X-Youtube-Client-Name").unwrap(), HeaderValue::from_str("85").unwrap()),
+            (HeaderName::from_str("X-Youtube-Client-Version").unwrap(), HeaderValue::from_str("2.0").unwrap()),
+            (HeaderName::from_str("Origin").unwrap(), HeaderValue::from_str("https://www.youtube.com").unwrap()),
+            (HeaderName::from_str("User-Agent").unwrap(), HeaderValue::from_str("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3513.0 Safari/537.36").unwrap()),
+            (HeaderName::from_str("Referer").unwrap(), HeaderValue::from_str("https://www.youtube.com/").unwrap()),
+            (HeaderName::from_str("Accept").unwrap(), HeaderValue::from_str("text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8").unwrap()),
+            (HeaderName::from_str("Accept-Language").unwrap(), HeaderValue::from_str("en-US,en;q=0.5").unwrap()),
+            (HeaderName::from_str("Accept-Encoding").unwrap(), HeaderValue::from_str("gzip, deflate").unwrap()),
+        ]),"AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8")
+        });
+
+        let response = self
+            .client
+            .post("https://www.youtube.com/youtubei/v1/player")
+            .headers(CONFIGS.0.clone())
+            .query(&[("key", CONFIGS.1)])
+            .json(&query)
+            .send()
+            .await
+            .map_err(VideoError::ReqwestMiddleware)?;
+
+        let response = response
+            .error_for_status()
+            .map_err(VideoError::Reqwest)?
+            .text()
+            .await?;
+
+        Ok(response)
+    }
 }
 
 async fn get_m3u8(
diff --git a/src/structs.rs b/src/structs.rs
@@ -987,3 +987,9 @@ pub struct ErrorScreen {
     #[serde(rename = "playerLegacyDesktopYpcOfferRenderer")]
     pub player_legacy_desktop_ypc_offer_renderer: Option<String>,
 }
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct YTConfig {
+    #[serde(rename = "STS")]
+    pub sts: Option<u64>,
+}
diff --git a/src/utils.rs b/src/utils.rs
@@ -2,6 +2,7 @@ use boa_engine::{Context, Source};
 use once_cell::sync::Lazy;
 use rand::Rng;
 use regex::Regex;
+use scraper::{Html, Selector};
 use serde::{Deserialize, Serialize};
 use std::{
     borrow::Cow,
@@ -20,7 +21,7 @@ use crate::{
     info_extras::{get_author, get_chapters, get_dislikes, get_likes, get_storyboards},
     structs::{
         Embed, PlayerResponse, StreamingDataFormat, StringUtils, VideoDetails, VideoError,
-        VideoFormat, VideoOptions, VideoQuality, VideoSearchOptions,
+        VideoFormat, VideoOptions, VideoQuality, VideoSearchOptions, YTConfig,
     },
 };
 
@@ -500,7 +501,7 @@ fn ncode(
     n_transfrom_cache: &mut HashMap<String, String>,
 ) -> String {
     let components: serde_json::value::Map<String, serde_json::Value> =
-        serde_qs::from_str(&decode(url).unwrap_or(Cow::Borrowed(url))).unwrap();
+        serde_qs::from_str(&decode(url).unwrap_or(Cow::Borrowed(url))).unwrap_or_default();
 
     let n_transform_value = match components.get("n").and_then(serde_json::Value::as_str) {
         Some(val) if !n_transform_script_string.1.is_empty() => val,
@@ -534,7 +535,7 @@ fn ncode(
             .and_then(|result| {
                 result
                     .as_string()
-                    .map(|js_str| js_str.to_std_string().unwrap())
+                    .map(|js_str| js_str.to_std_string().unwrap_or_default())
             })
     }
 
@@ -888,6 +889,54 @@ pub fn is_age_restricted(media: &serde_json::Value) -> bool {
     age_restricted
 }
 
+#[cfg_attr(feature = "performance_analysis", flamer::flame)]
+pub fn is_age_restricted_from_html(player_response: &PlayerResponse, html: &str) -> bool {
+    if !player_response
+        .micro_format
+        .as_ref()
+        .and_then(|x| x.player_micro_format_renderer.clone())
+        .and_then(|x| x.is_family_safe)
+        .unwrap_or(true)
+    {
+        return true;
+    }
+
+    let document = Html::parse_document(html);
+    let metas_selector = Selector::parse("meta").unwrap();
+
+    // <meta property="og:restrictions:age" content="18+">
+    let og_restrictions_age = document
+        .select(&metas_selector)
+        .filter(|x| {
+            x.attr("itemprop")
+                .or(x.attr("name"))
+                .or(x.attr("property"))
+                .or(x.attr("id"))
+                .or(x.attr("http-equiv"))
+                == Some("og:restrictions:age")
+        })
+        .map(|x| x.attr("content").unwrap_or("").to_string())
+        .next()
+        .unwrap_or(String::from(""));
+
+    // <meta itemprop="isFamilyFriendly" content="true">
+    let is_family_friendly = document
+        .select(&metas_selector)
+        .filter(|x| {
+            x.attr("itemprop")
+                .or(x.attr("name"))
+                .or(x.attr("property"))
+                .or(x.attr("id"))
+                .or(x.attr("http-equiv"))
+                == Some("isFamilyFriendly")
+        })
+        .map(|x| x.attr("content").unwrap_or("").to_string())
+        .next()
+        .unwrap_or(String::from(""));
+
+    is_family_friendly == "false" || og_restrictions_age == "18+"
+}
+
 #[cfg_attr(feature = "performance_analysis", flamer::flame)]
 pub fn is_rental(player_response: &PlayerResponse) -> bool {
     if player_response.playability_status.is_none() {
@@ -946,6 +995,17 @@ pub fn is_private_video(player_response: &PlayerResponse) -> bool {
         .unwrap_or(false)
 }
 
+pub fn get_ytconfig(html: &str) -> Result<YTConfig, VideoError> {
+    static PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r#"ytcfg\.set\((\{.*\})\)"#).unwrap());
+    match PATTERN.captures(html) {
+        Some(c) => Ok(
+            serde_json::from_str::<YTConfig>(c.get(1).map_or("", |m| m.as_str()))
+                .map_err(|_x| VideoError::VideoSourceNotFound)?,
+        ),
+        None => Err(VideoError::VideoSourceNotFound),
+    }
+}
+
 type CacheFunctions = Lazy<RwLock<Option<(String, Vec<(String, String)>)>>>;
 static FUNCTIONS: CacheFunctions = Lazy::new(|| RwLock::new(None));
 
