Skip to content

[YouTube] Extract trends from A/B tested "Videos" tab and fix extraction of trends name from A/B tested new title design #1045

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 19, 2023
Merged

[YouTube] Extract trends from A/B tested "Videos" tab and fix extraction of trends name from A/B tested new title design #1045

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
24eba62
fix: extract YouTube trends from new "Videos" tab
Theta-Dev Apr 2, 2023
0addb98
tests: update mocks
Theta-Dev Apr 16, 2023
3673d4a
fix: YouTube trending name extraction
Theta-Dev May 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 27 additions & 10 deletions ...va/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeTrendingExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,15 @@

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.stream.Stream;

import javax.annotation.Nonnull;

public class YoutubeTrendingExtractor extends KioskExtractor<StreamInfoItem> {
private JsonObject initialData;

private static final String VIDEOS_TAB_PARAMS = "4gIOGgxtb3N0X3BvcHVsYXI%3D";

public YoutubeTrendingExtractor(final StreamingService service,
final ListLinkHandler linkHandler,
final String kioskId) {
Expand All @@ -60,6 +63,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
final byte[] body = JsonWriter.string(prepareDesktopJsonBuilder(getExtractorLocalization(),
getExtractorContentCountry())
.value("browseId", "FEtrending")
.value("params", VIDEOS_TAB_PARAMS)
.done())
.getBytes(StandardCharsets.UTF_8);
// @formatter:on
Expand All @@ -81,6 +85,8 @@ public String getName() throws ParsingException {
name = getTextAtKey(header.getObject("feedTabbedHeaderRenderer"), "title");
} else if (header.has("c4TabbedHeaderRenderer")) {
name = getTextAtKey(header.getObject("c4TabbedHeaderRenderer"), "title");
} else if (header.has("pageHeaderRenderer")) {
name = getTextAtKey(header.getObject("pageHeaderRenderer"), "pageTitle");
}

if (isNullOrEmpty(name)) {
Expand All @@ -94,7 +100,10 @@ public String getName() throws ParsingException {
public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final TimeAgoParser timeAgoParser = getTimeAgoParser();
final JsonObject tabContent = getTrendingTabContent();
final JsonObject tab = getTrendingTab();
final JsonObject tabContent = tab.getObject("content");
final boolean isVideoTab = tab.getObject("endpoint").getObject("browseEndpoint")
.getString("params", "").equals(VIDEOS_TAB_PARAMS);

if (tabContent.has("richGridRenderer")) {
tabContent.getObject("richGridRenderer")
Expand All @@ -110,7 +119,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
.forEachOrdered(videoRenderer -> collector.commit(
new YoutubeStreamInfoItemExtractor(videoRenderer, timeAgoParser)));
} else if (tabContent.has("sectionListRenderer")) {
tabContent.getObject("sectionListRenderer")
final Stream<JsonObject> shelves = tabContent.getObject("sectionListRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
Expand All @@ -120,11 +129,19 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
.stream())
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.map(content -> content.getObject("shelfRenderer"))
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends
.filter(shelfRenderer -> !shelfRenderer.has("title"))
.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
.map(content -> content.getObject("shelfRenderer"));

final Stream<JsonObject> items;
if (isVideoTab) {
// The first shelf of the Videos tab contains the normal trends
items = shelves.findFirst().stream();
} else {
// Filter Trending shorts and Recently trending sections which have a title,
// contrary to normal trends
items = shelves.filter(shelfRenderer -> !shelfRenderer.has("title"));
}

items.flatMap(shelfRenderer -> shelfRenderer.getObject("content")
.getObject("expandedShelfContentsRenderer")
.getArray("items")
.stream())
Expand All @@ -138,7 +155,7 @@ public InfoItemsPage<StreamInfoItem> getInitialPage() throws ParsingException {
return new InfoItemsPage<>(collector, null);
}

private JsonObject getTrendingTabContent() throws ParsingException {
private JsonObject getTrendingTab() throws ParsingException {
return initialData.getObject("contents")
.getObject("twoColumnBrowseResultsRenderer")
.getArray("tabs")
Expand All @@ -150,7 +167,7 @@ private JsonObject getTrendingTabContent() throws ParsingException {
.filter(tabRenderer -> tabRenderer.has("content"))
// There should be at most one tab selected
.findFirst()
.orElseThrow(() -> new ParsingException("Could not get \"Now\" trending tab"))
.getObject("content");
.orElseThrow(() ->
new ParsingException("Could not get \"Now\" or \"Videos\" trending tab"));
}
}
21 changes: 12 additions & 9 deletions extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"Referer": [
"https://www.youtube.com"
],
"Referer": [
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
Expand All @@ -29,22 +29,25 @@
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 22 Nov 2022 10:40:53 GMT"
"Sun, 16 Apr 2023 17:42:25 GMT"
],
"expires": [
"Tue, 22 Nov 2022 10:40:53 GMT"
"Sun, 16 Apr 2023 17:42:25 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
Expand All @@ -59,9 +62,9 @@
"ESF"
],
"set-cookie": [
"YSC\u003dKWKE6LaMJTE; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:53 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+649; expires\u003dThu, 21-Nov-2024 10:40:53 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003deNq1o5_KNzg; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:42:25 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+549; expires\u003dTue, 15-Apr-2025 17:42:25 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
Expand Down
17 changes: 10 additions & 7 deletions extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_1.json
View file
Open in desktop

Large diffs are not rendered by default.

60 changes: 49 additions & 11 deletions extractor/src/test/resources/org/schabi/newpipe/extractor/kiosk/generated_mock_2.json
View file
Open in desktop

Large diffs are not rendered by default.

21 changes: 12 additions & 9 deletions .../schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_0.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@
"httpMethod": "GET",
"url": "https://www.youtube.com/sw.js",
"headers": {
"Origin": [
"Referer": [
"https://www.youtube.com"
],
"Referer": [
"Origin": [
"https://www.youtube.com"
],
"Accept-Language": [
Expand All @@ -29,22 +29,25 @@
"https://www.youtube.com"
],
"alt-svc": [
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000,h3-Q050\u003d\":443\"; ma\u003d2592000,h3-Q046\u003d\":443\"; ma\u003d2592000,h3-Q043\u003d\":443\"; ma\u003d2592000,quic\u003d\":443\"; ma\u003d2592000; v\u003d\"46,43\""
"h3\u003d\":443\"; ma\u003d2592000,h3-29\u003d\":443\"; ma\u003d2592000"
],
"cache-control": [
"private, max-age\u003d0"
],
"content-type": [
"text/javascript; charset\u003dutf-8"
],
"cross-origin-opener-policy-report-only": [
"cross-origin-opener-policy": [
"same-origin; report-to\u003d\"youtube_main\""
],
"date": [
"Tue, 22 Nov 2022 10:40:26 GMT"
"Sun, 16 Apr 2023 17:35:48 GMT"
],
"expires": [
"Tue, 22 Nov 2022 10:40:26 GMT"
"Sun, 16 Apr 2023 17:35:48 GMT"
],
"origin-trial": [
"AvC9UlR6RDk2crliDsFl66RWLnTbHrDbp+DiY6AYz/PNQ4G4tdUTjrHYr2sghbkhGQAVxb7jaPTHpEVBz0uzQwkAAAB4eyJvcmlnaW4iOiJodHRwczovL3lvdXR1YmUuY29tOjQ0MyIsImZlYXR1cmUiOiJXZWJWaWV3WFJlcXVlc3RlZFdpdGhEZXByZWNhdGlvbiIsImV4cGlyeSI6MTcxOTUzMjc5OSwiaXNTdWJkb21haW4iOnRydWV9"
],
"p3p": [
"CP\u003d\"This is not a P3P policy! See http://support.google.com/accounts/answer/151657?hl\u003den-GB for more info.\""
Expand All @@ -59,9 +62,9 @@
"ESF"
],
"set-cookie": [
"YSC\u003daSSq4mC6HTI; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dWed, 26-Feb-2020 10:40:26 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+953; expires\u003dThu, 21-Nov-2024 10:40:26 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
"YSC\u003dKx2AfujDxNk; Domain\u003d.youtube.com; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"VISITOR_INFO1_LIVE\u003d; Domain\u003d.youtube.com; Expires\u003dMon, 20-Jul-2020 17:35:48 GMT; Path\u003d/; Secure; HttpOnly; SameSite\u003dnone",
"CONSENT\u003dPENDING+605; expires\u003dTue, 15-Apr-2025 17:35:48 GMT; path\u003d/; domain\u003d.youtube.com; Secure"
],
"strict-transport-security": [
"max-age\u003d31536000"
Expand Down
15 changes: 9 additions & 6 deletions .../schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_1.json
View file
Open in desktop

Large diffs are not rendered by default.

60 changes: 49 additions & 11 deletions .../schabi/newpipe/extractor/services/youtube/extractor/kiosk/trending/generated_mock_2.json
View file
Open in desktop

Large diffs are not rendered by default.