Skip to content

Commit fbe9e62

Browse files
StypoxStypox
authored
Merge pull request #1168 from AudricV/yt_upd-cver-rm-keys-and-do-fixes
[YouTube] Update clients versions, restore access to some streams and more
2 parents 6c3c2e2 + 4e9e7cb commit fbe9e62

File tree

393 files changed

+15593
-15085
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

393 files changed

+15593
-15085
lines changed

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeMetaInfoHelper.java‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,16 +170,34 @@ private static void getEmergencyOneboxRenderer(
170170

171171
// usually an encouragement like "We are with you"
172172
final String title = getTextFromObjectOrThrow(r.getObject("title"), "title");
173+
173174
// usually a phone number
174-
final String action = getTextFromObjectOrThrow(r.getObject("actionText"), "action");
175+
final String action; // this variable is expected to start with "\n"
176+
if (r.has("actionText")) {
177+
action = "\n" + getTextFromObjectOrThrow(r.getObject("actionText"), "action");
178+
} else if (r.has("contacts")) {
179+
final JsonArray contacts = r.getArray("contacts");
180+
final StringBuilder stringBuilder = new StringBuilder();
181+
// Loop over contacts item from the first contact to the last one
182+
for (int i = 0; i < contacts.size(); i++) {
183+
stringBuilder.append("\n");
184+
stringBuilder.append(getTextFromObjectOrThrow(contacts.getObject(i)
185+
.getObject("actionText"), "contacts.actionText"));
186+
}
187+
action = stringBuilder.toString();
188+
} else {
189+
action = "";
190+
}
191+
175192
// usually details about the phone number
176193
final String details = getTextFromObjectOrThrow(r.getObject("detailsText"), "details");
194+
177195
// usually the name of an association
178196
final String urlText = getTextFromObjectOrThrow(r.getObject("navigationText"),
179197
"urlText");
180198

181199
metaInfo.setTitle(title);
182-
metaInfo.setContent(new Description(details + "\n" + action, Description.PLAIN_TEXT));
200+
metaInfo.setContent(new Description(details + action, Description.PLAIN_TEXT));
183201
metaInfo.addUrlText(urlText);
184202

185203
// usually the webpage of the association

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 98 additions & 146 deletions
Large diffs are not rendered by default.

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
3030
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
3131
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
32-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
3332
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
3433
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
3534
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
@@ -422,8 +421,8 @@ private Page getNextPageFrom(final JsonObject continuations,
422421
.done())
423422
.getBytes(StandardCharsets.UTF_8);
424423

425-
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
426-
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
424+
return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, null,
425+
channelIds, null, body);
427426
}
428427

429428
/**

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMixPlaylistExtractor.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
55
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
66
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
7-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
87
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
98
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
109
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
@@ -103,8 +102,8 @@ public void onFetchPage(@Nonnull final Downloader downloader)
103102
final var headers = getYouTubeHeaders();
104103

105104
final Response response = getDownloader().postWithContentTypeJson(
106-
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
107-
headers, body, localization);
105+
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body,
106+
localization);
108107

109108
initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
110109
playlistData = initialData
@@ -225,7 +224,8 @@ private Page getNextPageFrom(@Nonnull final JsonObject playlistJson,
225224
.done())
226225
.getBytes(StandardCharsets.UTF_8);
227226

228-
return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
227+
return new Page(YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null,
228+
cookies, body);
229229
}
230230

231231
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeMusicSearchExtractor.java‎

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
55
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
6+
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion;
67
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders;
78
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
89
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
@@ -25,10 +26,8 @@
2526
import org.schabi.newpipe.extractor.downloader.Downloader;
2627
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
2728
import org.schabi.newpipe.extractor.exceptions.ParsingException;
28-
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
2929
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
3030
import org.schabi.newpipe.extractor.search.SearchExtractor;
31-
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
3231
import org.schabi.newpipe.extractor.utils.JsonUtils;
3332

3433
import java.io.IOException;
@@ -52,10 +51,8 @@ public YoutubeMusicSearchExtractor(final StreamingService service,
5251
@Override
5352
public void onFetchPage(@Nonnull final Downloader downloader)
5453
throws IOException, ExtractionException {
55-
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
56-
57-
final String url = "https://music.youtube.com/youtubei/v1/search?key="
58-
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
54+
final String url = "https://music.youtube.com/youtubei/v1/search?"
55+
+ DISABLE_PRETTY_PRINT_PARAMETER;
5956

6057
final String params;
6158

@@ -86,7 +83,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
8683
.object("context")
8784
.object("client")
8885
.value("clientName", "WEB_REMIX")
89-
.value("clientVersion", youtubeMusicKeys[2])
86+
.value("clientVersion", getYoutubeMusicClientVersion())
9087
.value("hl", "en-GB")
9188
.value("gl", getExtractorContentCountry().getCountryCode())
9289
.value("platform", "DESKTOP")
@@ -206,15 +203,13 @@ public InfoItemsPage<InfoItem> getPage(final Page page)
206203

207204
final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());
208205

209-
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();
210-
211206
// @formatter:off
212207
final byte[] json = JsonWriter.string()
213208
.object()
214209
.object("context")
215210
.object("client")
216211
.value("clientName", "WEB_REMIX")
217-
.value("clientVersion", youtubeMusicKeys[2])
212+
.value("clientVersion", getYoutubeMusicClientVersion())
218213
.value("hl", "en-GB")
219214
.value("gl", getExtractorContentCountry().getCountryCode())
220215
.value("platform", "DESKTOP")
@@ -295,8 +290,7 @@ private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector,
295290
}
296291

297292
@Nullable
298-
private Page getNextPageFrom(final JsonArray continuations)
299-
throws IOException, ParsingException, ReCaptchaException {
293+
private Page getNextPageFrom(final JsonArray continuations) {
300294
if (isNullOrEmpty(continuations)) {
301295
return null;
302296
}
@@ -306,7 +300,6 @@ private Page getNextPageFrom(final JsonArray continuations)
306300
final String continuation = nextContinuationData.getString("continuation");
307301

308302
return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
309-
+ "&continuation=" + continuation + "&key="
310-
+ YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
303+
+ "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER);
311304
}
312305
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubePlaylistExtractor.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
55
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
66
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
7-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
87
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
98
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
109
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
@@ -387,8 +386,7 @@ private Page getNextPageFrom(final JsonArray contents)
387386
.done())
388387
.getBytes(StandardCharsets.UTF_8);
389388

390-
return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
391-
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
389+
return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, body);
392390
} else {
393391
return null;
394392
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeSearchExtractor.java‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
44
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
55
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
6-
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
76
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
87
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
98
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
@@ -247,8 +246,7 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
247246
}
248247

249248
@Nullable
250-
private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
251-
ExtractionException {
249+
private Page getNextPageFrom(final JsonObject continuationItemRenderer) {
252250
if (isNullOrEmpty(continuationItemRenderer)) {
253251
return null;
254252
}
@@ -257,8 +255,7 @@ private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws I
257255
.getObject("continuationCommand")
258256
.getString("token");
259257

260-
final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
261-
+ DISABLE_PRETTY_PRINT_PARAMETER;
258+
final String url = YOUTUBEI_V1_URL + "search?" + DISABLE_PRETTY_PRINT_PARAMETER;
262259

263260
return new Page(url, token);
264261
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -965,7 +965,7 @@ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentC
965965
.value(RACY_CHECK_OK, true)
966966
// Workaround getting streaming URLs which return 403 HTTP response code by
967967
// using some parameters for Android client requests
968-
.value("params", "CgIQBg")
968+
.value("params", "CgIIAQ%3D%3D")
969969
.done())
970970
.getBytes(StandardCharsets.UTF_8);
971971

‎extractor/src/test/java/org/schabi/newpipe/extractor/ExtractorAsserts.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,8 +171,17 @@ public static void assertContains(
171171
public static void assertTabsContain(@Nonnull final List<ListLinkHandler> tabs,
172172
@Nonnull final String... expectedTabs) {
173173
final Set<String> tabSet = tabs.stream()
174-
.map(linkHandler -> linkHandler.getContentFilters().get(0))
174+
.map(linkHandler -> {
175+
assertEquals(1, linkHandler.getContentFilters().size(),
176+
"Unexpected content filters for channel tab: "
177+
+ linkHandler.getContentFilters());
178+
return linkHandler.getContentFilters().get(0);
179+
})
175180
.collect(Collectors.toUnmodifiableSet());
181+
182+
assertEquals(expectedTabs.length, tabSet.size(),
183+
"Different amount of tabs returned:\nExpected: "
184+
+ Arrays.toString(expectedTabs) + "\nActual: " + tabSet);
176185
Arrays.stream(expectedTabs)
177186
.forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab),
178187
"Missing " + expectedTab + " tab (got " + tabSet + ")"));

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeChannelExtractorTest.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
2626
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
2727
import org.schabi.newpipe.extractor.exceptions.ParsingException;
28+
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
2829
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
2930
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
3031
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
@@ -540,7 +541,8 @@ public void testVerified() throws Exception {
540541
@Test
541542
@Override
542543
public void testTabs() throws Exception {
543-
assertTabsContain(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
544+
assertTabsContain(extractor.getTabs(),
545+
ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS, ChannelTabs.SHORTS);
544546
assertTrue(extractor.getTabs().stream()
545547
.filter(it -> ChannelTabs.VIDEOS.equals(it.getContentFilters().get(0)))
546548
.allMatch(ReadyChannelTabListLinkHandler.class::isInstance));
@@ -916,8 +918,10 @@ public void testVerified() throws Exception {
916918
@Test
917919
@Override
918920
public void testTabs() throws Exception {
919-
// Gaming topic channels tabs are not yet supported, so an empty list should be returned
920-
assertTrue(extractor.getTabs().isEmpty());
921+
// Gaming topic channels tabs are not yet supported
922+
// However, a Shorts tab like on other channel types is returned, so it is supported
923+
// Check that it is returned
924+
assertTabsContain(extractor.getTabs(), ChannelTabs.SHORTS);
921925
}
922926

923927
@Test

0 commit comments

Comments
 (0)