Improve stream extractor tests & various fixes

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCConferenceExtractor.java

@@ -13,6 +13,7 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems.MediaCCCStreamInfoItemExtractor;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 
@@ -71,8 +72,8 @@ public String getParentChannelAvatarUrl() throws ParsingException {
     @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() {
-        StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
-        JsonArray events = conferenceData.getArray("events");
+        final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
+        final JsonArray events = conferenceData.getArray("events");
         for (int i = 0; i < events.size(); i++) {
             collector.commit(new MediaCCCStreamInfoItemExtractor(events.getObject(i)));
         }
@@ -87,10 +88,11 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
+        final String conferenceUrl = MediaCCCConferenceLinkHandlerFactory.CONFERENCE_API_ENDPOINT + getId();
         try {
-            conferenceData = JsonParser.object().from(downloader.get(getUrl()).responseBody());
+            conferenceData = JsonParser.object().from(downloader.get(conferenceUrl).responseBody());
         } catch (JsonParserException jpe) {
-            throw new ExtractionException("Could not parse json returnd by url: " + getUrl());
+            throw new ExtractionException("Could not parse json returnd by url: " + conferenceUrl);
         }
     }
 
@@ -99,10 +101,4 @@ public void onFetchPage(@Nonnull final Downloader downloader)
     public String getName() throws ParsingException {
         return conferenceData.getString("title");
     }
-
-    @Nonnull
-    @Override
-    public String getOriginalUrl() {
-        return "https://media.ccc.de/c/" + conferenceData.getString("acronym");
-    }
 }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCParsingHelper.java

@@ -6,14 +6,17 @@
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.TimeZone;
 
 public final class MediaCCCParsingHelper {
     private MediaCCCParsingHelper() { }
 
     public static Calendar parseDateFrom(final String textualUploadDate) throws ParsingException {
-        Date date;
+        final Date date;
         try {
-            date = new SimpleDateFormat("yyyy-MM-dd").parse(textualUploadDate);
+            final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
+            sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+            date = sdf.parse(textualUploadDate);
         } catch (ParseException e) {
             throw new ParsingException("Could not parse date: \"" + textualUploadDate + "\"", e);
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/extractors/MediaCCCStreamExtractor.java

@@ -19,14 +19,18 @@
 import org.schabi.newpipe.extractor.stream.StreamType;
 import org.schabi.newpipe.extractor.stream.SubtitlesStream;
 import org.schabi.newpipe.extractor.stream.VideoStream;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCConferenceLinkHandlerFactory;
+import org.schabi.newpipe.extractor.services.media_ccc.linkHandler.MediaCCCStreamLinkHandlerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
 
 public class MediaCCCStreamExtractor extends StreamExtractor {
     private JsonObject data;
@@ -93,7 +97,7 @@ public long getDislikeCount() {
     @Nonnull
     @Override
     public String getUploaderUrl() {
-        return data.getString("conference_url");
+        return MediaCCCConferenceLinkHandlerFactory.CONFERENCE_PATH + getUploaderName();
     }
 
     @Nonnull
@@ -111,25 +115,25 @@ public String getUploaderAvatarUrl() {
 
     @Nonnull
     @Override
-    public String getSubChannelUrl() throws ParsingException {
+    public String getSubChannelUrl() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelName() throws ParsingException {
+    public String getSubChannelName() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getSubChannelAvatarUrl() throws ParsingException {
+    public String getSubChannelAvatarUrl() {
         return "";
     }
 
     @Nonnull
     @Override
-    public String getDashMpdUrl() throws ParsingException {
+    public String getDashMpdUrl() {
         return "";
     }
 
@@ -194,7 +198,7 @@ public List<VideoStream> getVideoStreams() throws ExtractionException {
 
     @Override
     public List<VideoStream> getVideoOnlyStreams() {
-        return null;
+        return Collections.emptyList();
     }
 
     @Nonnull
@@ -214,9 +218,10 @@ public StreamType getStreamType() {
         return StreamType.VIDEO_STREAM;
     }
 
+    @Nullable
     @Override
     public StreamInfoItemsCollector getRelatedStreams() {
-        return new StreamInfoItemsCollector(getServiceId());
+        return null;
     }
 
     @Override
@@ -227,14 +232,13 @@ public String getErrorMessage() {
     @Override
     public void onFetchPage(@Nonnull final Downloader downloader)
             throws IOException, ExtractionException {
+        final String videoUrl = MediaCCCStreamLinkHandlerFactory.VIDEO_API_ENDPOINT + getId();
         try {
-            data = JsonParser.object().from(
-                    downloader.get(getLinkHandler().getUrl()).responseBody());
+            data = JsonParser.object().from(downloader.get(videoUrl).responseBody());
             conferenceData = JsonParser.object()
-                    .from(downloader.get(getUploaderUrl()).responseBody());
+                    .from(downloader.get(data.getString("conference_url")).responseBody());
         } catch (JsonParserException jpe) {
-            throw new ExtractionException("Could not parse json returned by url: "
-                    + getLinkHandler().getUrl(), jpe);
+            throw new ExtractionException("Could not parse json returned by url: " + videoUrl, jpe);
         }
     }
 
@@ -250,21 +254,25 @@ public String getOriginalUrl() {
         return data.getString("frontend_link");
     }
 
+    @Nonnull
     @Override
     public String getHost() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getPrivacy() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getCategory() {
         return "";
     }
 
+    @Nonnull
     @Override
     public String getLicence() {
         return "";
@@ -278,7 +286,7 @@ public Locale getLanguageInfo() {
     @Nonnull
     @Override
     public List<String> getTags() {
-        return new ArrayList<>();
+        return Arrays.asList(data.getArray("tags").toArray(new String[0]));
     }
 
     @Nonnull

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCConferenceLinkHandlerFactory.java

@@ -7,30 +7,26 @@
 import java.util.List;
 
 public class MediaCCCConferenceLinkHandlerFactory extends ListLinkHandlerFactory {
+    public static final String CONFERENCE_API_ENDPOINT = "https://api.media.ccc.de/public/conferences/";
+    public static final String CONFERENCE_PATH = "https://media.ccc.de/c/";
+    private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/conferences/)|(?:media\\.ccc\\.de/[bc]/))([^/?&#]*)";
+
     @Override
-    public String getUrl(final String id, final List<String> contentFilter, final String sortFilter)
-            throws ParsingException {
-        return "https://media.ccc.de/public/conferences/" + id;
+    public String getUrl(final String id,
+                         final List<String> contentFilter,
+                         final String sortFilter) throws ParsingException {
+        return CONFERENCE_PATH + id;
     }
 
     @Override
     public String getId(final String url) throws ParsingException {
-        if (url.startsWith("https://media.ccc.de/public/conferences/")
-                || url.startsWith("https://api.media.ccc.de/public/conferences/")) {
-            return url.replaceFirst("https://(api\\.)?media\\.ccc\\.de/public/conferences/", "");
-        } else if (url.startsWith("https://media.ccc.de/c/")) {
-            return Parser.matchGroup1("https://media.ccc.de/c/([^?#]*)", url);
-        } else if (url.startsWith("https://media.ccc.de/b/")) {
-            return Parser.matchGroup1("https://media.ccc.de/b/([^?#]*)", url);
-        }
-        throw new ParsingException("Could not get id from url: " + url);
+        return Parser.matchGroup1(ID_PATTERN, url);
     }
 
     @Override
     public boolean onAcceptUrl(final String url) {
         try {
-            getId(url);
-            return true;
+            return getId(url) != null;
         } catch (ParsingException e) {
             return false;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/media_ccc/linkHandler/MediaCCCStreamLinkHandlerFactory.java

@@ -2,54 +2,27 @@
 
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.LinkHandlerFactory;
-import org.schabi.newpipe.extractor.utils.Utils;
-
-import java.net.MalformedURLException;
-import java.net.URL;
+import org.schabi.newpipe.extractor.utils.Parser;
 
 public class MediaCCCStreamLinkHandlerFactory extends LinkHandlerFactory {
-    @Override
-    public String getId(final String urlString) throws ParsingException {
-        if (urlString.startsWith("https://media.ccc.de/public/events/")
-                && !urlString.contains("?q=")) {
-            return urlString.substring(35); //remove …/public/events part
-        }
+    public static final String VIDEO_API_ENDPOINT = "https://api.media.ccc.de/public/events/";
+    private static final String VIDEO_PATH = "https://media.ccc.de/v/";
+    private static final String ID_PATTERN = "(?:(?:(?:api\\.)?media\\.ccc\\.de/public/events/)|(?:media\\.ccc\\.de/v/))([^/?&#]*)";
 
-        if (urlString.startsWith("https://api.media.ccc.de/public/events/")
-                && !urlString.contains("?q=")) {
-            return urlString.substring(39); //remove api…/public/events part
-        }
-
-        URL url;
-        try {
-            url = Utils.stringToURL(urlString);
-        } catch (MalformedURLException e) {
-            throw new IllegalArgumentException("The given URL is not valid");
-        }
-
-        String path = url.getPath();
-        // remove leading "/" of URL-path if URL-path is given
-        if (!path.isEmpty()) {
-            path = path.substring(1);
-        }
-
-        if (path.startsWith("v/")) {
-            return path.substring(2);
-        }
-
-        throw new ParsingException("Could not get id from url: " + url);
+    @Override
+    public String getId(final String url) throws ParsingException {
+        return Parser.matchGroup1(ID_PATTERN, url);
     }
 
     @Override
     public String getUrl(final String id) throws ParsingException {
-        return "https://media.ccc.de/public/events/" + id;
+        return VIDEO_PATH + id;
     }
 
     @Override
     public boolean onAcceptUrl(final String url) {
         try {
-            getId(url);
-            return true;
+            return getId(url) != null;
         } catch (ParsingException e) {
             return false;
         }

extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeAccountExtractor.java

@@ -13,13 +13,16 @@
 import org.schabi.newpipe.extractor.exceptions.ParsingException;
 import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
 import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
+import org.schabi.newpipe.extractor.services.peertube.linkHandler.PeertubeChannelLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamInfoItem;
 import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
 import org.schabi.newpipe.extractor.utils.JsonUtils;
 import org.schabi.newpipe.extractor.utils.Utils;
 
 import java.io.IOException;
 
+import javax.annotation.Nonnull;
+
 import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
 import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
 import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;
@@ -85,14 +88,16 @@ public String getParentChannelAvatarUrl() {
         return "";
     }
 
+    @Nonnull
     @Override
     public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
-        final String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
-        return getPage(new Page(pageUrl));
+        return getPage(new Page(
+                getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE));
     }
 
     @Override
-    public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException, ExtractionException {
+    public InfoItemsPage<StreamInfoItem> getPage(final Page page)
+            throws IOException, ExtractionException {
         if (page == null || isNullOrEmpty(page.getUrl())) {
             throw new IllegalArgumentException("Page doesn't contain an URL");
         }
@@ -122,8 +127,16 @@ public InfoItemsPage<StreamInfoItem> getPage(final Page page) throws IOException
     }
 
     @Override
-    public void onFetchPage(final Downloader downloader) throws IOException, ExtractionException {
-        final Response response = downloader.get(getUrl());
+    public void onFetchPage(@Nonnull final Downloader downloader)
+            throws IOException, ExtractionException {
+        String accountUrl = baseUrl + PeertubeChannelLinkHandlerFactory.API_ENDPOINT;
+        if (getId().contains("accounts/")) {
+            accountUrl += getId();
+        } else {
+            accountUrl += "accounts/" + getId();
+        }
+
+        final Response response = downloader.get(accountUrl);
         if (response != null && response.responseBody() != null) {
             setInitialData(response.responseBody());
         } else {
@@ -140,13 +153,9 @@ private void setInitialData(final String responseBody) throws ExtractionExceptio
         if (json == null) throw new ExtractionException("Unable to extract PeerTube account data");
     }
 
+    @Nonnull
     @Override
     public String getName() throws ParsingException {
         return JsonUtils.getString(json, "displayName");
     }
-
-    @Override
-    public String getOriginalUrl() throws ParsingException {
-        return baseUrl + "/" + getId();
-    }
 }