From a1479d04df090cae02b19fc48fbcb94b55ee72a7 Mon Sep 17 00:00:00 2001 From: Christian Schabesberger Date: Sun, 21 Feb 2016 21:05:16 +0100 Subject: [PATCH] fixed some searchengine errors --- .../youtube/YoutubeSearchEngineTest.java | 6 + .../YoutubeStreamExtractorDefaultTest.java | 6 +- .../YoutubeStreamExtractorGemaTest.java | 4 +- .../newpipe/VideoItemDetailFragment.java | 22 +++- .../schabi/newpipe/VideoItemListFragment.java | 69 +--------- ...xception.java => ExtractionException.java} | 12 +- .../newpipe/extractor/ParsingException.java | 2 +- .../newpipe/extractor/SearchEngine.java | 4 +- .../newpipe/extractor/StreamExtractor.java | 2 +- .../newpipe/extractor/StreamingService.java | 2 +- .../schabi/newpipe/extractor/VideoInfo.java | 18 +-- .../services/youtube/YoutubeSearchEngine.java | 124 +++++++++++++----- .../services/youtube/YoutubeService.java | 4 +- .../youtube/YoutubeStreamExtractor.java | 57 ++++---- .../youtube/YoutubeVideoUrlIdHandler.java | 2 +- app/src/main/res/values/strings.xml | 1 + 16 files changed, 183 insertions(+), 152 deletions(-) rename app/src/main/java/org/schabi/newpipe/extractor/{ExctractionException.java => ExtractionException.java} (74%) diff --git a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeSearchEngineTest.java b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeSearchEngineTest.java index b6fa10fcf..7ab6cacf3 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeSearchEngineTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeSearchEngineTest.java @@ -81,6 +81,12 @@ public class YoutubeSearchEngineTest extends AndroidTestCase { } } + public void testViewCount() { + for(VideoPreviewInfo i : result.resultList) { + assertTrue(Long.toString(i.view_count), i.view_count != -1); + } + } + public void testIfSuggestionsAreReplied() { assertEquals(suggestionReply.size() > 0, true); } diff --git a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorDefaultTest.java b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorDefaultTest.java index 058fc6e24..7790acf29 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorDefaultTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorDefaultTest.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.ExctractionException; +import org.schabi.newpipe.extractor.ExtractionException; import org.schabi.newpipe.extractor.ParsingException; import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.VideoInfo; @@ -33,7 +33,7 @@ import java.io.IOException; public class YoutubeStreamExtractorDefaultTest extends AndroidTestCase { private YoutubeStreamExtractor extractor; - public void setUp() throws IOException, ExctractionException { + public void setUp() throws IOException, ExtractionException { /* some anonymus video test extractor = new YoutubeStreamExtractor("https://www.youtube.com/watch?v=FmG385_uUys", new Downloader()); */ @@ -47,7 +47,7 @@ public class YoutubeStreamExtractorDefaultTest extends AndroidTestCase { extractor.getTimeStamp() <= 0); } - public void testGetValidTimeStamp() throws ExctractionException, IOException { + public void testGetValidTimeStamp() throws ExtractionException, IOException { YoutubeStreamExtractor extractor = new YoutubeStreamExtractor("https://youtu.be/FmG385_uUys?t=174", new Downloader()); assertTrue(Integer.toString(extractor.getTimeStamp()), diff --git a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorGemaTest.java b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorGemaTest.java index 740e07a8a..a4bdec252 100644 --- a/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorGemaTest.java +++ b/app/src/androidTest/java/org/schabi/newpipe/extractor/youtube/YoutubeStreamExtractorGemaTest.java @@ -3,7 +3,7 @@ package org.schabi.newpipe.services.youtube; import android.test.AndroidTestCase; import org.schabi.newpipe.Downloader; -import org.schabi.newpipe.extractor.ExctractionException; +import org.schabi.newpipe.extractor.ExtractionException; import org.schabi.newpipe.extractor.services.youtube.YoutubeStreamExtractor; import java.io.IOException; @@ -35,7 +35,7 @@ public class YoutubeStreamExtractorGemaTest extends AndroidTestCase { // Deaktivate this Test Case bevore uploading it githup, otherwise CI will fail. private static final boolean testActive = false; - public void testGemaError() throws IOException, ExctractionException { + public void testGemaError() throws IOException, ExtractionException { if(testActive) { try { new YoutubeStreamExtractor("https://www.youtube.com/watch?v=3O1_3zBUKM8", diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java index 89791bc2b..ecbdfe121 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemDetailFragment.java @@ -147,6 +147,13 @@ public class VideoItemDetailFragment extends Fragment { onErrorBlockedByGema(); } }); + } catch(YoutubeStreamExtractor.LiveStreamException e) { + h.post(new Runnable() { + @Override + public void run() { + onNotSpecifiedContentErrorWithMessage(R.string.live_streams_not_supported); + } + }); } // ---------------------------------------- catch(StreamExtractor.ContentNotAvailableException e) { @@ -186,8 +193,10 @@ public class VideoItemDetailFragment extends Fragment { @Override public void onLoadingFailed(String imageUri, View view, FailReason failReason) { - Toast.makeText(VideoItemDetailFragment.this.getActivity(), - R.string.could_not_load_thumbnails, Toast.LENGTH_LONG).show(); + if(getContext() != null) { + Toast.makeText(VideoItemDetailFragment.this.getActivity(), + R.string.could_not_load_thumbnails, Toast.LENGTH_LONG).show(); + } failReason.getCause().printStackTrace(); } @@ -672,6 +681,15 @@ public class VideoItemDetailFragment extends Fragment { .show(); } + private void onNotSpecifiedContentErrorWithMessage(int resourceId) { + ImageView thumbnailView = (ImageView) activity.findViewById(R.id.detailThumbnailView); + progressBar.setVisibility(View.GONE); + thumbnailView.setImageBitmap(BitmapFactory.decodeResource( + getResources(), R.drawable.not_available_monkey)); + Toast.makeText(activity, resourceId, Toast.LENGTH_LONG) + .show(); + } + private boolean useStream(VideoInfo.VideoStream stream, Vector streams) { for(VideoInfo.VideoStream i : streams) { if(i.resolution.equals(stream.resolution)) { diff --git a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java index 7a75fa38e..999f6875b 100644 --- a/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java +++ b/app/src/main/java/org/schabi/newpipe/VideoItemListFragment.java @@ -16,7 +16,8 @@ import android.widget.Toast; import java.io.IOException; import java.util.List; -import org.schabi.newpipe.extractor.ExctractionException; +import org.schabi.newpipe.extractor.ExtractionException; +import org.schabi.newpipe.extractor.ParsingException; import org.schabi.newpipe.extractor.VideoPreviewInfo; import org.schabi.newpipe.extractor.SearchEngine; import org.schabi.newpipe.extractor.StreamingService; @@ -108,14 +109,14 @@ public class VideoItemListFragment extends ListFragment { SearchEngine.Result result = engine.search(query, page, searchLanguage, new Downloader()); - Log.i(TAG, "language code passed:\""+searchLanguage+"\""); + //Log.i(TAG, "language code passed:\""+searchLanguage+"\""); if(runs) { h.post(new ResultRunnable(result, requestId)); } } catch(IOException e) { postNewErrorToast(h, R.string.network_error); e.printStackTrace(); - } catch(ExctractionException ce) { + } catch(ExtractionException ce) { postNewErrorToast(h, R.string.parsing_error); ce.printStackTrace(); } catch(Exception e) { @@ -124,67 +125,7 @@ public class VideoItemListFragment extends ListFragment { } } } -/* -<<< - private class LoadThumbsRunnable implements Runnable { - private final Vector thumbnailUrlList = new Vector<>(); - private final Vector downloadedList; - final Handler h = new Handler(); - private volatile boolean run = true; - private final int requestId; - public LoadThumbsRunnable(Vector videoList, - Vector downloadedList, int requestId) { - for(VideoPreviewInfo item : videoList) { - thumbnailUrlList.add(item.thumbnail_url); - } - this.downloadedList = downloadedList; - this.requestId = requestId; - } - public void terminate() { - run = false; - } - public boolean isRunning() { - return run; - } - @Override - public void run() { - for(int i = 0; i < thumbnailUrlList.size() && run; i++) { - if(!downloadedList.get(i)) { - Bitmap thumbnail; - try { - //todo: make bitmaps not bypass tor - thumbnail = BitmapFactory.decodeStream( - new URL(thumbnailUrlList.get(i)).openConnection().getInputStream()); - h.post(new SetThumbnailRunnable(i, thumbnail, requestId)); - } catch (Exception e) { - e.printStackTrace(); - } - } - } - } - } - private class SetThumbnailRunnable implements Runnable { - private final int index; - private final Bitmap thumbnail; - private final int requestId; - public SetThumbnailRunnable(int index, Bitmap thumbnail, int requestId) { - this.index = index; - this.thumbnail = thumbnail; - this.requestId = requestId; - } - @Override - public void run() { - if(requestId == currentRequestId) { - videoListAdapter.updateDownloadedThumbnailList(index); - videoListAdapter.setThumbnail(index, thumbnail); - } - } - } - -======= ->>>>>>> 6d1b4652fc98e5c2d5e19b0f98ba38a731137a70 -*/ public void present(List videoList) { mode = PRESENT_VIDEOS_MODE; setListShown(true); @@ -378,7 +319,7 @@ public class VideoItemListFragment extends ListFragment { @Override public void run() { setListShown(true); - Toast.makeText(getActivity(), getString(R.string.network_error), + Toast.makeText(getActivity(), getString(stringResource), Toast.LENGTH_SHORT).show(); } }); diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ExctractionException.java b/app/src/main/java/org/schabi/newpipe/extractor/ExtractionException.java similarity index 74% rename from app/src/main/java/org/schabi/newpipe/extractor/ExctractionException.java rename to app/src/main/java/org/schabi/newpipe/extractor/ExtractionException.java index f279730fc..68ded9c26 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/ExctractionException.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/ExtractionException.java @@ -4,7 +4,7 @@ package org.schabi.newpipe.extractor; * Created by Christian Schabesberger on 30.01.16. * * Copyright (C) Christian Schabesberger 2016 - * ExctractionException.java is part of NewPipe. + * ExtractionException.java is part of NewPipe. * * NewPipe is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,18 +20,18 @@ package org.schabi.newpipe.extractor; * along with NewPipe. If not, see . */ -public class ExctractionException extends Exception { - public ExctractionException() {} +public class ExtractionException extends Exception { + public ExtractionException() {} - public ExctractionException(String message) { + public ExtractionException(String message) { super(message); } - public ExctractionException(Throwable cause) { + public ExtractionException(Throwable cause) { super(cause); } - public ExctractionException(String message, Throwable cause) { + public ExtractionException(String message, Throwable cause) { super(message, cause); } } \ No newline at end of file diff --git a/app/src/main/java/org/schabi/newpipe/extractor/ParsingException.java b/app/src/main/java/org/schabi/newpipe/extractor/ParsingException.java index dd916afd0..c978743ba 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/ParsingException.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/ParsingException.java @@ -21,7 +21,7 @@ package org.schabi.newpipe.extractor; */ -public class ParsingException extends ExctractionException { +public class ParsingException extends ExtractionException { public ParsingException() {} public ParsingException(String message) { super(message); diff --git a/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java b/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java index fe8aaa152..be176d39e 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/SearchEngine.java @@ -34,9 +34,9 @@ public interface SearchEngine { } ArrayList suggestionList(String query, Downloader dl) - throws ExctractionException, IOException; + throws ExtractionException, IOException; //Result search(String query, int page); Result search(String query, int page, String contentCountry, Downloader dl) - throws ExctractionException, IOException; + throws ExtractionException, IOException; } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/StreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/StreamExtractor.java index 70129de61..a9b8c8e3a 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/StreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/StreamExtractor.java @@ -28,7 +28,7 @@ import java.util.List; @SuppressWarnings("ALL") public interface StreamExtractor { - public class ExctractorInitException extends ExctractionException { + public class ExctractorInitException extends ExtractionException { public ExctractorInitException() {} public ExctractorInitException(String message) { super(message); diff --git a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java index f2144f581..88e2757e4 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/StreamingService.java @@ -28,7 +28,7 @@ public interface StreamingService { } ServiceInfo getServiceInfo(); StreamExtractor getExtractorInstance(String url, Downloader downloader) - throws IOException, ExctractionException; + throws IOException, ExtractionException; SearchEngine getSearchEngineInstance(); VideoUrlIdHandler getUrlIdHandler(); diff --git a/app/src/main/java/org/schabi/newpipe/extractor/VideoInfo.java b/app/src/main/java/org/schabi/newpipe/extractor/VideoInfo.java index 9135c82a9..85cec6178 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/VideoInfo.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/VideoInfo.java @@ -31,7 +31,7 @@ public class VideoInfo extends AbstractVideoInfo { /**Fills out the video info fields which are common to all services. * Probably needs to be overridden by subclasses*/ public static VideoInfo getVideoInfo(StreamExtractor extractor, Downloader downloader) - throws ExctractionException, IOException { + throws ExtractionException, IOException { VideoInfo videoInfo = new VideoInfo(); videoInfo = extractImportantData(videoInfo, extractor, downloader); @@ -43,7 +43,7 @@ public class VideoInfo extends AbstractVideoInfo { private static VideoInfo extractImportantData( VideoInfo videoInfo, StreamExtractor extractor, Downloader downloader) - throws ExctractionException, IOException { + throws ExtractionException, IOException { /* ---- importand data, withoug the video can't be displayed goes here: ---- */ // if one of these is not available an exception is ment to be thrown directly into the frontend. @@ -62,7 +62,7 @@ public class VideoInfo extends AbstractVideoInfo { private static VideoInfo extractStreams( VideoInfo videoInfo, StreamExtractor extractor, Downloader downloader) - throws ExctractionException, IOException { + throws ExtractionException, IOException { /* ---- stream extraction goes here ---- */ // At least one type of stream has to be available, // otherwise an exception will be thrown directly into the frontend. @@ -70,14 +70,14 @@ public class VideoInfo extends AbstractVideoInfo { try { videoInfo.dashMpdUrl = extractor.getDashMpdUrl(); } catch(Exception e) { - videoInfo.addException(new ExctractionException("Couldn't get Dash manifest", e)); + videoInfo.addException(new ExtractionException("Couldn't get Dash manifest", e)); } /* Load and extract audio */ try { videoInfo.audio_streams = extractor.getAudioStreams(); } catch(Exception e) { - videoInfo.addException(new ExctractionException("Couldn't get audio streams", e)); + videoInfo.addException(new ExtractionException("Couldn't get audio streams", e)); } // also try to get streams from the dashMpd if(videoInfo.dashMpdUrl != null && !videoInfo.dashMpdUrl.isEmpty()) { @@ -91,7 +91,7 @@ public class VideoInfo extends AbstractVideoInfo { DashMpdParser.getAudioStreams(videoInfo.dashMpdUrl, downloader)); } catch(Exception e) { videoInfo.addException( - new ExctractionException("Couldn't get audio streams from dash mpd", e)); + new ExtractionException("Couldn't get audio streams from dash mpd", e)); } } /* Extract video stream url*/ @@ -99,14 +99,14 @@ public class VideoInfo extends AbstractVideoInfo { videoInfo.video_streams = extractor.getVideoStreams(); } catch (Exception e) { videoInfo.addException( - new ExctractionException("Couldn't get video streams", e)); + new ExtractionException("Couldn't get video streams", e)); } /* Extract video only stream url*/ try { videoInfo.video_only_streams = extractor.getVideoOnlyStreams(); } catch(Exception e) { videoInfo.addException( - new ExctractionException("Couldn't get video only streams", e)); + new ExtractionException("Couldn't get video only streams", e)); } // either dash_mpd audio_only or video has to be available, otherwise we didn't get a stream, @@ -114,7 +114,7 @@ public class VideoInfo extends AbstractVideoInfo { if((videoInfo.video_streams == null || videoInfo.video_streams.isEmpty()) && (videoInfo.audio_streams == null || videoInfo.audio_streams.isEmpty()) && (videoInfo.dashMpdUrl == null || videoInfo.dashMpdUrl.isEmpty())) { - throw new ExctractionException("Could not get any stream. See error variable to get further details."); + throw new ExtractionException("Could not get any stream. See error variable to get further details."); } return videoInfo; diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java index 7e5871959..fd7528926 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeSearchEngine.java @@ -7,6 +7,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.schabi.newpipe.extractor.Downloader; +import org.schabi.newpipe.extractor.Parser; import org.schabi.newpipe.extractor.ParsingException; import org.schabi.newpipe.extractor.SearchEngine; import org.schabi.newpipe.extractor.VideoPreviewInfo; @@ -18,8 +19,6 @@ import org.xml.sax.SAXException; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -101,44 +100,40 @@ public class YoutubeSearchEngine implements SearchEngine { // video item type } else if (!((el = item.select("div[class*=\"yt-lockup-video\"").first()) == null)) { VideoPreviewInfo resultItem = new VideoPreviewInfo(); - Element dl = el.select("h3").first().select("a").first(); - resultItem.webpage_url = dl.attr("abs:href"); + + // importand information + resultItem.webpage_url = getWebpageUrl(item); + resultItem.id = (new YoutubeVideoUrlIdHandler()).getVideoId(resultItem.webpage_url); + resultItem.title = getTitle(item); + + // optional iformation + //todo: make this a proper error handling try { - Pattern p = Pattern.compile("v=([0-9a-zA-Z-]*)"); - Matcher m = p.matcher(resultItem.webpage_url); - resultItem.id = m.group(1); + resultItem.duration = getDuration(item); } catch (Exception e) { - //e.printStackTrace(); + e.printStackTrace(); } - resultItem.title = dl.text(); - - resultItem.duration = item.select("span[class=\"video-time\"]").first().text(); - - resultItem.uploader = item.select("div[class=\"yt-lockup-byline\"]").first() - .select("a").first() - .text(); - resultItem.upload_date = item.select("div[class=\"yt-lockup-meta\"]").first() - .select("li").first() - .text(); - - //todo: test against view_count - String viewCountInfo = item.select("div[class=\"yt-lockup-meta\"]").first() - .select("li").get(1) - .text(); - viewCountInfo = viewCountInfo.substring(0, viewCountInfo.indexOf(' ')); - viewCountInfo = viewCountInfo.replaceAll("[,.]", ""); - viewCountInfo = viewCountInfo.replaceAll("\\s",""); - resultItem.view_count = Long.parseLong(viewCountInfo); - - Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() - .select("img").first(); - resultItem.thumbnail_url = te.attr("abs:src"); - // Sometimes youtube sends links to gif files which somehow seem to not exist - // anymore. Items with such gif also offer a secondary image source. So we are going - // to use that if we've caught such an item. - if (resultItem.thumbnail_url.contains(".gif")) { - resultItem.thumbnail_url = te.attr("abs:data-thumb"); + try { + resultItem.uploader = getUploader(item); + } catch (Exception e) { + e.printStackTrace(); } + try { + resultItem.upload_date = getUploadDate(item); + } catch (Exception e) { + e.printStackTrace(); + } + try { + resultItem.view_count = getViewCount(item); + } catch (Exception e) { + e.printStackTrace(); + } + try { + resultItem.thumbnail_url = getThumbnailUrl(item); + } catch (Exception e) { + e.printStackTrace(); + } + result.resultList.add(resultItem); } else { //noinspection ConstantConditions @@ -207,4 +202,61 @@ public class YoutubeSearchEngine implements SearchEngine { } } + private String getWebpageUrl(Element item) { + Element el = item.select("div[class*=\"yt-lockup-video\"").first(); + Element dl = el.select("h3").first().select("a").first(); + return dl.attr("abs:href"); + } + + private String getTitle(Element item) { + Element el = item.select("div[class*=\"yt-lockup-video\"").first(); + Element dl = el.select("h3").first().select("a").first(); + return dl.text(); + } + + private String getDuration(Element item) { + try { + return item.select("span[class=\"video-time\"]").first().text(); + } catch(Exception e) { + e.printStackTrace(); + } + return ""; + } + + private String getUploader(Element item) { + return item.select("div[class=\"yt-lockup-byline\"]").first() + .select("a").first() + .text(); + } + + private String getUploadDate(Element item) { + return item.select("div[class=\"yt-lockup-meta\"]").first() + .select("li").first() + .text(); + } + + private long getViewCount(Element item) throws Parser.RegexException{ + String output; + String input = item.select("div[class=\"yt-lockup-meta\"]").first() + .select("li").get(1) + .text(); + output = Parser.matchGroup1("([0-9,\\. ])", input).replace(" ", ""); + + return Long.parseLong(output); + } + + private String getThumbnailUrl(Element item) { + String url; + Element te = item.select("div[class=\"yt-thumb video-thumb\"]").first() + .select("img").first(); + url = te.attr("abs:src"); + // Sometimes youtube sends links to gif files which somehow seem to not exist + // anymore. Items with such gif also offer a secondary image source. So we are going + // to use that if we've caught such an item. + if (url.contains(".gif")) { + url = te.attr("abs:data-thumb"); + } + + return url; + } } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java index 2f9e1319a..768337043 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java @@ -1,6 +1,6 @@ package org.schabi.newpipe.extractor.services.youtube; -import org.schabi.newpipe.extractor.ExctractionException; +import org.schabi.newpipe.extractor.ExtractionException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.StreamExtractor; import org.schabi.newpipe.extractor.StreamingService; @@ -39,7 +39,7 @@ public class YoutubeService implements StreamingService { } @Override public StreamExtractor getExtractorInstance(String url, Downloader downloader) - throws ExctractionException, IOException { + throws ExtractionException, IOException { VideoUrlIdHandler urlIdHandler = new YoutubeVideoUrlIdHandler(); if(urlIdHandler.acceptUrl(url)) { return new YoutubeStreamExtractor(url, downloader) ; diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java index 0e17d1c9a..6e7f25a56 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamExtractor.java @@ -10,7 +10,7 @@ import org.jsoup.nodes.Element; import org.mozilla.javascript.Context; import org.mozilla.javascript.Function; import org.mozilla.javascript.ScriptableObject; -import org.schabi.newpipe.extractor.ExctractionException; +import org.schabi.newpipe.extractor.ExtractionException; import org.schabi.newpipe.extractor.Downloader; import org.schabi.newpipe.extractor.Parser; import org.schabi.newpipe.extractor.ParsingException; @@ -47,6 +47,33 @@ import java.util.Vector; public class YoutubeStreamExtractor implements StreamExtractor { + // exceptions + + public class DecryptException extends ParsingException { + DecryptException(Throwable cause) { + super(cause); + } + DecryptException(String message, Throwable cause) { + super(message, cause); + } + } + + // special content not available exceptions + + public class GemaException extends ContentNotAvailableException { + GemaException(String message) { + super(message); + } + } + + public class LiveStreamException extends ContentNotAvailableException { + LiveStreamException() { + super(); + } + } + + // ---------------- + // Sometimes if the html page of youtube is already downloaded, youtube web page will internally // download the /get_video_info page. Since a certain date dashmpd url is only available over // this /get_video_info page, so we always need to download this one to. @@ -138,25 +165,6 @@ public class YoutubeStreamExtractor implements StreamExtractor { throw new ParsingException("itag=" + Integer.toString(itag) + " not supported"); } - public class DecryptException extends ParsingException { - DecryptException(Throwable cause) { - super(cause); - } - DecryptException(String message, Throwable cause) { - super(message, cause); - } - } - - // special content not available exceptions - - public class GemaException extends ContentNotAvailableException { - GemaException(String message) { - super(message); - } - } - - // ---------------- - private static final String TAG = YoutubeStreamExtractor.class.toString(); private final Document doc; private JSONObject playerArgs; @@ -173,7 +181,7 @@ public class YoutubeStreamExtractor implements StreamExtractor { private Downloader downloader; - public YoutubeStreamExtractor(String pageUrl, Downloader dl) throws ExctractionException, IOException { + public YoutubeStreamExtractor(String pageUrl, Downloader dl) throws ExtractionException, IOException { //most common videoInfo fields are now set in our superclass, for all services downloader = dl; this.pageUrl = pageUrl; @@ -183,11 +191,13 @@ public class YoutubeStreamExtractor implements StreamExtractor { JSONObject ytPlayerConfig; //attempt to load the youtube js player JSON arguments + String ps; //used to determine if this is a livestream or not try { ytPlayerConfigRaw = Parser.matchGroup1("ytplayer.config\\s*=\\s*(\\{.*?\\});", pageContent); ytPlayerConfig = new JSONObject(ytPlayerConfigRaw); playerArgs = ytPlayerConfig.getJSONObject("args"); + ps = playerArgs.get("ps").toString(); } catch (Parser.RegexException e) { String errorReason = findErrorReason(doc); switch(errorReason) { @@ -199,7 +209,10 @@ public class YoutubeStreamExtractor implements StreamExtractor { throw new ContentNotAvailableException("Content not available", e); } } catch (JSONException e) { - throw new ParsingException("Could not parse yt player config"); + throw new ParsingException("Could not parse yt player config", e); + } + if (ps.equals("live")) { + throw new LiveStreamException(); } diff --git a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeVideoUrlIdHandler.java b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeVideoUrlIdHandler.java index 591d981c7..d94d71d4e 100644 --- a/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeVideoUrlIdHandler.java +++ b/app/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeVideoUrlIdHandler.java @@ -37,7 +37,7 @@ public class YoutubeVideoUrlIdHandler implements VideoUrlIdHandler { @SuppressWarnings("WeakerAccess") @Override public String getVideoId(String url) throws ParsingException { - String id = ""; + String id; if(url.contains("youtube")) { if(url.contains("attribution_link")) { diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 33a59d813..52c6ab978 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -84,6 +84,7 @@ Content not available. Blocked by GEMA. Could not setup download menu. + This is a LIVE STREAM. These are not yet supported.