From 855202e40e09af1cb5fb372d4a2d05a61b3a9bb2 Mon Sep 17 00:00:00 2001 From: Gavin Johnson Date: Mon, 16 Jan 2023 15:40:38 -0800 Subject: [PATCH 1/8] added youtube playlist import; initial commit Signed-off-by: Gavin Johnson --- locales/en-US.json | 1 + src/invidious/user/imports.cr | 85 +++++++++++++++++++++++ src/invidious/views/user/data_control.ecr | 5 ++ 3 files changed, 91 insertions(+) diff --git a/locales/en-US.json b/locales/en-US.json index 12955665..c30a90db 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -33,6 +33,7 @@ "Import": "Import", "Import Invidious data": "Import Invidious JSON data", "Import YouTube subscriptions": "Import YouTube/OPML subscriptions", + "Import YouTube playlist": "Import YouTube playlist (.csv)", "Import FreeTube subscriptions (.db)": "Import FreeTube subscriptions (.db)", "Import NewPipe subscriptions (.json)": "Import NewPipe subscriptions (.json)", "Import NewPipe data (.zip)": "Import NewPipe data (.zip)", diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 20ae0d47..870d083e 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,6 +30,75 @@ struct Invidious::User return subscriptions end + # Parse a youtube CSV playlist file and create the playlist + #NEW - Done + def parse_playlist_export_csv(user : User, csv_content : String) + rows = CSV.new(csv_content, headers: false) + if rows.size >= 2 + title = rows[1][4]?.try &.as_s?.try + descripton = rows[1][5]?.try &.as_s?.try + visibility = rows[1][6]?.try &.as_s?.try + + if visibility.compare("Public", case_insensitive: true) == 0 + privacy = PlaylistPrivacy:Public + else + privacy = PlaylistPrivacy:Private + end + + if title && privacy && user + playlist = create_playlist(title, privacy, user) + end + + if playlist && descripton + Invidious::Database::Playlists.update_description(playlist.id, description) + end + end + + return playlist + end + + # Parse a youtube CSV playlist file and add videos from it to a playlist + #NEW - done + def parse_playlist_videos_export_csv(playlist : Playlist, csv_content : String) + rows = CSV.new(csv_content, headers: false) + if rows.size >= 5 + offset = env.params.query["index"]?.try &.to_i? || 0 + row_counter = 0 + rows.each do |row| + if row_counter >= 4 + video_id = row[0]?.try &.as_s?.try + end + row_counter += 1 + next if !video_id + + begin + video = get_video(video_id) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + end + + videos = get_playlist_videos(playlist, offset: offset) + end + + return videos + end + # ------------------- # Invidious # ------------------- @@ -149,6 +218,22 @@ struct Invidious::User return true end + # Import playlist from Youtube + # Returns success status + #NEW + def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool + extension = filename.split(".").last + + if extension == "csv" || type == "text/csv" + playlist = parse_playlist_export_csv(user, body) + playlist = parse_playlist_videos_export_csv(playlist, body) + else + return false + end + + return true + end + # ------------------- # Freetube # ------------------- diff --git a/src/invidious/views/user/data_control.ecr b/src/invidious/views/user/data_control.ecr index a451159f..0f8e8dae 100644 --- a/src/invidious/views/user/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr @@ -21,6 +21,11 @@ +
+ + +
+
From 96344f28b4b842e915325aef64bc93fc9fc55387 Mon Sep 17 00:00:00 2001 From: Gavin Johnson Date: Sat, 28 Jan 2023 09:26:16 -0800 Subject: [PATCH 2/8] added youtube playlist import functionality. fixes issue #2114 Signed-off-by: Gavin Johnson --- locales/en-US.json | 2 +- src/invidious/routes/preferences.cr | 10 ++ src/invidious/user/imports.cr | 129 +++++++++++----------- src/invidious/views/feeds/playlists.ecr | 13 ++- src/invidious/views/user/data_control.ecr | 4 +- 5 files changed, 86 insertions(+), 72 deletions(-) diff --git a/locales/en-US.json b/locales/en-US.json index c30a90db..8f1ec58d 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -33,7 +33,7 @@ "Import": "Import", "Import Invidious data": "Import Invidious JSON data", "Import YouTube subscriptions": "Import YouTube/OPML subscriptions", - "Import YouTube playlist": "Import YouTube playlist (.csv)", + "Import YouTube playlist (.csv)": "Import YouTube playlist (.csv)", "Import FreeTube subscriptions (.db)": "Import FreeTube subscriptions (.db)", "Import NewPipe subscriptions (.json)": "Import NewPipe subscriptions (.json)", "Import NewPipe data (.zip)": "Import NewPipe data (.zip)", diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 570cba69..adac0068 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -310,6 +310,16 @@ module Invidious::Routes::PreferencesRoute response: error_template(415, "Invalid subscription file uploaded") ) end + # Gavin Johnson (thtmnisamnstr), 20230127: Call the Youtube playlist import function + when "import_youtube_pl" + filename = part.filename || "" + success = Invidious::User::Import.from_youtube_pl(user, body, filename, type) + + if !success + haltf(env, status_code: 415, + response: error_template(415, "Invalid playlist file uploaded") + ) + end when "import_freetube" Invidious::User::Import.from_freetube(user, body) when "import_newpipe_subscriptions" diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 870d083e..fa1bbe7f 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,75 +30,72 @@ struct Invidious::User return subscriptions end - # Parse a youtube CSV playlist file and create the playlist - #NEW - Done + # Gavin Johnson (thtmnisamnstr), 20230127: Parse a youtube CSV playlist file and create the playlist def parse_playlist_export_csv(user : User, csv_content : String) - rows = CSV.new(csv_content, headers: false) - if rows.size >= 2 - title = rows[1][4]?.try &.as_s?.try - descripton = rows[1][5]?.try &.as_s?.try - visibility = rows[1][6]?.try &.as_s?.try - - if visibility.compare("Public", case_insensitive: true) == 0 - privacy = PlaylistPrivacy:Public - else - privacy = PlaylistPrivacy:Private - end + rows = CSV.new(csv_content, headers: true) + row_counter = 0 + playlist = uninitialized InvidiousPlaylist + title = uninitialized String + description = uninitialized String + visibility = uninitialized String + rows.each do |row| + if row_counter == 0 + title = row[4] + description = row[5] + visibility = row[6] - if title && privacy && user - playlist = create_playlist(title, privacy, user) - end + if visibility.compare("Public", case_insensitive: true) == 0 + privacy = PlaylistPrivacy::Public + else + privacy = PlaylistPrivacy::Private + end + + if title && privacy && user + playlist = create_playlist(title, privacy, user) + end + + if playlist && description + Invidious::Database::Playlists.update_description(playlist.id, description) + end - if playlist && descripton - Invidious::Database::Playlists.update_description(playlist.id, description) + row_counter += 1 + end + if row_counter > 0 && row_counter < 3 + row_counter += 1 + end + if row_counter >= 3 + if playlist + video_id = row[0] + row_counter += 1 + next if !video_id + + begin + video = get_video(video_id) + rescue ex + next + end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + end end end return playlist end - # Parse a youtube CSV playlist file and add videos from it to a playlist - #NEW - done - def parse_playlist_videos_export_csv(playlist : Playlist, csv_content : String) - rows = CSV.new(csv_content, headers: false) - if rows.size >= 5 - offset = env.params.query["index"]?.try &.to_i? || 0 - row_counter = 0 - rows.each do |row| - if row_counter >= 4 - video_id = row[0]?.try &.as_s?.try - end - row_counter += 1 - next if !video_id - - begin - video = get_video(video_id) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - Invidious::Database::PlaylistVideos.insert(playlist_video) - Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) - end - - videos = get_playlist_videos(playlist, offset: offset) - end - - return videos - end - # ------------------- # Invidious # ------------------- @@ -218,20 +215,20 @@ struct Invidious::User return true end - # Import playlist from Youtube - # Returns success status - #NEW + # Gavin Johnson (thtmnisamnstr), 20230127: Import playlist from Youtube export. Returns success status. def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last if extension == "csv" || type == "text/csv" playlist = parse_playlist_export_csv(user, body) - playlist = parse_playlist_videos_export_csv(playlist, body) + if playlist + return true + else + return false + end else return false end - - return true end # ------------------- diff --git a/src/invidious/views/feeds/playlists.ecr b/src/invidious/views/feeds/playlists.ecr index a59344c4..05a48ce3 100644 --- a/src/invidious/views/feeds/playlists.ecr +++ b/src/invidious/views/feeds/playlists.ecr @@ -5,14 +5,21 @@ <%= rendered "components/feed_menu" %>
-
+

<%= translate(locale, "user_created_playlists", %(#{items_created.size})) %>

-
-

+
+

<%= translate(locale, "Create playlist") %>

+
+

+ + <%= translate(locale, "Import/export") %> + +

+

diff --git a/src/invidious/views/user/data_control.ecr b/src/invidious/views/user/data_control.ecr index 0f8e8dae..27654b40 100644 --- a/src/invidious/views/user/data_control.ecr +++ b/src/invidious/views/user/data_control.ecr @@ -8,7 +8,7 @@ <%= translate(locale, "Import") %>
- +
@@ -22,7 +22,7 @@
- +
From 5c7bda66ae90f3aef559a0269e56156a359814a3 Mon Sep 17 00:00:00 2001 From: Gavin Johnson Date: Sat, 28 Jan 2023 09:55:36 -0800 Subject: [PATCH 3/8] removed comments Signed-off-by: Gavin Johnson --- src/invidious/user/imports.cr | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index fa1bbe7f..77009538 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,7 +30,6 @@ struct Invidious::User return subscriptions end - # Gavin Johnson (thtmnisamnstr), 20230127: Parse a youtube CSV playlist file and create the playlist def parse_playlist_export_csv(user : User, csv_content : String) rows = CSV.new(csv_content, headers: true) row_counter = 0 @@ -215,7 +214,6 @@ struct Invidious::User return true end - # Gavin Johnson (thtmnisamnstr), 20230127: Import playlist from Youtube export. Returns success status. def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool extension = filename.split(".").last From 72d0c9e40971b5460048f8906914fbef55289236 Mon Sep 17 00:00:00 2001 From: Gavin Johnson Date: Sat, 28 Jan 2023 09:57:28 -0800 Subject: [PATCH 4/8] removed comments Signed-off-by: Gavin Johnson --- src/invidious/routes/preferences.cr | 1 - 1 file changed, 1 deletion(-) diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index adac0068..abe0f34e 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -310,7 +310,6 @@ module Invidious::Routes::PreferencesRoute response: error_template(415, "Invalid subscription file uploaded") ) end - # Gavin Johnson (thtmnisamnstr), 20230127: Call the Youtube playlist import function when "import_youtube_pl" filename = part.filename || "" success = Invidious::User::Import.from_youtube_pl(user, body, filename, type) From 6f01d6eacf0719e8569a338e5a44615f159c5120 Mon Sep 17 00:00:00 2001 From: thtmnisamnstr Date: Fri, 10 Feb 2023 12:00:02 -0800 Subject: [PATCH 5/8] ran crystal tool format. it should fix some CI issues Signed-off-by: thtmnisamnstr --- src/invidious/user/imports.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 77009538..aa87ca99 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -48,11 +48,11 @@ struct Invidious::User else privacy = PlaylistPrivacy::Private end - + if title && privacy && user - playlist = create_playlist(title, privacy, user) + playlist = create_playlist(title, privacy, user) end - + if playlist && description Invidious::Database::Playlists.update_description(playlist.id, description) end From b3eea6ab3ebdb1618916b02041b22e0e238e8a7d Mon Sep 17 00:00:00 2001 From: thtmnisamnstr Date: Thu, 23 Feb 2023 15:55:38 -0800 Subject: [PATCH 6/8] improved import algorithm, fixed a referer issue from the playlists page after deleting a playlist Signed-off-by: thtmnisamnstr --- src/invidious/user/imports.cr | 55 +++++++++++++------------ src/invidious/views/feeds/playlists.ecr | 4 +- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index aa87ca99..7fddcc4c 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,43 +30,44 @@ struct Invidious::User return subscriptions end - def parse_playlist_export_csv(user : User, csv_content : String) - rows = CSV.new(csv_content, headers: true) - row_counter = 0 + def parse_playlist_export_csv(user : User, raw_input : String) playlist = uninitialized InvidiousPlaylist title = uninitialized String description = uninitialized String visibility = uninitialized String - rows.each do |row| - if row_counter == 0 - title = row[4] - description = row[5] - visibility = row[6] + privacy = uninitialized PlaylistPrivacy - if visibility.compare("Public", case_insensitive: true) == 0 - privacy = PlaylistPrivacy::Public - else - privacy = PlaylistPrivacy::Private - end + # Split the input into head and body content + raw_head, raw_body = raw_input.split("\n\n", limit: 2, remove_empty: true) - if title && privacy && user - playlist = create_playlist(title, privacy, user) - end + # Create the playlist from the head content + csv_head = CSV.new(raw_head, headers: true) + csv_head.next + title = csv_head[4] + description = csv_head[5] + visibility = csv_head[6] + + if visibility.compare("Public", case_insensitive: true) == 0 + privacy = PlaylistPrivacy::Public + else + privacy = PlaylistPrivacy::Private + end - if playlist && description - Invidious::Database::Playlists.update_description(playlist.id, description) - end + if title && privacy && user + playlist = create_playlist(title, privacy, user) + end - row_counter += 1 - end - if row_counter > 0 && row_counter < 3 - row_counter += 1 - end - if row_counter >= 3 + if playlist && description + Invidious::Database::Playlists.update_description(playlist.id, description) + end + + # Add each video to the playlist from the body content + CSV.each_row(raw_body) do |row| + if row.size >= 1 + video_id = row[0] if playlist - video_id = row[0] - row_counter += 1 next if !video_id + next if video_id == "Video Id" begin video = get_video(video_id) diff --git a/src/invidious/views/feeds/playlists.ecr b/src/invidious/views/feeds/playlists.ecr index 05a48ce3..43173355 100644 --- a/src/invidious/views/feeds/playlists.ecr +++ b/src/invidious/views/feeds/playlists.ecr @@ -10,12 +10,12 @@

- <%= translate(locale, "Create playlist") %> + "><%= translate(locale, "Create playlist") %>

- + "> <%= translate(locale, "Import/export") %>

From 3341929060bb20c086974a282fd1e33029f685f3 Mon Sep 17 00:00:00 2001 From: thtmnisamnstr Date: Tue, 7 Mar 2023 15:46:36 -0800 Subject: [PATCH 7/8] removed unnecessary conditionals and uninitialized variable declarations Signed-off-by: thtmnisamnstr --- src/invidious/user/imports.cr | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 7fddcc4c..757f5b13 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -31,12 +31,6 @@ struct Invidious::User end def parse_playlist_export_csv(user : User, raw_input : String) - playlist = uninitialized InvidiousPlaylist - title = uninitialized String - description = uninitialized String - visibility = uninitialized String - privacy = uninitialized PlaylistPrivacy - # Split the input into head and body content raw_head, raw_body = raw_input.split("\n\n", limit: 2, remove_empty: true) @@ -53,13 +47,8 @@ struct Invidious::User privacy = PlaylistPrivacy::Private end - if title && privacy && user - playlist = create_playlist(title, privacy, user) - end - - if playlist && description - Invidious::Database::Playlists.update_description(playlist.id, description) - end + playlist = create_playlist(title, privacy, user) + Invidious::Database::Playlists.update_description(playlist.id, description) # Add each video to the playlist from the body content CSV.each_row(raw_body) do |row| From fffdaa1410db7f9b5c67b1b47d401a2744e7b220 Mon Sep 17 00:00:00 2001 From: thtmnisamnstr Date: Mon, 3 Apr 2023 17:07:58 -0700 Subject: [PATCH 8/8] Updated csv reading as per feedback and ran Signed-off-by: thtmnisamnstr --- src/invidious/user/imports.cr | 53 +++++++++++++++++------------------ 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 757f5b13..673991f7 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -40,7 +40,7 @@ struct Invidious::User title = csv_head[4] description = csv_head[5] visibility = csv_head[6] - + if visibility.compare("Public", case_insensitive: true) == 0 privacy = PlaylistPrivacy::Public else @@ -51,34 +51,33 @@ struct Invidious::User Invidious::Database::Playlists.update_description(playlist.id, description) # Add each video to the playlist from the body content - CSV.each_row(raw_body) do |row| - if row.size >= 1 - video_id = row[0] - if playlist - next if !video_id - next if video_id == "Video Id" + csv_body = CSV.new(raw_body, headers: true) + csv_body.each do |row| + video_id = row[0] + if playlist + next if !video_id + next if video_id == "Video Id" - begin - video = get_video(video_id) - rescue ex - next - end - - playlist_video = PlaylistVideo.new({ - title: video.title, - id: video.id, - author: video.author, - ucid: video.ucid, - length_seconds: video.length_seconds, - published: video.published, - plid: playlist.id, - live_now: video.live_now, - index: Random::Secure.rand(0_i64..Int64::MAX), - }) - - Invidious::Database::PlaylistVideos.insert(playlist_video) - Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) + begin + video = get_video(video_id) + rescue ex + next end + + playlist_video = PlaylistVideo.new({ + title: video.title, + id: video.id, + author: video.author, + ucid: video.ucid, + length_seconds: video.length_seconds, + published: video.published, + plid: playlist.id, + live_now: video.live_now, + index: Random::Secure.rand(0_i64..Int64::MAX), + }) + + Invidious::Database::PlaylistVideos.insert(playlist_video) + Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index) end end