From cf13c11236de6c4e373b110730530c8e1f305900 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Thu, 10 Feb 2022 22:16:40 -0600 Subject: [PATCH] Migrations tweaks --- src/invidious.cr | 4 +- src/invidious/{ => database}/migration.cr | 6 +-- .../migrations/0001_create_channels_table.cr} | 4 +- .../migrations/0002_create_videos_table.cr} | 4 +- .../0003_create_channel_videos_table.cr} | 4 +- .../migrations/0004_create_users_table.cr} | 4 +- .../0005_create_session_ids_table.cr} | 4 +- .../migrations/0006_create_nonces_table.cr} | 4 +- .../0007_create_annotations_table.cr} | 4 +- .../migrations/0008_create_playlists_table.cr | 50 +++++++++++++++++++ .../0009_create_playlist_videos_table.cr | 27 ++++++++++ .../migrations/0010_make_videos_unlogged.cr | 11 ++++ src/invidious/database/migrator.cr | 42 ++++++++++++++++ .../migrations/0007_create_playlists_table.cr | 47 ----------------- .../0008_create_playlist_videos_table.cr | 27 ---------- src/invidious/migrator.cr | 41 --------------- 16 files changed, 149 insertions(+), 134 deletions(-) rename src/invidious/{ => database}/migration.cr (78%) rename src/invidious/{migrations/0000_create_channels_table.cr => database/migrations/0001_create_channels_table.cr} (92%) rename src/invidious/{migrations/0001_create_videos_table.cr => database/migrations/0002_create_videos_table.cr} (91%) rename src/invidious/{migrations/0002_create_channel_videos_table.cr => database/migrations/0003_create_channel_videos_table.cr} (94%) rename src/invidious/{migrations/0003_create_users_table.cr => database/migrations/0004_create_users_table.cr} (93%) rename src/invidious/{migrations/0004_create_session_ids_table.cr => database/migrations/0005_create_session_ids_table.cr} (92%) rename src/invidious/{migrations/0005_create_nonces_table.cr => database/migrations/0006_create_nonces_table.cr} (91%) rename src/invidious/{migrations/0006_create_annotations_table.cr => database/migrations/0007_create_annotations_table.cr} (88%) create mode 100644 src/invidious/database/migrations/0008_create_playlists_table.cr create mode 100644 src/invidious/database/migrations/0009_create_playlist_videos_table.cr create mode 100644 src/invidious/database/migrations/0010_make_videos_unlogged.cr create mode 100644 src/invidious/database/migrator.cr delete mode 100644 src/invidious/migrations/0007_create_playlists_table.cr delete mode 100644 src/invidious/migrations/0008_create_playlist_videos_table.cr delete mode 100644 src/invidious/migrator.cr diff --git a/src/invidious.cr b/src/invidious.cr index 6ec5f3a5..e8ad03ef 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,6 +27,7 @@ require "compress/zip" require "protodec/utils" require "./invidious/database/*" +require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/*" @@ -34,7 +35,6 @@ require "./invidious/channels/*" require "./invidious/user/*" require "./invidious/routes/**" require "./invidious/jobs/**" -require "./invidious/migrations/*" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) @@ -113,7 +113,7 @@ OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mo LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Run migrations -Invidious::Migrator.new(PG_DB).migrate +Invidious::Database::Migrator.new(PG_DB).migrate # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/migration.cr b/src/invidious/database/migration.cr similarity index 78% rename from src/invidious/migration.cr rename to src/invidious/database/migration.cr index a4eec1c5..921d8f38 100644 --- a/src/invidious/migration.cr +++ b/src/invidious/database/migration.cr @@ -1,6 +1,6 @@ -abstract class Invidious::Migration +abstract class Invidious::Database::Migration macro inherited - Invidious::Migrator.migrations << self + Migrator.migrations << self end @@version : Int64? @@ -33,6 +33,6 @@ abstract class Invidious::Migration end private def track(conn : DB::Connection) - conn.exec("INSERT INTO #{Invidious::Migrator::MIGRATIONS_TABLE}(version) VALUES ($1)", version) + conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version) end end diff --git a/src/invidious/migrations/0000_create_channels_table.cr b/src/invidious/database/migrations/0001_create_channels_table.cr similarity index 92% rename from src/invidious/migrations/0000_create_channels_table.cr rename to src/invidious/database/migrations/0001_create_channels_table.cr index 1f8f18e2..a1362bcf 100644 --- a/src/invidious/migrations/0000_create_channels_table.cr +++ b/src/invidious/database/migrations/0001_create_channels_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateChannelsTable < Migration - version 0 + version 1 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0001_create_videos_table.cr b/src/invidious/database/migrations/0002_create_videos_table.cr similarity index 91% rename from src/invidious/migrations/0001_create_videos_table.cr rename to src/invidious/database/migrations/0002_create_videos_table.cr index cdc9993f..c2ac84f8 100644 --- a/src/invidious/migrations/0001_create_videos_table.cr +++ b/src/invidious/database/migrations/0002_create_videos_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateVideosTable < Migration - version 1 + version 2 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0002_create_channel_videos_table.cr b/src/invidious/database/migrations/0003_create_channel_videos_table.cr similarity index 94% rename from src/invidious/migrations/0002_create_channel_videos_table.cr rename to src/invidious/database/migrations/0003_create_channel_videos_table.cr index 737abad4..c9b62e4c 100644 --- a/src/invidious/migrations/0002_create_channel_videos_table.cr +++ b/src/invidious/database/migrations/0003_create_channel_videos_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateChannelVideosTable < Migration - version 2 + version 3 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0003_create_users_table.cr b/src/invidious/database/migrations/0004_create_users_table.cr similarity index 93% rename from src/invidious/migrations/0003_create_users_table.cr rename to src/invidious/database/migrations/0004_create_users_table.cr index d91cca8d..a13ba15f 100644 --- a/src/invidious/migrations/0003_create_users_table.cr +++ b/src/invidious/database/migrations/0004_create_users_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateUsersTable < Migration - version 3 + version 4 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0004_create_session_ids_table.cr b/src/invidious/database/migrations/0005_create_session_ids_table.cr similarity index 92% rename from src/invidious/migrations/0004_create_session_ids_table.cr rename to src/invidious/database/migrations/0005_create_session_ids_table.cr index 9ef00f78..13c2228d 100644 --- a/src/invidious/migrations/0004_create_session_ids_table.cr +++ b/src/invidious/database/migrations/0005_create_session_ids_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateSessionIdsTable < Migration - version 4 + version 5 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0005_create_nonces_table.cr b/src/invidious/database/migrations/0006_create_nonces_table.cr similarity index 91% rename from src/invidious/migrations/0005_create_nonces_table.cr rename to src/invidious/database/migrations/0006_create_nonces_table.cr index 4b1220e6..cf1229e1 100644 --- a/src/invidious/migrations/0005_create_nonces_table.cr +++ b/src/invidious/database/migrations/0006_create_nonces_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateNoncesTable < Migration - version 5 + version 6 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0006_create_annotations_table.cr b/src/invidious/database/migrations/0007_create_annotations_table.cr similarity index 88% rename from src/invidious/migrations/0006_create_annotations_table.cr rename to src/invidious/database/migrations/0007_create_annotations_table.cr index 86f21dd9..dcecbc3b 100644 --- a/src/invidious/migrations/0006_create_annotations_table.cr +++ b/src/invidious/database/migrations/0007_create_annotations_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateAnnotationsTable < Migration - version 6 + version 7 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/database/migrations/0008_create_playlists_table.cr b/src/invidious/database/migrations/0008_create_playlists_table.cr new file mode 100644 index 00000000..6aa16e1a --- /dev/null +++ b/src/invidious/database/migrations/0008_create_playlists_table.cr @@ -0,0 +1,50 @@ +module Invidious::Database::Migrations + class CreatePlaylistsTable < Migration + version 8 + + def up(conn : DB::Connection) + if !privacy_type_exists?(conn) + conn.exec <<-SQL + CREATE TYPE public.privacy AS ENUM + ( + 'Public', + 'Unlisted', + 'Private' + ); + SQL + end + + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlists + ( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON public.playlists TO current_user; + SQL + end + + private def privacy_type_exists?(conn : DB::Connection) : Bool + request = <<-SQL + SELECT 1 AS one + FROM pg_type + INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace + WHERE pg_namespace.nspname = 'public' + AND pg_type.typname = 'privacy' + LIMIT 1; + SQL + + !conn.query_one?(request, as: Int32).nil? + end + end +end diff --git a/src/invidious/database/migrations/0009_create_playlist_videos_table.cr b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr new file mode 100644 index 00000000..84938b9b --- /dev/null +++ b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr @@ -0,0 +1,27 @@ +module Invidious::Database::Migrations + class CreatePlaylistVideosTable < Migration + version 9 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlist_videos + ( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.playlist_videos TO current_user; + SQL + end + end +end diff --git a/src/invidious/database/migrations/0010_make_videos_unlogged.cr b/src/invidious/database/migrations/0010_make_videos_unlogged.cr new file mode 100644 index 00000000..f5d19683 --- /dev/null +++ b/src/invidious/database/migrations/0010_make_videos_unlogged.cr @@ -0,0 +1,11 @@ +module Invidious::Database::Migrations + class MakeVideosUnlogged < Migration + version 10 + + def up(conn : DB::Connection) + conn.exec <<-SQL + ALTER TABLE public.videos SET UNLOGGED; + SQL + end + end +end diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr new file mode 100644 index 00000000..2cd869c9 --- /dev/null +++ b/src/invidious/database/migrator.cr @@ -0,0 +1,42 @@ +class Invidious::Database::Migrator + MIGRATIONS_TABLE = "public.invidious_migrations" + + class_getter migrations = [] of Invidious::Database::Migration.class + + def initialize(@db : DB::Database) + end + + def migrate + versions = load_versions + + ran_migration = false + load_migrations.sort_by(&.version) + .each do |migration| + next if versions.includes?(migration.version) + + puts "Running migration: #{migration.class.name}" + migration.migrate + ran_migration = true + end + + puts "No migrations to run." unless ran_migration + end + + private def load_migrations : Array(Invidious::Database::Migration) + self.class.migrations.map(&.new(@db)) + end + + private def load_versions : Array(Int64) + create_migrations_table + @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) + end + + private def create_migrations_table + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( + id bigserial PRIMARY KEY, + version bigint NOT NULL + ) + SQL + end +end diff --git a/src/invidious/migrations/0007_create_playlists_table.cr b/src/invidious/migrations/0007_create_playlists_table.cr deleted file mode 100644 index 81217365..00000000 --- a/src/invidious/migrations/0007_create_playlists_table.cr +++ /dev/null @@ -1,47 +0,0 @@ -module Invidious::Migrations - class CreatePlaylistsTable < Migration - version 7 - - def up(conn : DB::Connection) - conn.exec <<-SQL - DO - $$ - BEGIN - IF NOT EXISTS (SELECT * - FROM pg_type typ - INNER JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace - WHERE nsp.nspname = 'public' - AND typ.typname = 'privacy') THEN - CREATE TYPE public.privacy AS ENUM - ( - 'Public', - 'Unlisted', - 'Private' - ); - END IF; - END; - $$ - LANGUAGE plpgsql; - SQL - - conn.exec <<-SQL - CREATE TABLE IF NOT EXISTS public.playlists - ( - title text, - id text primary key, - author text, - description text, - video_count integer, - created timestamptz, - updated timestamptz, - privacy privacy, - index int8[] - ); - SQL - - conn.exec <<-SQL - GRANT ALL ON public.playlists TO current_user; - SQL - end - end -end diff --git a/src/invidious/migrations/0008_create_playlist_videos_table.cr b/src/invidious/migrations/0008_create_playlist_videos_table.cr deleted file mode 100644 index 80fa6b5f..00000000 --- a/src/invidious/migrations/0008_create_playlist_videos_table.cr +++ /dev/null @@ -1,27 +0,0 @@ -module Invidious::Migrations - class CreatePlaylistVideosTable < Migration - version 8 - - def up(conn : DB::Connection) - conn.exec <<-SQL - CREATE TABLE IF NOT EXISTS public.playlist_videos - ( - title text, - id text, - author text, - ucid text, - length_seconds integer, - published timestamptz, - plid text references playlists(id), - index int8, - live_now boolean, - PRIMARY KEY (index,plid) - ); - SQL - - conn.exec <<-SQL - GRANT ALL ON TABLE public.playlist_videos TO current_user; - SQL - end - end -end diff --git a/src/invidious/migrator.cr b/src/invidious/migrator.cr deleted file mode 100644 index dc6880b9..00000000 --- a/src/invidious/migrator.cr +++ /dev/null @@ -1,41 +0,0 @@ -class Invidious::Migrator - MIGRATIONS_TABLE = "invidious_migrations" - - class_getter migrations = [] of Invidious::Migration.class - - def initialize(@db : DB::Database) - end - - def migrate - run_migrations = load_run_migrations - migrations = load_migrations.sort_by(&.version) - migrations_to_run = migrations.reject { |migration| run_migrations.includes?(migration.version) } - if migrations.empty? - puts "No migrations to run." - return - end - - migrations_to_run.each do |migration| - puts "Running migration: #{migration.class.name}" - migration.migrate - end - end - - private def load_migrations : Array(Invidious::Migration) - self.class.migrations.map(&.new(@db)) - end - - private def load_run_migrations : Array(Int64) - create_migrations_table - @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) - end - - private def create_migrations_table - @db.exec <<-SQL - CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( - id bigserial PRIMARY KEY, - version bigint NOT NULL - ) - SQL - end -end