From 8ec992a8a31742a82de38a0aa5eeb509362da9b4 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Wed, 9 Feb 2022 00:50:32 -0600 Subject: [PATCH 1/4] Add custom migration implementation --- src/invidious.cr | 3 ++ src/invidious/migration.cr | 38 +++++++++++++++ .../migrations/0000_create_channels_table.cr | 30 ++++++++++++ .../migrations/0001_create_videos_table.cr | 28 +++++++++++ .../0002_create_channel_videos_table.cr | 35 ++++++++++++++ .../migrations/0003_create_users_table.cr | 34 ++++++++++++++ .../0004_create_session_ids_table.cr | 28 +++++++++++ .../migrations/0005_create_nonces_table.cr | 27 +++++++++++ .../0006_create_annotations_table.cr | 20 ++++++++ .../migrations/0007_create_playlists_table.cr | 47 +++++++++++++++++++ .../0008_create_playlist_videos_table.cr | 27 +++++++++++ src/invidious/migrator.cr | 41 ++++++++++++++++ 12 files changed, 358 insertions(+) create mode 100644 src/invidious/migration.cr create mode 100644 src/invidious/migrations/0000_create_channels_table.cr create mode 100644 src/invidious/migrations/0001_create_videos_table.cr create mode 100644 src/invidious/migrations/0002_create_channel_videos_table.cr create mode 100644 src/invidious/migrations/0003_create_users_table.cr create mode 100644 src/invidious/migrations/0004_create_session_ids_table.cr create mode 100644 src/invidious/migrations/0005_create_nonces_table.cr create mode 100644 src/invidious/migrations/0006_create_annotations_table.cr create mode 100644 src/invidious/migrations/0007_create_playlists_table.cr create mode 100644 src/invidious/migrations/0008_create_playlist_videos_table.cr create mode 100644 src/invidious/migrator.cr diff --git a/src/invidious.cr b/src/invidious.cr index 1ff70905..6ec5f3a5 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -34,6 +34,7 @@ require "./invidious/channels/*" require "./invidious/user/*" require "./invidious/routes/**" require "./invidious/jobs/**" +require "./invidious/migrations/*" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) @@ -111,6 +112,8 @@ end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) +# Run migrations +Invidious::Migrator.new(PG_DB).migrate # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/migration.cr b/src/invidious/migration.cr new file mode 100644 index 00000000..a4eec1c5 --- /dev/null +++ b/src/invidious/migration.cr @@ -0,0 +1,38 @@ +abstract class Invidious::Migration + macro inherited + Invidious::Migrator.migrations << self + end + + @@version : Int64? + + def self.version(version : Int32 | Int64) + @@version = version.to_i64 + end + + getter? completed = false + + def initialize(@db : DB::Database) + end + + abstract def up(conn : DB::Connection) + + def migrate + # migrator already ignores completed migrations + # but this is an extra check to make sure a migration doesn't run twice + return if completed? + + @db.transaction do |txn| + up(txn.connection) + track(txn.connection) + @completed = true + end + end + + def version : Int64 + @@version.not_nil! + end + + private def track(conn : DB::Connection) + conn.exec("INSERT INTO #{Invidious::Migrator::MIGRATIONS_TABLE}(version) VALUES ($1)", version) + end +end diff --git a/src/invidious/migrations/0000_create_channels_table.cr b/src/invidious/migrations/0000_create_channels_table.cr new file mode 100644 index 00000000..1f8f18e2 --- /dev/null +++ b/src/invidious/migrations/0000_create_channels_table.cr @@ -0,0 +1,30 @@ +module Invidious::Migrations + class CreateChannelsTable < Migration + version 0 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channels + ( + id text NOT NULL, + author text, + updated timestamp with time zone, + deleted boolean, + subscribed timestamp with time zone, + CONSTRAINT channels_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channels TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channels_id_idx + ON public.channels + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0001_create_videos_table.cr b/src/invidious/migrations/0001_create_videos_table.cr new file mode 100644 index 00000000..cdc9993f --- /dev/null +++ b/src/invidious/migrations/0001_create_videos_table.cr @@ -0,0 +1,28 @@ +module Invidious::Migrations + class CreateVideosTable < Migration + version 1 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE UNLOGGED TABLE IF NOT EXISTS public.videos + ( + id text NOT NULL, + info text, + updated timestamp with time zone, + CONSTRAINT videos_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS id_idx + ON public.videos + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0002_create_channel_videos_table.cr b/src/invidious/migrations/0002_create_channel_videos_table.cr new file mode 100644 index 00000000..737abad4 --- /dev/null +++ b/src/invidious/migrations/0002_create_channel_videos_table.cr @@ -0,0 +1,35 @@ +module Invidious::Migrations + class CreateChannelVideosTable < Migration + version 2 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channel_videos + ( + id text NOT NULL, + title text, + published timestamp with time zone, + updated timestamp with time zone, + ucid text, + author text, + length_seconds integer, + live_now boolean, + premiere_timestamp timestamp with time zone, + views bigint, + CONSTRAINT channel_videos_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channel_videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx + ON public.channel_videos + USING btree + (ucid COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0003_create_users_table.cr b/src/invidious/migrations/0003_create_users_table.cr new file mode 100644 index 00000000..d91cca8d --- /dev/null +++ b/src/invidious/migrations/0003_create_users_table.cr @@ -0,0 +1,34 @@ +module Invidious::Migrations + class CreateUsersTable < Migration + version 3 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.users + ( + updated timestamp with time zone, + notifications text[], + subscriptions text[], + email text NOT NULL, + preferences text, + password text, + token text, + watched text[], + feed_needs_update boolean, + CONSTRAINT users_email_key UNIQUE (email) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.users TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx + ON public.users + USING btree + (lower(email) COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0004_create_session_ids_table.cr b/src/invidious/migrations/0004_create_session_ids_table.cr new file mode 100644 index 00000000..9ef00f78 --- /dev/null +++ b/src/invidious/migrations/0004_create_session_ids_table.cr @@ -0,0 +1,28 @@ +module Invidious::Migrations + class CreateSessionIdsTable < Migration + version 4 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.session_ids + ( + id text NOT NULL, + email text, + issued timestamp with time zone, + CONSTRAINT session_ids_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.session_ids TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS session_ids_id_idx + ON public.session_ids + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0005_create_nonces_table.cr b/src/invidious/migrations/0005_create_nonces_table.cr new file mode 100644 index 00000000..4b1220e6 --- /dev/null +++ b/src/invidious/migrations/0005_create_nonces_table.cr @@ -0,0 +1,27 @@ +module Invidious::Migrations + class CreateNoncesTable < Migration + version 5 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.nonces + ( + nonce text, + expire timestamp with time zone, + CONSTRAINT nonces_id_key UNIQUE (nonce) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.nonces TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS nonces_nonce_idx + ON public.nonces + USING btree + (nonce COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/migrations/0006_create_annotations_table.cr b/src/invidious/migrations/0006_create_annotations_table.cr new file mode 100644 index 00000000..86f21dd9 --- /dev/null +++ b/src/invidious/migrations/0006_create_annotations_table.cr @@ -0,0 +1,20 @@ +module Invidious::Migrations + class CreateAnnotationsTable < Migration + version 6 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.annotations + ( + id text NOT NULL, + annotations xml, + CONSTRAINT annotations_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.annotations TO current_user; + SQL + end + end +end diff --git a/src/invidious/migrations/0007_create_playlists_table.cr b/src/invidious/migrations/0007_create_playlists_table.cr new file mode 100644 index 00000000..81217365 --- /dev/null +++ b/src/invidious/migrations/0007_create_playlists_table.cr @@ -0,0 +1,47 @@ +module Invidious::Migrations + class CreatePlaylistsTable < Migration + version 7 + + def up(conn : DB::Connection) + conn.exec <<-SQL + DO + $$ + BEGIN + IF NOT EXISTS (SELECT * + FROM pg_type typ + INNER JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace + WHERE nsp.nspname = 'public' + AND typ.typname = 'privacy') THEN + CREATE TYPE public.privacy AS ENUM + ( + 'Public', + 'Unlisted', + 'Private' + ); + END IF; + END; + $$ + LANGUAGE plpgsql; + SQL + + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlists + ( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON public.playlists TO current_user; + SQL + end + end +end diff --git a/src/invidious/migrations/0008_create_playlist_videos_table.cr b/src/invidious/migrations/0008_create_playlist_videos_table.cr new file mode 100644 index 00000000..80fa6b5f --- /dev/null +++ b/src/invidious/migrations/0008_create_playlist_videos_table.cr @@ -0,0 +1,27 @@ +module Invidious::Migrations + class CreatePlaylistVideosTable < Migration + version 8 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlist_videos + ( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.playlist_videos TO current_user; + SQL + end + end +end diff --git a/src/invidious/migrator.cr b/src/invidious/migrator.cr new file mode 100644 index 00000000..dc6880b9 --- /dev/null +++ b/src/invidious/migrator.cr @@ -0,0 +1,41 @@ +class Invidious::Migrator + MIGRATIONS_TABLE = "invidious_migrations" + + class_getter migrations = [] of Invidious::Migration.class + + def initialize(@db : DB::Database) + end + + def migrate + run_migrations = load_run_migrations + migrations = load_migrations.sort_by(&.version) + migrations_to_run = migrations.reject { |migration| run_migrations.includes?(migration.version) } + if migrations.empty? + puts "No migrations to run." + return + end + + migrations_to_run.each do |migration| + puts "Running migration: #{migration.class.name}" + migration.migrate + end + end + + private def load_migrations : Array(Invidious::Migration) + self.class.migrations.map(&.new(@db)) + end + + private def load_run_migrations : Array(Int64) + create_migrations_table + @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) + end + + private def create_migrations_table + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( + id bigserial PRIMARY KEY, + version bigint NOT NULL + ) + SQL + end +end From cf13c11236de6c4e373b110730530c8e1f305900 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Thu, 10 Feb 2022 22:16:40 -0600 Subject: [PATCH 2/4] Migrations tweaks --- src/invidious.cr | 4 +- src/invidious/{ => database}/migration.cr | 6 +-- .../migrations/0001_create_channels_table.cr} | 4 +- .../migrations/0002_create_videos_table.cr} | 4 +- .../0003_create_channel_videos_table.cr} | 4 +- .../migrations/0004_create_users_table.cr} | 4 +- .../0005_create_session_ids_table.cr} | 4 +- .../migrations/0006_create_nonces_table.cr} | 4 +- .../0007_create_annotations_table.cr} | 4 +- .../migrations/0008_create_playlists_table.cr | 50 +++++++++++++++++++ .../0009_create_playlist_videos_table.cr | 27 ++++++++++ .../migrations/0010_make_videos_unlogged.cr | 11 ++++ src/invidious/database/migrator.cr | 42 ++++++++++++++++ .../migrations/0007_create_playlists_table.cr | 47 ----------------- .../0008_create_playlist_videos_table.cr | 27 ---------- src/invidious/migrator.cr | 41 --------------- 16 files changed, 149 insertions(+), 134 deletions(-) rename src/invidious/{ => database}/migration.cr (78%) rename src/invidious/{migrations/0000_create_channels_table.cr => database/migrations/0001_create_channels_table.cr} (92%) rename src/invidious/{migrations/0001_create_videos_table.cr => database/migrations/0002_create_videos_table.cr} (91%) rename src/invidious/{migrations/0002_create_channel_videos_table.cr => database/migrations/0003_create_channel_videos_table.cr} (94%) rename src/invidious/{migrations/0003_create_users_table.cr => database/migrations/0004_create_users_table.cr} (93%) rename src/invidious/{migrations/0004_create_session_ids_table.cr => database/migrations/0005_create_session_ids_table.cr} (92%) rename src/invidious/{migrations/0005_create_nonces_table.cr => database/migrations/0006_create_nonces_table.cr} (91%) rename src/invidious/{migrations/0006_create_annotations_table.cr => database/migrations/0007_create_annotations_table.cr} (88%) create mode 100644 src/invidious/database/migrations/0008_create_playlists_table.cr create mode 100644 src/invidious/database/migrations/0009_create_playlist_videos_table.cr create mode 100644 src/invidious/database/migrations/0010_make_videos_unlogged.cr create mode 100644 src/invidious/database/migrator.cr delete mode 100644 src/invidious/migrations/0007_create_playlists_table.cr delete mode 100644 src/invidious/migrations/0008_create_playlist_videos_table.cr delete mode 100644 src/invidious/migrator.cr diff --git a/src/invidious.cr b/src/invidious.cr index 6ec5f3a5..e8ad03ef 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,6 +27,7 @@ require "compress/zip" require "protodec/utils" require "./invidious/database/*" +require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/*" @@ -34,7 +35,6 @@ require "./invidious/channels/*" require "./invidious/user/*" require "./invidious/routes/**" require "./invidious/jobs/**" -require "./invidious/migrations/*" CONFIG = Config.load HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) @@ -113,7 +113,7 @@ OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mo LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Run migrations -Invidious::Migrator.new(PG_DB).migrate +Invidious::Database::Migrator.new(PG_DB).migrate # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/migration.cr b/src/invidious/database/migration.cr similarity index 78% rename from src/invidious/migration.cr rename to src/invidious/database/migration.cr index a4eec1c5..921d8f38 100644 --- a/src/invidious/migration.cr +++ b/src/invidious/database/migration.cr @@ -1,6 +1,6 @@ -abstract class Invidious::Migration +abstract class Invidious::Database::Migration macro inherited - Invidious::Migrator.migrations << self + Migrator.migrations << self end @@version : Int64? @@ -33,6 +33,6 @@ abstract class Invidious::Migration end private def track(conn : DB::Connection) - conn.exec("INSERT INTO #{Invidious::Migrator::MIGRATIONS_TABLE}(version) VALUES ($1)", version) + conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version) end end diff --git a/src/invidious/migrations/0000_create_channels_table.cr b/src/invidious/database/migrations/0001_create_channels_table.cr similarity index 92% rename from src/invidious/migrations/0000_create_channels_table.cr rename to src/invidious/database/migrations/0001_create_channels_table.cr index 1f8f18e2..a1362bcf 100644 --- a/src/invidious/migrations/0000_create_channels_table.cr +++ b/src/invidious/database/migrations/0001_create_channels_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateChannelsTable < Migration - version 0 + version 1 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0001_create_videos_table.cr b/src/invidious/database/migrations/0002_create_videos_table.cr similarity index 91% rename from src/invidious/migrations/0001_create_videos_table.cr rename to src/invidious/database/migrations/0002_create_videos_table.cr index cdc9993f..c2ac84f8 100644 --- a/src/invidious/migrations/0001_create_videos_table.cr +++ b/src/invidious/database/migrations/0002_create_videos_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateVideosTable < Migration - version 1 + version 2 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0002_create_channel_videos_table.cr b/src/invidious/database/migrations/0003_create_channel_videos_table.cr similarity index 94% rename from src/invidious/migrations/0002_create_channel_videos_table.cr rename to src/invidious/database/migrations/0003_create_channel_videos_table.cr index 737abad4..c9b62e4c 100644 --- a/src/invidious/migrations/0002_create_channel_videos_table.cr +++ b/src/invidious/database/migrations/0003_create_channel_videos_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateChannelVideosTable < Migration - version 2 + version 3 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0003_create_users_table.cr b/src/invidious/database/migrations/0004_create_users_table.cr similarity index 93% rename from src/invidious/migrations/0003_create_users_table.cr rename to src/invidious/database/migrations/0004_create_users_table.cr index d91cca8d..a13ba15f 100644 --- a/src/invidious/migrations/0003_create_users_table.cr +++ b/src/invidious/database/migrations/0004_create_users_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateUsersTable < Migration - version 3 + version 4 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0004_create_session_ids_table.cr b/src/invidious/database/migrations/0005_create_session_ids_table.cr similarity index 92% rename from src/invidious/migrations/0004_create_session_ids_table.cr rename to src/invidious/database/migrations/0005_create_session_ids_table.cr index 9ef00f78..13c2228d 100644 --- a/src/invidious/migrations/0004_create_session_ids_table.cr +++ b/src/invidious/database/migrations/0005_create_session_ids_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateSessionIdsTable < Migration - version 4 + version 5 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0005_create_nonces_table.cr b/src/invidious/database/migrations/0006_create_nonces_table.cr similarity index 91% rename from src/invidious/migrations/0005_create_nonces_table.cr rename to src/invidious/database/migrations/0006_create_nonces_table.cr index 4b1220e6..cf1229e1 100644 --- a/src/invidious/migrations/0005_create_nonces_table.cr +++ b/src/invidious/database/migrations/0006_create_nonces_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateNoncesTable < Migration - version 5 + version 6 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/migrations/0006_create_annotations_table.cr b/src/invidious/database/migrations/0007_create_annotations_table.cr similarity index 88% rename from src/invidious/migrations/0006_create_annotations_table.cr rename to src/invidious/database/migrations/0007_create_annotations_table.cr index 86f21dd9..dcecbc3b 100644 --- a/src/invidious/migrations/0006_create_annotations_table.cr +++ b/src/invidious/database/migrations/0007_create_annotations_table.cr @@ -1,6 +1,6 @@ -module Invidious::Migrations +module Invidious::Database::Migrations class CreateAnnotationsTable < Migration - version 6 + version 7 def up(conn : DB::Connection) conn.exec <<-SQL diff --git a/src/invidious/database/migrations/0008_create_playlists_table.cr b/src/invidious/database/migrations/0008_create_playlists_table.cr new file mode 100644 index 00000000..6aa16e1a --- /dev/null +++ b/src/invidious/database/migrations/0008_create_playlists_table.cr @@ -0,0 +1,50 @@ +module Invidious::Database::Migrations + class CreatePlaylistsTable < Migration + version 8 + + def up(conn : DB::Connection) + if !privacy_type_exists?(conn) + conn.exec <<-SQL + CREATE TYPE public.privacy AS ENUM + ( + 'Public', + 'Unlisted', + 'Private' + ); + SQL + end + + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlists + ( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON public.playlists TO current_user; + SQL + end + + private def privacy_type_exists?(conn : DB::Connection) : Bool + request = <<-SQL + SELECT 1 AS one + FROM pg_type + INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace + WHERE pg_namespace.nspname = 'public' + AND pg_type.typname = 'privacy' + LIMIT 1; + SQL + + !conn.query_one?(request, as: Int32).nil? + end + end +end diff --git a/src/invidious/database/migrations/0009_create_playlist_videos_table.cr b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr new file mode 100644 index 00000000..84938b9b --- /dev/null +++ b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr @@ -0,0 +1,27 @@ +module Invidious::Database::Migrations + class CreatePlaylistVideosTable < Migration + version 9 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlist_videos + ( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.playlist_videos TO current_user; + SQL + end + end +end diff --git a/src/invidious/database/migrations/0010_make_videos_unlogged.cr b/src/invidious/database/migrations/0010_make_videos_unlogged.cr new file mode 100644 index 00000000..f5d19683 --- /dev/null +++ b/src/invidious/database/migrations/0010_make_videos_unlogged.cr @@ -0,0 +1,11 @@ +module Invidious::Database::Migrations + class MakeVideosUnlogged < Migration + version 10 + + def up(conn : DB::Connection) + conn.exec <<-SQL + ALTER TABLE public.videos SET UNLOGGED; + SQL + end + end +end diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr new file mode 100644 index 00000000..2cd869c9 --- /dev/null +++ b/src/invidious/database/migrator.cr @@ -0,0 +1,42 @@ +class Invidious::Database::Migrator + MIGRATIONS_TABLE = "public.invidious_migrations" + + class_getter migrations = [] of Invidious::Database::Migration.class + + def initialize(@db : DB::Database) + end + + def migrate + versions = load_versions + + ran_migration = false + load_migrations.sort_by(&.version) + .each do |migration| + next if versions.includes?(migration.version) + + puts "Running migration: #{migration.class.name}" + migration.migrate + ran_migration = true + end + + puts "No migrations to run." unless ran_migration + end + + private def load_migrations : Array(Invidious::Database::Migration) + self.class.migrations.map(&.new(@db)) + end + + private def load_versions : Array(Int64) + create_migrations_table + @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) + end + + private def create_migrations_table + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( + id bigserial PRIMARY KEY, + version bigint NOT NULL + ) + SQL + end +end diff --git a/src/invidious/migrations/0007_create_playlists_table.cr b/src/invidious/migrations/0007_create_playlists_table.cr deleted file mode 100644 index 81217365..00000000 --- a/src/invidious/migrations/0007_create_playlists_table.cr +++ /dev/null @@ -1,47 +0,0 @@ -module Invidious::Migrations - class CreatePlaylistsTable < Migration - version 7 - - def up(conn : DB::Connection) - conn.exec <<-SQL - DO - $$ - BEGIN - IF NOT EXISTS (SELECT * - FROM pg_type typ - INNER JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace - WHERE nsp.nspname = 'public' - AND typ.typname = 'privacy') THEN - CREATE TYPE public.privacy AS ENUM - ( - 'Public', - 'Unlisted', - 'Private' - ); - END IF; - END; - $$ - LANGUAGE plpgsql; - SQL - - conn.exec <<-SQL - CREATE TABLE IF NOT EXISTS public.playlists - ( - title text, - id text primary key, - author text, - description text, - video_count integer, - created timestamptz, - updated timestamptz, - privacy privacy, - index int8[] - ); - SQL - - conn.exec <<-SQL - GRANT ALL ON public.playlists TO current_user; - SQL - end - end -end diff --git a/src/invidious/migrations/0008_create_playlist_videos_table.cr b/src/invidious/migrations/0008_create_playlist_videos_table.cr deleted file mode 100644 index 80fa6b5f..00000000 --- a/src/invidious/migrations/0008_create_playlist_videos_table.cr +++ /dev/null @@ -1,27 +0,0 @@ -module Invidious::Migrations - class CreatePlaylistVideosTable < Migration - version 8 - - def up(conn : DB::Connection) - conn.exec <<-SQL - CREATE TABLE IF NOT EXISTS public.playlist_videos - ( - title text, - id text, - author text, - ucid text, - length_seconds integer, - published timestamptz, - plid text references playlists(id), - index int8, - live_now boolean, - PRIMARY KEY (index,plid) - ); - SQL - - conn.exec <<-SQL - GRANT ALL ON TABLE public.playlist_videos TO current_user; - SQL - end - end -end diff --git a/src/invidious/migrator.cr b/src/invidious/migrator.cr deleted file mode 100644 index dc6880b9..00000000 --- a/src/invidious/migrator.cr +++ /dev/null @@ -1,41 +0,0 @@ -class Invidious::Migrator - MIGRATIONS_TABLE = "invidious_migrations" - - class_getter migrations = [] of Invidious::Migration.class - - def initialize(@db : DB::Database) - end - - def migrate - run_migrations = load_run_migrations - migrations = load_migrations.sort_by(&.version) - migrations_to_run = migrations.reject { |migration| run_migrations.includes?(migration.version) } - if migrations.empty? - puts "No migrations to run." - return - end - - migrations_to_run.each do |migration| - puts "Running migration: #{migration.class.name}" - migration.migrate - end - end - - private def load_migrations : Array(Invidious::Migration) - self.class.migrations.map(&.new(@db)) - end - - private def load_run_migrations : Array(Int64) - create_migrations_table - @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) - end - - private def create_migrations_table - @db.exec <<-SQL - CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( - id bigserial PRIMARY KEY, - version bigint NOT NULL - ) - SQL - end -end From 59654289cb7c024d3cf7c9d00d38cf4453bc48df Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Fri, 11 Feb 2022 22:43:16 -0600 Subject: [PATCH 3/4] Run migrations through CLI instead of when app starts --- src/invidious.cr | 9 ++++++++- src/invidious/database/migrator.cr | 7 +++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index e8ad03ef..25ee7c78 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -102,6 +102,10 @@ Kemal.config.extra_options do |parser| puts SOFTWARE.to_pretty_json exit end + parser.on("--migrate", "Run any migrations") do + Invidious::Database::Migrator.new(PG_DB).migrate + exit + end end Kemal::CLI.new ARGV @@ -113,7 +117,10 @@ OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mo LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) # Run migrations -Invidious::Database::Migrator.new(PG_DB).migrate +if Invidious::Database::Migrator.new(PG_DB).pending_migrations? + puts "There are pending migrations. Run `invidious --migrate` to apply the migrations." + exit 46 +end # Check table integrity Invidious::Database.check_integrity(CONFIG) diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr index 2cd869c9..660c3203 100644 --- a/src/invidious/database/migrator.cr +++ b/src/invidious/database/migrator.cr @@ -22,6 +22,13 @@ class Invidious::Database::Migrator puts "No migrations to run." unless ran_migration end + def pending_migrations? : Bool + versions = load_versions + + load_migrations.sort_by(&.version) + .any? { |migration| !versions.includes?(migration.version) } + end + private def load_migrations : Array(Invidious::Database::Migration) self.class.migrations.map(&.new(@db)) end From bf054dfda5ac6d1d3c4ab40b44a3bbb45ca132a3 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Sat, 12 Feb 2022 09:20:43 -0600 Subject: [PATCH 4/4] Do not check for pending migrations on app start This is so that we don't break deploys with this PR. Instead we only ship the 'invidious --migrate' cli command and let people test that. Maybe even ship a new migration that wouldn't break apps that don't run the migrations. Then we roll out the functionality that requires migrations. --- src/invidious.cr | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 25ee7c78..04b18a65 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -116,11 +116,6 @@ end OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mode: "a") LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) -# Run migrations -if Invidious::Database::Migrator.new(PG_DB).pending_migrations? - puts "There are pending migrations. Run `invidious --migrate` to apply the migrations." - exit 46 -end # Check table integrity Invidious::Database.check_integrity(CONFIG)