diff --git a/src/invidious.cr b/src/invidious.cr index 1bdf3097..abc459b7 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -27,6 +27,7 @@ require "compress/zip" require "protodec/utils" require "./invidious/database/*" +require "./invidious/database/migrations/*" require "./invidious/helpers/*" require "./invidious/yt_backend/*" require "./invidious/frontend/*" @@ -102,6 +103,10 @@ Kemal.config.extra_options do |parser| puts SOFTWARE.to_pretty_json exit end + parser.on("--migrate", "Run any migrations") do + Invidious::Database::Migrator.new(PG_DB).migrate + exit + end end Kemal::CLI.new ARGV diff --git a/src/invidious/database/migration.cr b/src/invidious/database/migration.cr new file mode 100644 index 00000000..921d8f38 --- /dev/null +++ b/src/invidious/database/migration.cr @@ -0,0 +1,38 @@ +abstract class Invidious::Database::Migration + macro inherited + Migrator.migrations << self + end + + @@version : Int64? + + def self.version(version : Int32 | Int64) + @@version = version.to_i64 + end + + getter? completed = false + + def initialize(@db : DB::Database) + end + + abstract def up(conn : DB::Connection) + + def migrate + # migrator already ignores completed migrations + # but this is an extra check to make sure a migration doesn't run twice + return if completed? + + @db.transaction do |txn| + up(txn.connection) + track(txn.connection) + @completed = true + end + end + + def version : Int64 + @@version.not_nil! + end + + private def track(conn : DB::Connection) + conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version) + end +end diff --git a/src/invidious/database/migrations/0001_create_channels_table.cr b/src/invidious/database/migrations/0001_create_channels_table.cr new file mode 100644 index 00000000..a1362bcf --- /dev/null +++ b/src/invidious/database/migrations/0001_create_channels_table.cr @@ -0,0 +1,30 @@ +module Invidious::Database::Migrations + class CreateChannelsTable < Migration + version 1 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channels + ( + id text NOT NULL, + author text, + updated timestamp with time zone, + deleted boolean, + subscribed timestamp with time zone, + CONSTRAINT channels_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channels TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channels_id_idx + ON public.channels + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0002_create_videos_table.cr b/src/invidious/database/migrations/0002_create_videos_table.cr new file mode 100644 index 00000000..c2ac84f8 --- /dev/null +++ b/src/invidious/database/migrations/0002_create_videos_table.cr @@ -0,0 +1,28 @@ +module Invidious::Database::Migrations + class CreateVideosTable < Migration + version 2 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE UNLOGGED TABLE IF NOT EXISTS public.videos + ( + id text NOT NULL, + info text, + updated timestamp with time zone, + CONSTRAINT videos_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS id_idx + ON public.videos + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0003_create_channel_videos_table.cr b/src/invidious/database/migrations/0003_create_channel_videos_table.cr new file mode 100644 index 00000000..c9b62e4c --- /dev/null +++ b/src/invidious/database/migrations/0003_create_channel_videos_table.cr @@ -0,0 +1,35 @@ +module Invidious::Database::Migrations + class CreateChannelVideosTable < Migration + version 3 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.channel_videos + ( + id text NOT NULL, + title text, + published timestamp with time zone, + updated timestamp with time zone, + ucid text, + author text, + length_seconds integer, + live_now boolean, + premiere_timestamp timestamp with time zone, + views bigint, + CONSTRAINT channel_videos_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.channel_videos TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx + ON public.channel_videos + USING btree + (ucid COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0004_create_users_table.cr b/src/invidious/database/migrations/0004_create_users_table.cr new file mode 100644 index 00000000..a13ba15f --- /dev/null +++ b/src/invidious/database/migrations/0004_create_users_table.cr @@ -0,0 +1,34 @@ +module Invidious::Database::Migrations + class CreateUsersTable < Migration + version 4 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.users + ( + updated timestamp with time zone, + notifications text[], + subscriptions text[], + email text NOT NULL, + preferences text, + password text, + token text, + watched text[], + feed_needs_update boolean, + CONSTRAINT users_email_key UNIQUE (email) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.users TO current_user; + SQL + + conn.exec <<-SQL + CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx + ON public.users + USING btree + (lower(email) COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0005_create_session_ids_table.cr b/src/invidious/database/migrations/0005_create_session_ids_table.cr new file mode 100644 index 00000000..13c2228d --- /dev/null +++ b/src/invidious/database/migrations/0005_create_session_ids_table.cr @@ -0,0 +1,28 @@ +module Invidious::Database::Migrations + class CreateSessionIdsTable < Migration + version 5 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.session_ids + ( + id text NOT NULL, + email text, + issued timestamp with time zone, + CONSTRAINT session_ids_pkey PRIMARY KEY (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.session_ids TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS session_ids_id_idx + ON public.session_ids + USING btree + (id COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0006_create_nonces_table.cr b/src/invidious/database/migrations/0006_create_nonces_table.cr new file mode 100644 index 00000000..cf1229e1 --- /dev/null +++ b/src/invidious/database/migrations/0006_create_nonces_table.cr @@ -0,0 +1,27 @@ +module Invidious::Database::Migrations + class CreateNoncesTable < Migration + version 6 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.nonces + ( + nonce text, + expire timestamp with time zone, + CONSTRAINT nonces_id_key UNIQUE (nonce) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.nonces TO current_user; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS nonces_nonce_idx + ON public.nonces + USING btree + (nonce COLLATE pg_catalog."default"); + SQL + end + end +end diff --git a/src/invidious/database/migrations/0007_create_annotations_table.cr b/src/invidious/database/migrations/0007_create_annotations_table.cr new file mode 100644 index 00000000..dcecbc3b --- /dev/null +++ b/src/invidious/database/migrations/0007_create_annotations_table.cr @@ -0,0 +1,20 @@ +module Invidious::Database::Migrations + class CreateAnnotationsTable < Migration + version 7 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.annotations + ( + id text NOT NULL, + annotations xml, + CONSTRAINT annotations_id_key UNIQUE (id) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.annotations TO current_user; + SQL + end + end +end diff --git a/src/invidious/database/migrations/0008_create_playlists_table.cr b/src/invidious/database/migrations/0008_create_playlists_table.cr new file mode 100644 index 00000000..6aa16e1a --- /dev/null +++ b/src/invidious/database/migrations/0008_create_playlists_table.cr @@ -0,0 +1,50 @@ +module Invidious::Database::Migrations + class CreatePlaylistsTable < Migration + version 8 + + def up(conn : DB::Connection) + if !privacy_type_exists?(conn) + conn.exec <<-SQL + CREATE TYPE public.privacy AS ENUM + ( + 'Public', + 'Unlisted', + 'Private' + ); + SQL + end + + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlists + ( + title text, + id text primary key, + author text, + description text, + video_count integer, + created timestamptz, + updated timestamptz, + privacy privacy, + index int8[] + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON public.playlists TO current_user; + SQL + end + + private def privacy_type_exists?(conn : DB::Connection) : Bool + request = <<-SQL + SELECT 1 AS one + FROM pg_type + INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace + WHERE pg_namespace.nspname = 'public' + AND pg_type.typname = 'privacy' + LIMIT 1; + SQL + + !conn.query_one?(request, as: Int32).nil? + end + end +end diff --git a/src/invidious/database/migrations/0009_create_playlist_videos_table.cr b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr new file mode 100644 index 00000000..84938b9b --- /dev/null +++ b/src/invidious/database/migrations/0009_create_playlist_videos_table.cr @@ -0,0 +1,27 @@ +module Invidious::Database::Migrations + class CreatePlaylistVideosTable < Migration + version 9 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.playlist_videos + ( + title text, + id text, + author text, + ucid text, + length_seconds integer, + published timestamptz, + plid text references playlists(id), + index int8, + live_now boolean, + PRIMARY KEY (index,plid) + ); + SQL + + conn.exec <<-SQL + GRANT ALL ON TABLE public.playlist_videos TO current_user; + SQL + end + end +end diff --git a/src/invidious/database/migrations/0010_make_videos_unlogged.cr b/src/invidious/database/migrations/0010_make_videos_unlogged.cr new file mode 100644 index 00000000..f5d19683 --- /dev/null +++ b/src/invidious/database/migrations/0010_make_videos_unlogged.cr @@ -0,0 +1,11 @@ +module Invidious::Database::Migrations + class MakeVideosUnlogged < Migration + version 10 + + def up(conn : DB::Connection) + conn.exec <<-SQL + ALTER TABLE public.videos SET UNLOGGED; + SQL + end + end +end diff --git a/src/invidious/database/migrator.cr b/src/invidious/database/migrator.cr new file mode 100644 index 00000000..660c3203 --- /dev/null +++ b/src/invidious/database/migrator.cr @@ -0,0 +1,49 @@ +class Invidious::Database::Migrator + MIGRATIONS_TABLE = "public.invidious_migrations" + + class_getter migrations = [] of Invidious::Database::Migration.class + + def initialize(@db : DB::Database) + end + + def migrate + versions = load_versions + + ran_migration = false + load_migrations.sort_by(&.version) + .each do |migration| + next if versions.includes?(migration.version) + + puts "Running migration: #{migration.class.name}" + migration.migrate + ran_migration = true + end + + puts "No migrations to run." unless ran_migration + end + + def pending_migrations? : Bool + versions = load_versions + + load_migrations.sort_by(&.version) + .any? { |migration| !versions.includes?(migration.version) } + end + + private def load_migrations : Array(Invidious::Database::Migration) + self.class.migrations.map(&.new(@db)) + end + + private def load_versions : Array(Int64) + create_migrations_table + @db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64) + end + + private def create_migrations_table + @db.exec <<-SQL + CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} ( + id bigserial PRIMARY KEY, + version bigint NOT NULL + ) + SQL + end +end