Merge pull request #2878 from matthewmcgarvey/migrations

Add custom migration implementation
This commit is contained in:
Samantaz Fox 2022-03-11 20:49:26 +01:00 committed by GitHub
commit 55da1e3e92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 382 additions and 0 deletions

View File

@ -27,6 +27,7 @@ require "compress/zip"
require "protodec/utils" require "protodec/utils"
require "./invidious/database/*" require "./invidious/database/*"
require "./invidious/database/migrations/*"
require "./invidious/helpers/*" require "./invidious/helpers/*"
require "./invidious/yt_backend/*" require "./invidious/yt_backend/*"
require "./invidious/frontend/*" require "./invidious/frontend/*"
@ -102,6 +103,10 @@ Kemal.config.extra_options do |parser|
puts SOFTWARE.to_pretty_json puts SOFTWARE.to_pretty_json
exit exit
end end
parser.on("--migrate", "Run any migrations") do
Invidious::Database::Migrator.new(PG_DB).migrate
exit
end
end end
Kemal::CLI.new ARGV Kemal::CLI.new ARGV

View File

@ -0,0 +1,38 @@
abstract class Invidious::Database::Migration
macro inherited
Migrator.migrations << self
end
@@version : Int64?
def self.version(version : Int32 | Int64)
@@version = version.to_i64
end
getter? completed = false
def initialize(@db : DB::Database)
end
abstract def up(conn : DB::Connection)
def migrate
# migrator already ignores completed migrations
# but this is an extra check to make sure a migration doesn't run twice
return if completed?
@db.transaction do |txn|
up(txn.connection)
track(txn.connection)
@completed = true
end
end
def version : Int64
@@version.not_nil!
end
private def track(conn : DB::Connection)
conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version)
end
end

View File

@ -0,0 +1,30 @@
module Invidious::Database::Migrations
class CreateChannelsTable < Migration
version 1
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.channels
(
id text NOT NULL,
author text,
updated timestamp with time zone,
deleted boolean,
subscribed timestamp with time zone,
CONSTRAINT channels_id_key UNIQUE (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.channels TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS channels_id_idx
ON public.channels
USING btree
(id COLLATE pg_catalog."default");
SQL
end
end
end

View File

@ -0,0 +1,28 @@
module Invidious::Database::Migrations
class CreateVideosTable < Migration
version 2
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE UNLOGGED TABLE IF NOT EXISTS public.videos
(
id text NOT NULL,
info text,
updated timestamp with time zone,
CONSTRAINT videos_pkey PRIMARY KEY (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.videos TO current_user;
SQL
conn.exec <<-SQL
CREATE UNIQUE INDEX IF NOT EXISTS id_idx
ON public.videos
USING btree
(id COLLATE pg_catalog."default");
SQL
end
end
end

View File

@ -0,0 +1,35 @@
module Invidious::Database::Migrations
class CreateChannelVideosTable < Migration
version 3
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.channel_videos
(
id text NOT NULL,
title text,
published timestamp with time zone,
updated timestamp with time zone,
ucid text,
author text,
length_seconds integer,
live_now boolean,
premiere_timestamp timestamp with time zone,
views bigint,
CONSTRAINT channel_videos_id_key UNIQUE (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.channel_videos TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS channel_videos_ucid_idx
ON public.channel_videos
USING btree
(ucid COLLATE pg_catalog."default");
SQL
end
end
end

View File

@ -0,0 +1,34 @@
module Invidious::Database::Migrations
class CreateUsersTable < Migration
version 4
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.users
(
updated timestamp with time zone,
notifications text[],
subscriptions text[],
email text NOT NULL,
preferences text,
password text,
token text,
watched text[],
feed_needs_update boolean,
CONSTRAINT users_email_key UNIQUE (email)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.users TO current_user;
SQL
conn.exec <<-SQL
CREATE UNIQUE INDEX IF NOT EXISTS email_unique_idx
ON public.users
USING btree
(lower(email) COLLATE pg_catalog."default");
SQL
end
end
end

View File

@ -0,0 +1,28 @@
module Invidious::Database::Migrations
class CreateSessionIdsTable < Migration
version 5
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.session_ids
(
id text NOT NULL,
email text,
issued timestamp with time zone,
CONSTRAINT session_ids_pkey PRIMARY KEY (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.session_ids TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS session_ids_id_idx
ON public.session_ids
USING btree
(id COLLATE pg_catalog."default");
SQL
end
end
end

View File

@ -0,0 +1,27 @@
module Invidious::Database::Migrations
class CreateNoncesTable < Migration
version 6
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.nonces
(
nonce text,
expire timestamp with time zone,
CONSTRAINT nonces_id_key UNIQUE (nonce)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.nonces TO current_user;
SQL
conn.exec <<-SQL
CREATE INDEX IF NOT EXISTS nonces_nonce_idx
ON public.nonces
USING btree
(nonce COLLATE pg_catalog."default");
SQL
end
end
end

View File

@ -0,0 +1,20 @@
module Invidious::Database::Migrations
class CreateAnnotationsTable < Migration
version 7
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.annotations
(
id text NOT NULL,
annotations xml,
CONSTRAINT annotations_id_key UNIQUE (id)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.annotations TO current_user;
SQL
end
end
end

View File

@ -0,0 +1,50 @@
module Invidious::Database::Migrations
class CreatePlaylistsTable < Migration
version 8
def up(conn : DB::Connection)
if !privacy_type_exists?(conn)
conn.exec <<-SQL
CREATE TYPE public.privacy AS ENUM
(
'Public',
'Unlisted',
'Private'
);
SQL
end
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlists
(
title text,
id text primary key,
author text,
description text,
video_count integer,
created timestamptz,
updated timestamptz,
privacy privacy,
index int8[]
);
SQL
conn.exec <<-SQL
GRANT ALL ON public.playlists TO current_user;
SQL
end
private def privacy_type_exists?(conn : DB::Connection) : Bool
request = <<-SQL
SELECT 1 AS one
FROM pg_type
INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace
WHERE pg_namespace.nspname = 'public'
AND pg_type.typname = 'privacy'
LIMIT 1;
SQL
!conn.query_one?(request, as: Int32).nil?
end
end
end

View File

@ -0,0 +1,27 @@
module Invidious::Database::Migrations
class CreatePlaylistVideosTable < Migration
version 9
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlist_videos
(
title text,
id text,
author text,
ucid text,
length_seconds integer,
published timestamptz,
plid text references playlists(id),
index int8,
live_now boolean,
PRIMARY KEY (index,plid)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.playlist_videos TO current_user;
SQL
end
end
end

View File

@ -0,0 +1,11 @@
module Invidious::Database::Migrations
class MakeVideosUnlogged < Migration
version 10
def up(conn : DB::Connection)
conn.exec <<-SQL
ALTER TABLE public.videos SET UNLOGGED;
SQL
end
end
end

View File

@ -0,0 +1,49 @@
class Invidious::Database::Migrator
MIGRATIONS_TABLE = "public.invidious_migrations"
class_getter migrations = [] of Invidious::Database::Migration.class
def initialize(@db : DB::Database)
end
def migrate
versions = load_versions
ran_migration = false
load_migrations.sort_by(&.version)
.each do |migration|
next if versions.includes?(migration.version)
puts "Running migration: #{migration.class.name}"
migration.migrate
ran_migration = true
end
puts "No migrations to run." unless ran_migration
end
def pending_migrations? : Bool
versions = load_versions
load_migrations.sort_by(&.version)
.any? { |migration| !versions.includes?(migration.version) }
end
private def load_migrations : Array(Invidious::Database::Migration)
self.class.migrations.map(&.new(@db))
end
private def load_versions : Array(Int64)
create_migrations_table
@db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64)
end
private def create_migrations_table
@db.exec <<-SQL
CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} (
id bigserial PRIMARY KEY,
version bigint NOT NULL
)
SQL
end
end