Migrations tweaks

This commit is contained in:
matthewmcgarvey 2022-02-10 22:16:40 -06:00
parent 8ec992a8a3
commit cf13c11236
16 changed files with 149 additions and 134 deletions

View File

@ -27,6 +27,7 @@ require "compress/zip"
require "protodec/utils" require "protodec/utils"
require "./invidious/database/*" require "./invidious/database/*"
require "./invidious/database/migrations/*"
require "./invidious/helpers/*" require "./invidious/helpers/*"
require "./invidious/yt_backend/*" require "./invidious/yt_backend/*"
require "./invidious/*" require "./invidious/*"
@ -34,7 +35,6 @@ require "./invidious/channels/*"
require "./invidious/user/*" require "./invidious/user/*"
require "./invidious/routes/**" require "./invidious/routes/**"
require "./invidious/jobs/**" require "./invidious/jobs/**"
require "./invidious/migrations/*"
CONFIG = Config.load CONFIG = Config.load
HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32)
@ -113,7 +113,7 @@ OUTPUT = CONFIG.output.upcase == "STDOUT" ? STDOUT : File.open(CONFIG.output, mo
LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level) LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
# Run migrations # Run migrations
Invidious::Migrator.new(PG_DB).migrate Invidious::Database::Migrator.new(PG_DB).migrate
# Check table integrity # Check table integrity
Invidious::Database.check_integrity(CONFIG) Invidious::Database.check_integrity(CONFIG)

View File

@ -1,6 +1,6 @@
abstract class Invidious::Migration abstract class Invidious::Database::Migration
macro inherited macro inherited
Invidious::Migrator.migrations << self Migrator.migrations << self
end end
@@version : Int64? @@version : Int64?
@ -33,6 +33,6 @@ abstract class Invidious::Migration
end end
private def track(conn : DB::Connection) private def track(conn : DB::Connection)
conn.exec("INSERT INTO #{Invidious::Migrator::MIGRATIONS_TABLE}(version) VALUES ($1)", version) conn.exec("INSERT INTO #{Migrator::MIGRATIONS_TABLE} (version) VALUES ($1)", version)
end end
end end

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateChannelsTable < Migration class CreateChannelsTable < Migration
version 0 version 1
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateVideosTable < Migration class CreateVideosTable < Migration
version 1 version 2
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateChannelVideosTable < Migration class CreateChannelVideosTable < Migration
version 2 version 3
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateUsersTable < Migration class CreateUsersTable < Migration
version 3 version 4
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateSessionIdsTable < Migration class CreateSessionIdsTable < Migration
version 4 version 5
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateNoncesTable < Migration class CreateNoncesTable < Migration
version 5 version 6
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -1,6 +1,6 @@
module Invidious::Migrations module Invidious::Database::Migrations
class CreateAnnotationsTable < Migration class CreateAnnotationsTable < Migration
version 6 version 7
def up(conn : DB::Connection) def up(conn : DB::Connection)
conn.exec <<-SQL conn.exec <<-SQL

View File

@ -0,0 +1,50 @@
module Invidious::Database::Migrations
class CreatePlaylistsTable < Migration
version 8
def up(conn : DB::Connection)
if !privacy_type_exists?(conn)
conn.exec <<-SQL
CREATE TYPE public.privacy AS ENUM
(
'Public',
'Unlisted',
'Private'
);
SQL
end
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlists
(
title text,
id text primary key,
author text,
description text,
video_count integer,
created timestamptz,
updated timestamptz,
privacy privacy,
index int8[]
);
SQL
conn.exec <<-SQL
GRANT ALL ON public.playlists TO current_user;
SQL
end
private def privacy_type_exists?(conn : DB::Connection) : Bool
request = <<-SQL
SELECT 1 AS one
FROM pg_type
INNER JOIN pg_namespace ON pg_namespace.oid = pg_type.typnamespace
WHERE pg_namespace.nspname = 'public'
AND pg_type.typname = 'privacy'
LIMIT 1;
SQL
!conn.query_one?(request, as: Int32).nil?
end
end
end

View File

@ -0,0 +1,27 @@
module Invidious::Database::Migrations
class CreatePlaylistVideosTable < Migration
version 9
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlist_videos
(
title text,
id text,
author text,
ucid text,
length_seconds integer,
published timestamptz,
plid text references playlists(id),
index int8,
live_now boolean,
PRIMARY KEY (index,plid)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.playlist_videos TO current_user;
SQL
end
end
end

View File

@ -0,0 +1,11 @@
module Invidious::Database::Migrations
class MakeVideosUnlogged < Migration
version 10
def up(conn : DB::Connection)
conn.exec <<-SQL
ALTER TABLE public.videos SET UNLOGGED;
SQL
end
end
end

View File

@ -0,0 +1,42 @@
class Invidious::Database::Migrator
MIGRATIONS_TABLE = "public.invidious_migrations"
class_getter migrations = [] of Invidious::Database::Migration.class
def initialize(@db : DB::Database)
end
def migrate
versions = load_versions
ran_migration = false
load_migrations.sort_by(&.version)
.each do |migration|
next if versions.includes?(migration.version)
puts "Running migration: #{migration.class.name}"
migration.migrate
ran_migration = true
end
puts "No migrations to run." unless ran_migration
end
private def load_migrations : Array(Invidious::Database::Migration)
self.class.migrations.map(&.new(@db))
end
private def load_versions : Array(Int64)
create_migrations_table
@db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64)
end
private def create_migrations_table
@db.exec <<-SQL
CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} (
id bigserial PRIMARY KEY,
version bigint NOT NULL
)
SQL
end
end

View File

@ -1,47 +0,0 @@
module Invidious::Migrations
class CreatePlaylistsTable < Migration
version 7
def up(conn : DB::Connection)
conn.exec <<-SQL
DO
$$
BEGIN
IF NOT EXISTS (SELECT *
FROM pg_type typ
INNER JOIN pg_namespace nsp ON nsp.oid = typ.typnamespace
WHERE nsp.nspname = 'public'
AND typ.typname = 'privacy') THEN
CREATE TYPE public.privacy AS ENUM
(
'Public',
'Unlisted',
'Private'
);
END IF;
END;
$$
LANGUAGE plpgsql;
SQL
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlists
(
title text,
id text primary key,
author text,
description text,
video_count integer,
created timestamptz,
updated timestamptz,
privacy privacy,
index int8[]
);
SQL
conn.exec <<-SQL
GRANT ALL ON public.playlists TO current_user;
SQL
end
end
end

View File

@ -1,27 +0,0 @@
module Invidious::Migrations
class CreatePlaylistVideosTable < Migration
version 8
def up(conn : DB::Connection)
conn.exec <<-SQL
CREATE TABLE IF NOT EXISTS public.playlist_videos
(
title text,
id text,
author text,
ucid text,
length_seconds integer,
published timestamptz,
plid text references playlists(id),
index int8,
live_now boolean,
PRIMARY KEY (index,plid)
);
SQL
conn.exec <<-SQL
GRANT ALL ON TABLE public.playlist_videos TO current_user;
SQL
end
end
end

View File

@ -1,41 +0,0 @@
class Invidious::Migrator
MIGRATIONS_TABLE = "invidious_migrations"
class_getter migrations = [] of Invidious::Migration.class
def initialize(@db : DB::Database)
end
def migrate
run_migrations = load_run_migrations
migrations = load_migrations.sort_by(&.version)
migrations_to_run = migrations.reject { |migration| run_migrations.includes?(migration.version) }
if migrations.empty?
puts "No migrations to run."
return
end
migrations_to_run.each do |migration|
puts "Running migration: #{migration.class.name}"
migration.migrate
end
end
private def load_migrations : Array(Invidious::Migration)
self.class.migrations.map(&.new(@db))
end
private def load_run_migrations : Array(Int64)
create_migrations_table
@db.query_all("SELECT version FROM #{MIGRATIONS_TABLE}", as: Int64)
end
private def create_migrations_table
@db.exec <<-SQL
CREATE TABLE IF NOT EXISTS #{MIGRATIONS_TABLE} (
id bigserial PRIMARY KEY,
version bigint NOT NULL
)
SQL
end
end