Add related videos and clean up video class

This commit is contained in:
Omar Roth 2018-01-06 20:39:24 -06:00
parent d1eb81b653
commit 060c4da96d
5 changed files with 180 additions and 164 deletions

View File

@ -1,37 +0,0 @@
-- Table: public.videos
-- DROP TABLE public.videos;
CREATE TABLE public.videos
(
last_updated timestamp with time zone,
video_id text COLLATE pg_catalog."default" NOT NULL,
video_info text COLLATE pg_catalog."default",
video_html text COLLATE pg_catalog."default",
views bigint,
likes integer,
dislikes integer,
rating double precision,
description text COLLATE pg_catalog."default",
CONSTRAINT videos_pkey PRIMARY KEY (video_id)
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
ALTER TABLE public.videos
OWNER to omar;
GRANT ALL ON TABLE public.videos TO kemal;
GRANT ALL ON TABLE public.videos TO omar;
-- Index: videos_video_id_idx
-- DROP INDEX public.videos_video_id_idx;
CREATE INDEX videos_video_id_idx
ON public.videos USING btree
(video_id COLLATE pg_catalog."default")
TABLESPACE pg_default;

View File

@ -5,85 +5,53 @@ require "pg"
require "xml"
require "time"
PG_DB = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
CONTEXT = OpenSSL::SSL::Context::Client.insecure
macro templated(filename)
render "src/views/#{{{filename}}}.ecr", "src/views/layout.ecr"
end
class Video
getter last_updated : Time
getter video_id : String
getter video_info : String
getter video_html : String
getter views : String
getter likes : Int32
getter dislikes : Int32
getter rating : Float64
getter description : String
module HTTPParamConverter
def self.from_rs(rs)
HTTP::Params.parse(rs.read(String))
end
end
def initialize(last_updated, video_id, video_info, video_html, views, likes, dislikes, rating, description)
@last_updated = last_updated
@video_id = video_id
@video_info = video_info
@video_html = video_html
@views = views
@likes = likes
@dislikes = dislikes
@rating = rating
@description = description
module XMLConverter
def self.from_rs(rs)
XML.parse(rs.read(String))
end
end
def initialize(id, info, html, updated)
@id = id
@info = info
@html = html
@updated = updated
end
def to_a
return [@last_updated, @video_id, @video_info, @video_html, @views, @likes, @dislikes, @rating, @description]
return [@id, @info, @html, @updated]
end
DB.mapping({
last_updated: Time,
video_id: String,
video_info: String,
video_html: String,
views: Int64,
likes: Int32,
dislikes: Int32,
rating: Float64,
description: String,
id: String,
info: {
type: HTTP::Params,
default: HTTP::Params.parse(""),
converter: Video::HTTPParamConverter,
},
html: {
type: XML::Node,
default: XML.parse(""),
converter: Video::XMLConverter,
},
updated: Time,
})
end
def get_video(video_id, context)
client = HTTP::Client.new("www.youtube.com", 443, context)
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
info = HTTP::Params.parse(video_info)
video_html = client.get("/watch?v=#{video_id}").body
html = XML.parse(video_html)
views = info["view_count"].to_i64
rating = info["avg_rating"].to_f64
likes = html.xpath_node(%q(//button[@title="I like this"]/span))
if likes
likes = likes.content.delete(",").to_i
else
likes = 1
end
dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
if dislikes
dislikes = dislikes.content.delete(",").to_i
else
dislikes = 1
end
description = html.xpath_node(%q(//p[@id="eow-description"]))
if description
description = description.to_xml
else
description = ""
end
video_record = Video.new(Time.now, video_id, video_info, video_html, views, likes, dislikes, rating, description)
return video_record
end
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
def ci_lower_bound(pos, n)
if n == 0
@ -97,15 +65,52 @@ def ci_lower_bound(pos, n)
return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n)
end
def fetch_video(id)
client = HTTP::Client.new("www.youtube.com", 443, CONTEXT)
info = client.get("/get_video_info?video_id=#{id}&el=info&ps=default&eurl=&gl=US&hl=en").body
info = HTTP::Params.parse(info)
html = client.get("/watch?v=#{id}").body
html = XML.parse(html)
if info["reason"]?
raise info["reason"]
end
video = Video.new(id, info, html, Time.now)
return video
end
def get_video(id)
if PG_DB.query_one?("SELECT EXISTS ( SELECT true FROM videos WHERE id = $1)", id, as: Bool)
video = PG_DB.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
# If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours)
if Time.now - video.updated > Time::Span.new(0, 5, 0, 0)
video = fetch_video(id)
end
else
video = fetch_video(id)
PG_DB.exec("INSERT INTO videos VALUES ($1, $2, $3, $4)", video.to_a)
end
return video
end
get "/" do |env|
templated "index"
end
pg = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
context = OpenSSL::SSL::Context::Client.insecure
get "/watch" do |env|
video_id = env.params.query["v"]
id = env.params.query["v"]
begin
video = get_video(id)
rescue ex
error_message = ex.message
next templated "error"
end
if env.params.query["listen"]? && env.params.query["listen"] == "true"
env.request.query_params.delete_all("listen")
@ -115,55 +120,60 @@ get "/watch" do |env|
listen = false
end
if pg.query_one?("select exists (select true from videos where video_id = $1)", video_id, as: Bool)
video_record = pg.query_one("select * from videos where video_id = $1", video_id, as: Video)
# If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours)
if Time.now - video_record.last_updated > Time::Span.new(0, 5, 0, 0)
video_record = get_video(video_id, context)
pg.exec("update videos set last_updated = $1, video_info = $3, video_html = $4,\
views = $5, likes = $6, dislikes = $7, rating = $8, description = $9 where video_id = $2",
video_record.to_a)
end
else
client = HTTP::Client.new("www.youtube.com", 443, context)
video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
info = HTTP::Params.parse(video_info)
if info["reason"]?
error_message = info["reason"]
next templated "error"
end
video_record = get_video(video_id, context)
pg.exec("insert into videos values ($1,$2,$3,$4,$5,$6,$7,$8, $9)", video_record.to_a)
end
# last_updated, video_id, video_info, video_html, views, likes, dislikes, rating
video_info = HTTP::Params.parse(video_record.video_info)
video_html = XML.parse(video_record.video_html)
fmt_stream = [] of HTTP::Params
video_info["url_encoded_fmt_stream_map"].split(",") do |string|
video.info["url_encoded_fmt_stream_map"].split(",") do |string|
fmt_stream << HTTP::Params.parse(string)
end
adaptive_fmts = [] of HTTP::Params
video_info["adaptive_fmts"].split(",") do |string|
adaptive_fmts << HTTP::Params.parse(string)
end
fmt_stream.reverse! # We want lowest quality first
related_videos = video_html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
if related_videos.empty?
related_videos = video_html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
adaptive_fmts = [] of HTTP::Params
video.info["adaptive_fmts"].split(",") do |string|
adaptive_fmts << HTTP::Params.parse(string)
end
likes = video_record.likes.to_f
dislikes = video_record.dislikes.to_f
views = video_record.views.to_f
related_videos = video.html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
if related_videos.empty?
related_videos = video.html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
end
related_videos_list = [] of Video
related_videos.each do |related_video|
related_id = related_video.content.split("=")[1]
begin
related_videos_list << get_video(related_id)
rescue ex
p "#{related_id}: #{ex.message}"
end
end
likes = video.html.xpath_node(%q(//button[@title="I like this"]/span))
if likes
likes = likes.content.delete(",").to_i
else
likes = 1
end
dislikes = video.html.xpath_node(%q(//button[@title="I dislike this"]/span))
if dislikes
dislikes = dislikes.content.delete(",").to_i
else
dislikes = 1
end
description = video.html.xpath_node(%q(//p[@id="eow-description"]))
if description
description = description.to_xml
else
description = ""
end
views = video.info["view_count"].to_i64
rating = video.info["avg_rating"].to_f64
likes = likes.to_f
dislikes = dislikes.to_f
views = views.to_f
engagement = ((dislikes + likes)/views * 100)
calculated_rating = (likes/(likes + dislikes) * 4 + 1)
@ -173,9 +183,9 @@ end
get "/search" do |env|
query = URI.escape(env.params.query["q"])
client = HTTP::Client.new("www.youtube.com", 443, context)
results_html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body
html = XML.parse(results_html)
client = HTTP::Client.new("www.youtube.com", 443, CONTEXT)
html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body
html = XML.parse(html)
videos = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-video-renderer))
channels = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-channel-renderer))
@ -185,6 +195,16 @@ get "/search" do |env|
channels = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-channel")]/div/div[contains(@class,"yt-lockup-thumbnail")]/a/@href))
end
videos_list = [] of Video
videos.each do |video|
id = video.content.split("=")[1]
begin
videos_list << get_video(id)
rescue ex
p "#{id}: #{ex.message}"
end
end
templated "search"
end

View File

@ -1,3 +1,3 @@
<% videos.each do |video| %>
<p><a href="<%= video.content %>"><%= video.content %></a></p>
<% videos_list.each do |video| %>
<p><a href="/watch?v=<%= video.id %>"><%= video.info["title"] %></a></p>
<% end %>

View File

@ -1,5 +1,5 @@
<title><%= video_info["title"] %> - Invidious</title>
<video style="width: 100%" poster="<%= video_info.has_key?("iurlhq720") ? video_info["iurlhq720"] : video_info["iurlmq"] %>" controls>
<title><%= video.info["title"] %> - Invidious</title>
<video style="width: 100%" poster="<%= video.info.has_key?("iurlhq720") ? video.info["iurlhq720"] : video.info["iurlmq"] %>" controls>
<% if listen %>
<% adaptive_fmts.each do |fmt| %>
<% url = fmt["url"] %>
@ -14,28 +14,29 @@
<% end %>
<% end %>
</video>
<h1><%= video_info["title"] %> <a href="/watch?<%= env.request.query %>">
<h1><%= video.info["title"] %> <a href="/watch?<%= env.request.query %>">
<i class="fa <%= listen ? "fa-video-camera" : "fa-volume-up" %>" aria-hidden="true"></i>
</a>
</h1>
<div class="pure-g">
<div class="pure-u-1 pure-u-md-1-5">
<p><i class="fa fa-eye" aria-hidden="true"></i> <%= video_record.views %></p>
<p><i class="fa fa-thumbs-up" aria-hidden="true"></i> <%= video_record.likes %></p>
<p><i class="fa fa-thumbs-down" aria-hidden="true"></i> <%= video_record.dislikes %></p>
<p>Wilson Score : <%= ci_lower_bound(video_record.likes, video_record.likes + video_record.dislikes).round(4) %></p>
<p>Rating : <%= video_record.rating.round(4) %> / 5</p>
<p><i class="fa fa-eye" aria-hidden="true"></i> <%= views.to_i %></p>
<p><i class="fa fa-thumbs-up" aria-hidden="true"></i> <%= likes.to_i %></p>
<p><i class="fa fa-thumbs-down" aria-hidden="true"></i> <%= dislikes.to_i %></p>
<p>Wilson Score : <%= ci_lower_bound(likes, likes + dislikes).round(4) %></p>
<p>Rating : <%= rating.round(4) %> / 5</p>
<!-- <p>Calculated Rating : <%= calculated_rating.round(4) %> / 5</p> -->
<p>Engagement : <%= engagement.round(2) %>%</p>
<p>Earnings : <%= video_info.has_key?("allowed_ads") ? "~$" + ((video_record.views.to_f / 500).round(2)).to_s : "Unmonetized" %></p>
<p>Allowed ads : <br><%= video_info.has_key?("allowed_ads") ? video_info["allowed_ads"] : "Unmonetized" %></p>
<p>Earnings : <%= video.info.has_key?("allowed_ads") ? "~$" + ((views.to_f / 500).round(2)).to_s : "Unmonetized" %></p>
<p>Allowed ads : <br><%= video.info.has_key?("allowed_ads") ? video.info["allowed_ads"] : "Unmonetized" %></p>
</div>
<div class="pure-u-1 pure-u-md-3-5">
<p><%= video_record.description %></p>
<p><%= video.info["author"] %></p>
<p><%= description %></p>
</div>
<div class="pure-u-1 pure-u-md-1-5">
<% related_videos.each do |video| %>
<p><a href="<%= video.content %>"><%= video.content %></a></p>
<% related_videos_list.each do |video| %>
<p><a href="/watch?v=<%= video.id %>"><%= video.info["title"] %></a></p>
<% end %>
</div>
</div>

32
videos.sql Normal file
View File

@ -0,0 +1,32 @@
-- Table: public.videos
-- DROP TABLE public.videos;
CREATE TABLE public.videos
(
id text COLLATE pg_catalog."default" NOT NULL,
info text COLLATE pg_catalog."default",
html text COLLATE pg_catalog."default",
updated timestamp with time zone,
CONSTRAINT videos_pkey PRIMARY KEY (id)
)
WITH (
OIDS = FALSE
)
TABLESPACE pg_default;
ALTER TABLE public.videos
OWNER to omar;
GRANT ALL ON TABLE public.videos TO kemal;
GRANT ALL ON TABLE public.videos TO omar;
-- Index: id_idx
-- DROP INDEX public.id_idx;
CREATE UNIQUE INDEX id_idx
ON public.videos USING btree
(id COLLATE pg_catalog."default")
TABLESPACE pg_default;