From 3830604e423fe8cefb6091fdded0b95ab2bb3cd1 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Wed, 3 Oct 2018 10:38:07 -0500 Subject: [PATCH] Try to speed up find_working_proxies --- src/invidious.cr | 23 +++++++++++++------- src/invidious/helpers/proxy.cr | 21 +++++++++++++++++- src/invidious/jobs.cr | 39 ++++++---------------------------- src/invidious/videos.cr | 9 ++++++++ 4 files changed, 51 insertions(+), 41 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 614b25c0..2e141c29 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -107,14 +107,14 @@ end proxies = {} of String => Array({ip: String, port: Int32}) if CONFIG.geo_bypass -spawn do - find_working_proxies(BYPASS_REGIONS) do |region, list| - if !list.empty? - proxies[region] = list + spawn do + find_working_proxies(BYPASS_REGIONS) do |region, list| + if !list.empty? + proxies[region] = list + end end end end -end before_all do |env| env.response.headers["X-XSS-Protection"] = "1; mode=block;" @@ -1876,6 +1876,8 @@ get "/api/v1/comments/:id" do |env| proxies.each do |region, list| spawn do + proxy_html = %() + list.each do |proxy| begin proxy_client = HTTPClient.new(YT_URL) @@ -1886,10 +1888,10 @@ get "/api/v1/comments/:id" do |env| proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) proxy_client.set_proxy(proxy) - proxy_html = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") + response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") proxy_headers = HTTP::Headers.new - proxy_headers["cookie"] = proxy_html.cookies.add_request_headers(headers)["cookie"] - proxy_html = proxy_html.body + proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"] + proxy_html = response.body if proxy_html.match(//) bypass_channel.send(nil) @@ -1901,6 +1903,11 @@ get "/api/v1/comments/:id" do |env| rescue ex end end + + # If none of the proxies we tried returned a valid response + if proxy_html.match(//) + bypass_channel.send(nil) + end end end diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr index 469bb236..8d7ba384 100644 --- a/src/invidious/helpers/proxy.cr +++ b/src/invidious/helpers/proxy.cr @@ -93,6 +93,25 @@ def get_proxies(country_code = "US") return get_nova_proxies(country_code) end +def filter_proxies(proxies) + proxies.select! do |proxy| + begin + client = HTTPClient.new(YT_URL) + client.read_timeout = 10.seconds + client.connect_timeout = 10.seconds + + proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) + client.set_proxy(proxy) + + client.head("/").status_code == 200 + rescue ex + false + end + end + + return proxies +end + def get_nova_proxies(country_code = "US") country_code = country_code.downcase client = HTTP::Client.new(URI.parse("https://www.proxynova.com")) @@ -127,7 +146,7 @@ def get_nova_proxies(country_code = "US") proxies << {ip: ip, port: port, score: score} end - proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse + # proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse return proxies end diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index b9feb1b8..e1c4e2d3 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -156,39 +156,14 @@ def update_decrypt_function end def find_working_proxies(regions) - proxy_channel = Channel({String, Array({ip: String, port: Int32})}).new + loop do + regions.each do |region| + proxies = get_proxies(region).first(20) + proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} } + # proxies = filter_proxies(proxies) - regions.each do |region| - spawn do - loop do - begin - proxies = get_proxies(region).first(20) - rescue ex - next proxy_channel.send({region, Array({ip: String, port: Int32}).new}) - end - - proxies.select! do |proxy| - begin - client = HTTPClient.new(YT_URL) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds - - proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) - client.set_proxy(proxy) - - client.get("/").status_code == 200 - rescue ex - false - end - end - proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} } - - proxy_channel.send({region, proxies}) - end + yield region, proxies + Fiber.yield end end - - loop do - yield proxy_channel.receive - end end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 95cd0033..42adb3b4 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -543,6 +543,10 @@ def fetch_video(id, proxies) proxies.each do |region, list| spawn do + info = HTTP::Params.new({ + "reason" => [info["reason"]], + }) + list.each do |proxy| begin client = HTTPClient.new(YT_URL) @@ -563,6 +567,11 @@ def fetch_video(id, proxies) rescue ex end end + + # If none of the proxies we tried returned a valid response + if info["reason"]? + bypass_channel.send(nil) + end end end