Try to speed up find_working_proxies

This commit is contained in:
Omar Roth 2018-10-03 10:38:07 -05:00
parent f83e9e6eb9
commit 3830604e42
4 changed files with 51 additions and 41 deletions

View File

@ -107,13 +107,13 @@ end
proxies = {} of String => Array({ip: String, port: Int32}) proxies = {} of String => Array({ip: String, port: Int32})
if CONFIG.geo_bypass if CONFIG.geo_bypass
spawn do spawn do
find_working_proxies(BYPASS_REGIONS) do |region, list| find_working_proxies(BYPASS_REGIONS) do |region, list|
if !list.empty? if !list.empty?
proxies[region] = list proxies[region] = list
end end
end end
end end
end end
before_all do |env| before_all do |env|
@ -1876,6 +1876,8 @@ get "/api/v1/comments/:id" do |env|
proxies.each do |region, list| proxies.each do |region, list|
spawn do spawn do
proxy_html = %(<meta itemprop="regionsAllowed" content="">)
list.each do |proxy| list.each do |proxy|
begin begin
proxy_client = HTTPClient.new(YT_URL) proxy_client = HTTPClient.new(YT_URL)
@ -1886,10 +1888,10 @@ get "/api/v1/comments/:id" do |env|
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
proxy_client.set_proxy(proxy) proxy_client.set_proxy(proxy)
proxy_html = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
proxy_headers = HTTP::Headers.new proxy_headers = HTTP::Headers.new
proxy_headers["cookie"] = proxy_html.cookies.add_request_headers(headers)["cookie"] proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
proxy_html = proxy_html.body proxy_html = response.body
if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/) if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
bypass_channel.send(nil) bypass_channel.send(nil)
@ -1901,6 +1903,11 @@ get "/api/v1/comments/:id" do |env|
rescue ex rescue ex
end end
end end
# If none of the proxies we tried returned a valid response
if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
bypass_channel.send(nil)
end
end end
end end

View File

@ -93,6 +93,25 @@ def get_proxies(country_code = "US")
return get_nova_proxies(country_code) return get_nova_proxies(country_code)
end end
def filter_proxies(proxies)
proxies.select! do |proxy|
begin
client = HTTPClient.new(YT_URL)
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
client.set_proxy(proxy)
client.head("/").status_code == 200
rescue ex
false
end
end
return proxies
end
def get_nova_proxies(country_code = "US") def get_nova_proxies(country_code = "US")
country_code = country_code.downcase country_code = country_code.downcase
client = HTTP::Client.new(URI.parse("https://www.proxynova.com")) client = HTTP::Client.new(URI.parse("https://www.proxynova.com"))
@ -127,7 +146,7 @@ def get_nova_proxies(country_code = "US")
proxies << {ip: ip, port: port, score: score} proxies << {ip: ip, port: port, score: score}
end end
proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse # proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse
return proxies return proxies
end end

View File

@ -156,39 +156,14 @@ def update_decrypt_function
end end
def find_working_proxies(regions) def find_working_proxies(regions)
proxy_channel = Channel({String, Array({ip: String, port: Int32})}).new loop do
regions.each do |region| regions.each do |region|
spawn do
loop do
begin
proxies = get_proxies(region).first(20) proxies = get_proxies(region).first(20)
rescue ex
next proxy_channel.send({region, Array({ip: String, port: Int32}).new})
end
proxies.select! do |proxy|
begin
client = HTTPClient.new(YT_URL)
client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
client.set_proxy(proxy)
client.get("/").status_code == 200
rescue ex
false
end
end
proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} } proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
# proxies = filter_proxies(proxies)
proxy_channel.send({region, proxies}) yield region, proxies
Fiber.yield
end end
end end
end
loop do
yield proxy_channel.receive
end
end end

View File

@ -543,6 +543,10 @@ def fetch_video(id, proxies)
proxies.each do |region, list| proxies.each do |region, list|
spawn do spawn do
info = HTTP::Params.new({
"reason" => [info["reason"]],
})
list.each do |proxy| list.each do |proxy|
begin begin
client = HTTPClient.new(YT_URL) client = HTTPClient.new(YT_URL)
@ -563,6 +567,11 @@ def fetch_video(id, proxies)
rescue ex rescue ex
end end
end end
# If none of the proxies we tried returned a valid response
if info["reason"]?
bypass_channel.send(nil)
end
end end
end end