Use QUIC for connections to YouTube

This commit is contained in:
Omar Roth 2019-11-18 17:28:32 -05:00
parent 236c172c6f
commit d46b26e3bc
No known key found for this signature in database
GPG Key ID: B8254FB7EC3D37F2
6 changed files with 73 additions and 1602 deletions

View File

@ -1,15 +1,24 @@
FROM alpine:edge FROM alpine:edge
RUN apk add --no-cache crystal shards libc-dev \ RUN apk add --no-cache crystal shards libc-dev \
yaml-dev libxml2-dev sqlite-dev zlib-dev openssl-dev yaml-dev libxml2-dev sqlite-dev zlib-dev curl && \
curl -Lo /etc/apk/keys/omarroth.rsa.pub https://github.com/omarroth/boringssl-alpine/releases/download/1.1.0-r0/omarroth.rsa.pub && \
curl -Lo boringssl-dev.apk https://github.com/omarroth/boringssl-alpine/releases/download/1.1.0-r0/boringssl-dev-1.1.0-r0.apk && \
curl -Lo lsquic.apk https://github.com/omarroth/lsquic-alpine/releases/download/2.6.3-r0/lsquic-2.6.3-r0.apk && \
apk update && \
apk add boringssl-dev.apk lsquic.apk && \
rm -rf /var/cache/apk/* boringssl-dev.apk lsquic.apk
WORKDIR /invidious WORKDIR /invidious
COPY ./shard.yml ./shard.yml COPY ./shard.yml ./shard.yml
RUN shards update && shards install RUN shards update && shards install
RUN cp /usr/lib/libcrypto.a ./lib/lsquic/src/lsquic/ext/libcrypto.a && \
cp /usr/lib/libssl.a ./lib/lsquic/src/lsquic/ext/libssl.a && \
cp /usr/lib/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
COPY ./src/ ./src/ COPY ./src/ ./src/
# TODO: .git folder is required for building this is destructive. # TODO: .git folder is required for building this is destructive.
# See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
COPY ./.git/ ./.git/ COPY ./.git/ ./.git/
RUN crystal build --release --warnings all --error-on-warnings \ RUN crystal build --release --warnings all --error-on-warnings \
# TODO: Remove next line, see https://github.com/crystal-lang/crystal/issues/7946 # TODO: Remove next line, see https://github.com/crystal-lang/crystal/issues/7946
-Dmusl \ -Dmusl \
./src/invidious.cr ./src/invidious.cr

View File

@ -24,6 +24,9 @@ dependencies:
protodec: protodec:
github: omarroth/protodec github: omarroth/protodec
version: ~> 0.1.2 version: ~> 0.1.2
lsquic:
github: omarroth/lsquic.cr
version: ~> 0.1.3
crystal: 0.31.1 crystal: 0.31.1

View File

@ -94,7 +94,7 @@ LOCALES = {
"zh-TW" => load_locale("zh-TW"), "zh-TW" => load_locale("zh-TW"),
} }
YT_POOL = HTTPPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05) YT_POOL = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05) YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
config = CONFIG config = CONFIG
@ -1435,6 +1435,7 @@ post "/login" do |env|
traceback = IO::Memory.new traceback = IO::Memory.new
# See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82 # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
# TODO: Convert to QUIC
begin begin
client = make_client(LOGIN_URL) client = make_client(LOGIN_URL)
headers = HTTP::Headers.new headers = HTTP::Headers.new
@ -1459,7 +1460,7 @@ post "/login" do |env|
headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8" headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
headers["Google-Accounts-XSRF"] = "1" headers["Google-Accounts-XSRF"] = "1"
headers["User-Agent"] = random_user_agent headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req)) response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
lookup_results = JSON.parse(response.body[5..-1]) lookup_results = JSON.parse(response.body[5..-1])
@ -4509,8 +4510,9 @@ get "/api/v1/search/suggestions" do |env|
query ||= "" query ||= ""
begin begin
client = make_client(URI.parse("https://suggestqueries.google.com")) response = QUIC::Client.get(
response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body "https://suggestqueries.google.com/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback"
).body
body = response[35..-2] body = response[35..-2]
body = JSON.parse(body).as_a body = JSON.parse(body).as_a

View File

@ -263,7 +263,7 @@ def bypass_captcha(captcha_key, logger)
# "proxyPort" => CONFIG.proxy_port, # "proxyPort" => CONFIG.proxy_port,
# "proxyLogin" => CONFIG.proxy_user, # "proxyLogin" => CONFIG.proxy_user,
# "proxyPassword" => CONFIG.proxy_pass, # "proxyPassword" => CONFIG.proxy_pass,
# "userAgent" => random_user_agent, # "userAgent" => "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
}, },
}.to_json).body) }.to_json).body)

View File

@ -90,20 +90,6 @@ class HTTPClient < HTTP::Client
return opts return opts
end end
def exec(request)
if self.host == "www.youtube.com"
request.headers["x-youtube-client-name"] ||= "1"
request.headers["x-youtube-client-version"] ||= "1.20180719"
request.headers["user-agent"] ||= random_user_agent
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
request.headers["accept-language"] ||= "en-us,en;q=0.5"
request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
end
super
end
end end
def get_proxies(country_code = "US") def get_proxies(country_code = "US")
@ -115,6 +101,7 @@ def filter_proxies(proxies)
proxies.select! do |proxy| proxies.select! do |proxy|
begin begin
client = HTTPClient.new(YT_URL) client = HTTPClient.new(YT_URL)
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
client.read_timeout = 10.seconds client.read_timeout = 10.seconds
client.connect_timeout = 10.seconds client.connect_timeout = 10.seconds

File diff suppressed because it is too large Load Diff