From 2eac23a0b37082eebbfa1436c2e6200210b6e0a9 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 16 Nov 2021 13:24:13 +0100 Subject: [PATCH 1/4] Temporary fix for #2612 Don't rely on the auto compression/decompression provided by the crystal stdlib. --- src/invidious/yt_backend/youtube_api.cr | 30 +++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 27f25036..0403cb56 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -404,19 +404,10 @@ module YoutubeAPI url = "#{endpoint}?key=#{client_config.api_key}" headers = HTTP::Headers{ - "Content-Type" => "application/json; charset=UTF-8", + "Content-Type" => "application/json; charset=UTF-8", + "Accept-Encoding" => "gzip, deflate", } - # The normal HTTP client automatically applies accept-encoding: gzip, - # and decompresses. However, explicitly applying it will remove this functionality. - # - # https://github.com/crystal-lang/crystal/issues/11252#issuecomment-929594741 - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - headers["Accept-Encoding"] = "gzip" - end - {% end %} - # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}") @@ -434,8 +425,23 @@ module YoutubeAPI ) end + # Decompress the body ourselves, given that auto-decompress is + # broken in the Crystal stdlib. + # Read more: + # - https://github.com/iv-org/invidious/issues/2612 + # - https://github.com/crystal-lang/crystal/issues/11354 + # + case headers["Content-Encoding"]? + when "gzip" + body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) + when "deflate" + body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) + else + body = response.body + end + # Convert result to Hash - initial_data = JSON.parse(response.body).as_h + initial_data = JSON.parse(body).as_h # Error handling if initial_data.has_key?("error") From dad8f9a0ce576ce3e938fbaa578d12179ca63553 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 16 Nov 2021 20:39:26 +0100 Subject: [PATCH 2/4] Fix typo Should be checking the returned headers, not the sent ones. --- src/invidious/yt_backend/youtube_api.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 0403cb56..4973e4de 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -431,7 +431,7 @@ module YoutubeAPI # - https://github.com/iv-org/invidious/issues/2612 # - https://github.com/crystal-lang/crystal/issues/11354 # - case headers["Content-Encoding"]? + case response.headers["Content-Encoding"]? when "gzip" body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) when "deflate" From 2c447a42f29a05b8067444338248e344b3705cd0 Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Tue, 16 Nov 2021 21:40:35 +0100 Subject: [PATCH 3/4] Make sure to only apply fix if QUIC is disabled --- src/invidious/yt_backend/youtube_api.cr | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 4973e4de..b26af8d1 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -425,19 +425,23 @@ module YoutubeAPI ) end - # Decompress the body ourselves, given that auto-decompress is - # broken in the Crystal stdlib. - # Read more: - # - https://github.com/iv-org/invidious/issues/2612 - # - https://github.com/crystal-lang/crystal/issues/11354 - # - case response.headers["Content-Encoding"]? - when "gzip" - body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) - when "deflate" - body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) - else + if {{ !flag?(:disable_quic) }} && CONFIG.use_quic body = response.body + else + # Decompress the body ourselves, when using HTTP::Client given that + # auto-decompress is broken in the Crystal stdlib. + # Read more: + # - https://github.com/iv-org/invidious/issues/2612 + # - https://github.com/crystal-lang/crystal/issues/11354 + # + case response.headers["Content-Encoding"]? + when "gzip" + body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) + when "deflate" + body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) + else + body = response.body + end end # Convert result to Hash From ba48f68fc30990437331791848efe896559f49cd Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Sun, 21 Nov 2021 18:16:05 +0100 Subject: [PATCH 4/4] allow multiple, successive content-encodings --- src/invidious/yt_backend/youtube_api.cr | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b26af8d1..977aea04 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -434,11 +434,22 @@ module YoutubeAPI # - https://github.com/iv-org/invidious/issues/2612 # - https://github.com/crystal-lang/crystal/issues/11354 # - case response.headers["Content-Encoding"]? - when "gzip" - body = Compress::Gzip::Reader.new(response.body_io, sync_close: true) - when "deflate" - body = Compress::Deflate::Reader.new(response.body_io, sync_close: true) + if encodings = response.headers["Content-Encoding"]? + io = response.body_io + + # Multiple encodings can be combined, and are listed in the order + # in which they were applied. E.g: "deflate, gzip" means that the + # content must be first "gunzipped", then "defated". + encodings.split(',').reverse.each do |enc| + case enc.strip(' ') + when "gzip" + io = Compress::Gzip::Reader.new(io, sync_close: true) + when "deflate" + io = Compress::Deflate::Reader.new(io, sync_close: true) + end + end + + body = io.gets_to_end else body = response.body end