From 7a03291442a4e35572c73a2fcfe809fa8e03063e Mon Sep 17 00:00:00 2001 From: Thomas J Faughnan Jr Date: Wed, 15 Nov 2023 22:12:00 -0500 Subject: [PATCH] Fix default User-Agent regression The recent HTTP client refactor in 14e25ab9fe09b9951b38e56af2bdff7a0737b280 introduced a bug in which the global default User-Agent is no longer used for requests. Unless a per-feed User-Agent exists, the Go standard library's default User-Agent is used, which looks something like "Go-http-client/1.1". To fix this, make RequestBuilder.WithUserAgent take an additional argument, the default User-Agent, which will be used if there is no per-feed User-Agent (i.e. it is an empty string). Fixes #2188 Fixes #2189 --- internal/api/subscription.go | 2 +- internal/reader/fetcher/request_builder.go | 4 ++-- internal/reader/handler/handler.go | 6 +++--- internal/reader/processor/processor.go | 4 ++-- internal/ui/subscription_submit.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/api/subscription.go b/internal/api/subscription.go index 9aca0313..03f7912d 100644 --- a/internal/api/subscription.go +++ b/internal/api/subscription.go @@ -37,7 +37,7 @@ func (h *handler) discoverSubscriptions(w http.ResponseWriter, r *http.Request) requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.WithUserAgent(subscriptionDiscoveryRequest.UserAgent) + requestBuilder.WithUserAgent(subscriptionDiscoveryRequest.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(subscriptionDiscoveryRequest.Cookie) requestBuilder.WithUsernameAndPassword(subscriptionDiscoveryRequest.Username, subscriptionDiscoveryRequest.Password) requestBuilder.UseProxy(subscriptionDiscoveryRequest.FetchViaProxy) diff --git a/internal/reader/fetcher/request_builder.go b/internal/reader/fetcher/request_builder.go index 6c11488b..6cda8465 100644 --- a/internal/reader/fetcher/request_builder.go +++ b/internal/reader/fetcher/request_builder.go @@ -53,11 +53,11 @@ func (r *RequestBuilder) WithLastModified(lastModified string) *RequestBuilder { return r } -func (r *RequestBuilder) WithUserAgent(userAgent string) *RequestBuilder { +func (r *RequestBuilder) WithUserAgent(userAgent string, defaultUserAgent string) *RequestBuilder { if userAgent != "" { r.headers.Set("User-Agent", userAgent) } else { - r.headers.Del("User-Agent") + r.headers.Set("User-Agent", defaultUserAgent) } return r } diff --git a/internal/reader/handler/handler.go b/internal/reader/handler/handler.go index 49ccdee4..559098f0 100644 --- a/internal/reader/handler/handler.go +++ b/internal/reader/handler/handler.go @@ -85,7 +85,7 @@ func CreateFeedFromSubscriptionDiscovery(store *storage.Storage, userID int64, f requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithUsernameAndPassword(feedCreationRequest.Username, feedCreationRequest.Password) - requestBuilder.WithUserAgent(feedCreationRequest.UserAgent) + requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feedCreationRequest.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) @@ -121,7 +121,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithUsernameAndPassword(feedCreationRequest.Username, feedCreationRequest.Password) - requestBuilder.WithUserAgent(feedCreationRequest.UserAgent) + requestBuilder.WithUserAgent(feedCreationRequest.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feedCreationRequest.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) @@ -230,7 +230,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithUsernameAndPassword(originalFeed.Username, originalFeed.Password) - requestBuilder.WithUserAgent(originalFeed.UserAgent) + requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(originalFeed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) diff --git a/internal/reader/processor/processor.go b/internal/reader/processor/processor.go index 2463e525..bad2f134 100644 --- a/internal/reader/processor/processor.go +++ b/internal/reader/processor/processor.go @@ -65,7 +65,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us startTime := time.Now() requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithUserAgent(feed.UserAgent) + requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) @@ -166,7 +166,7 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) websiteURL := getUrlFromEntry(feed, entry) requestBuilder := fetcher.NewRequestBuilder() - requestBuilder.WithUserAgent(feed.UserAgent) + requestBuilder.WithUserAgent(feed.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(feed.Cookie) requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) diff --git a/internal/ui/subscription_submit.go b/internal/ui/subscription_submit.go index f0de2079..31f287af 100644 --- a/internal/ui/subscription_submit.go +++ b/internal/ui/subscription_submit.go @@ -60,7 +60,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { requestBuilder := fetcher.NewRequestBuilder() requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout()) requestBuilder.WithProxy(config.Opts.HTTPClientProxy()) - requestBuilder.WithUserAgent(subscriptionForm.UserAgent) + requestBuilder.WithUserAgent(subscriptionForm.UserAgent, config.Opts.HTTPClientUserAgent()) requestBuilder.WithCookie(subscriptionForm.Cookie) requestBuilder.WithUsernameAndPassword(subscriptionForm.Username, subscriptionForm.Password) requestBuilder.UseProxy(subscriptionForm.FetchViaProxy)