rdelaage
/
miniflux-v2
mirror of https://github.com/miniflux/v2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'miniflux:main' into patch-1

This commit is contained in:
mcnesium 2024-04-19 15:43:39 +02:00 committed by GitHub
parent 47903f69ca 647c66e70a
commit abdc35b142
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
159 changed files with 11773 additions and 6089 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.devcontainer/docker-compose.yml
View File

@ -1,7 +1,7 @@
version: '3.8'
services:
app:
image: mcr.microsoft.com/devcontainers/go
image: mcr.microsoft.com/devcontainers/go:1.22
volumes:
- ..:/workspace:cached
command: sleep infinity
@ -24,7 +24,7 @@ services:
ports:
- 5432:5432
apprise:
image: caronc/apprise:latest
image: caronc/apprise:1.0
restart: unless-stopped
hostname: apprise
volumes:

3
.github/pull_request_template.md vendored
View File

@ -1,4 +1,7 @@
Do you follow the guidelines?
- [ ] I have tested my changes
- [ ] There is no breaking changes
- [ ] I really tested my changes and there is no regression
- [ ] Ideally, my commit messages use the same convention as the Go project: https://go.dev/doc/contribute#commit_messages
- [ ] I read this document: https://miniflux.app/faq.html#pull-request

92
.github/workflows/docker.yml vendored
View File

@ -8,35 +8,8 @@ on:
pull_request:
branches: [ main ]
jobs:
test-docker-images:
if: github.event.pull_request
name: Test Images
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Build Alpine image
uses: docker/build-push-action@v5
with:
context: .
file: ./packaging/docker/alpine/Dockerfile
push: false
tags: ${{ github.repository_owner }}/miniflux:alpine-dev
- name: Test Alpine Docker image
run: docker run --rm ${{ github.repository_owner }}/miniflux:alpine-dev miniflux -i
- name: Build Distroless image
uses: docker/build-push-action@v5
with:
context: .
file: ./packaging/docker/distroless/Dockerfile
push: false
tags: ${{ github.repository_owner }}/miniflux:distroless-dev
- name: Test Distroless Docker image
run: docker run --rm ${{ github.repository_owner }}/miniflux:distroless-dev miniflux -i
publish-docker-images:
if: ${{ ! github.event.pull_request }}
name: Publish Images
docker-images:
name: Docker Images
permissions:
packages: write
runs-on: ubuntu-latest
@ -46,33 +19,31 @@ jobs:
with:
fetch-depth: 0
- name: Generate Alpine Docker tag
id: docker_alpine_tag
run: |
DOCKER_IMAGE=${{ github.repository_owner }}/miniflux
DOCKER_VERSION=dev
if [ "${{ github.event_name }}" = "schedule" ]; then
DOCKER_VERSION=nightly
TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION}"
elif [[ $GITHUB_REF == refs/tags/* ]]; then
DOCKER_VERSION=${GITHUB_REF#refs/tags/}
TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION},docker.io/${DOCKER_IMAGE}:latest,ghcr.io/${DOCKER_IMAGE}:latest,quay.io/${DOCKER_IMAGE}:latest"
fi
echo ::set-output name=tags::${TAGS}
- name: Generate Alpine Docker tags
id: docker_alpine_tags
uses: docker/metadata-action@v5
with:
images: |
docker.io/${{ github.repository_owner }}/miniflux
ghcr.io/${{ github.repository_owner }}/miniflux
quay.io/${{ github.repository_owner }}/miniflux
tags: |
type=ref,event=pr
type=schedule,pattern=nightly
type=semver,pattern={{raw}}
- name: Generate Distroless Docker tag
id: docker_distroless_tag
run: |
DOCKER_IMAGE=${{ github.repository_owner }}/miniflux
DOCKER_VERSION=dev-distroless
if [ "${{ github.event_name }}" = "schedule" ]; then
DOCKER_VERSION=nightly-distroless
TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION}"
elif [[ $GITHUB_REF == refs/tags/* ]]; then
DOCKER_VERSION=${GITHUB_REF#refs/tags/}-distroless
TAGS="docker.io/${DOCKER_IMAGE}:${DOCKER_VERSION},ghcr.io/${DOCKER_IMAGE}:${DOCKER_VERSION},quay.io/${DOCKER_IMAGE}:${DOCKER_VERSION},docker.io/${DOCKER_IMAGE}:latest-distroless,ghcr.io/${DOCKER_IMAGE}:latest-distroless,quay.io/${DOCKER_IMAGE}:latest-distroless"
fi
echo ::set-output name=tags::${TAGS}
- name: Generate Distroless Docker tags
id: docker_distroless_tags
uses: docker/metadata-action@v5
with:
images: |
docker.io/${{ github.repository_owner }}/miniflux
ghcr.io/${{ github.repository_owner }}/miniflux
quay.io/${{ github.repository_owner }}/miniflux
tags: |
type=ref,event=pr,suffix=-distroless
type=schedule,pattern=nightly,suffix=-distroless
type=semver,pattern={{raw}},suffix=-distroless
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
@ -81,12 +52,14 @@ jobs:
uses: docker/setup-buildx-action@v3
- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to GitHub Container Registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: ghcr.io
@ -94,6 +67,7 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}
- name: Login to Quay Container Registry
if: github.event_name != 'pull_request'
uses: docker/login-action@v3
with:
registry: quay.io
@ -106,8 +80,8 @@ jobs:
context: .
file: ./packaging/docker/alpine/Dockerfile
platforms: linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64
push: true
tags: ${{ steps.docker_alpine_tag.outputs.tags }}
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker_alpine_tags.outputs.tags }}
- name: Build and Push Distroless images
uses: docker/build-push-action@v5
@ -115,5 +89,5 @@ jobs:
context: .
file: ./packaging/docker/distroless/Dockerfile
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.docker_distroless_tag.outputs.tags }}
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker_distroless_tags.outputs.tags }}

6
.github/workflows/linters.yml vendored
View File

@ -22,7 +22,7 @@ jobs:
run: eslint internal/ui/static/js/*.js
golangci:
name: Golang Linter
name: Golang Linters
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@ -32,8 +32,8 @@ jobs:
- run: "go vet ./..."
- uses: golangci/golangci-lint-action@v4
with:
args: --timeout 10m --skip-dirs tests --disable errcheck --enable sqlclosecheck --enable misspell --enable gofmt --enable goimports --enable whitespace
- uses: dominikh/staticcheck-action@v1.3.0
args: --timeout 10m --skip-dirs tests --disable errcheck --enable sqlclosecheck --enable misspell --enable gofmt --enable goimports --enable whitespace --enable gocritic
- uses: dominikh/staticcheck-action@v1.3.1
with:
version: "2023.1.7"
install-go: false

81
ChangeLog
View File

@ -1,3 +1,84 @@
Version 2.1.2 (March 30, 2024)
------------------------------
* `api`: rewrite API integration tests without build tags
* `ci`: add basic ESLinter checks
* `ci`: enable go-critic linter and fix various issues detected
* `ci`: fix JavaScript linter path in GitHub Actions
* `cli`: avoid misleading error message when creating an admin user automatically
* `config`: add `FILTER_ENTRY_MAX_AGE_DAYS` option
* `config`: bump the number of simultaneous workers
* `config`: rename `PROXY_*` options to `MEDIA_PROXY_*`
* `config`: use `crypto.GenerateRandomBytes` instead of doing it by hand
* `http/request`: refactor conditions to be more idiomatic
* `http/response`: remove legacy `X-XSS-Protection` header
* `integration/rssbrige`: fix rssbrige import
* `integration/shaarli`: factorize the header+payload concatenation as data
* `integration/shaarli`: no need to base64-encode then remove the padding when we can simply encode without padding
* `integration/shaarli`: the JWT token was declared as using HS256 as algorithm, but was using HS512
* `integration/webhook`: add category title to request body
* `locale`: update Turkish translations
* `man page`: sort config options in alphabetical order
* `mediaproxy`: reduce the internal indentation of `ProxifiedUrl` by inverting some conditions
* `mediaproxy`: simplify and refactor the package
* `model`: replace` Optional{Int,Int64,Float64}` with a generic function `OptionalNumber()`
* `model`: use struct embedding for `FeedCreationRequestFromSubscriptionDiscovery` to reduce code duplication
* `reader/atom`: avoid debug message when the date is empty
* `reader/atom`: change `if !a { a = } if !a {a = }` constructs into `if !a { a = ; if !a {a = }}` to reduce the number of comparisons and improve readability
* `reader/atom`: Move the population of the feed's entries into a new function, to make BuildFeed easier to understand/separate concerns/implementation details
* `reader/atom`: refactor Atom parser to use an adapter
* `reader/atom`: use `sort+compact` instead of `compact+sort` to remove duplicates
* `reader/atom`: when detecting the format, detect its version as well
* `reader/encoding`: inline a one-liner function
* `reader/handler`: fix force refresh feature
* `reader/json`: refactor JSON Feed parser to use an adapter
* `reader/media`: remove a superfluous error-check: `strconv.ParseInt` returns `0` when passed an empty string
* `reader/media`: simplify switch-case by moving a common condition above it
* `reader/processor`: compile block/keep regex only once per feed
* `reader/rdf`: refactor RDF parser to use an adapter
* `reader/rewrite`: inline some one-line functions
* `reader/rewrite`: simplify `removeClickbait`
* `reader/rewrite`: transform a free-standing function into a method
* `reader/rewrite`: use a proper constant instead of a magic number in `applyFuncOnTextContent`
* `reader/rss`: add support for `<media:category>` element
* `reader/rss`: don't add empty tags to RSS items
* `reader/rss`: refactor RSS parser to use a default namespace to avoid some limitations of the Go XML parser
* `reader/rss`: refactor RSS Parser to use an adapter
* `reader/rss`: remove some duplicated code in RSS parser
* `reader`: ensure that enclosure URLs are always absolute
* `reader`: move iTunes and GooglePlay XML definitions to their own packages
* `reader`: parse podcast categories
* `reader`: remove trailing space in `SiteURL` and `FeedURL`
* `storage`: do not store empty tags
* `storage`: simplify `removeDuplicates()` to use a `sort`+`compact` construct instead of doing it by hand with a hashmap
* `storage`: Use plain strings concatenation instead of building an array and then joining it
* `timezone`: make sure the tests pass when the timezone database is not installed on the host
* `ui/css`: align `min-width` with the other `min-width` values
* `ui/css`: fix regression: "Add to Home Screen" button is unreadable
* `ui/js`: don't use lambdas to return a function, use directly the function instead
* `ui/js`: enable trusted-types
* `ui/js`: fix download button loading label
* `ui/js`: fix JavaScript error on the login page when the user not authenticated
* `ui/js`: inline one-line functions
* `ui/js`: inline some `querySelectorAll` calls
* `ui/js`: reduce the scope of some variables
* `ui/js`: remove a hack for "Chrome 67 and earlier" since it was released in 2018
* `ui/js`: replace `DomHelper.findParent` with `.closest`
* `ui/js`: replace `let` with `const`
* `ui/js`: simplify `DomHelper.getVisibleElements` by using a `filter` instead of a loop with an index
* `ui/js`: use a `Set` instead of an array in a `KeyboardHandler`'s member
* `ui/js`: use some ternaries where it makes sense
* `ui/static`: make use of `HashFromBytes` everywhere
* `ui/static`: set minifier ECMAScript version
* `ui`: add keyboard shortcuts for scrolling to top/bottom of the item list
* `ui`: add media player control playback speed
* `ui`: remove unused variables and improve JSON decoding in `saveEnclosureProgression()`
* `validator`: display an error message on edit feed page when the feed URL is not unique
* Bump `github.com/coreos/go-oidc/v3` from `3.9.0` to `3.10.0`
* Bump `github.com/go-webauthn/webauthn` from `0.10.1` to `0.10.2`
* Bump `github.com/tdewolff/minify/v2` from `2.20.18` to `2.20.19`
* Bump `google.golang.org/protobuf` from `1.32.0` to `1.33.0`
Version 2.1.1 (March 10, 2024)
-----------------------------

8
Makefile
View File

@ -101,7 +101,7 @@ windows-x86:
@ GOOS=windows GOARCH=386 go build -ldflags=$(LD_FLAGS) -o $(APP)-$@.exe main.go
run:
@ LOG_DATE_TIME=1 DEBUG=1 RUN_MIGRATIONS=1 CREATE_ADMIN=1 ADMIN_USERNAME=admin ADMIN_PASSWORD=test123 go run main.go
@ LOG_DATE_TIME=1 LOG_LEVEL=debug RUN_MIGRATIONS=1 CREATE_ADMIN=1 ADMIN_USERNAME=admin ADMIN_PASSWORD=test123 go run main.go
clean:
@ rm -f $(APP)-* $(APP) $(APP)*.rpm $(APP)*.deb $(APP)*.exe
@ -128,7 +128,11 @@ integration-test:
./miniflux-test >/tmp/miniflux.log 2>&1 & echo "$$!" > "/tmp/miniflux.pid"
while ! nc -z localhost 8080; do sleep 1; done
go test -v -tags=integration -count=1 miniflux.app/v2/internal/tests
TEST_MINIFLUX_BASE_URL=http://127.0.0.1:8080 \
TEST_MINIFLUX_ADMIN_USERNAME=admin \
TEST_MINIFLUX_ADMIN_PASSWORD=test123 \
go test -v -count=1 ./internal/api
clean-integration-test:
@ kill -9 `cat /tmp/miniflux.pid`

57
client/client.go
View File

@ -18,16 +18,44 @@ type Client struct {
}
// New returns a new Miniflux client.
// Deprecated: use NewClient instead.
func New(endpoint string, credentials ...string) *Client {
// Web gives "API Endpoint = https://miniflux.app/v1/", it doesn't work (/v1/v1/me)
return NewClient(endpoint, credentials...)
}
// NewClient returns a new Miniflux client.
func NewClient(endpoint string, credentials ...string) *Client {
// Trim trailing slashes and /v1 from the endpoint.
endpoint = strings.TrimSuffix(endpoint, "/")
endpoint = strings.TrimSuffix(endpoint, "/v1")
// trim to https://miniflux.app
if len(credentials) == 2 {
switch len(credentials) {
case 2:
return &Client{request: &request{endpoint: endpoint, username: credentials[0], password: credentials[1]}}
case 1:
return &Client{request: &request{endpoint: endpoint, apiKey: credentials[0]}}
default:
return &Client{request: &request{endpoint: endpoint}}
}
return &Client{request: &request{endpoint: endpoint, apiKey: credentials[0]}}
}
// Healthcheck checks if the application is up and running.
func (c *Client) Healthcheck() error {
body, err := c.request.Get("/healthcheck")
if err != nil {
return fmt.Errorf("miniflux: unable to perform healthcheck: %w", err)
}
defer body.Close()
responseBodyContent, err := io.ReadAll(body)
if err != nil {
return fmt.Errorf("miniflux: unable to read healthcheck response: %w", err)
}
if string(responseBodyContent) != "OK" {
return fmt.Errorf("miniflux: invalid healthcheck response: %q", responseBodyContent)
}
return nil
}
// Version returns the version of the Miniflux instance.
@ -528,6 +556,25 @@ func (c *Client) SaveEntry(entryID int64) error {
return err
}
// FetchEntryOriginalContent fetches the original content of an entry using the scraper.
func (c *Client) FetchEntryOriginalContent(entryID int64) (string, error) {
body, err := c.request.Get(fmt.Sprintf("/v1/entries/%d/fetch-content", entryID))
if err != nil {
return "", err
}
defer body.Close()
var response struct {
Content string `json:"content"`
}
if err := json.NewDecoder(body).Decode(&response); err != nil {
return "", fmt.Errorf("miniflux: response error (%v)", err)
}
return response.Content, nil
}
// FetchCounters fetches feed counters.
func (c *Client) FetchCounters() (*FeedCounters, error) {
body, err := c.request.Get("/v1/feeds/counters")

2
client/doc.go
View File

@ -12,7 +12,7 @@ This code snippet fetch the list of users:
miniflux "miniflux.app/v2/client"
)
client := miniflux.New("https://api.example.org", "admin", "secret")
client := miniflux.NewClient("https://api.example.org", "admin", "secret")
users, err := client.Users()
if err != nil {
fmt.Println(err)

50
client/model.go
View File

@ -41,6 +41,7 @@ type User struct {
DefaultHomePage string `json:"default_home_page"`
CategoriesSortingOrder string `json:"categories_sorting_order"`
MarkReadOnView bool `json:"mark_read_on_view"`
MediaPlaybackRate float64 `json:"media_playback_rate"`
}
func (u User) String() string {
@ -58,28 +59,29 @@ type UserCreationRequest struct {
// UserModificationRequest represents the request to update a user.
type UserModificationRequest struct {
Username *string `json:"username"`
Password *string `json:"password"`
IsAdmin *bool `json:"is_admin"`
Theme *string `json:"theme"`
Language *string `json:"language"`
Timezone *string `json:"timezone"`
EntryDirection *string `json:"entry_sorting_direction"`
EntryOrder *string `json:"entry_sorting_order"`
Stylesheet *string `json:"stylesheet"`
GoogleID *string `json:"google_id"`
OpenIDConnectID *string `json:"openid_connect_id"`
EntriesPerPage *int `json:"entries_per_page"`
KeyboardShortcuts *bool `json:"keyboard_shortcuts"`
ShowReadingTime *bool `json:"show_reading_time"`
EntrySwipe *bool `json:"entry_swipe"`
GestureNav *string `json:"gesture_nav"`
DisplayMode *string `json:"display_mode"`
DefaultReadingSpeed *int `json:"default_reading_speed"`
CJKReadingSpeed *int `json:"cjk_reading_speed"`
DefaultHomePage *string `json:"default_home_page"`
CategoriesSortingOrder *string `json:"categories_sorting_order"`
MarkReadOnView *bool `json:"mark_read_on_view"`
Username *string `json:"username"`
Password *string `json:"password"`
IsAdmin *bool `json:"is_admin"`
Theme *string `json:"theme"`
Language *string `json:"language"`
Timezone *string `json:"timezone"`
EntryDirection *string `json:"entry_sorting_direction"`
EntryOrder *string `json:"entry_sorting_order"`
Stylesheet *string `json:"stylesheet"`
GoogleID *string `json:"google_id"`
OpenIDConnectID *string `json:"openid_connect_id"`
EntriesPerPage *int `json:"entries_per_page"`
KeyboardShortcuts *bool `json:"keyboard_shortcuts"`
ShowReadingTime *bool `json:"show_reading_time"`
EntrySwipe *bool `json:"entry_swipe"`
GestureNav *string `json:"gesture_nav"`
DisplayMode *string `json:"display_mode"`
DefaultReadingSpeed *int `json:"default_reading_speed"`
CJKReadingSpeed *int `json:"cjk_reading_speed"`
DefaultHomePage *string `json:"default_home_page"`
CategoriesSortingOrder *string `json:"categories_sorting_order"`
MarkReadOnView *bool `json:"mark_read_on_view"`
MediaPlaybackRate *float64 `json:"media_playback_rate"`
}
// Users represents a list of users.
@ -290,3 +292,7 @@ type VersionResponse struct {
Arch string `json:"arch"`
OS string `json:"os"`
}
func SetOptionalField[T any](value T) *T {
return &value
}

5
client/request.go
View File

@ -26,6 +26,7 @@ var (
ErrForbidden = errors.New("miniflux: access forbidden")
ErrServerError = errors.New("miniflux: internal server error")
ErrNotFound = errors.New("miniflux: resource not found")
ErrBadRequest = errors.New("miniflux: bad request")
)
type errorResponse struct {
@ -124,10 +125,10 @@ func (r *request) execute(method, path string, data interface{}) (io.ReadCloser,
var resp errorResponse
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&resp); err != nil {
return nil, fmt.Errorf("miniflux: bad request error (%v)", err)
return nil, fmt.Errorf("%w (%v)", ErrBadRequest, err)
}
return nil, fmt.Errorf("miniflux: bad request (%s)", resp.ErrorMessage)
return nil, fmt.Errorf("%w (%s)", ErrBadRequest, resp.ErrorMessage)
}
if response.StatusCode > 400 {

32
go.mod
View File

@ -5,23 +5,24 @@ module miniflux.app/v2
require (
github.com/PuerkitoBio/goquery v1.9.1
github.com/abadojack/whatlanggo v1.0.1
github.com/coreos/go-oidc/v3 v3.9.0
github.com/go-webauthn/webauthn v0.10.1
github.com/coreos/go-oidc/v3 v3.10.0
github.com/go-webauthn/webauthn v0.10.2
github.com/gorilla/mux v1.8.1
github.com/lib/pq v1.10.9
github.com/prometheus/client_golang v1.19.0
github.com/tdewolff/minify/v2 v2.20.18
github.com/yuin/goldmark v1.7.0
golang.org/x/crypto v0.21.0
golang.org/x/net v0.22.0
golang.org/x/oauth2 v0.18.0
golang.org/x/term v0.18.0
github.com/tdewolff/minify/v2 v2.20.19
github.com/yuin/goldmark v1.7.1
golang.org/x/crypto v0.22.0
golang.org/x/net v0.24.0
golang.org/x/oauth2 v0.19.0
golang.org/x/term v0.19.0
golang.org/x/text v0.14.0
mvdan.cc/xurls/v2 v2.5.0
)
require (
github.com/go-webauthn/x v0.1.8 // indirect
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
github.com/go-webauthn/x v0.1.9 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
github.com/google/go-tpm v0.9.0 // indirect
)
@ -29,9 +30,8 @@ require (
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/fxamacker/cbor/v2 v2.5.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/fxamacker/cbor/v2 v2.6.0 // indirect
github.com/go-jose/go-jose/v4 v4.0.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
@ -39,10 +39,8 @@ require (
github.com/prometheus/procfs v0.12.0 // indirect
github.com/tdewolff/parse/v2 v2.7.12 // indirect
github.com/x448/float16 v0.8.4 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/protobuf v1.32.0 // indirect
golang.org/x/sys v0.19.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
)
go 1.22

83
go.sum
View File

@ -8,27 +8,20 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo=
github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/coreos/go-oidc/v3 v3.10.0 h1:tDnXHnLyiTVyT/2zLDGj09pFPkhND8Gl8lnTRhoEaJU=
github.com/coreos/go-oidc/v3 v3.10.0/go.mod h1:5j11xcw0D3+SGxn6Z/WFADsgcWVMyNAlSQupk0KK3ac=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
github.com/go-webauthn/webauthn v0.10.1 h1:+RFKj4yHPy282teiiy5sqTYPfRilzBpJyedrz9KsNFE=
github.com/go-webauthn/webauthn v0.10.1/go.mod h1:a7BwAtrSMkeuJXtIKz433Av99nAv01pdfzB0a9xkDnI=
github.com/go-webauthn/x v0.1.8 h1:f1C6k1AyUlDvnIzWSW+G9rN9nbp1hhLXZagUtyxZ8nc=
github.com/go-webauthn/x v0.1.8/go.mod h1:i8UNlGVt3oy6oAFcP4SZB1djZLx/4pbekCbWowjTaJg=
github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw=
github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA=
github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4=
github.com/go-webauthn/webauthn v0.10.2/go.mod h1:Gd1IDsGAybuvK1NkwUTLbGmeksxuRJjVN2PE/xsPxHs=
github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE=
github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk=
@ -51,12 +44,10 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tdewolff/minify/v2 v2.20.18 h1:y+s6OzlZwFqApgNXWNtaMuEMEPbHT72zrCyb9Az35Xo=
github.com/tdewolff/minify/v2 v2.20.18/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tdewolff/minify/v2 v2.20.19 h1:tX0SR0LUrIqGoLjXnkIzRSIbKJ7PaNnSENLD4CyH6Xo=
github.com/tdewolff/minify/v2 v2.20.19/go.mod h1:ulkFoeAVWMLEyjuDz1ZIWOA31g5aWOawCFRp9R/MudM=
github.com/tdewolff/parse/v2 v2.7.12 h1:tgavkHc2ZDEQVKy1oWxwIyh5bP4F5fEh/JmBwPP/3LQ=
github.com/tdewolff/parse/v2 v2.7.12/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA=
github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
@ -65,13 +56,12 @@ github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzv
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/goldmark v1.7.0 h1:EfOIvIMZIzHdB/R/zVrikYLPPwJlfMcNczJFMs1m6sA=
github.com/yuin/goldmark v1.7.0/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=
github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -79,11 +69,10 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg=
golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -94,22 +83,17 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
@ -119,15 +103,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8=

2321
internal/api/api_integration_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

12
internal/api/entry.go
View File

@ -15,8 +15,8 @@ import (
"miniflux.app/v2/internal/http/request"
"miniflux.app/v2/internal/http/response/json"
"miniflux.app/v2/internal/integration"
"miniflux.app/v2/internal/mediaproxy"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxy"
"miniflux.app/v2/internal/reader/processor"
"miniflux.app/v2/internal/reader/readingtime"
"miniflux.app/v2/internal/storage"
@ -36,14 +36,14 @@ func (h *handler) getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b
return
}
entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content)
proxyOption := config.Opts.ProxyOption()
entry.Content = mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entry.Content)
proxyOption := config.Opts.MediaProxyMode()
for i := range entry.Enclosures {
if proxyOption == "all" || proxyOption != "none" && !urllib.IsHTTPS(entry.Enclosures[i].URL) {
for _, mediaType := range config.Opts.ProxyMediaTypes() {
for _, mediaType := range config.Opts.MediaProxyResourceTypes() {
if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") {
entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL)
entry.Enclosures[i].URL = mediaproxy.ProxifyAbsoluteURL(h.router, r.Host, entry.Enclosures[i].URL)
break
}
}
@ -164,7 +164,7 @@ func (h *handler) findEntries(w http.ResponseWriter, r *http.Request, feedID int
}
for i := range entries {
entries[i].Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entries[i].Content)
entries[i].Content = mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entries[i].Content)
}
json.OK(w, r, &entriesResponse{Total: count, Entries: entries})

2
internal/api/feed.go
View File

@ -115,7 +115,7 @@ func (h *handler) updateFeed(w http.ResponseWriter, r *http.Request) {
return
}
if validationErr := validator.ValidateFeedModification(h.store, userID, &feedModificationRequest); validationErr != nil {
if validationErr := validator.ValidateFeedModification(h.store, userID, originalFeed.ID, &feedModificationRequest); validationErr != nil {
json.BadRequest(w, r, validationErr.Error())
return
}

2
internal/cli/ask_credentials.go
View File

@ -16,7 +16,7 @@ func askCredentials() (string, string) {
fd := int(os.Stdin.Fd())
if !term.IsTerminal(fd) {
printErrorAndExit(fmt.Errorf("this is not a terminal, exiting"))
printErrorAndExit(fmt.Errorf("this is not an interactive terminal, exiting"))
}
fmt.Print("Enter Username: ")

7
internal/cli/cli.go
View File

@ -23,7 +23,7 @@ const (
flagVersionHelp = "Show application version"
flagMigrateHelp = "Run SQL migrations"
flagFlushSessionsHelp = "Flush all sessions (disconnect users)"
flagCreateAdminHelp = "Create admin user"
flagCreateAdminHelp = "Create an admin user from an interactive terminal"
flagResetPasswordHelp = "Reset user password"
flagResetFeedErrorsHelp = "Clear all feed errors for all users"
flagDebugModeHelp = "Show debug logs"
@ -191,7 +191,7 @@ func Parse() {
}
if flagCreateAdmin {
createAdmin(store)
createAdminUserFromInteractiveTerminal(store)
return
}
@ -211,9 +211,8 @@ func Parse() {
printErrorAndExit(err)
}
// Create admin user and start the daemon.
if config.Opts.CreateAdmin() {
createAdmin(store)
createAdminUserFromEnvironmentVariables(store)
}
if flagRefreshFeeds {

28
internal/cli/create_admin.go
View File

@ -12,15 +12,20 @@ import (
"miniflux.app/v2/internal/validator"
)
func createAdmin(store *storage.Storage) {
userCreationRequest := &model.UserCreationRequest{
Username: config.Opts.AdminUsername(),
Password: config.Opts.AdminPassword(),
IsAdmin: true,
}
func createAdminUserFromEnvironmentVariables(store *storage.Storage) {
createAdminUser(store, config.Opts.AdminUsername(), config.Opts.AdminPassword())
}
if userCreationRequest.Username == "" || userCreationRequest.Password == "" {
userCreationRequest.Username, userCreationRequest.Password = askCredentials()
func createAdminUserFromInteractiveTerminal(store *storage.Storage) {
username, password := askCredentials()
createAdminUser(store, username, password)
}
func createAdminUser(store *storage.Storage, username, password string) {
userCreationRequest := &model.UserCreationRequest{
Username: username,
Password: password,
IsAdmin: true,
}
if store.UserExists(userCreationRequest.Username) {
@ -34,7 +39,12 @@ func createAdmin(store *storage.Storage) {
printErrorAndExit(validationErr.Error())
}
if _, err := store.CreateUser(userCreationRequest); err != nil {
if user, err := store.CreateUser(userCreationRequest); err != nil {
printErrorAndExit(err)
} else {
slog.Info("Created new admin user",
slog.String("username", user.Username),
slog.Int64("user_id", user.ID),
)
}
}

229
internal/config/config_test.go
View File

@ -4,6 +4,7 @@
package config // import "miniflux.app/v2/internal/config"
import (
"bytes"
"os"
"testing"
)
@ -1442,9 +1443,9 @@ func TestPocketConsumerKeyFromUserPrefs(t *testing.T) {
}
}
func TestProxyOption(t *testing.T) {
func TestMediaProxyMode(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("MEDIA_PROXY_MODE", "all")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -1453,14 +1454,14 @@ func TestProxyOption(t *testing.T) {
}
expected := "all"
result := opts.ProxyOption()
result := opts.MediaProxyMode()
if result != expected {
t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected)
t.Fatalf(`Unexpected MEDIA_PROXY_MODE value, got %q instead of %q`, result, expected)
}
}
func TestDefaultProxyOptionValue(t *testing.T) {
func TestDefaultMediaProxyModeValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
@ -1469,17 +1470,17 @@ func TestDefaultProxyOptionValue(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultProxyOption
result := opts.ProxyOption()
expected := defaultMediaProxyMode
result := opts.MediaProxyMode()
if result != expected {
t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected)
t.Fatalf(`Unexpected MEDIA_PROXY_MODE value, got %q instead of %q`, result, expected)
}
}
func TestProxyMediaTypes(t *testing.T) {
func TestMediaProxyResourceTypes(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_MEDIA_TYPES", "image,audio")
os.Setenv("MEDIA_PROXY_RESOURCE_TYPES", "image,audio")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -1489,25 +1490,25 @@ func TestProxyMediaTypes(t *testing.T) {
expected := []string{"audio", "image"}
if len(expected) != len(opts.ProxyMediaTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
if len(expected) != len(opts.MediaProxyResourceTypes()) {
t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.ProxyMediaTypes() {
for _, mediaType := range opts.MediaProxyResourceTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
}
}
func TestProxyMediaTypesWithDuplicatedValues(t *testing.T) {
func TestMediaProxyResourceTypesWithDuplicatedValues(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_MEDIA_TYPES", "image,audio, image")
os.Setenv("MEDIA_PROXY_RESOURCE_TYPES", "image,audio, image")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -1516,23 +1517,119 @@ func TestProxyMediaTypesWithDuplicatedValues(t *testing.T) {
}
expected := []string{"audio", "image"}
if len(expected) != len(opts.ProxyMediaTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
if len(expected) != len(opts.MediaProxyResourceTypes()) {
t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.ProxyMediaTypes() {
for _, mediaType := range opts.MediaProxyResourceTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
}
}
func TestProxyImagesOptionBackwardCompatibility(t *testing.T) {
func TestDefaultMediaProxyResourceTypes(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{"image"}
if len(expected) != len(opts.MediaProxyResourceTypes()) {
t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.MediaProxyResourceTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected MEDIA_PROXY_RESOURCE_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
}
}
func TestMediaProxyHTTPClientTimeout(t *testing.T) {
os.Clearenv()
os.Setenv("MEDIA_PROXY_HTTP_CLIENT_TIMEOUT", "24")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 24
result := opts.MediaProxyHTTPClientTimeout()
if result != expected {
t.Fatalf(`Unexpected MEDIA_PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
func TestDefaultMediaProxyHTTPClientTimeoutValue(t *testing.T) {
os.Clearenv()
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultMediaProxyHTTPClientTimeout
result := opts.MediaProxyHTTPClientTimeout()
if result != expected {
t.Fatalf(`Unexpected MEDIA_PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
func TestMediaProxyCustomURL(t *testing.T) {
os.Clearenv()
os.Setenv("MEDIA_PROXY_CUSTOM_URL", "http://example.org/proxy")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := "http://example.org/proxy"
result := opts.MediaCustomProxyURL()
if result != expected {
t.Fatalf(`Unexpected MEDIA_PROXY_CUSTOM_URL value, got %q instead of %q`, result, expected)
}
}
func TestMediaProxyPrivateKey(t *testing.T) {
os.Clearenv()
os.Setenv("MEDIA_PROXY_PRIVATE_KEY", "foobar")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []byte("foobar")
result := opts.MediaProxyPrivateKey()
if !bytes.Equal(result, expected) {
t.Fatalf(`Unexpected MEDIA_PROXY_PRIVATE_KEY value, got %q instead of %q`, result, expected)
}
}
func TestProxyImagesOptionForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGES", "all")
@ -1543,30 +1640,31 @@ func TestProxyImagesOptionBackwardCompatibility(t *testing.T) {
}
expected := []string{"image"}
if len(expected) != len(opts.ProxyMediaTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
if len(expected) != len(opts.MediaProxyResourceTypes()) {
t.Fatalf(`Unexpected PROXY_IMAGES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.ProxyMediaTypes() {
for _, mediaType := range opts.MediaProxyResourceTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
t.Fatalf(`Unexpected PROXY_IMAGES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
}
expectedProxyOption := "all"
result := opts.ProxyOption()
result := opts.MediaProxyMode()
if result != expectedProxyOption {
t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expectedProxyOption)
}
}
func TestDefaultProxyMediaTypes(t *testing.T) {
func TestProxyImageURLForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_IMAGE_URL", "http://example.org/proxy")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -1574,25 +1672,73 @@ func TestDefaultProxyMediaTypes(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{"image"}
expected := "http://example.org/proxy"
result := opts.MediaCustomProxyURL()
if result != expected {
t.Fatalf(`Unexpected PROXY_IMAGE_URL value, got %q instead of %q`, result, expected)
}
}
if len(expected) != len(opts.ProxyMediaTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
func TestProxyURLOptionForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_URL", "http://example.org/proxy")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := "http://example.org/proxy"
result := opts.MediaCustomProxyURL()
if result != expected {
t.Fatalf(`Unexpected PROXY_URL value, got %q instead of %q`, result, expected)
}
}
func TestProxyMediaTypesOptionForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_MEDIA_TYPES", "image,audio")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := []string{"audio", "image"}
if len(expected) != len(opts.MediaProxyResourceTypes()) {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
resultMap := make(map[string]bool)
for _, mediaType := range opts.ProxyMediaTypes() {
for _, mediaType := range opts.MediaProxyResourceTypes() {
resultMap[mediaType] = true
}
for _, mediaType := range expected {
if !resultMap[mediaType] {
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected)
t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.MediaProxyResourceTypes(), expected)
}
}
}
func TestProxyHTTPClientTimeout(t *testing.T) {
func TestProxyOptionForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := "all"
result := opts.MediaProxyMode()
if result != expected {
t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected)
}
}
func TestProxyHTTPClientTimeoutOptionForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_HTTP_CLIENT_TIMEOUT", "24")
@ -1601,29 +1747,26 @@ func TestProxyHTTPClientTimeout(t *testing.T) {
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := 24
result := opts.ProxyHTTPClientTimeout()
result := opts.MediaProxyHTTPClientTimeout()
if result != expected {
t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
}
}
func TestDefaultProxyHTTPClientTimeoutValue(t *testing.T) {
func TestProxyPrivateKeyOptionForBackwardCompatibility(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_PRIVATE_KEY", "foobar")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultProxyHTTPClientTimeout
result := opts.ProxyHTTPClientTimeout()
if result != expected {
t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected)
expected := []byte("foobar")
result := opts.MediaProxyPrivateKey()
if !bytes.Equal(result, expected) {
t.Fatalf(`Unexpected PROXY_PRIVATE_KEY value, got %q instead of %q`, result, expected)
}
}

83
internal/config/options.go
View File

@ -27,7 +27,7 @@ const (
defaultBaseURL = "http://localhost"
defaultRootURL = "http://localhost"
defaultBasePath = ""
defaultWorkerPoolSize = 5
defaultWorkerPoolSize = 16
defaultPollingFrequency = 60
defaultForceRefreshInterval = 30
defaultBatchSize = 100
@ -51,10 +51,11 @@ const (
defaultCleanupArchiveUnreadDays = 180
defaultCleanupArchiveBatchSize = 10000
defaultCleanupRemoveSessionsDays = 30
defaultProxyHTTPClientTimeout = 120
defaultProxyOption = "http-only"
defaultProxyMediaTypes = "image"
defaultProxyUrl = ""
defaultMediaProxyHTTPClientTimeout = 120
defaultMediaProxyMode = "http-only"
defaultMediaResourceTypes = "image"
defaultMediaProxyURL = ""
defaultFilterEntryMaxAgeDays = 0
defaultFetchOdyseeWatchTime = false
defaultFetchYouTubeWatchTime = false
defaultYouTubeEmbedUrlOverride = "https://www.youtube-nocookie.com/embed/"
@ -135,12 +136,13 @@ type Options struct {
createAdmin bool
adminUsername string
adminPassword string
proxyHTTPClientTimeout int
proxyOption string
proxyMediaTypes []string
proxyUrl string
mediaProxyHTTPClientTimeout int
mediaProxyMode string
mediaProxyResourceTypes []string
mediaProxyCustomURL string
fetchOdyseeWatchTime bool
fetchYouTubeWatchTime bool
filterEntryMaxAgeDays int
youTubeEmbedUrlOverride string
oauth2UserCreationAllowed bool
oauth2ClientID string
@ -165,7 +167,7 @@ type Options struct {
metricsPassword string
watchdog bool
invidiousInstance string
proxyPrivateKey []byte
mediaProxyPrivateKey []byte
webAuthn bool
}
@ -209,10 +211,11 @@ func NewOptions() *Options {
pollingParsingErrorLimit: defaultPollingParsingErrorLimit,
workerPoolSize: defaultWorkerPoolSize,
createAdmin: defaultCreateAdmin,
proxyHTTPClientTimeout: defaultProxyHTTPClientTimeout,
proxyOption: defaultProxyOption,
proxyMediaTypes: []string{defaultProxyMediaTypes},
proxyUrl: defaultProxyUrl,
mediaProxyHTTPClientTimeout: defaultMediaProxyHTTPClientTimeout,
mediaProxyMode: defaultMediaProxyMode,
mediaProxyResourceTypes: []string{defaultMediaResourceTypes},
mediaProxyCustomURL: defaultMediaProxyURL,
filterEntryMaxAgeDays: defaultFilterEntryMaxAgeDays,
fetchOdyseeWatchTime: defaultFetchOdyseeWatchTime,
fetchYouTubeWatchTime: defaultFetchYouTubeWatchTime,
youTubeEmbedUrlOverride: defaultYouTubeEmbedUrlOverride,
@ -239,7 +242,7 @@ func NewOptions() *Options {
metricsPassword: defaultMetricsPassword,
watchdog: defaultWatchdog,
invidiousInstance: defaultInvidiousInstance,
proxyPrivateKey: crypto.GenerateRandomBytes(16),
mediaProxyPrivateKey: crypto.GenerateRandomBytes(16),
webAuthn: defaultWebAuthn,
}
}
@ -489,24 +492,29 @@ func (o *Options) FetchOdyseeWatchTime() bool {
return o.fetchOdyseeWatchTime
}
// ProxyOption returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy.
func (o *Options) ProxyOption() string {
return o.proxyOption
// MediaProxyMode returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy.
func (o *Options) MediaProxyMode() string {
return o.mediaProxyMode
}
// ProxyMediaTypes returns a slice of media types to proxy.
func (o *Options) ProxyMediaTypes() []string {
return o.proxyMediaTypes
// MediaProxyResourceTypes returns a slice of resource types to proxy.
func (o *Options) MediaProxyResourceTypes() []string {
return o.mediaProxyResourceTypes
}
// ProxyUrl returns a string of a URL to use to proxy image requests
func (o *Options) ProxyUrl() string {
return o.proxyUrl
// MediaCustomProxyURL returns the custom proxy URL for medias.
func (o *Options) MediaCustomProxyURL() string {
return o.mediaProxyCustomURL
}
// ProxyHTTPClientTimeout returns the time limit in seconds before the proxy HTTP client cancel the request.
func (o *Options) ProxyHTTPClientTimeout() int {
return o.proxyHTTPClientTimeout
// MediaProxyHTTPClientTimeout returns the time limit in seconds before the proxy HTTP client cancel the request.
func (o *Options) MediaProxyHTTPClientTimeout() int {
return o.mediaProxyHTTPClientTimeout
}
// MediaProxyPrivateKey returns the private key used by the media proxy.
func (o *Options) MediaProxyPrivateKey() []byte {
return o.mediaProxyPrivateKey
}
// HasHTTPService returns true if the HTTP service is enabled.
@ -602,16 +610,16 @@ func (o *Options) InvidiousInstance() string {
return o.invidiousInstance
}
// ProxyPrivateKey returns the private key used by the media proxy
func (o *Options) ProxyPrivateKey() []byte {
return o.proxyPrivateKey
}
// WebAuthn returns true if WebAuthn logins are supported
func (o *Options) WebAuthn() bool {
return o.webAuthn
}
// FilterEntryMaxAgeDays returns the number of days after which entries should be retained.
func (o *Options) FilterEntryMaxAgeDays() int {
return o.filterEntryMaxAgeDays
}
// SortedOptions returns options as a list of key value pairs, sorted by keys.
func (o *Options) SortedOptions(redactSecret bool) []*Option {
var keyValues = map[string]interface{}{
@ -637,6 +645,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"DISABLE_HSTS": !o.hsts,
"DISABLE_HTTP_SERVICE": !o.httpService,
"DISABLE_SCHEDULER_SERVICE": !o.schedulerService,
"FILTER_ENTRY_MAX_AGE_DAYS": o.filterEntryMaxAgeDays,
"FETCH_YOUTUBE_WATCH_TIME": o.fetchYouTubeWatchTime,
"FETCH_ODYSEE_WATCH_TIME": o.fetchOdyseeWatchTime,
"HTTPS": o.HTTPS,
@ -671,11 +680,11 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"FORCE_REFRESH_INTERVAL": o.forceRefreshInterval,
"POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit,
"POLLING_SCHEDULER": o.pollingScheduler,
"PROXY_HTTP_CLIENT_TIMEOUT": o.proxyHTTPClientTimeout,
"PROXY_MEDIA_TYPES": o.proxyMediaTypes,
"PROXY_OPTION": o.proxyOption,
"PROXY_PRIVATE_KEY": redactSecretValue(string(o.proxyPrivateKey), redactSecret),
"PROXY_URL": o.proxyUrl,
"MEDIA_PROXY_HTTP_CLIENT_TIMEOUT": o.mediaProxyHTTPClientTimeout,
"MEDIA_PROXY_RESOURCE_TYPES": o.mediaProxyResourceTypes,
"MEDIA_PROXY_MODE": o.mediaProxyMode,
"MEDIA_PROXY_PRIVATE_KEY": redactSecretValue(string(o.mediaProxyPrivateKey), redactSecret),
"MEDIA_PROXY_CUSTOM_URL": o.mediaProxyCustomURL,
"ROOT_URL": o.rootURL,
"RUN_MIGRATIONS": o.runMigrations,
"SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval,

45
internal/config/parser.go
View File

@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"io"
"log/slog"
"net/url"
"os"
"strconv"
@ -87,6 +88,7 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.logFormat = parsedValue
}
case "DEBUG":
slog.Warn("The DEBUG environment variable is deprecated, use LOG_LEVEL instead")
parsedValue := parseBool(value, defaultDebug)
if parsedValue {
p.opts.logLevel = "debug"
@ -112,6 +114,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.databaseMinConns = parseInt(value, defaultDatabaseMinConns)
case "DATABASE_CONNECTION_LIFETIME":
p.opts.databaseConnectionLifetime = parseInt(value, defaultDatabaseConnectionLifetime)
case "FILTER_ENTRY_MAX_AGE_DAYS":
p.opts.filterEntryMaxAgeDays = parseInt(value, defaultFilterEntryMaxAgeDays)
case "RUN_MIGRATIONS":
p.opts.runMigrations = parseBool(value, defaultRunMigrations)
case "DISABLE_HSTS":
@ -158,20 +162,41 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.schedulerRoundRobinMinInterval = parseInt(value, defaultSchedulerRoundRobinMinInterval)
case "POLLING_PARSING_ERROR_LIMIT":
p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
// kept for compatibility purpose
case "PROXY_IMAGES":
p.opts.proxyOption = parseString(value, defaultProxyOption)
slog.Warn("The PROXY_IMAGES environment variable is deprecated, use MEDIA_PROXY_MODE instead")
p.opts.mediaProxyMode = parseString(value, defaultMediaProxyMode)
case "PROXY_HTTP_CLIENT_TIMEOUT":
p.opts.proxyHTTPClientTimeout = parseInt(value, defaultProxyHTTPClientTimeout)
slog.Warn("The PROXY_HTTP_CLIENT_TIMEOUT environment variable is deprecated, use MEDIA_PROXY_HTTP_CLIENT_TIMEOUT instead")
p.opts.mediaProxyHTTPClientTimeout = parseInt(value, defaultMediaProxyHTTPClientTimeout)
case "MEDIA_PROXY_HTTP_CLIENT_TIMEOUT":
p.opts.mediaProxyHTTPClientTimeout = parseInt(value, defaultMediaProxyHTTPClientTimeout)
case "PROXY_OPTION":
p.opts.proxyOption = parseString(value, defaultProxyOption)
slog.Warn("The PROXY_OPTION environment variable is deprecated, use MEDIA_PROXY_MODE instead")
p.opts.mediaProxyMode = parseString(value, defaultMediaProxyMode)
case "MEDIA_PROXY_MODE":
p.opts.mediaProxyMode = parseString(value, defaultMediaProxyMode)
case "PROXY_MEDIA_TYPES":
p.opts.proxyMediaTypes = parseStringList(value, []string{defaultProxyMediaTypes})
// kept for compatibility purpose
slog.Warn("The PROXY_MEDIA_TYPES environment variable is deprecated, use MEDIA_PROXY_RESOURCE_TYPES instead")
p.opts.mediaProxyResourceTypes = parseStringList(value, []string{defaultMediaResourceTypes})
case "MEDIA_PROXY_RESOURCE_TYPES":
p.opts.mediaProxyResourceTypes = parseStringList(value, []string{defaultMediaResourceTypes})
case "PROXY_IMAGE_URL":
p.opts.proxyUrl = parseString(value, defaultProxyUrl)
slog.Warn("The PROXY_IMAGE_URL environment variable is deprecated, use MEDIA_PROXY_CUSTOM_URL instead")
p.opts.mediaProxyCustomURL = parseString(value, defaultMediaProxyURL)
case "PROXY_URL":
p.opts.proxyUrl = parseString(value, defaultProxyUrl)
slog.Warn("The PROXY_URL environment variable is deprecated, use MEDIA_PROXY_CUSTOM_URL instead")
p.opts.mediaProxyCustomURL = parseString(value, defaultMediaProxyURL)
case "PROXY_PRIVATE_KEY":
slog.Warn("The PROXY_PRIVATE_KEY environment variable is deprecated, use MEDIA_PROXY_PRIVATE_KEY instead")
randomKey := make([]byte, 16)
rand.Read(randomKey)
p.opts.mediaProxyPrivateKey = parseBytes(value, randomKey)
case "MEDIA_PROXY_PRIVATE_KEY":
randomKey := make([]byte, 16)
rand.Read(randomKey)
p.opts.mediaProxyPrivateKey = parseBytes(value, randomKey)
case "MEDIA_PROXY_CUSTOM_URL":
p.opts.mediaProxyCustomURL = parseString(value, defaultMediaProxyURL)
case "CREATE_ADMIN":
p.opts.createAdmin = parseBool(value, defaultCreateAdmin)
case "ADMIN_USERNAME":
@ -244,10 +269,6 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.watchdog = parseBool(value, defaultWatchdog)
case "INVIDIOUS_INSTANCE":
p.opts.invidiousInstance = parseString(value, defaultInvidiousInstance)
case "PROXY_PRIVATE_KEY":
randomKey := make([]byte, 16)
rand.Read(randomKey)
p.opts.proxyPrivateKey = parseBytes(value, randomKey)
case "WEBAUTHN":
p.opts.webAuthn = parseBool(value, defaultWebAuthn)
}

17
internal/database/migrations.go
View File

@ -871,4 +871,21 @@ var migrations = []func(tx *sql.Tx) error{
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
sql := `ALTER TABLE users ADD COLUMN media_playback_rate numeric default 1;`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
// the WHERE part speed-up the request a lot
sql := `UPDATE entries SET tags = array_remove(tags, '') WHERE '' = ANY(tags);`
_, err = tx.Exec(sql)
return err
},
func(tx *sql.Tx) (err error) {
// Entry URLs can exceeds btree maximum size
// Checking entry existence is now using entries_feed_id_status_hash_idx index
_, err = tx.Exec(`DROP INDEX entries_feed_url_idx`)
return err
},
}

4
internal/fever/handler.go
View File

@ -13,8 +13,8 @@ import (
"miniflux.app/v2/internal/http/request"
"miniflux.app/v2/internal/http/response/json"
"miniflux.app/v2/internal/integration"
"miniflux.app/v2/internal/mediaproxy"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxy"
"miniflux.app/v2/internal/storage"
"github.com/gorilla/mux"
@ -324,7 +324,7 @@ func (h *handler) handleItems(w http.ResponseWriter, r *http.Request) {
FeedID: entry.FeedID,
Title: entry.Title,
Author: entry.Author,
HTML: proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content),
HTML: mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entry.Content),
URL: entry.URL,
IsSaved: isSaved,
IsRead: isRead,

49
internal/googlereader/handler.go
View File

@ -18,8 +18,8 @@ import (
"miniflux.app/v2/internal/http/response/json"
"miniflux.app/v2/internal/http/route"
"miniflux.app/v2/internal/integration"
"miniflux.app/v2/internal/mediaproxy"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/proxy"
"miniflux.app/v2/internal/reader/fetcher"
mff "miniflux.app/v2/internal/reader/handler"
mfs "miniflux.app/v2/internal/reader/subscription"
@ -265,9 +265,10 @@ func getStreamFilterModifiers(r *http.Request) (RequestModifiers, error) {
}
func getStream(streamID string, userID int64) (Stream, error) {
if strings.HasPrefix(streamID, FeedPrefix) {
switch {
case strings.HasPrefix(streamID, FeedPrefix):
return Stream{Type: FeedStream, ID: strings.TrimPrefix(streamID, FeedPrefix)}, nil
} else if strings.HasPrefix(streamID, fmt.Sprintf(UserStreamPrefix, userID)) || strings.HasPrefix(streamID, StreamPrefix) {
case strings.HasPrefix(streamID, fmt.Sprintf(UserStreamPrefix, userID)) || strings.HasPrefix(streamID, StreamPrefix):
id := strings.TrimPrefix(streamID, fmt.Sprintf(UserStreamPrefix, userID))
id = strings.TrimPrefix(id, StreamPrefix)
switch id {
@ -288,15 +289,15 @@ func getStream(streamID string, userID int64) (Stream, error) {
default:
return Stream{NoStream, ""}, fmt.Errorf("googlereader: unknown stream with id: %s", id)
}
} else if strings.HasPrefix(streamID, fmt.Sprintf(UserLabelPrefix, userID)) || strings.HasPrefix(streamID, LabelPrefix) {
case strings.HasPrefix(streamID, fmt.Sprintf(UserLabelPrefix, userID)) || strings.HasPrefix(streamID, LabelPrefix):
id := strings.TrimPrefix(streamID, fmt.Sprintf(UserLabelPrefix, userID))
id = strings.TrimPrefix(id, LabelPrefix)
return Stream{LabelStream, id}, nil
} else if streamID == "" {
case streamID == "":
return Stream{NoStream, ""}, nil
default:
return Stream{NoStream, ""}, fmt.Errorf("googlereader: unknown stream type: %s", streamID)
}
return Stream{NoStream, ""}, fmt.Errorf("googlereader: unknown stream type: %s", streamID)
}
func getStreams(streamIDs []string, userID int64) ([]Stream, error) {
@ -382,7 +383,7 @@ func getItemIDs(r *http.Request) ([]int64, error) {
return itemIDs, nil
}
func checkOutputFormat(w http.ResponseWriter, r *http.Request) error {
func checkOutputFormat(r *http.Request) error {
var output string
if r.Method == http.MethodPost {
err := r.ParseForm()
@ -736,11 +737,12 @@ func getFeed(stream Stream, store *storage.Storage, userID int64) (*model.Feed,
}
func getOrCreateCategory(category Stream, store *storage.Storage, userID int64) (*model.Category, error) {
if category.ID == "" {
switch {
case category.ID == "":
return store.FirstCategory(userID)
} else if store.CategoryTitleExists(userID, category.ID) {
case store.CategoryTitleExists(userID, category.ID):
return store.CategoryByTitle(userID, category.ID)
} else {
default:
catRequest := model.CategoryRequest{
Title: category.ID,
}
@ -764,7 +766,7 @@ func subscribe(newFeed Stream, category Stream, title string, store *storage.Sto
}
created, localizedError := mff.CreateFeed(store, userID, &feedRequest)
if err != nil {
if localizedError != nil {
return nil, localizedError.Error()
}
@ -908,7 +910,7 @@ func (h *handler) streamItemContentsHandler(w http.ResponseWriter, r *http.Reque
slog.Int64("user_id", userID),
)
if err := checkOutputFormat(w, r); err != nil {
if err := checkOutputFormat(r); err != nil {
json.BadRequest(w, r, err)
return
}
@ -1001,14 +1003,14 @@ func (h *handler) streamItemContentsHandler(w http.ResponseWriter, r *http.Reque
categories = append(categories, userStarred)
}
entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content)
proxyOption := config.Opts.ProxyOption()
entry.Content = mediaproxy.RewriteDocumentWithAbsoluteProxyURL(h.router, r.Host, entry.Content)
proxyOption := config.Opts.MediaProxyMode()
for i := range entry.Enclosures {
if proxyOption == "all" || proxyOption != "none" && !urllib.IsHTTPS(entry.Enclosures[i].URL) {
for _, mediaType := range config.Opts.ProxyMediaTypes() {
for _, mediaType := range config.Opts.MediaProxyResourceTypes() {
if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") {
entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL)
entry.Enclosures[i].URL = mediaproxy.ProxifyAbsoluteURL(h.router, r.Host, entry.Enclosures[i].URL)
break
}
}
@ -1170,7 +1172,7 @@ func (h *handler) tagListHandler(w http.ResponseWriter, r *http.Request) {
slog.String("user_agent", r.UserAgent()),
)
if err := checkOutputFormat(w, r); err != nil {
if err := checkOutputFormat(r); err != nil {
json.BadRequest(w, r, err)
return
}
@ -1205,7 +1207,7 @@ func (h *handler) subscriptionListHandler(w http.ResponseWriter, r *http.Request
slog.String("user_agent", r.UserAgent()),
)
if err := checkOutputFormat(w, r); err != nil {
if err := checkOutputFormat(r); err != nil {
json.BadRequest(w, r, err)
return
}
@ -1224,7 +1226,7 @@ func (h *handler) subscriptionListHandler(w http.ResponseWriter, r *http.Request
URL: feed.FeedURL,
Categories: []subscriptionCategory{{fmt.Sprintf(UserLabelPrefix, userID) + feed.Category.Title, feed.Category.Title, "folder"}},
HTMLURL: feed.SiteURL,
IconURL: "", //TODO Icons are only base64 encode in DB yet
IconURL: "", // TODO: Icons are base64 encoded in the DB.
})
}
json.OK(w, r, result)
@ -1251,7 +1253,7 @@ func (h *handler) userInfoHandler(w http.ResponseWriter, r *http.Request) {
slog.String("user_agent", r.UserAgent()),
)
if err := checkOutputFormat(w, r); err != nil {
if err := checkOutputFormat(r); err != nil {
json.BadRequest(w, r, err)
return
}
@ -1276,7 +1278,7 @@ func (h *handler) streamItemIDsHandler(w http.ResponseWriter, r *http.Request) {
slog.Int64("user_id", userID),
)
if err := checkOutputFormat(w, r); err != nil {
if err := checkOutputFormat(r); err != nil {
json.BadRequest(w, r, err)
return
}
@ -1477,8 +1479,7 @@ func (h *handler) handleFeedStreamHandler(w http.ResponseWriter, r *http.Request
if len(rm.ExcludeTargets) > 0 {
for _, s := range rm.ExcludeTargets {
switch s.Type {
case ReadStream:
if s.Type == ReadStream {
builder.WithoutStatus(model.EntryStatusRead)
}
}

32
internal/http/request/context.go
View File

@ -37,14 +37,10 @@ const (
func WebAuthnSessionData(r *http.Request) *model.WebAuthnSession {
if v := r.Context().Value(WebAuthnDataContextKey); v != nil {
value, valid := v.(model.WebAuthnSession)
if !valid {
return nil
if value, valid := v.(model.WebAuthnSession); valid {
return &value
}
return &value
}
return nil
}
@ -151,39 +147,27 @@ func ClientIP(r *http.Request) string {
func getContextStringValue(r *http.Request, key ContextKey) string {
if v := r.Context().Value(key); v != nil {
value, valid := v.(string)
if !valid {
return ""
if value, valid := v.(string); valid {
return value
}
return value
}
return ""
}
func getContextBoolValue(r *http.Request, key ContextKey) bool {
if v := r.Context().Value(key); v != nil {
value, valid := v.(bool)
if !valid {
return false
if value, valid := v.(bool); valid {
return value
}
return value
}
return false
}
func getContextInt64Value(r *http.Request, key ContextKey) int64 {
if v := r.Context().Value(key); v != nil {
value, valid := v.(int64)
if !valid {
return 0
if value, valid := v.(int64); valid {
return value
}
return value
}
return 0
}

1
internal/http/response/builder.go
View File

@ -96,7 +96,6 @@ func (b *Builder) Write() {
}
func (b *Builder) writeHeaders() {
b.headers["X-XSS-Protection"] = "1; mode=block"
b.headers["X-Content-Type-Options"] = "nosniff"
b.headers["X-Frame-Options"] = "DENY"
b.headers["Referrer-Policy"] = "no-referrer"

1
internal/http/response/builder_test.go
View File

@ -28,7 +28,6 @@ func TestResponseHasCommonHeaders(t *testing.T) {
resp := w.Result()
headers := map[string]string{
"X-XSS-Protection": "1; mode=block",
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
}

2
internal/integration/matrixbot/matrixbot.go
View File

@ -10,7 +10,7 @@ import (
"miniflux.app/v2/internal/model"
)
// PushEntry pushes entries to matrix chat using integration settings provided
// PushEntries pushes entries to matrix chat using integration settings provided
func PushEntries(feed *model.Feed, entries model.Entries, matrixBaseURL, matrixUsername, matrixPassword, matrixRoomID string) error {
client := NewClient(matrixBaseURL)
discovery, err := client.DiscoverEndpoints()

2
internal/integration/rssbridge/rssbridge.go
View File

@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package rssbridge // import "miniflux.app/integration/rssbridge"
package rssbridge // import "miniflux.app/v2/internal/integration/rssbridge"
import (
"encoding/json"

12
internal/integration/shaarli/shaarli.go
View File

@ -11,7 +11,6 @@ import (
"encoding/json"
"fmt"
"net/http"
"strings"
"time"
"miniflux.app/v2/internal/urllib"
@ -74,14 +73,15 @@ func (c *Client) CreateLink(entryURL, entryTitle string) error {
}
func (c *Client) generateBearerToken() string {
header := strings.TrimRight(base64.URLEncoding.EncodeToString([]byte(`{"typ":"JWT", "alg":"HS256"}`)), "=")
payload := strings.TrimRight(base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"iat": %d}`, time.Now().Unix()))), "=")
header := base64.RawURLEncoding.EncodeToString([]byte(`{"typ":"JWT","alg":"HS512"}`))
payload := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf(`{"iat":%d}`, time.Now().Unix())))
data := header + "." + payload
mac := hmac.New(sha512.New, []byte(c.apiSecret))
mac.Write([]byte(header + "." + payload))
signature := strings.TrimRight(base64.URLEncoding.EncodeToString(mac.Sum(nil)), "=")
mac.Write([]byte(data))
signature := base64.RawURLEncoding.EncodeToString(mac.Sum(nil))
return header + "." + payload + "." + signature
return data + "." + signature
}
type addLinkRequest struct {

23
internal/integration/webhook/webhook.go
View File

@ -57,6 +57,7 @@ func (c *Client) SendSaveEntryWebhookEvent(entry *model.Entry) error {
ID: entry.Feed.ID,
UserID: entry.Feed.UserID,
CategoryID: entry.Feed.Category.ID,
Category: &WebhookCategory{ID: entry.Feed.Category.ID, Title: entry.Feed.Category.Title},
FeedURL: entry.Feed.FeedURL,
SiteURL: entry.Feed.SiteURL,
Title: entry.Feed.Title,
@ -94,13 +95,13 @@ func (c *Client) SendNewEntriesWebhookEvent(feed *model.Feed, entries model.Entr
Tags: entry.Tags,
})
}
return c.makeRequest(NewEntriesEventType, &WebhookNewEntriesEvent{
EventType: NewEntriesEventType,
Feed: &WebhookFeed{
ID: feed.ID,
UserID: feed.UserID,
CategoryID: feed.Category.ID,
Category: &WebhookCategory{ID: feed.Category.ID, Title: feed.Category.Title},
FeedURL: feed.FeedURL,
SiteURL: feed.SiteURL,
Title: feed.Title,
@ -145,13 +146,19 @@ func (c *Client) makeRequest(eventType string, payload any) error {
}
type WebhookFeed struct {
ID int64 `json:"id"`
UserID int64 `json:"user_id"`
CategoryID int64 `json:"category_id"`
FeedURL string `json:"feed_url"`
SiteURL string `json:"site_url"`
Title string `json:"title"`
CheckedAt time.Time `json:"checked_at"`
ID int64 `json:"id"`
UserID int64 `json:"user_id"`
CategoryID int64 `json:"category_id"`
Category *WebhookCategory `json:"category,omitempty"`
FeedURL string `json:"feed_url"`
SiteURL string `json:"site_url"`
Title string `json:"title"`
CheckedAt time.Time `json:"checked_at"`
}
type WebhookCategory struct {
ID int64 `json:"id"`
Title string `json:"title"`
}
type WebhookEntry struct {

9
internal/locale/translations/de_DE.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Zum Abonnement gehen",
"page.keyboard_shortcuts.go_to_previous_page": "Zur vorherigen Seite gehen",
"page.keyboard_shortcuts.go_to_next_page": "Zur nächsten Seite gehen",
"page.keyboard_shortcuts.go_to_bottom_item": "Gehen Sie zum untersten Element",
"page.keyboard_shortcuts.go_to_top_item": "Zum obersten Artikel gehen",
"page.keyboard_shortcuts.open_item": "Gewählten Artikel öffnen",
"page.keyboard_shortcuts.open_original": "Original-Artikel öffnen",
"page.keyboard_shortcuts.open_original_same_window": "Öffne den Original-Link in der aktuellen Registerkarte",
@ -256,6 +258,7 @@
"alert.no_bookmark": "Es existiert derzeit kein Lesezeichen.",
"alert.no_category": "Es ist keine Kategorie vorhanden.",
"alert.no_category_entry": "Es befindet sich kein Artikel in dieser Kategorie.",
"alert.no_tag_entry": "Es gibt keine Artikel, die diesem Tag entsprechen.",
"alert.no_feed_entry": "Es existiert kein Artikel für dieses Abonnement.",
"alert.no_feed": "Es sind keine Abonnements vorhanden.",
"alert.no_feed_in_category": "Für diese Kategorie gibt es kein Abonnement.",
@ -505,7 +508,7 @@
"error.http_body_read": "Der HTTP-Inhalt kann nicht gelesen werden: %v",
"error.http_empty_response_body": "Der Inhalt der HTTP-Antwort ist leer.",
"error.http_empty_response": "Die HTTP-Antwort ist leer. Vielleicht versucht die Webseite, sich vor Bots zu schützen?",
"error.tls_error": "TLS-Fehler: %v. Wenn Sie mögen, können Sie versuchen die TLS-Verifizierung in den Einstellungen des Abonnements zu deaktivieren.",
"error.tls_error": "TLS-Fehler: %q. Wenn Sie mögen, können Sie versuchen die TLS-Verifizierung in den Einstellungen des Abonnements zu deaktivieren.",
"error.network_operation": "Miniflux kann die Webseite aufgrund eines Netzwerk-Fehlers nicht erreichen: %v",
"error.network_timeout": "Die Webseite ist zu langsam und die Anfrage ist abgelaufen: %v.",
"error.http_client_error": "HTTP-Client-Fehler: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Dieses Abonnement kann nicht gelesen werden: %v.",
"error.feed_not_found": "Dieses Abonnement existiert nicht oder gehört nicht zu diesem Benutzer.",
"error.unable_to_detect_rssbridge": "Abonnement kann nicht durch RSS-Bridge erkannt werden: %v.",
"error.feed_format_not_detected": "Das Format des Abonnements kann nicht erkannt werden: %v."
"error.feed_format_not_detected": "Das Format des Abonnements kann nicht erkannt werden: %v.",
"form.prefs.label.media_playback_rate": "Wiedergabegeschwindigkeit von Audio/Video",
"error.settings_media_playback_rate_range": "Die Wiedergabegeschwindigkeit liegt außerhalb des Bereichs"
}

9
internal/locale/translations/el_EL.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Πηγαίνετε στη ροή",
"page.keyboard_shortcuts.go_to_previous_page": "Μετάβαση στην προηγούμενη σελίδα",
"page.keyboard_shortcuts.go_to_next_page": "Μετάβαση στην επόμενη σελίδα",
"page.keyboard_shortcuts.go_to_bottom_item": "Μετάβαση στο κάτω στοιχείο",
"page.keyboard_shortcuts.go_to_top_item": "Μετάβαση στο επάνω στοιχείο",
"page.keyboard_shortcuts.open_item": "Άνοιγμα επιλεγμένου στοιχείου",
"page.keyboard_shortcuts.open_original": "Άνοιγμα αρχικού συνδέσμου",
"page.keyboard_shortcuts.open_original_same_window": "Άνοιγμα αρχικού συνδέσμου στην τρέχουσα καρτέλα",
@ -256,6 +258,7 @@
"alert.no_bookmark": "Δεν υπάρχει σελιδοδείκτης αυτή τη στιγμή.",
"alert.no_category": "Δεν υπάρχει κατηγορία.",
"alert.no_category_entry": "Δεν υπάρχουν άρθρα σε αυτήν την κατηγορία.",
"alert.no_tag_entry": "Δεν υπάρχουν αντικείμενα που να ταιριάζουν με αυτή την ετικέτα.",
"alert.no_feed_entry": "Δεν υπάρχουν άρθρα για αυτήν τη ροή.",
"alert.no_feed": "Δεν έχετε συνδρομές.",
"alert.no_feed_in_category": "Δεν υπάρχει συνδρομή για αυτήν την κατηγορία.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Ταχύτητα αναπαραγωγής του ήχου/βίντεο",
"error.settings_media_playback_rate_range": "Η ταχύτητα αναπαραγωγής είναι εκτός εύρους"
}

11
internal/locale/translations/en_US.json
View File

@ -176,6 +176,8 @@
"page.keyboard_shortcuts.go_to_previous_item": "Go to previous item",
"page.keyboard_shortcuts.go_to_next_item": "Go to next item",
"page.keyboard_shortcuts.go_to_feed": "Go to feed",
"page.keyboard_shortcuts.go_to_top_item": "Go to top item",
"page.keyboard_shortcuts.go_to_bottom_item": "Go to bottom item",
"page.keyboard_shortcuts.go_to_previous_page": "Go to previous page",
"page.keyboard_shortcuts.go_to_next_page": "Go to next page",
"page.keyboard_shortcuts.open_item": "Open selected item",
@ -215,7 +217,7 @@
"page.settings.webauthn.last_seen_on": "Last Used",
"page.settings.webauthn.register": "Register passkey",
"page.settings.webauthn.register.error": "Unable to register passkey",
"page.settings.webauthn.delete" : [
"page.settings.webauthn.delete": [
"Remove %d passkey",
"Remove %d passkeys"
],
@ -256,6 +258,7 @@
"alert.no_bookmark": "There are no starred entries.",
"alert.no_category": "There is no category.",
"alert.no_category_entry": "There are no entries in this category.",
"alert.no_tag_entry": "There are no entries matching this tag.",
"alert.no_feed_entry": "There are no entries for this feed.",
"alert.no_feed": "You don’t have any feeds.",
"alert.no_feed_in_category": "There is no feed for this category.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Playback speed of the audio/video",
"error.settings_media_playback_rate_range": "Playback speed is out of range"
}

9
internal/locale/translations/es_ES.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Ir a la fuente",
"page.keyboard_shortcuts.go_to_previous_page": "Ir al página anterior",
"page.keyboard_shortcuts.go_to_next_page": "Ir al página siguiente",
"page.keyboard_shortcuts.go_to_bottom_item": "Ir al elemento inferior",
"page.keyboard_shortcuts.go_to_top_item": "Ir al elemento superior",
"page.keyboard_shortcuts.open_item": "Abrir el elemento seleccionado",
"page.keyboard_shortcuts.open_original": "Abrir el enlace original",
"page.keyboard_shortcuts.open_original_same_window": "Abrir enlace original en la pestaña actual",
@ -256,6 +258,7 @@
"alert.no_bookmark": "No hay marcador en este momento.",
"alert.no_category": "No hay categoría.",
"alert.no_category_entry": "No hay artículos en esta categoría.",
"alert.no_tag_entry": "No hay artículos con esta etiqueta.",
"alert.no_feed_entry": "No hay artículos para esta fuente.",
"alert.no_feed": "No tienes fuentes.",
"alert.no_feed_in_category": "No hay fuentes para esta categoría.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Velocidad de reproducción del audio/vídeo",
"error.settings_media_playback_rate_range": "La velocidad de reproducción está fuera de rango"
}

9
internal/locale/translations/fi_FI.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Siirry syötteeseen",
"page.keyboard_shortcuts.go_to_previous_page": "Siirry edelliselle sivulle",
"page.keyboard_shortcuts.go_to_next_page": "Siirry seuraavalle sivulle",
"page.keyboard_shortcuts.go_to_bottom_item": "Siirry alimpaan kohtaan",
"page.keyboard_shortcuts.go_to_top_item": "Siirry alkuun",
"page.keyboard_shortcuts.open_item": "Avaa valittu kohde",
"page.keyboard_shortcuts.open_original": "Avaa alkuperäinen linkki",
"page.keyboard_shortcuts.open_original_same_window": "Avaa alkuperäinen linkki nykyisessä välilehdessä",
@ -256,6 +258,7 @@
"alert.no_bookmark": "Tällä hetkellä ei ole kirjanmerkkiä.",
"alert.no_category": "Ei ole kategoriaa.",
"alert.no_category_entry": "Tässä kategoriassa ei ole artikkeleita.",
"alert.no_tag_entry": "Tätä tunnistetta vastaavia merkintöjä ei ole.",
"alert.no_feed_entry": "Tässä syötteessä ei ole artikkeleita.",
"alert.no_feed": "Sinulla ei ole tilauksia.",
"alert.no_feed_in_category": "Tälle kategorialle ei ole tilausta.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Äänen/videon toistonopeus",
"error.settings_media_playback_rate_range": "Toistonopeus on alueen ulkopuolella"
}

11
internal/locale/translations/fr_FR.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Voir abonnement",
"page.keyboard_shortcuts.go_to_previous_page": "Page précédente",
"page.keyboard_shortcuts.go_to_next_page": "Page suivante",
"page.keyboard_shortcuts.go_to_bottom_item": "Aller à l'élément du bas",
"page.keyboard_shortcuts.go_to_top_item": "Aller à l'élément supérieur",
"page.keyboard_shortcuts.open_item": "Ouvrir élément sélectionné",
"page.keyboard_shortcuts.open_original": "Ouvrir le lien original",
"page.keyboard_shortcuts.open_original_same_window": "Ouvrir le lien original dans l'onglet en cours",
@ -215,7 +217,7 @@
"page.settings.webauthn.last_seen_on": "Dernière utilisation",
"page.settings.webauthn.register": "Enregister une nouvelle clé d’accès",
"page.settings.webauthn.register.error": "Impossible d'enregistrer la clé d’accès",
"page.settings.webauthn.delete" : [
"page.settings.webauthn.delete": [
"Supprimer %d clé d’accès",
"Supprimer %d clés d’accès"
],
@ -256,6 +258,7 @@
"alert.no_bookmark": "Il n'y a aucun favoris pour le moment.",
"alert.no_category": "Il n'y a aucune catégorie.",
"alert.no_category_entry": "Il n'y a aucun article dans cette catégorie.",
"alert.no_tag_entry": "Il n'y a aucun article correspondant à ce tag.",
"alert.no_feed_entry": "Il n'y a aucun article pour cet abonnement.",
"alert.no_feed": "Vous n'avez aucun abonnement.",
"alert.no_feed_in_category": "Il n'y a pas d'abonnement pour cette catégorie.",
@ -505,7 +508,7 @@
"error.http_body_read": "Impossible de lire le corps de la réponse HTTP : %v.",
"error.http_empty_response_body": "Le corps de la réponse HTTP est vide.",
"error.http_empty_response": "La réponse HTTP est vide. Peut-être que ce site web bloque Miniflux avec une protection anti-bot ?",
"error.tls_error": "Erreur TLS : %v. Vous pouvez désactiver la vérification TLS dans les paramètres de l'abonnement.",
"error.tls_error": "Erreur TLS : %q. Vous pouvez désactiver la vérification TLS dans les paramètres de l'abonnement.",
"error.network_operation": "Miniflux n'est pas en mesure de se connecter à ce site web à cause d'un problème réseau : %v.",
"error.network_timeout": "Ce site web est trop lent à répondre : %v.",
"error.http_client_error": "Erreur du client HTTP : %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Impossible d'analyser ce flux : %v.",
"error.feed_not_found": "Impossible de trouver ce flux.",
"error.unable_to_detect_rssbridge": "Impossible de détecter un flux RSS en utilisant RSS-Bridge: %v.",
"error.feed_format_not_detected": "Impossible de détecter le format du flux : %v."
"error.feed_format_not_detected": "Impossible de détecter le format du flux : %v.",
"form.prefs.label.media_playback_rate": "Vitesse de lecture de l'audio/vidéo",
"error.settings_media_playback_rate_range": "La vitesse de lecture est hors limites"
}

9
internal/locale/translations/hi_IN.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "फ़ीड पर जाएं",
"page.keyboard_shortcuts.go_to_previous_page": "पिछले पृष्ठ पर जाएं",
"page.keyboard_shortcuts.go_to_next_page": "अगले पेज पर जाएं",
"page.keyboard_shortcuts.go_to_bottom_item": "निचले आइटम पर जाएँ",
"page.keyboard_shortcuts.go_to_top_item": "शीर्ष आइटम पर जाएँ",
"page.keyboard_shortcuts.open_item": "चयनित आइटम खोलें",
"page.keyboard_shortcuts.open_original": "मूल लिंक खोलें",
"page.keyboard_shortcuts.open_original_same_window": "वर्तमान टैब में मूल लिंक खोलें",
@ -256,6 +258,7 @@
"alert.no_bookmark": "इस समय कोई बुकमार्क नहीं है",
"alert.no_category": "कोई श्रेणी नहीं है।",
"alert.no_category_entry": "इस श्रेणी में कोई विषय-वस्तु नहीं है।",
"alert.no_tag_entry": "इस टैग से मेल खाती कोई प्रविष्टियाँ नहीं हैं।",
"alert.no_feed_entry": "इस फ़ीड के लिए कोई विषय-वस्तु नहीं है।",
"alert.no_feed": "आपके पास कोई सदस्यता नहीं है।",
"alert.no_feed_in_category": "इस श्रेणी के लिए कोई सदस्यता नहीं है।",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "ऑडियो/वीडियो की प्लेबैक गति",
"error.settings_media_playback_rate_range": "प्लेबैक गति सीमा से बाहर है"
}

9
internal/locale/translations/id_ID.json
View File

@ -169,6 +169,8 @@
"page.keyboard_shortcuts.go_to_feed": "Ke umpan",
"page.keyboard_shortcuts.go_to_previous_page": "Ke halaman sebelumnya",
"page.keyboard_shortcuts.go_to_next_page": "Ke halaman berikutnya",
"page.keyboard_shortcuts.go_to_bottom_item": "Pergi ke item paling bawah",
"page.keyboard_shortcuts.go_to_top_item": "Pergi ke item teratas",
"page.keyboard_shortcuts.open_item": "Buka entri yang dipilih",
"page.keyboard_shortcuts.open_original": "Buka tautan asli",
"page.keyboard_shortcuts.open_original_same_window": "Buka tautan asli di bilah saat ini",
@ -246,6 +248,7 @@
"alert.no_bookmark": "Tidak ada markah.",
"alert.no_category": "Tidak ada kategori.",
"alert.no_category_entry": "Tidak ada artikel di kategori ini.",
"alert.no_tag_entry": "Tidak ada entri yang cocok dengan tag ini.",
"alert.no_feed_entry": "Tidak ada artikel di umpan ini.",
"alert.no_feed": "Anda tidak memiliki langganan.",
"alert.no_feed_in_category": "Tidak ada langganan untuk kategori ini.",
@ -488,7 +491,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -507,5 +510,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Kecepatan pemutaran audio/video",
"error.settings_media_playback_rate_range": "Kecepatan pemutaran di luar jangkauan"
}

9
internal/locale/translations/it_IT.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Mostra il feed",
"page.keyboard_shortcuts.go_to_previous_page": "Mostra la pagina precedente",
"page.keyboard_shortcuts.go_to_next_page": "Mostra la pagina successiva",
"page.keyboard_shortcuts.go_to_bottom_item": "Vai all'elemento in fondo",
"page.keyboard_shortcuts.go_to_top_item": "Vai all'elemento principale",
"page.keyboard_shortcuts.open_item": "Apri l'articolo selezionato",
"page.keyboard_shortcuts.open_original": "Apri la pagina web originale",
"page.keyboard_shortcuts.open_original_same_window": "Apri il link originale nella scheda corrente",
@ -256,6 +258,7 @@
"alert.no_bookmark": "Nessun preferito disponibile.",
"alert.no_category": "Nessuna categoria disponibile.",
"alert.no_category_entry": "Questa categoria non contiene alcun articolo.",
"alert.no_tag_entry": "Non ci sono voci corrispondenti a questo tag.",
"alert.no_feed_entry": "Questo feed non contiene alcun articolo.",
"alert.no_feed": "Nessun feed disponibile.",
"alert.no_feed_in_category": "Non esiste un abbonamento per questa categoria.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Velocità di riproduzione dell'audio/video",
"error.settings_media_playback_rate_range": "La velocità di riproduzione non rientra nell'intervallo"
}

9
internal/locale/translations/ja_JP.json
View File

@ -169,6 +169,8 @@
"page.keyboard_shortcuts.go_to_feed": "フィード",
"page.keyboard_shortcuts.go_to_previous_page": "前のページ",
"page.keyboard_shortcuts.go_to_next_page": "次のページ",
"page.keyboard_shortcuts.go_to_bottom_item": "一番下の項目に移動",
"page.keyboard_shortcuts.go_to_top_item": "先頭の項目に移動",
"page.keyboard_shortcuts.open_item": "選択されたアイテムを開く",
"page.keyboard_shortcuts.open_original": "オリジナルのリンクを開く",
"page.keyboard_shortcuts.open_original_same_window": "現在のタブでオリジナルのリンクを開く",
@ -246,6 +248,7 @@
"alert.no_bookmark": "現在星付きはありません。",
"alert.no_category": "カテゴリが存在しません。",
"alert.no_category_entry": "このカテゴリには記事がありません。",
"alert.no_tag_entry": "このタグに一致するエントリーはありません。",
"alert.no_feed_entry": "このフィードには記事がありません。",
"alert.no_feed": "何も購読していません。",
"alert.no_feed_in_category": "このカテゴリには購読中のフィードがありません。",
@ -488,7 +491,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -507,5 +510,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "オーディオ/ビデオの再生速度",
"error.settings_media_playback_rate_range": "再生速度が範囲外"
}

9
internal/locale/translations/nl_NL.json
View File

@ -179,6 +179,8 @@
"page.keyboard_shortcuts.go_to_feed": "Ga naar feed",
"page.keyboard_shortcuts.go_to_previous_page": "Vorige pagina",
"page.keyboard_shortcuts.go_to_next_page": "Volgende pagina",
"page.keyboard_shortcuts.go_to_bottom_item": "Ga naar het onderste item",
"page.keyboard_shortcuts.go_to_top_item": "Ga naar het bovenste item",
"page.keyboard_shortcuts.open_item": "Open geselecteerde link",
"page.keyboard_shortcuts.open_original": "Open originele link",
"page.keyboard_shortcuts.open_original_same_window": "Oorspronkelijke koppeling op huidig tabblad openen",
@ -256,6 +258,7 @@
"alert.no_bookmark": "Er zijn op dit moment geen favorieten.",
"alert.no_category": "Er zijn geen categorieën.",
"alert.no_category_entry": "Deze categorie bevat geen feeds.",
"alert.no_tag_entry": "Er zijn geen items die overeenkomen met deze tag.",
"alert.no_feed_entry": "Er zijn geen artikelen in deze feed.",
"alert.no_feed": "Je hebt nog geen feeds geabboneerd staan.",
"alert.no_feed_in_category": "Er is geen abonnement voor deze categorie.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Afspeelsnelheid van de audio/video",
"error.settings_media_playback_rate_range": "Afspeelsnelheid is buiten bereik"
}

9
internal/locale/translations/pl_PL.json
View File

@ -187,6 +187,8 @@
"page.keyboard_shortcuts.go_to_feed": "Przejdź do subskrypcji",
"page.keyboard_shortcuts.go_to_previous_page": "Przejdź do poprzedniej strony",
"page.keyboard_shortcuts.go_to_next_page": "Przejdź do następnej strony",
"page.keyboard_shortcuts.go_to_bottom_item": "Przejdź do dolnego elementu",
"page.keyboard_shortcuts.go_to_top_item": "Przejdź do najwyższego elementu",
"page.keyboard_shortcuts.open_item": "Otwórz zaznaczony artykuł",
"page.keyboard_shortcuts.open_original": "Otwórz oryginalny artykuł",
"page.keyboard_shortcuts.open_original_same_window": "Otwórz oryginalny link w bieżącej karcie",
@ -266,6 +268,7 @@
"alert.no_bookmark": "Obecnie nie ma żadnych zakładek.",
"alert.no_category": "Nie ma żadnej kategorii!",
"alert.no_category_entry": "W tej kategorii nie ma żadnych artykułów",
"alert.no_tag_entry": "Nie ma wpisów pasujących do tego tagu.",
"alert.no_feed_entry": "Nie ma artykułu dla tego kanału.",
"alert.no_feed": "Nie masz żadnej subskrypcji.",
"alert.no_feed_in_category": "Nie ma subskrypcji dla tej kategorii.",
@ -522,7 +525,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -541,5 +544,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Prędkość odtwarzania audio/wideo",
"error.settings_media_playback_rate_range": "Prędkość odtwarzania jest poza zakresem"
}

9
internal/locale/translations/pt_BR.json
View File

@ -178,6 +178,8 @@
"page.keyboard_shortcuts.go_to_feed": "Ir a fonte",
"page.keyboard_shortcuts.go_to_previous_page": "Ir a página anterior",
"page.keyboard_shortcuts.go_to_next_page": "Ir a página seguinte",
"page.keyboard_shortcuts.go_to_bottom_item": "Ir para o item inferior",
"page.keyboard_shortcuts.go_to_top_item": "Ir para o item superior",
"page.keyboard_shortcuts.open_item": "Abrir o item selecionado",
"page.keyboard_shortcuts.open_original": "Abrir o conteúdo original",
"page.keyboard_shortcuts.open_original_same_window": "Abrir o conteúdo original na janela atual",
@ -256,6 +258,7 @@
"alert.no_bookmark": "Não há favorito neste momento.",
"alert.no_category": "Não há categoria.",
"alert.no_category_entry": "Não há itens nesta categoria.",
"alert.no_tag_entry": "Não há itens que correspondam a esta etiqueta.",
"alert.no_feed_entry": "Não há itens nessa fonte.",
"alert.no_feed": "Não há inscrições.",
"alert.no_feed_in_category": "Não há inscrições nessa categoria.",
@ -505,7 +508,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -524,5 +527,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Velocidade de reprodução do áudio/vídeo",
"error.settings_media_playback_rate_range": "A velocidade de reprodução está fora do intervalo"
}

9
internal/locale/translations/ru_RU.json
View File

@ -187,6 +187,8 @@
"page.keyboard_shortcuts.go_to_feed": "Перейти к подписке",
"page.keyboard_shortcuts.go_to_previous_page": "Перейти к предыдущей странице",
"page.keyboard_shortcuts.go_to_next_page": "Перейти к следующей странице",
"page.keyboard_shortcuts.go_to_bottom_item": "Перейти к нижнему элементу",
"page.keyboard_shortcuts.go_to_top_item": "Перейти к верхнему элементу",
"page.keyboard_shortcuts.open_item": "Открыть выбранный элемент",
"page.keyboard_shortcuts.open_original_same_window": "Открыть оригинальную ссылку в текущей вкладке",
"page.keyboard_shortcuts.open_original": "Открыть оригинальную ссылку",
@ -266,6 +268,7 @@
"alert.no_bookmark": "Избранное отсутствует.",
"alert.no_category": "Категории отсутствуют.",
"alert.no_category_entry": "В этой категории нет статей.",
"alert.no_tag_entry": "Нет записей, соответствующих этому тегу.",
"alert.no_feed_entry": "В этой подписке отсутствуют статьи.",
"alert.no_feed": "У вас нет ни одной подписки.",
"alert.no_feed_in_category": "Для этой категории нет подписки.",
@ -522,7 +525,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -541,5 +544,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Скорость воспроизведения аудио/видео",
"error.settings_media_playback_rate_range": "Скорость воспроизведения выходит за пределы диапазона"
}

1024
internal/locale/translations/tr_TR.json
View File

File diff suppressed because it is too large Load Diff

9
internal/locale/translations/uk_UA.json
View File

@ -187,6 +187,8 @@
"page.keyboard_shortcuts.go_to_feed": "Перейти до стрічки",
"page.keyboard_shortcuts.go_to_previous_page": "Перейти до попередньої сторінки",
"page.keyboard_shortcuts.go_to_next_page": "Перейти до наступної сторінки",
"page.keyboard_shortcuts.go_to_bottom_item": "Перейти до нижнього пункту",
"page.keyboard_shortcuts.go_to_top_item": "Перейти до верхнього пункту",
"page.keyboard_shortcuts.open_item": "Відкрити виділений запис",
"page.keyboard_shortcuts.open_original": "Відкрити оригінальне посилання",
"page.keyboard_shortcuts.open_original_same_window": "Відкрити оригінальне посилання в поточній вкладці",
@ -266,6 +268,7 @@
"alert.no_bookmark": "Наразі закладки відсутні.",
"alert.no_category": "Немає категорії.",
"alert.no_category_entry": "У цій категорії немає записів.",
"alert.no_tag_entry": "Немає записів, що відповідають цьому тегу.",
"alert.no_feed_entry": "У цій стрічці немає записів.",
"alert.no_feed": "У вас немає підписок.",
"alert.no_feed_in_category": "У цій категорії немає підписок.",
@ -522,7 +525,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -541,5 +544,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "Швидкість відтворення аудіо/відео",
"error.settings_media_playback_rate_range": "Швидкість відтворення виходить за межі діапазону"
}

15
internal/locale/translations/zh_CN.json
View File

@ -169,6 +169,8 @@
"page.keyboard_shortcuts.go_to_feed": "转到源页面",
"page.keyboard_shortcuts.go_to_previous_page": "上一页",
"page.keyboard_shortcuts.go_to_next_page": "下一页",
"page.keyboard_shortcuts.go_to_bottom_item": "转到底部项目",
"page.keyboard_shortcuts.go_to_top_item": "转到顶部项目",
"page.keyboard_shortcuts.open_item": "打开选定的文章",
"page.keyboard_shortcuts.open_original": "打开原始链接",
"page.keyboard_shortcuts.open_original_same_window": "在当前标签页中打开原始链接",
@ -246,6 +248,7 @@
"alert.no_bookmark": "目前没有收藏",
"alert.no_category": "目前没有分类",
"alert.no_category_entry": "该分类下没有文章",
"alert.no_tag_entry": "没有与此标签匹配的条目。",
"alert.no_feed_entry": "该源中没有文章",
"alert.no_feed": "目前没有源",
"alert.no_history": "目前没有历史",
@ -406,9 +409,9 @@
"form.integration.omnivore_activate": "保存文章到 Omnivore",
"form.integration.omnivore_url": "Omnivore API 端点",
"form.integration.omnivore_api_key": "Omnivore API 密钥",
"form.integration.espial_activate": "保存文章到 Espial",
"form.integration.espial_endpoint": "Espial API 端点",
"form.integration.espial_api_key": "Espial API 密钥",
"form.integration.espial_activate": "保存文章到 Espial",
"form.integration.espial_endpoint": "Espial API 端点",
"form.integration.espial_api_key": "Espial API 密钥",
"form.integration.espial_tags": "Espial 标签",
"form.integration.readwise_activate": "保存文章到 Readwise Reader",
"form.integration.readwise_api_key": "Readwise Reader Access Token",
@ -488,7 +491,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -507,5 +510,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "音频/视频的播放速度",
"error.settings_media_playback_rate_range": "播放速度超出范围"
}

9
internal/locale/translations/zh_TW.json
View File

@ -169,6 +169,8 @@
"page.keyboard_shortcuts.go_to_feed": "轉到Feed頁面",
"page.keyboard_shortcuts.go_to_previous_page": "上一頁",
"page.keyboard_shortcuts.go_to_next_page": "下一頁",
"page.keyboard_shortcuts.go_to_bottom_item": "转到底部项目",
"page.keyboard_shortcuts.go_to_top_item": "转到顶部项目",
"page.keyboard_shortcuts.open_item": "開啟選定的文章",
"page.keyboard_shortcuts.open_original": "開啟原始連結",
"page.keyboard_shortcuts.open_original_same_window": "在當前標籤頁中開啟原始連結",
@ -246,6 +248,7 @@
"alert.no_bookmark": "目前沒有收藏",
"alert.no_category": "目前沒有分類",
"alert.no_category_entry": "該分類下沒有文章",
"alert.no_tag_entry": "沒有與此標籤相符的條目。",
"alert.no_feed_entry": "該Feed中沒有文章",
"alert.no_feed": "目前沒有Feed",
"alert.no_history": "目前沒有歷史",
@ -488,7 +491,7 @@
"error.http_body_read": "Unable to read the HTTP body: %v.",
"error.http_empty_response_body": "The HTTP response body is empty.",
"error.http_empty_response": "The HTTP response is empty. Perhaps, this website is using a bot protection mechanism?",
"error.tls_error": "TLS error: %v. You could disable TLS verification in the feed settings if you would like.",
"error.tls_error": "TLS error: %q. You could disable TLS verification in the feed settings if you would like.",
"error.network_operation": "Miniflux is not able to reach this website due to a network error: %v.",
"error.network_timeout": "This website is too slow and the request timed out: %v",
"error.http_client_error": "HTTP client error: %v.",
@ -507,5 +510,7 @@
"error.unable_to_parse_feed": "Unable to parse this feed: %v.",
"error.feed_not_found": "This feed does not exist or does not belong to this user.",
"error.unable_to_detect_rssbridge": "Unable to detect feed using RSS-Bridge: %v.",
"error.feed_format_not_detected": "Unable to detect feed format: %v."
"error.feed_format_not_detected": "Unable to detect feed format: %v.",
"form.prefs.label.media_playback_rate": "音訊/視訊的播放速度",
"error.settings_media_playback_rate_range": "播放速度超出範圍"
}

180
internal/proxy/media_proxy_test.go → internal/mediaproxy/media_proxy_test.go
View File

@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package proxy // import "miniflux.app/v2/internal/proxy"
package mediaproxy // import "miniflux.app/v2/internal/mediaproxy"
import (
"net/http"
@ -29,11 +29,11 @@ func TestProxyFilterWithHttpDefault(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -53,11 +53,11 @@ func TestProxyFilterWithHttpsDefault(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -76,11 +76,11 @@ func TestProxyFilterWithHttpNever(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := input
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -99,11 +99,11 @@ func TestProxyFilterWithHttpsNever(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := input
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -124,11 +124,11 @@ func TestProxyFilterWithHttpAlways(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -149,11 +149,87 @@ func TestProxyFilterWithHttpsAlways(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="/proxy/LdPNR1GBDigeeNp2ArUQRyZsVqT_PWLfHGjYFrrWWIY=/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
func TestAbsoluteProxyFilterWithHttpsAlways(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input)
expected := `<p><img src="http://localhost/proxy/LdPNR1GBDigeeNp2ArUQRyZsVqT_PWLfHGjYFrrWWIY=/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
func TestAbsoluteProxyFilterWithHttpsScheme(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_PRIVATE_KEY", "test")
os.Setenv("HTTPS", "1")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input)
expected := `<p><img src="https://localhost/proxy/LdPNR1GBDigeeNp2ArUQRyZsVqT_PWLfHGjYFrrWWIY=/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
func TestAbsoluteProxyFilterWithHttpsAlwaysAndAudioTag(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "audio")
os.Setenv("PROXY_PRIVATE_KEY", "test")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<audio src="https://website/folder/audio.mp3"></audio>`
output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input)
expected := `<audio src="http://localhost/proxy/EmBTvmU5B17wGuONkeknkptYopW_Tl6Y6_W8oYbN_Xs=/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9hdWRpby5tcDM="></audio>`
if expected != output {
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -174,11 +250,61 @@ func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="https://proxy-example/proxy/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
func TestProxyFilterWithHttpsAlwaysAndIncorrectCustomProxyServer(t *testing.T) {
os.Clearenv()
os.Setenv("PROXY_OPTION", "all")
os.Setenv("PROXY_MEDIA_TYPES", "image")
os.Setenv("PROXY_URL", "http://:8080example.com")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
func TestAbsoluteProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) {
os.Clearenv()
os.Setenv("MEDIA_PROXY_MODE", "all")
os.Setenv("MEDIA_PROXY_RESOURCE_TYPES", "image")
os.Setenv("MEDIA_PROXY_CUSTOM_URL", "https://proxy-example/proxy")
var err error
parser := config.NewParser()
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
r := mux.NewRouter()
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := RewriteDocumentWithAbsoluteProxyURL(r, "localhost", input)
expected := `<p><img src="https://proxy-example/proxy/aHR0cHM6Ly93ZWJzaXRlL2ZvbGRlci9pbWFnZS5wbmc=" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -198,11 +324,11 @@ func TestProxyFilterWithHttpInvalid(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="http://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -222,11 +348,11 @@ func TestProxyFilterWithHttpsInvalid(t *testing.T) {
r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy")
input := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
expected := `<p><img src="https://website/folder/image.png" alt="Test"/></p>`
if expected != output {
t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected)
t.Errorf(`Not expected output: got %q instead of %q`, output, expected)
}
}
@ -248,7 +374,7 @@ func TestProxyFilterWithSrcset(t *testing.T) {
input := `<p><img src="http://website/folder/image.png" srcset="http://website/folder/image2.png 656w, http://website/folder/image3.png 360w" alt="test"></p>`
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/QgAmrJWiAud_nNAsz3F8OTxaIofwAiO36EDzH_YfMzo=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w" alt="test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -273,7 +399,7 @@ func TestProxyFilterWithEmptySrcset(t *testing.T) {
input := `<p><img src="http://website/folder/image.png" srcset="" alt="test"></p>`
expected := `<p><img src="/proxy/okK5PsdNY8F082UMQEAbLPeUFfbe2WnNfInNmR9T4WA=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlLnBuZw==" srcset="" alt="test"/></p>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -298,7 +424,7 @@ func TestProxyFilterWithPictureSource(t *testing.T) {
input := `<picture><source srcset="http://website/folder/image2.png 656w, http://website/folder/image3.png 360w, https://website/some,image.png 2x"></picture>`
expected := `<picture><source srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, /proxy/QgAmrJWiAud_nNAsz3F8OTxaIofwAiO36EDzH_YfMzo=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMy5wbmc= 360w, /proxy/ZIw0hv8WhSTls5aSqhnFaCXlUrKIqTnBRaY0-NaLnds=/aHR0cHM6Ly93ZWJzaXRlL3NvbWUsaW1hZ2UucG5n 2x"/></picture>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -323,7 +449,7 @@ func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) {
input := `<picture><source srcset="http://website/folder/image2.png 656w, https://website/some,image.png 2x"></picture>`
expected := `<picture><source srcset="/proxy/aY5Hb4urDnUCly2vTJ7ExQeeaVS-52O7kjUr2v9VrAs=/aHR0cDovL3dlYnNpdGUvZm9sZGVyL2ltYWdlMi5wbmc= 656w, https://website/some,image.png 2x"/></picture>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -347,7 +473,7 @@ func TestProxyWithImageDataURL(t *testing.T) {
input := `<img src="data:image/gif;base64,test">`
expected := `<img src="data:image/gif;base64,test"/>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -371,7 +497,7 @@ func TestProxyWithImageSourceDataURL(t *testing.T) {
input := `<picture><source srcset="data:image/gif;base64,test"/></picture>`
expected := `<picture><source srcset="data:image/gif;base64,test"/></picture>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -396,7 +522,7 @@ func TestProxyFilterWithVideo(t *testing.T) {
input := `<video poster="https://example.com/img.png" src="https://example.com/video.mp4"></video>`
expected := `<video poster="/proxy/aDFfroYL57q5XsojIzATT6OYUCkuVSPXYJQAVrotnLw=/aHR0cHM6Ly9leGFtcGxlLmNvbS9pbWcucG5n" src="/proxy/0y3LR8zlx8S8qJkj1qWFOO6x3a-5yf2gLWjGIJV5yyc=/aHR0cHM6Ly9leGFtcGxlLmNvbS92aWRlby5tcDQ="></video>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)
@ -421,7 +547,7 @@ func TestProxyFilterVideoPoster(t *testing.T) {
input := `<video poster="https://example.com/img.png" src="https://example.com/video.mp4"></video>`
expected := `<video poster="/proxy/aDFfroYL57q5XsojIzATT6OYUCkuVSPXYJQAVrotnLw=/aHR0cHM6Ly9leGFtcGxlLmNvbS9pbWcucG5n" src="https://example.com/video.mp4"></video>`
output := ProxyRewriter(r, input)
output := RewriteDocumentWithRelativeProxyURL(r, input)
if expected != output {
t.Errorf(`Not expected output: got %s`, output)

28
internal/proxy/media_proxy.go → internal/mediaproxy/rewriter.go
View File

@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package proxy // import "miniflux.app/v2/internal/proxy"
package mediaproxy // import "miniflux.app/v2/internal/mediaproxy"
import (
"strings"
@ -16,31 +16,29 @@ import (
type urlProxyRewriter func(router *mux.Router, url string) string
// ProxyRewriter replaces media URLs with internal proxy URLs.
func ProxyRewriter(router *mux.Router, data string) string {
return genericProxyRewriter(router, ProxifyURL, data)
func RewriteDocumentWithRelativeProxyURL(router *mux.Router, htmlDocument string) string {
return genericProxyRewriter(router, ProxifyRelativeURL, htmlDocument)
}
// AbsoluteProxyRewriter do the same as ProxyRewriter except it uses absolute URLs.
func AbsoluteProxyRewriter(router *mux.Router, host, data string) string {
func RewriteDocumentWithAbsoluteProxyURL(router *mux.Router, host, htmlDocument string) string {
proxifyFunction := func(router *mux.Router, url string) string {
return AbsoluteProxifyURL(router, host, url)
return ProxifyAbsoluteURL(router, host, url)
}
return genericProxyRewriter(router, proxifyFunction, data)
return genericProxyRewriter(router, proxifyFunction, htmlDocument)
}
func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string {
proxyOption := config.Opts.ProxyOption()
func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, htmlDocument string) string {
proxyOption := config.Opts.MediaProxyMode()
if proxyOption == "none" {
return data
return htmlDocument
}
doc, err := goquery.NewDocumentFromReader(strings.NewReader(data))
doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlDocument))
if err != nil {
return data
return htmlDocument
}
for _, mediaType := range config.Opts.ProxyMediaTypes() {
for _, mediaType := range config.Opts.MediaProxyResourceTypes() {
switch mediaType {
case "image":
doc.Find("img, picture source").Each(func(i int, img *goquery.Selection) {
@ -91,7 +89,7 @@ func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter,
output, err := doc.Find("body").First().Html()
if err != nil {
return data
return htmlDocument
}
return output

70
internal/mediaproxy/url.go Normal file
View File

@ -0,0 +1,70 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package mediaproxy // import "miniflux.app/v2/internal/mediaproxy"
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"log/slog"
"net/url"
"path"
"miniflux.app/v2/internal/http/route"
"github.com/gorilla/mux"
"miniflux.app/v2/internal/config"
)
func ProxifyRelativeURL(router *mux.Router, mediaURL string) string {
if mediaURL == "" {
return ""
}
if customProxyURL := config.Opts.MediaCustomProxyURL(); customProxyURL != "" {
return proxifyURLWithCustomProxy(mediaURL, customProxyURL)
}
mac := hmac.New(sha256.New, config.Opts.MediaProxyPrivateKey())
mac.Write([]byte(mediaURL))
digest := mac.Sum(nil)
return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(mediaURL)))
}
func ProxifyAbsoluteURL(router *mux.Router, host, mediaURL string) string {
if mediaURL == "" {
return ""
}
if customProxyURL := config.Opts.MediaCustomProxyURL(); customProxyURL != "" {
return proxifyURLWithCustomProxy(mediaURL, customProxyURL)
}
proxifiedUrl := ProxifyRelativeURL(router, mediaURL)
scheme := "http"
if config.Opts.HTTPS {
scheme = "https"
}
return scheme + "://" + host + proxifiedUrl
}
func proxifyURLWithCustomProxy(mediaURL, customProxyURL string) string {
if customProxyURL == "" {
return mediaURL
}
proxyUrl, err := url.Parse(customProxyURL)
if err != nil {
slog.Error("Incorrect custom media proxy URL",
slog.String("custom_proxy_url", customProxyURL),
slog.Any("error", err),
)
return mediaURL
}
proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(mediaURL)))
return proxyUrl.String()
}

12
internal/model/enclosure.go
View File

@ -2,7 +2,6 @@
// SPDX-License-Identifier: Apache-2.0
package model // import "miniflux.app/v2/internal/model"
import "strings"
// Enclosure represents an attachment.
type Enclosure struct {
@ -17,15 +16,8 @@ type Enclosure struct {
// Html5MimeType will modify the actual MimeType to allow direct playback from HTML5 player for some kind of MimeType
func (e Enclosure) Html5MimeType() string {
if strings.HasPrefix(e.MimeType, "video") {
switch e.MimeType {
// Solution from this stackoverflow discussion:
// https://stackoverflow.com/questions/15277147/m4v-mimetype-video-mp4-or-video-m4v/66945470#66945470
// tested at the time of this commit (06/2023) on latest Firefox & Vivaldi on this feed
// https://www.florenceporcel.com/podcast/lfhdu.xml
case "video/m4v":
return "video/x-m4v"
}
if e.MimeType == "video/m4v" {
return "video/x-m4v"
}
return e.MimeType
}

20
internal/model/feed.go
View File

@ -159,25 +159,7 @@ type FeedCreationRequestFromSubscriptionDiscovery struct {
ETag string
LastModified string
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Cookie string `json:"cookie"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
Disabled bool `json:"disabled"`
NoMediaPlayer bool `json:"no_media_player"`
IgnoreHTTPCache bool `json:"ignore_http_cache"`
AllowSelfSignedCertificates bool `json:"allow_self_signed_certificates"`
FetchViaProxy bool `json:"fetch_via_proxy"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
BlocklistRules string `json:"blocklist_rules"`
KeeplistRules string `json:"keeplist_rules"`
HideGlobally bool `json:"hide_globally"`
UrlRewriteRules string `json:"urlrewrite_rules"`
DisableHTTP2 bool `json:"disable_http2"`
FeedCreationRequest
}
// FeedModificationRequest represents the request to update a feed.

28
internal/model/model.go
View File

@ -3,26 +3,20 @@
package model // import "miniflux.app/v2/internal/model"
// OptionalString populates an optional string field.
type Number interface {
int | int64 | float64
}
func OptionalNumber[T Number](value T) *T {
if value > 0 {
return &value
}
return nil
}
func OptionalString(value string) *string {
if value != "" {
return &value
}
return nil
}
// OptionalInt populates an optional int field.
func OptionalInt(value int) *int {
if value > 0 {
return &value
}
return nil
}
// OptionalInt64 populates an optional int64 field.
func OptionalInt64(value int64) *int64 {
if value > 0 {
return &value
}
return nil
}

50
internal/model/user.go
View File

@ -35,6 +35,7 @@ type User struct {
DefaultHomePage string `json:"default_home_page"`
CategoriesSortingOrder string `json:"categories_sorting_order"`
MarkReadOnView bool `json:"mark_read_on_view"`
MediaPlaybackRate float64 `json:"media_playback_rate"`
}
// UserCreationRequest represents the request to create a user.
@ -48,28 +49,29 @@ type UserCreationRequest struct {
// UserModificationRequest represents the request to update a user.
type UserModificationRequest struct {
Username *string `json:"username"`
Password *string `json:"password"`
Theme *string `json:"theme"`
Language *string `json:"language"`
Timezone *string `json:"timezone"`
EntryDirection *string `json:"entry_sorting_direction"`
EntryOrder *string `json:"entry_sorting_order"`
Stylesheet *string `json:"stylesheet"`
GoogleID *string `json:"google_id"`
OpenIDConnectID *string `json:"openid_connect_id"`
EntriesPerPage *int `json:"entries_per_page"`
IsAdmin *bool `json:"is_admin"`
KeyboardShortcuts *bool `json:"keyboard_shortcuts"`
ShowReadingTime *bool `json:"show_reading_time"`
EntrySwipe *bool `json:"entry_swipe"`
GestureNav *string `json:"gesture_nav"`
DisplayMode *string `json:"display_mode"`
DefaultReadingSpeed *int `json:"default_reading_speed"`
CJKReadingSpeed *int `json:"cjk_reading_speed"`
DefaultHomePage *string `json:"default_home_page"`
CategoriesSortingOrder *string `json:"categories_sorting_order"`
MarkReadOnView *bool `json:"mark_read_on_view"`
Username *string `json:"username"`
Password *string `json:"password"`
Theme *string `json:"theme"`
Language *string `json:"language"`
Timezone *string `json:"timezone"`
EntryDirection *string `json:"entry_sorting_direction"`
EntryOrder *string `json:"entry_sorting_order"`
Stylesheet *string `json:"stylesheet"`
GoogleID *string `json:"google_id"`
OpenIDConnectID *string `json:"openid_connect_id"`
EntriesPerPage *int `json:"entries_per_page"`
IsAdmin *bool `json:"is_admin"`
KeyboardShortcuts *bool `json:"keyboard_shortcuts"`
ShowReadingTime *bool `json:"show_reading_time"`
EntrySwipe *bool `json:"entry_swipe"`
GestureNav *string `json:"gesture_nav"`
DisplayMode *string `json:"display_mode"`
DefaultReadingSpeed *int `json:"default_reading_speed"`
CJKReadingSpeed *int `json:"cjk_reading_speed"`
DefaultHomePage *string `json:"default_home_page"`
CategoriesSortingOrder *string `json:"categories_sorting_order"`
MarkReadOnView *bool `json:"mark_read_on_view"`
MediaPlaybackRate *float64 `json:"media_playback_rate"`
}
// Patch updates the User object with the modification request.
@ -161,6 +163,10 @@ func (u *UserModificationRequest) Patch(user *User) {
if u.MarkReadOnView != nil {
user.MarkReadOnView = *u.MarkReadOnView
}
if u.MediaPlaybackRate != nil {
user.MediaPlaybackRate = *u.MediaPlaybackRate
}
}
// UseTimezone converts last login date to the given timezone.

69
internal/proxy/proxy.go
View File

@ -1,69 +0,0 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package proxy // import "miniflux.app/v2/internal/proxy"
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"net/url"
"path"
"miniflux.app/v2/internal/http/route"
"github.com/gorilla/mux"
"miniflux.app/v2/internal/config"
)
// ProxifyURL generates a relative URL for a proxified resource.
func ProxifyURL(router *mux.Router, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write([]byte(link))
digest := mac.Sum(nil)
return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
}
proxyUrl, err := url.Parse(proxyImageUrl)
if err != nil {
return ""
}
proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link)))
return proxyUrl.String()
}
return ""
}
// AbsoluteProxifyURL generates an absolute URL for a proxified resource.
func AbsoluteProxifyURL(router *mux.Router, host, link string) string {
if link != "" {
proxyImageUrl := config.Opts.ProxyUrl()
if proxyImageUrl == "" {
mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey())
mac.Write([]byte(link))
digest := mac.Sum(nil)
path := route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link)))
if config.Opts.HTTPS {
return "https://" + host + path
} else {
return "http://" + host + path
}
}
proxyUrl, err := url.Parse(proxyImageUrl)
if err != nil {
return ""
}
proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link)))
return proxyUrl.String()
}
return ""
}

216
internal/reader/atom/atom_03.go
View File

@ -6,158 +6,114 @@ package atom // import "miniflux.app/v2/internal/reader/atom"
import (
"encoding/base64"
"html"
"log/slog"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
// Specs: http://web.archive.org/web/20060811235523/http://www.mnot.net/drafts/draft-nottingham-atom-format-02.html
type atom03Feed struct {
ID string `xml:"id"`
Title atom03Text `xml:"title"`
Author atomPerson `xml:"author"`
Links atomLinks `xml:"link"`
Entries []atom03Entry `xml:"entry"`
type Atom03Feed struct {
Version string `xml:"version,attr"`
// The "atom:id" element's content conveys a permanent, globally unique identifier for the feed.
// It MUST NOT change over time, even if the feed is relocated. atom:feed elements MAY contain an atom:id element,
// but MUST NOT contain more than one. The content of this element, when present, MUST be a URI.
ID string `xml:"http://purl.org/atom/ns# id"`
// The "atom:title" element is a Content construct that conveys a human-readable title for the feed.
// atom:feed elements MUST contain exactly one atom:title element.
// If the feed describes a Web resource, its content SHOULD be the same as that resource's title.
Title Atom03Content `xml:"http://purl.org/atom/ns# title"`
// The "atom:link" element is a Link construct that conveys a URI associated with the feed.
// The nature of the relationship as well as the link itself is determined by the element's content.
// atom:feed elements MUST contain at least one atom:link element with a rel attribute value of "alternate".
// atom:feed elements MUST NOT contain more than one atom:link element with a rel attribute value of "alternate" that has the same type attribute value.
// atom:feed elements MAY contain additional atom:link elements beyond those described above.
Links AtomLinks `xml:"http://purl.org/atom/ns# link"`
// The "atom:author" element is a Person construct that indicates the default author of the feed.
// atom:feed elements MUST contain exactly one atom:author element,
// UNLESS all of the atom:feed element's child atom:entry elements contain an atom:author element.
// atom:feed elements MUST NOT contain more than one atom:author element.
Author AtomPerson `xml:"http://purl.org/atom/ns# author"`
// The "atom:entry" element's represents an individual entry that is contained by the feed.
// atom:feed elements MAY contain one or more atom:entry elements.
Entries []Atom03Entry `xml:"http://purl.org/atom/ns# entry"`
}
func (a *atom03Feed) Transform(baseURL string) *model.Feed {
var err error
type Atom03Entry struct {
// The "atom:id" element's content conveys a permanent, globally unique identifier for the entry.
// It MUST NOT change over time, even if other representations of the entry (such as a web representation pointed to by the entry's atom:link element) are relocated.
// If the same entry is syndicated in two atom:feeds published by the same entity, the entry's atom:id MUST be the same in both feeds.
ID string `xml:"id"`
feed := new(model.Feed)
// The "atom:title" element is a Content construct that conveys a human-readable title for the entry.
// atom:entry elements MUST have exactly one "atom:title" element.
// If an entry describes a Web resource, its content SHOULD be the same as that resource's title.
Title Atom03Content `xml:"title"`
feedURL := a.Links.firstLinkWithRelation("self")
feed.FeedURL, err = urllib.AbsoluteURL(baseURL, feedURL)
if err != nil {
feed.FeedURL = feedURL
}
// The "atom:modified" element is a Date construct that indicates the time that the entry was last modified.
// atom:entry elements MUST contain an atom:modified element, but MUST NOT contain more than one.
// The content of an atom:modified element MUST have a time zone whose value SHOULD be "UTC".
Modified string `xml:"modified"`
siteURL := a.Links.originalLink()
feed.SiteURL, err = urllib.AbsoluteURL(baseURL, siteURL)
if err != nil {
feed.SiteURL = siteURL
}
// The "atom:issued" element is a Date construct that indicates the time that the entry was issued.
// atom:entry elements MUST contain an atom:issued element, but MUST NOT contain more than one.
// The content of an atom:issued element MAY omit a time zone.
Issued string `xml:"issued"`
feed.Title = a.Title.String()
if feed.Title == "" {
feed.Title = feed.SiteURL
}
// The "atom:created" element is a Date construct that indicates the time that the entry was created.
// atom:entry elements MAY contain an atom:created element, but MUST NOT contain more than one.
// The content of an atom:created element MUST have a time zone whose value SHOULD be "UTC".
// If atom:created is not present, its content MUST considered to be the same as that of atom:modified.
Created string `xml:"created"`
for _, entry := range a.Entries {
item := entry.Transform()
entryURL, err := urllib.AbsoluteURL(feed.SiteURL, item.URL)
if err == nil {
item.URL = entryURL
}
// The "atom:link" element is a Link construct that conveys a URI associated with the entry.
// The nature of the relationship as well as the link itself is determined by the element's content.
// atom:entry elements MUST contain at least one atom:link element with a rel attribute value of "alternate".
// atom:entry elements MUST NOT contain more than one atom:link element with a rel attribute value of "alternate" that has the same type attribute value.
// atom:entry elements MAY contain additional atom:link elements beyond those described above.
Links AtomLinks `xml:"link"`
if item.Author == "" {
item.Author = a.Author.String()
}
// The "atom:summary" element is a Content construct that conveys a short summary, abstract or excerpt of the entry.
// atom:entry elements MAY contain an atom:created element, but MUST NOT contain more than one.
Summary Atom03Content `xml:"summary"`
if item.Title == "" {
item.Title = sanitizer.TruncateHTML(item.Content, 100)
}
// The "atom:content" element is a Content construct that conveys the content of the entry.
// atom:entry elements MAY contain one or more atom:content elements.
Content Atom03Content `xml:"content"`
if item.Title == "" {
item.Title = item.URL
}
feed.Entries = append(feed.Entries, item)
}
return feed
// The "atom:author" element is a Person construct that indicates the default author of the entry.
// atom:entry elements MUST contain exactly one atom:author element,
// UNLESS the atom:feed element containing them contains an atom:author element itself.
// atom:entry elements MUST NOT contain more than one atom:author element.
Author AtomPerson `xml:"author"`
}
type atom03Entry struct {
ID string `xml:"id"`
Title atom03Text `xml:"title"`
Modified string `xml:"modified"`
Issued string `xml:"issued"`
Created string `xml:"created"`
Links atomLinks `xml:"link"`
Summary atom03Text `xml:"summary"`
Content atom03Text `xml:"content"`
Author atomPerson `xml:"author"`
}
type Atom03Content struct {
// Content constructs MAY have a "type" attribute, whose value indicates the media type of the content.
// When present, this attribute's value MUST be a registered media type [RFC2045].
// If not present, its value MUST be considered to be "text/plain".
Type string `xml:"type,attr"`
func (a *atom03Entry) Transform() *model.Entry {
entry := model.NewEntry()
entry.URL = a.Links.originalLink()
entry.Date = a.entryDate()
entry.Author = a.Author.String()
entry.Hash = a.entryHash()
entry.Content = a.entryContent()
entry.Title = a.entryTitle()
return entry
}
// Content constructs MAY have a "mode" attribute, whose value indicates the method used to encode the content.
// When present, this attribute's value MUST be listed below.
// If not present, its value MUST be considered to be "xml".
//
// "xml": A mode attribute with the value "xml" indicates that the element's content is inline xml (for example, namespace-qualified XHTML).
//
// "escaped": A mode attribute with the value "escaped" indicates that the element's content is an escaped string.
// Processors MUST unescape the element's content before considering it as content of the indicated media type.
//
// "base64": A mode attribute with the value "base64" indicates that the element's content is base64-encoded [RFC2045].
// Processors MUST decode the element's content before considering it as content of the the indicated media type.
Mode string `xml:"mode,attr"`
func (a *atom03Entry) entryTitle() string {
return sanitizer.StripTags(a.Title.String())
}
func (a *atom03Entry) entryContent() string {
content := a.Content.String()
if content != "" {
return content
}
summary := a.Summary.String()
if summary != "" {
return summary
}
return ""
}
func (a *atom03Entry) entryDate() time.Time {
dateText := ""
for _, value := range []string{a.Issued, a.Modified, a.Created} {
if value != "" {
dateText = value
break
}
}
if dateText != "" {
result, err := date.Parse(dateText)
if err != nil {
slog.Debug("Unable to parse date from Atom 0.3 feed",
slog.String("date", dateText),
slog.String("id", a.ID),
slog.Any("error", err),
)
return time.Now()
}
return result
}
return time.Now()
}
func (a *atom03Entry) entryHash() string {
for _, value := range []string{a.ID, a.Links.originalLink()} {
if value != "" {
return crypto.Hash(value)
}
}
return ""
}
type atom03Text struct {
Type string `xml:"type,attr"`
Mode string `xml:"mode,attr"`
CharData string `xml:",chardata"`
InnerXML string `xml:",innerxml"`
}
func (a *atom03Text) String() string {
func (a *Atom03Content) Content() string {
content := ""
switch {

115
internal/reader/atom/atom_03_adapter.go Normal file
View File

@ -0,0 +1,115 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package atom // import "miniflux.app/v2/internal/reader/atom"
import (
"log/slog"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
type Atom03Adapter struct {
atomFeed *Atom03Feed
}
func NewAtom03Adapter(atomFeed *Atom03Feed) *Atom03Adapter {
return &Atom03Adapter{atomFeed}
}
func (a *Atom03Adapter) BuildFeed(baseURL string) *model.Feed {
feed := new(model.Feed)
// Populate the feed URL.
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
if feedURL != "" {
if absoluteFeedURL, err := urllib.AbsoluteURL(baseURL, feedURL); err == nil {
feed.FeedURL = absoluteFeedURL
}
} else {
feed.FeedURL = baseURL
}
// Populate the site URL.
siteURL := a.atomFeed.Links.OriginalLink()
if siteURL != "" {
if absoluteSiteURL, err := urllib.AbsoluteURL(baseURL, siteURL); err == nil {
feed.SiteURL = absoluteSiteURL
}
} else {
feed.SiteURL = baseURL
}
// Populate the feed title.
feed.Title = a.atomFeed.Title.Content()
if feed.Title == "" {
feed.Title = feed.SiteURL
}
for _, atomEntry := range a.atomFeed.Entries {
entry := model.NewEntry()
// Populate the entry URL.
entry.URL = atomEntry.Links.OriginalLink()
if entry.URL != "" {
if absoluteEntryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL); err == nil {
entry.URL = absoluteEntryURL
}
}
// Populate the entry content.
entry.Content = atomEntry.Content.Content()
if entry.Content == "" {
entry.Content = atomEntry.Summary.Content()
}
// Populate the entry title.
entry.Title = atomEntry.Title.Content()
if entry.Title == "" {
entry.Title = sanitizer.TruncateHTML(entry.Content, 100)
}
if entry.Title == "" {
entry.Title = entry.URL
}
// Populate the entry author.
entry.Author = atomEntry.Author.PersonName()
if entry.Author == "" {
entry.Author = a.atomFeed.Author.PersonName()
}
// Populate the entry date.
for _, value := range []string{atomEntry.Issued, atomEntry.Modified, atomEntry.Created} {
if parsedDate, err := date.Parse(value); err == nil {
entry.Date = parsedDate
break
} else {
slog.Debug("Unable to parse date from Atom 0.3 feed",
slog.String("date", value),
slog.String("id", atomEntry.ID),
slog.Any("error", err),
)
}
}
if entry.Date.IsZero() {
entry.Date = time.Now()
}
// Generate the entry hash.
for _, value := range []string{atomEntry.ID, atomEntry.Links.OriginalLink()} {
if value != "" {
entry.Hash = crypto.Hash(value)
break
}
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}

40
internal/reader/atom/atom_03_test.go
View File

@ -27,7 +27,7 @@ func TestParseAtom03(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/atom.xml", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -36,7 +36,7 @@ func TestParseAtom03(t *testing.T) {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
if feed.FeedURL != "http://diveintomark.org/" {
if feed.FeedURL != "http://diveintomark.org/atom.xml" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
@ -74,6 +74,28 @@ func TestParseAtom03(t *testing.T) {
}
}
func TestParseAtom03WithoutSiteURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed version="0.3" xmlns="http://purl.org/atom/ns#">
<modified>2003-12-13T18:30:02Z</modified>
<author><name>Mark Pilgrim</name></author>
<entry>
<title>Atom 0.3 snapshot</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/2003/12/13/atom03"/>
<id>tag:diveintomark.org,2003:3.2397</id>
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/atom.xml", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "http://diveintomark.org/atom.xml" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseAtom03WithoutFeedTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed version="0.3" xmlns="http://purl.org/atom/ns#">
@ -87,7 +109,7 @@ func TestParseAtom03WithoutFeedTitle(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -110,7 +132,7 @@ func TestParseAtom03WithoutEntryTitleButWithLink(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -138,7 +160,7 @@ func TestParseAtom03WithoutEntryTitleButWithSummary(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -166,7 +188,7 @@ func TestParseAtom03WithoutEntryTitleButWithXMLContent(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -197,7 +219,7 @@ func TestParseAtom03WithSummaryOnly(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -228,7 +250,7 @@ func TestParseAtom03WithXMLContent(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}
@ -259,7 +281,7 @@ func TestParseAtom03WithBase64Content(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://diveintomark.org/", bytes.NewReader([]byte(data)), "0.3")
if err != nil {
t.Fatal(err)
}

410
internal/reader/atom/atom_10.go
View File

@ -6,286 +6,200 @@ package atom // import "miniflux.app/v2/internal/reader/atom"
import (
"encoding/xml"
"html"
"log/slog"
"strconv"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/media"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
// The "atom:feed" element is the document (i.e., top-level) element of
// an Atom Feed Document, acting as a container for metadata and data
// associated with the feed. Its element children consist of metadata
// elements followed by zero or more atom:entry child elements.
//
// Specs:
// https://tools.ietf.org/html/rfc4287
// https://validator.w3.org/feed/docs/atom.html
type atom10Feed struct {
XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"`
ID string `xml:"id"`
Title atom10Text `xml:"title"`
Authors atomAuthors `xml:"author"`
Icon string `xml:"icon"`
Links atomLinks `xml:"link"`
Entries []atom10Entry `xml:"entry"`
type Atom10Feed struct {
XMLName xml.Name `xml:"http://www.w3.org/2005/Atom feed"`
// The "atom:id" element conveys a permanent, universally unique
// identifier for an entry or feed.
//
// Its content MUST be an IRI, as defined by [RFC3987]. Note that the
// definition of "IRI" excludes relative references. Though the IRI
// might use a dereferencable scheme, Atom Processors MUST NOT assume it
// can be dereferenced.
//
// atom:feed elements MUST contain exactly one atom:id element.
ID string `xml:"http://www.w3.org/2005/Atom id"`
// The "atom:title" element is a Text construct that conveys a human-
// readable title for an entry or feed.
//
// atom:feed elements MUST contain exactly one atom:title element.
Title Atom10Text `xml:"http://www.w3.org/2005/Atom title"`
// The "atom:author" element is a Person construct that indicates the
// author of the entry or feed.
//
// atom:feed elements MUST contain one or more atom:author elements,
// unless all of the atom:feed element's child atom:entry elements
// contain at least one atom:author element.
Authors AtomPersons `xml:"http://www.w3.org/2005/Atom author"`
// The "atom:icon" element's content is an IRI reference [RFC3987] that
// identifies an image that provides iconic visual identification for a
// feed.
//
// atom:feed elements MUST NOT contain more than one atom:icon element.
Icon string `xml:"http://www.w3.org/2005/Atom icon"`
// The "atom:logo" element's content is an IRI reference [RFC3987] that
// identifies an image that provides visual identification for a feed.
//
// atom:feed elements MUST NOT contain more than one atom:logo element.
Logo string `xml:"http://www.w3.org/2005/Atom logo"`
// atom:feed elements SHOULD contain one atom:link element with a rel
// attribute value of "self". This is the preferred URI for
// retrieving Atom Feed Documents representing this Atom feed.
//
// atom:feed elements MUST NOT contain more than one atom:link
// element with a rel attribute value of "alternate" that has the
// same combination of type and hreflang attribute values.
Links AtomLinks `xml:"http://www.w3.org/2005/Atom link"`
// The "atom:category" element conveys information about a category
// associated with an entry or feed. This specification assigns no
// meaning to the content (if any) of this element.
//
// atom:feed elements MAY contain any number of atom:category
// elements.
Categories AtomCategories `xml:"http://www.w3.org/2005/Atom category"`
Entries []Atom10Entry `xml:"http://www.w3.org/2005/Atom entry"`
}
func (a *atom10Feed) Transform(baseURL string) *model.Feed {
var err error
type Atom10Entry struct {
// The "atom:id" element conveys a permanent, universally unique
// identifier for an entry or feed.
//
// Its content MUST be an IRI, as defined by [RFC3987]. Note that the
// definition of "IRI" excludes relative references. Though the IRI
// might use a dereferencable scheme, Atom Processors MUST NOT assume it
// can be dereferenced.
//
// atom:entry elements MUST contain exactly one atom:id element.
ID string `xml:"http://www.w3.org/2005/Atom id"`
feed := new(model.Feed)
// The "atom:title" element is a Text construct that conveys a human-
// readable title for an entry or feed.
//
// atom:entry elements MUST contain exactly one atom:title element.
Title Atom10Text `xml:"http://www.w3.org/2005/Atom title"`
feedURL := a.Links.firstLinkWithRelation("self")
feed.FeedURL, err = urllib.AbsoluteURL(baseURL, feedURL)
if err != nil {
feed.FeedURL = feedURL
}
// The "atom:published" element is a Date construct indicating an
// instant in time associated with an event early in the life cycle of
// the entry.
Published string `xml:"http://www.w3.org/2005/Atom published"`
siteURL := a.Links.originalLink()
feed.SiteURL, err = urllib.AbsoluteURL(baseURL, siteURL)
if err != nil {
feed.SiteURL = siteURL
}
// The "atom:updated" element is a Date construct indicating the most
// recent instant in time when an entry or feed was modified in a way
// the publisher considers significant. Therefore, not all
// modifications necessarily result in a changed atom:updated value.
//
// atom:entry elements MUST contain exactly one atom:updated element.
Updated string `xml:"http://www.w3.org/2005/Atom updated"`
feed.Title = html.UnescapeString(a.Title.String())
if feed.Title == "" {
feed.Title = feed.SiteURL
}
// atom:entry elements MUST NOT contain more than one atom:link
// element with a rel attribute value of "alternate" that has the
// same combination of type and hreflang attribute values.
Links AtomLinks `xml:"http://www.w3.org/2005/Atom link"`
feed.IconURL = strings.TrimSpace(a.Icon)
// atom:entry elements MUST contain an atom:summary element in either
// of the following cases:
// * the atom:entry contains an atom:content that has a "src"
// attribute (and is thus empty).
// * the atom:entry contains content that is encoded in Base64;
// i.e., the "type" attribute of atom:content is a MIME media type
// [MIMEREG], but is not an XML media type [RFC3023], does not
// begin with "text/", and does not end with "/xml" or "+xml".
//
// atom:entry elements MUST NOT contain more than one atom:summary
// element.
Summary Atom10Text `xml:"http://www.w3.org/2005/Atom summary"`
for _, entry := range a.Entries {
item := entry.Transform()
entryURL, err := urllib.AbsoluteURL(feed.SiteURL, item.URL)
if err == nil {
item.URL = entryURL
}
// atom:entry elements MUST NOT contain more than one atom:content
// element.
Content Atom10Text `xml:"http://www.w3.org/2005/Atom content"`
if item.Author == "" {
item.Author = a.Authors.String()
}
// The "atom:author" element is a Person construct that indicates the
// author of the entry or feed.
//
// atom:entry elements MUST contain one or more atom:author elements
Authors AtomPersons `xml:"http://www.w3.org/2005/Atom author"`
if item.Title == "" {
item.Title = sanitizer.TruncateHTML(item.Content, 100)
}
// The "atom:category" element conveys information about a category
// associated with an entry or feed. This specification assigns no
// meaning to the content (if any) of this element.
//
// atom:entry elements MAY contain any number of atom:category
// elements.
Categories AtomCategories `xml:"http://www.w3.org/2005/Atom category"`
if item.Title == "" {
item.Title = item.URL
}
feed.Entries = append(feed.Entries, item)
}
return feed
}
type atom10Entry struct {
ID string `xml:"id"`
Title atom10Text `xml:"title"`
Published string `xml:"published"`
Updated string `xml:"updated"`
Links atomLinks `xml:"link"`
Summary atom10Text `xml:"summary"`
Content atom10Text `xml:"http://www.w3.org/2005/Atom content"`
Authors atomAuthors `xml:"author"`
Categories []atom10Category `xml:"category"`
media.Element
}
func (a *atom10Entry) Transform() *model.Entry {
entry := model.NewEntry()
entry.URL = a.Links.originalLink()
entry.Date = a.entryDate()
entry.Author = a.Authors.String()
entry.Hash = a.entryHash()
entry.Content = a.entryContent()
entry.Title = a.entryTitle()
entry.Enclosures = a.entryEnclosures()
entry.CommentsURL = a.entryCommentsURL()
entry.Tags = a.entryCategories()
return entry
}
func (a *atom10Entry) entryTitle() string {
return html.UnescapeString(a.Title.String())
}
func (a *atom10Entry) entryContent() string {
content := a.Content.String()
if content != "" {
return content
}
summary := a.Summary.String()
if summary != "" {
return summary
}
mediaDescription := a.FirstMediaDescription()
if mediaDescription != "" {
return mediaDescription
}
return ""
}
// Note: The published date represents the original creation date for YouTube feeds.
// Example:
// <published>2019-01-26T08:02:28+00:00</published>
// <updated>2019-01-29T07:27:27+00:00</updated>
func (a *atom10Entry) entryDate() time.Time {
dateText := a.Published
if dateText == "" {
dateText = a.Updated
}
if dateText != "" {
result, err := date.Parse(dateText)
if err != nil {
slog.Debug("Unable to parse date from Atom 0.3 feed",
slog.String("date", dateText),
slog.String("id", a.ID),
slog.Any("error", err),
)
return time.Now()
}
return result
}
return time.Now()
}
func (a *atom10Entry) entryHash() string {
for _, value := range []string{a.ID, a.Links.originalLink()} {
if value != "" {
return crypto.Hash(value)
}
}
return ""
}
func (a *atom10Entry) entryEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
duplicates := make(map[string]bool)
for _, mediaThumbnail := range a.AllMediaThumbnails() {
if _, found := duplicates[mediaThumbnail.URL]; !found {
duplicates[mediaThumbnail.URL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: mediaThumbnail.URL,
MimeType: mediaThumbnail.MimeType(),
Size: mediaThumbnail.Size(),
})
}
}
for _, link := range a.Links {
if strings.EqualFold(link.Rel, "enclosure") {
if link.URL == "" {
continue
}
if _, found := duplicates[link.URL]; !found {
duplicates[link.URL] = true
length, _ := strconv.ParseInt(link.Length, 10, 0)
enclosures = append(enclosures, &model.Enclosure{URL: link.URL, MimeType: link.Type, Size: length})
}
}
}
for _, mediaContent := range a.AllMediaContents() {
if _, found := duplicates[mediaContent.URL]; !found {
duplicates[mediaContent.URL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: mediaContent.URL,
MimeType: mediaContent.MimeType(),
Size: mediaContent.Size(),
})
}
}
for _, mediaPeerLink := range a.AllMediaPeerLinks() {
if _, found := duplicates[mediaPeerLink.URL]; !found {
duplicates[mediaPeerLink.URL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: mediaPeerLink.URL,
MimeType: mediaPeerLink.MimeType(),
Size: mediaPeerLink.Size(),
})
}
}
return enclosures
}
func (r *atom10Entry) entryCategories() []string {
categoryList := make([]string, 0)
for _, atomCategory := range r.Categories {
if strings.TrimSpace(atomCategory.Label) != "" {
categoryList = append(categoryList, strings.TrimSpace(atomCategory.Label))
} else {
categoryList = append(categoryList, strings.TrimSpace(atomCategory.Term))
}
}
return categoryList
}
// See https://tools.ietf.org/html/rfc4685#section-4
// If the type attribute of the atom:link is omitted, its value is assumed to be "application/atom+xml".
// We accept only HTML or XHTML documents for now since the intention is to have the same behavior as RSS.
func (a *atom10Entry) entryCommentsURL() string {
commentsURL := a.Links.firstLinkWithRelationAndType("replies", "text/html", "application/xhtml+xml")
if urllib.IsAbsoluteURL(commentsURL) {
return commentsURL
}
return ""
}
type atom10Text struct {
Type string `xml:"type,attr"`
CharData string `xml:",chardata"`
InnerXML string `xml:",innerxml"`
XHTMLRootElement atomXHTMLRootElement `xml:"http://www.w3.org/1999/xhtml div"`
}
type atom10Category struct {
Term string `xml:"term,attr"`
Label string `xml:"label,attr"`
media.MediaItemElement
}
// A Text construct contains human-readable text, usually in small
// quantities. The content of Text constructs is Language-Sensitive.
// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1
// Text: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1.1.1
// HTML: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1.1.2
// XHTML: https://datatracker.ietf.org/doc/html/rfc4287#section-3.1.1.3
func (a *atom10Text) String() string {
type Atom10Text struct {
Type string `xml:"type,attr"`
CharData string `xml:",chardata"`
InnerXML string `xml:",innerxml"`
XHTMLRootElement AtomXHTMLRootElement `xml:"http://www.w3.org/1999/xhtml div"`
}
func (a *Atom10Text) Body() string {
var content string
switch {
case a.Type == "", a.Type == "text", a.Type == "text/plain":
if strings.HasPrefix(strings.TrimSpace(a.InnerXML), `<![CDATA[`) {
content = html.EscapeString(a.CharData)
} else {
content = a.InnerXML
}
case a.Type == "xhtml":
var root = a.XHTMLRootElement
if root.XMLName.Local == "div" {
content = root.InnerXML
} else {
content = a.InnerXML
}
default:
if strings.EqualFold(a.Type, "xhtml") {
content = a.xhtmlContent()
} else {
content = a.CharData
}
return strings.TrimSpace(content)
}
type atomXHTMLRootElement struct {
func (a *Atom10Text) Title() string {
var content string
switch {
case strings.EqualFold(a.Type, "xhtml"):
content = a.xhtmlContent()
case strings.Contains(a.InnerXML, "<![CDATA["):
content = html.UnescapeString(a.CharData)
default:
content = a.CharData
}
content = sanitizer.StripTags(content)
return strings.TrimSpace(content)
}
func (a *Atom10Text) xhtmlContent() string {
if a.XHTMLRootElement.XMLName.Local == "div" {
return a.XHTMLRootElement.InnerXML
}
return a.InnerXML
}
type AtomXHTMLRootElement struct {
XMLName xml.Name `xml:"div"`
InnerXML string `xml:",innerxml"`
}

254
internal/reader/atom/atom_10_adapter.go Normal file
View File

@ -0,0 +1,254 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package atom // import "miniflux.app/v2/internal/reader/atom"
import (
"log/slog"
"slices"
"sort"
"strconv"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
type Atom10Adapter struct {
atomFeed *Atom10Feed
}
func NewAtom10Adapter(atomFeed *Atom10Feed) *Atom10Adapter {
return &Atom10Adapter{atomFeed}
}
func (a *Atom10Adapter) BuildFeed(baseURL string) *model.Feed {
feed := new(model.Feed)
// Populate the feed URL.
feedURL := a.atomFeed.Links.firstLinkWithRelation("self")
if feedURL != "" {
if absoluteFeedURL, err := urllib.AbsoluteURL(baseURL, feedURL); err == nil {
feed.FeedURL = absoluteFeedURL
}
} else {
feed.FeedURL = baseURL
}
// Populate the site URL.
siteURL := a.atomFeed.Links.OriginalLink()
if siteURL != "" {
if absoluteSiteURL, err := urllib.AbsoluteURL(baseURL, siteURL); err == nil {
feed.SiteURL = absoluteSiteURL
}
} else {
feed.SiteURL = baseURL
}
// Populate the feed title.
feed.Title = a.atomFeed.Title.Body()
if feed.Title == "" {
feed.Title = feed.SiteURL
}
// Populate the feed icon.
if a.atomFeed.Icon != "" {
if absoluteIconURL, err := urllib.AbsoluteURL(feed.SiteURL, a.atomFeed.Icon); err == nil {
feed.IconURL = absoluteIconURL
}
} else if a.atomFeed.Logo != "" {
if absoluteLogoURL, err := urllib.AbsoluteURL(feed.SiteURL, a.atomFeed.Logo); err == nil {
feed.IconURL = absoluteLogoURL
}
}
feed.Entries = a.populateEntries(feed.SiteURL)
return feed
}
func (a *Atom10Adapter) populateEntries(siteURL string) model.Entries {
entries := make(model.Entries, 0, len(a.atomFeed.Entries))
for _, atomEntry := range a.atomFeed.Entries {
entry := model.NewEntry()
// Populate the entry URL.
entry.URL = atomEntry.Links.OriginalLink()
if entry.URL != "" {
if absoluteEntryURL, err := urllib.AbsoluteURL(siteURL, entry.URL); err == nil {
entry.URL = absoluteEntryURL
}
}
// Populate the entry content.
entry.Content = atomEntry.Content.Body()
if entry.Content == "" {
entry.Content = atomEntry.Summary.Body()
if entry.Content == "" {
entry.Content = atomEntry.FirstMediaDescription()
}
}
// Populate the entry title.
entry.Title = atomEntry.Title.Title()
if entry.Title == "" {
entry.Title = sanitizer.TruncateHTML(entry.Content, 100)
if entry.Title == "" {
entry.Title = entry.URL
}
}
// Populate the entry author.
authors := atomEntry.Authors.PersonNames()
if len(authors) == 0 {
authors = a.atomFeed.Authors.PersonNames()
}
sort.Strings(authors)
authors = slices.Compact(authors)
entry.Author = strings.Join(authors, ", ")
// Populate the entry date.
for _, value := range []string{atomEntry.Published, atomEntry.Updated} {
if value != "" {
if parsedDate, err := date.Parse(value); err != nil {
slog.Debug("Unable to parse date from Atom 1.0 feed",
slog.String("date", value),
slog.String("url", entry.URL),
slog.Any("error", err),
)
} else {
entry.Date = parsedDate
break
}
}
}
if entry.Date.IsZero() {
entry.Date = time.Now()
}
// Populate categories.
categories := atomEntry.Categories.CategoryNames()
if len(categories) == 0 {
categories = a.atomFeed.Categories.CategoryNames()
}
sort.Strings(categories)
entry.Tags = slices.Compact(categories)
// Populate the commentsURL if defined.
// See https://tools.ietf.org/html/rfc4685#section-4
// If the type attribute of the atom:link is omitted, its value is assumed to be "application/atom+xml".
// We accept only HTML or XHTML documents for now since the intention is to have the same behavior as RSS.
commentsURL := atomEntry.Links.firstLinkWithRelationAndType("replies", "text/html", "application/xhtml+xml")
if urllib.IsAbsoluteURL(commentsURL) {
entry.CommentsURL = commentsURL
}
// Generate the entry hash.
for _, value := range []string{atomEntry.ID, atomEntry.Links.OriginalLink()} {
if value != "" {
entry.Hash = crypto.Hash(value)
break
}
}
// Populate the entry enclosures.
uniqueEnclosuresMap := make(map[string]bool)
for _, mediaThumbnail := range atomEntry.AllMediaThumbnails() {
mediaURL := strings.TrimSpace(mediaThumbnail.URL)
if mediaURL == "" {
continue
}
if _, found := uniqueEnclosuresMap[mediaURL]; !found {
if mediaAbsoluteURL, err := urllib.AbsoluteURL(siteURL, mediaURL); err != nil {
slog.Debug("Unable to build absolute URL for media thumbnail",
slog.String("url", mediaThumbnail.URL),
slog.String("site_url", siteURL),
slog.Any("error", err),
)
} else {
uniqueEnclosuresMap[mediaAbsoluteURL] = true
entry.Enclosures = append(entry.Enclosures, &model.Enclosure{
URL: mediaAbsoluteURL,
MimeType: mediaThumbnail.MimeType(),
Size: mediaThumbnail.Size(),
})
}
}
}
for _, link := range atomEntry.Links.findAllLinksWithRelation("enclosure") {
absoluteEnclosureURL, err := urllib.AbsoluteURL(siteURL, link.Href)
if err != nil {
slog.Debug("Unable to resolve absolute URL for enclosure",
slog.String("enclosure_url", link.Href),
slog.String("entry_url", entry.URL),
slog.Any("error", err),
)
} else {
if _, found := uniqueEnclosuresMap[absoluteEnclosureURL]; !found {
uniqueEnclosuresMap[absoluteEnclosureURL] = true
length, _ := strconv.ParseInt(link.Length, 10, 0)
entry.Enclosures = append(entry.Enclosures, &model.Enclosure{
URL: absoluteEnclosureURL,
MimeType: link.Type,
Size: length,
})
}
}
}
for _, mediaContent := range atomEntry.AllMediaContents() {
mediaURL := strings.TrimSpace(mediaContent.URL)
if mediaURL == "" {
continue
}
if mediaAbsoluteURL, err := urllib.AbsoluteURL(siteURL, mediaURL); err != nil {
slog.Debug("Unable to build absolute URL for media content",
slog.String("url", mediaContent.URL),
slog.String("site_url", siteURL),
slog.Any("error", err),
)
} else {
if _, found := uniqueEnclosuresMap[mediaAbsoluteURL]; !found {
uniqueEnclosuresMap[mediaAbsoluteURL] = true
entry.Enclosures = append(entry.Enclosures, &model.Enclosure{
URL: mediaAbsoluteURL,
MimeType: mediaContent.MimeType(),
Size: mediaContent.Size(),
})
}
}
}
for _, mediaPeerLink := range atomEntry.AllMediaPeerLinks() {
mediaURL := strings.TrimSpace(mediaPeerLink.URL)
if mediaURL == "" {
continue
}
if mediaAbsoluteURL, err := urllib.AbsoluteURL(siteURL, mediaURL); err != nil {
slog.Debug("Unable to build absolute URL for media peer link",
slog.String("url", mediaPeerLink.URL),
slog.String("site_url", siteURL),
slog.Any("error", err),
)
} else {
if _, found := uniqueEnclosuresMap[mediaAbsoluteURL]; !found {
uniqueEnclosuresMap[mediaAbsoluteURL] = true
entry.Enclosures = append(entry.Enclosures, &model.Enclosure{
URL: mediaAbsoluteURL,
MimeType: mediaPeerLink.MimeType(),
Size: mediaPeerLink.Size(),
})
}
}
}
entries = append(entries, entry)
}
return entries
}

458
internal/reader/atom/atom_10_test.go
View File

@ -12,7 +12,6 @@ import (
func TestParseAtomSample(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
@ -20,7 +19,6 @@ func TestParseAtomSample(t *testing.T) {
<name>John Doe</name>
</author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
@ -28,10 +26,9 @@ func TestParseAtomSample(t *testing.T) {
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data)))
feed, err := Parse("http://example.org/feed.xml", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -93,7 +90,7 @@ func TestParseFeedWithoutTitle(t *testing.T) {
<updated>2003-12-13T18:30:02Z</updated>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -123,7 +120,7 @@ func TestParseEntryWithoutTitleButWithURL(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -154,7 +151,7 @@ func TestParseEntryWithoutTitleButWithSummary(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -187,7 +184,7 @@ func TestParseEntryWithoutTitleButWithXHTMLContent(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -206,7 +203,7 @@ func TestParseFeedURL(t *testing.T) {
<updated>2003-12-13T18:30:02Z</updated>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -220,12 +217,31 @@ func TestParseFeedURL(t *testing.T) {
}
}
func TestParseFeedWithRelativeURL(t *testing.T) {
func TestParseFeedWithRelativeFeedURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link rel="alternate" type="text/html" href="https://example.org/"/>
<link rel="self" type="application/atom+xml" href="/feed"/>
<updated>2003-12-13T18:30:02Z</updated>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/feed" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
}
func TestParseFeedWithRelativeSiteURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="/blog/atom.xml" rel="self" type="application/atom+xml"/>
<link href="/blog"/>
<link href="/blog "/>
<entry>
<title>Test</title>
@ -238,21 +254,53 @@ func TestParseFeedWithRelativeURL(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/blog/atom.xml" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
t.Errorf("Incorrect feed URL, got: %q", feed.FeedURL)
}
if feed.SiteURL != "https://example.org/blog" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
t.Errorf("Incorrect site URL, got: %q", feed.SiteURL)
}
if feed.Entries[0].URL != "https://example.org/blog/article.html" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
t.Errorf("Incorrect entry URL, got: %q", feed.Entries[0].URL)
}
}
func TestParseFeedSiteURLWithTrailingSpace(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<link href="http://example.org "/>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "http://example.org" {
t.Errorf("Incorrect site URL, got: %q", feed.SiteURL)
}
}
func TestParseFeedWithFeedURLWithTrailingSpace(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<link href="/blog/atom.xml " rel="self" type="application/atom+xml"/>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/blog/atom.xml" {
t.Errorf("Incorrect site URL, got: %q", feed.FeedURL)
}
}
@ -272,7 +320,7 @@ func TestParseEntryWithRelativeURL(t *testing.T) {
</feed>`
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -298,7 +346,7 @@ func TestParseEntryURLWithTextHTMLType(t *testing.T) {
</feed>`
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -324,7 +372,7 @@ func TestParseEntryURLWithNoRelAndNoType(t *testing.T) {
</feed>`
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -350,7 +398,7 @@ func TestParseEntryURLWithAlternateRel(t *testing.T) {
</feed>`
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.net/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -378,7 +426,7 @@ func TestParseEntryTitleWithWhitespaces(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -412,7 +460,7 @@ func TestParseEntryWithPlainTextTitle(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -420,7 +468,7 @@ func TestParseEntryWithPlainTextTitle(t *testing.T) {
expected := `AT&T bought by SBC!`
for i := range 2 {
if feed.Entries[i].Title != expected {
t.Errorf("Incorrect title for entry #%d, got: %q", i, feed.Entries[i].Title)
t.Errorf("Incorrect title for entry #%d, got: %q instead of %q", i, feed.Entries[i].Title, expected)
}
}
}
@ -430,45 +478,32 @@ func TestParseEntryWithHTMLTitle(t *testing.T) {
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<title type="html">&lt;code&gt;Test&lt;/code&gt; Test</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<title type="html">&lt;code&gt;Code&lt;/code&gt; Test</title>
<link href="http://example.org/z"/>
</entry>
<entry>
<title type="html"><![CDATA[Test &#8220;Test&#8221;]]></title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<title type="html"><![CDATA[Test with &#8220;unicode quote&#8221;]]></title>
<link href="http://example.org/b"/>
</entry>
<entry>
<title>
<![CDATA[Entry title with space around CDATA]]>
</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<link href="http://example.org/c"/>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Title != "<code>Test</code> Test" {
if feed.Entries[0].Title != "Code Test" {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title)
}
if feed.Entries[1].Title != "Test “Test”" {
if feed.Entries[1].Title != "Test with “unicode quote”" {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[1].Title)
}
@ -497,13 +532,13 @@ func TestParseEntryWithXHTMLTitle(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Title != `This is <b>XHTML</b> content.` {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[1].Title)
if feed.Entries[0].Title != `This is XHTML content.` {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title)
}
}
@ -524,7 +559,7 @@ func TestParseEntryWithEmptyXHTMLTitle(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -551,7 +586,7 @@ func TestParseEntryWithXHTMLTitleWithoutDiv(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -577,7 +612,7 @@ func TestParseEntryWithNumericCharacterReferenceTitle(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -603,12 +638,12 @@ func TestParseEntryWithDoubleEncodedEntitiesTitle(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Title != `&#39;AT&amp;T&#39;` {
if feed.Entries[0].Title != `'AT&T'` {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title)
}
}
@ -629,7 +664,7 @@ func TestParseEntryWithXHTMLSummary(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -644,39 +679,33 @@ func TestParseEntryWithHTMLSummary(t *testing.T) {
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<title type="html">Example</title>
<title type="html">Example 1</title>
<link href="http://example.org/1"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary type="html">&lt;code&gt;std::unique_ptr&amp;lt;S&amp;gt;&lt;/code&gt;</summary>
<summary type="html">&lt;code&gt;std::unique_ptr&amp;lt;S&amp;gt; myvar;&lt;/code&gt;</summary>
</entry>
<entry>
<title type="html">Example</title>
<title type="html">Example 2</title>
<link href="http://example.org/2"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary type="text/html">&lt;code&gt;std::unique_ptr&amp;lt;S&amp;gt;&lt;/code&gt;</summary>
<summary type="text/html">&lt;code&gt;std::unique_ptr&amp;lt;S&amp;gt; myvar;&lt;/code&gt;</summary>
</entry>
<entry>
<title type="html">Example</title>
<title type="html">Example 3</title>
<link href="http://example.org/3"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary type="html"><![CDATA[<code>std::unique_ptr&lt;S&gt;</code>]]></summary>
<summary type="html"><![CDATA[<code>std::unique_ptr&lt;S&gt; myvar;</code>]]></summary>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
expected := `<code>std::unique_ptr&lt;S&gt;</code>`
if len(feed.Entries) != 3 {
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
expected := `<code>std::unique_ptr&lt;S&gt; myvar;</code>`
for i := range 3 {
if feed.Entries[i].Content != expected {
t.Errorf("Incorrect content for entry #%d, got: %q", i, feed.Entries[i].Content)
@ -723,12 +752,12 @@ func TestParseEntryWithTextSummary(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
expected := `AT&amp;T &lt;S&gt;`
expected := `AT&T <S>`
for i := range 4 {
if feed.Entries[i].Content != expected {
t.Errorf("Incorrect content for entry #%d, got: %q", i, feed.Entries[i].Content)
@ -747,7 +776,7 @@ func TestParseEntryWithTextContent(t *testing.T) {
<link href="http://example.org/a"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<content>AT&amp;T &lt;S&gt;</content>
<content>AT&amp;T &lt;strong&gt;Strong Element&lt;/strong&gt;</content>
</entry>
<entry>
@ -755,7 +784,7 @@ func TestParseEntryWithTextContent(t *testing.T) {
<link href="http://example.org/b"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<content type="text">AT&amp;T &lt;S&gt;</content>
<content type="text">AT&amp;T &lt;strong&gt;Strong Element&lt;/strong&gt;</content>
</entry>
<entry>
@ -763,7 +792,7 @@ func TestParseEntryWithTextContent(t *testing.T) {
<link href="http://example.org/c"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<content type="text/plain">AT&amp;T &lt;S&gt;</content>
<content type="text/plain">AT&amp;T &lt;strong&gt;Strong Element&lt;/strong&gt;</content>
</entry>
<entry>
@ -771,20 +800,20 @@ func TestParseEntryWithTextContent(t *testing.T) {
<link href="http://example.org/d"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<content><![CDATA[AT&T <S>]]></content>
<content><![CDATA[AT&T <strong>Strong Element</strong>]]></content>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
expected := `AT&amp;T &lt;S&gt;`
expected := `AT&T <strong>Strong Element</strong>`
for i := range 4 {
if feed.Entries[i].Content != expected {
t.Errorf("Incorrect content for entry #%d, got: %q", i, feed.Entries[i].Content)
t.Errorf("Incorrect content for entry #%d, got: %q instead of %q", i, feed.Entries[i].Content, expected)
}
}
}
@ -821,7 +850,7 @@ func TestParseEntryWithHTMLContent(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -852,7 +881,7 @@ func TestParseEntryWithXHTMLContent(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -881,7 +910,7 @@ func TestParseEntryWithAuthorName(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -910,7 +939,7 @@ func TestParseEntryWithoutAuthorName(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -925,7 +954,6 @@ func TestParseEntryWithMultipleAuthors(t *testing.T) {
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
@ -938,10 +966,9 @@ func TestParseEntryWithMultipleAuthors(t *testing.T) {
<name>Bob</name>
</author>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -951,7 +978,7 @@ func TestParseEntryWithMultipleAuthors(t *testing.T) {
}
}
func TestParseEntryWithoutAuthor(t *testing.T) {
func TestParseFeedWithEntryWithoutAuthor(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
@ -959,17 +986,15 @@ func TestParseEntryWithoutAuthor(t *testing.T) {
<author>
<name>John Doe</name>
</author>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -990,17 +1015,18 @@ func TestParseFeedWithMultipleAuthors(t *testing.T) {
<author>
<name>Bob</name>
</author>
<author>
<name>Bob</name>
</author>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1015,17 +1041,15 @@ func TestParseFeedWithoutAuthor(t *testing.T) {
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1075,13 +1099,13 @@ func TestParseEntryWithEnclosures(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
@ -1116,6 +1140,89 @@ func TestParseEntryWithEnclosures(t *testing.T) {
}
}
func TestParseEntryWithRelativeEnclosureURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<id>https://www.example.org/myfeed</id>
<title>My Podcast Feed</title>
<link href="https://example.org" />
<link rel="self" href="https://example.org/myfeed" />
<entry>
<id>https://www.example.org/entries/1</id>
<title>Atom 1.0</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="https://www.example.org/entries/1" />
<link rel="enclosure"
type="audio/mpeg"
title="MP3"
href=" /myaudiofile.mp3 "
length="1234" />
</content>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "https://example.org/myaudiofile.mp3" {
t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL)
}
}
func TestParseEntryWithDuplicateEnclosureURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<id>http://www.example.org/myfeed</id>
<title>My Podcast Feed</title>
<link href="http://example.org" />
<link rel="self" href="http://example.org/myfeed" />
<entry>
<id>http://www.example.org/entries/1</id>
<title>Atom 1.0</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="http://www.example.org/entries/1" />
<link rel="enclosure"
type="audio/mpeg"
title="MP3"
href="http://www.example.org/myaudiofile.mp3"
length="1234" />
<link rel="enclosure"
type="audio/mpeg"
title="MP3"
href=" http://www.example.org/myaudiofile.mp3 "
length="1234" />
</content>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL)
}
}
func TestParseEntryWithoutEnclosureURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -1135,7 +1242,7 @@ func TestParseEntryWithoutEnclosureURL(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1168,7 +1275,7 @@ func TestParseEntryWithPublished(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1194,7 +1301,7 @@ func TestParseEntryWithPublishedAndUpdated(t *testing.T) {
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1206,7 +1313,7 @@ func TestParseEntryWithPublishedAndUpdated(t *testing.T) {
func TestParseInvalidXml(t *testing.T) {
data := `garbage`
_, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
_, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err == nil {
t.Error("Parse should returns an error")
}
@ -1221,7 +1328,7 @@ func TestParseTitleWithSingleQuote(t *testing.T) {
</feed>
`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1240,7 +1347,7 @@ func TestParseTitleWithEncodedSingleQuote(t *testing.T) {
</feed>
`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1259,7 +1366,7 @@ func TestParseTitleWithSingleQuoteAndHTMLType(t *testing.T) {
</feed>
`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1278,7 +1385,7 @@ func TestParseWithHTMLEntity(t *testing.T) {
</feed>
`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1297,7 +1404,7 @@ func TestParseWithInvalidCharacterEntity(t *testing.T) {
</feed>
`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1310,44 +1417,41 @@ func TestParseWithInvalidCharacterEntity(t *testing.T) {
func TestParseMediaGroup(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
<id>http://www.example.org/myfeed</id>
<id>https://www.example.org/myfeed</id>
<title>My Video Feed</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="http://example.org" />
<link rel="self" href="http://example.org/myfeed" />
<link href="https://example.org" />
<link rel="self" href="https://example.org/myfeed" />
<entry>
<id>http://www.example.org/entries/1</id>
<id>https://www.example.org/entries/1</id>
<title>Some Video</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="http://www.example.org/entries/1" />
<link href="https://www.example.org/entries/1" />
<media:group>
<media:title>Another title</media:title>
<media:content url="https://www.youtube.com/v/abcd" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://example.org/thumbnail.jpg" width="480" height="360"/>
<media:content url=" /v/efg " type="application/x-shockwave-flash" width="640" height="390"/>
<media:content url=" " type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://www.example.org/duplicate-thumbnail.jpg" width="480" height="360"/>
<media:thumbnail url="https://www.example.org/duplicate-thumbnail.jpg" width="480" height="360"/>
<media:thumbnail url=" /thumbnail2.jpg " width="480" height="360"/>
<media:thumbnail url=" " width="480" height="360"/>
<media:description>Some description
A website: http://example.org/</media:description>
</media:group>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Content != `Some description<br>A website: <a href="http://example.org/">http://example.org/</a>` {
t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content)
}
if len(feed.Entries[0].Enclosures) != 2 {
if len(feed.Entries[0].Enclosures) != 4 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
@ -1356,8 +1460,10 @@ A website: http://example.org/</media:description>
mimeType string
size int64
}{
{"https://example.org/thumbnail.jpg", "image/*", 0},
{"https://www.example.org/duplicate-thumbnail.jpg", "image/*", 0},
{"https://example.org/thumbnail2.jpg", "image/*", 0},
{"https://www.youtube.com/v/abcd", "application/x-shockwave-flash", 0},
{"https://example.org/v/efg", "application/x-shockwave-flash", 0},
}
for index, enclosure := range feed.Entries[0].Enclosures {
@ -1378,42 +1484,41 @@ A website: http://example.org/</media:description>
func TestParseMediaElements(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/">
<id>http://www.example.org/myfeed</id>
<id>https://www.example.org/myfeed</id>
<title>My Video Feed</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="http://example.org" />
<link rel="self" href="http://example.org/myfeed" />
<link href="https://example.org" />
<link rel="self" href="https://example.org/myfeed" />
<entry>
<id>http://www.example.org/entries/1</id>
<id>https://www.example.org/entries/1</id>
<title>Some Video</title>
<updated>2005-07-15T12:00:00Z</updated>
<link href="http://www.example.org/entries/1" />
<link href="https://www.example.org/entries/1" />
<media:title>Another title</media:title>
<media:content url="https://www.youtube.com/v/abcd" type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://example.org/thumbnail.jpg" width="480" height="360"/>
<media:content url=" /relative/media.mp4 " type="application/x-shockwave-flash" width="640" height="390"/>
<media:content url=" " type="application/x-shockwave-flash" width="640" height="390"/>
<media:thumbnail url="https://example.org/duplicated-thumbnail.jpg" width="480" height="360"/>
<media:thumbnail url=" https://example.org/duplicated-thumbnail.jpg " width="480" height="360"/>
<media:thumbnail url=" " width="480" height="360"/>
<media:peerLink type="application/x-bittorrent" href=" http://www.example.org/sampleFile.torrent " />
<media:peerLink type="application/x-bittorrent" href=" /sampleFile2.torrent" />
<media:peerLink type="application/x-bittorrent" href=" " />
<media:description>Some description
A website: http://example.org/</media:description>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Content != `Some description<br>A website: <a href="http://example.org/">http://example.org/</a>` {
t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content)
}
if len(feed.Entries[0].Enclosures) != 2 {
if len(feed.Entries[0].Enclosures) != 5 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
@ -1422,8 +1527,11 @@ A website: http://example.org/</media:description>
mimeType string
size int64
}{
{"https://example.org/thumbnail.jpg", "image/*", 0},
{"https://example.org/duplicated-thumbnail.jpg", "image/*", 0},
{"https://www.youtube.com/v/abcd", "application/x-shockwave-flash", 0},
{"https://example.org/relative/media.mp4", "application/x-shockwave-flash", 0},
{"http://www.example.org/sampleFile.torrent", "application/x-bittorrent", 0},
{"https://example.org/sampleFile2.torrent", "application/x-bittorrent", 0},
}
for index, enclosure := range feed.Entries[0].Enclosures {
@ -1467,7 +1575,7 @@ func TestParseRepliesLinkRelationWithHTMLType(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1511,7 +1619,7 @@ func TestParseRepliesLinkRelationWithXHTMLType(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1550,7 +1658,7 @@ func TestParseRepliesLinkRelationWithNoType(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1590,7 +1698,7 @@ func TestAbsoluteCommentsURL(t *testing.T) {
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
@ -1608,51 +1716,73 @@ func TestAbsoluteCommentsURL(t *testing.T) {
}
}
func TestParseFeedWithCategories(t *testing.T) {
func TestParseItemWithCategories(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<author>
<name>Alice</name>
</author>
<author>
<name>Bob</name>
</author>
<entry>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<link href="http://www.example.org/entries/1" />
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
<category term='Tech' />
<category term='ZZZZ' />
<category term='Technology' label='Science' />
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries[0].Tags) != 2 {
t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags))
t.Fatalf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags))
}
expected := "Tech"
expected := "Science"
result := feed.Entries[0].Tags[0]
if result != expected {
t.Errorf("Incorrect entry category, got %q instead of %q", result, expected)
}
expected = "Science"
expected = "ZZZZ"
result = feed.Entries[0].Tags[1]
if result != expected {
t.Errorf("Incorrect entry category, got %q instead of %q", result, expected)
}
}
func TestParseFeedWithCategories(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<category term='Test' label='Some Label' />
<category term='Test' label='Some Label' />
<category term='Test' label='Some Label' />
<entry>
<link href="http://www.example.org/entries/1" />
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}
if len(feed.Entries[0].Tags) != 1 {
t.Fatalf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags))
}
expected := "Some Label"
result := feed.Entries[0].Tags[0]
if result != expected {
t.Errorf("Incorrect entry category, got %q instead of %q", result, expected)
}
}
func TestParseFeedWithIconURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -1661,7 +1791,7 @@ func TestParseFeedWithIconURL(t *testing.T) {
<icon>http://example.org/icon.png</icon>
</feed>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)), "10")
if err != nil {
t.Fatal(err)
}

126
internal/reader/atom/atom_common.go
View File

@ -3,77 +3,91 @@
package atom // import "miniflux.app/v2/internal/reader/atom"
import "strings"
import (
"strings"
)
type atomPerson struct {
Name string `xml:"name"`
// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-3.2
type AtomPerson struct {
// The "atom:name" element's content conveys a human-readable name for the author.
// It MAY be the name of a corporation or other entity no individual authors can be named.
// Person constructs MUST contain exactly one "atom:name" element, whose content MUST be a string.
Name string `xml:"name"`
// The "atom:email" element's content conveys an e-mail address associated with the Person construct.
// Person constructs MAY contain an atom:email element, but MUST NOT contain more than one.
// Its content MUST be an e-mail address [RFC2822].
// Ordering of the element children of Person constructs MUST NOT be considered significant.
Email string `xml:"email"`
}
func (a *atomPerson) String() string {
name := ""
switch {
case a.Name != "":
name = a.Name
case a.Email != "":
name = a.Email
func (a *AtomPerson) PersonName() string {
name := strings.TrimSpace(a.Name)
if name != "" {
return name
}
return strings.TrimSpace(name)
return strings.TrimSpace(a.Email)
}
type atomAuthors []*atomPerson
type AtomPersons []*AtomPerson
func (a atomAuthors) String() string {
var authors []string
func (a AtomPersons) PersonNames() []string {
var names []string
authorNamesMap := make(map[string]bool)
for _, person := range a {
authors = append(authors, person.String())
personName := person.PersonName()
if _, ok := authorNamesMap[personName]; !ok {
names = append(names, personName)
authorNamesMap[personName] = true
}
}
return strings.Join(authors, ", ")
return names
}
type atomLink struct {
URL string `xml:"href,attr"`
// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-4.2.7
type AtomLink struct {
Href string `xml:"href,attr"`
Type string `xml:"type,attr"`
Rel string `xml:"rel,attr"`
Length string `xml:"length,attr"`
Title string `xml:"title,attr"`
}
type atomLinks []*atomLink
type AtomLinks []*AtomLink
func (a atomLinks) originalLink() string {
func (a AtomLinks) OriginalLink() string {
for _, link := range a {
if strings.EqualFold(link.Rel, "alternate") {
return strings.TrimSpace(link.URL)
return strings.TrimSpace(link.Href)
}
if link.Rel == "" && (link.Type == "" || link.Type == "text/html") {
return strings.TrimSpace(link.URL)
return strings.TrimSpace(link.Href)
}
}
return ""
}
func (a atomLinks) firstLinkWithRelation(relation string) string {
func (a AtomLinks) firstLinkWithRelation(relation string) string {
for _, link := range a {
if strings.EqualFold(link.Rel, relation) {
return strings.TrimSpace(link.URL)
return strings.TrimSpace(link.Href)
}
}
return ""
}
func (a atomLinks) firstLinkWithRelationAndType(relation string, contentTypes ...string) string {
func (a AtomLinks) firstLinkWithRelationAndType(relation string, contentTypes ...string) string {
for _, link := range a {
if strings.EqualFold(link.Rel, relation) {
for _, contentType := range contentTypes {
if strings.EqualFold(link.Type, contentType) {
return strings.TrimSpace(link.URL)
return strings.TrimSpace(link.Href)
}
}
}
@ -81,3 +95,61 @@ func (a atomLinks) firstLinkWithRelationAndType(relation string, contentTypes ..
return ""
}
func (a AtomLinks) findAllLinksWithRelation(relation string) []*AtomLink {
var links []*AtomLink
for _, link := range a {
if strings.EqualFold(link.Rel, relation) {
link.Href = strings.TrimSpace(link.Href)
if link.Href != "" {
links = append(links, link)
}
}
}
return links
}
// The "atom:category" element conveys information about a category
// associated with an entry or feed. This specification assigns no
// meaning to the content (if any) of this element.
//
// Specs: https://datatracker.ietf.org/doc/html/rfc4287#section-4.2.2
type AtomCategory struct {
// The "term" attribute is a string that identifies the category to
// which the entry or feed belongs. Category elements MUST have a
// "term" attribute.
Term string `xml:"term,attr"`
// The "scheme" attribute is an IRI that identifies a categorization
// scheme. Category elements MAY have a "scheme" attribute.
Scheme string `xml:"scheme,attr"`
// The "label" attribute provides a human-readable label for display in
// end-user applications. The content of the "label" attribute is
// Language-Sensitive. Entities such as "&amp;" and "&lt;" represent
// their corresponding characters ("&" and "<", respectively), not
// markup. Category elements MAY have a "label" attribute.
Label string `xml:"label,attr"`
}
type AtomCategories []AtomCategory
func (ac AtomCategories) CategoryNames() []string {
var categories []string
for _, category := range ac {
label := strings.TrimSpace(category.Label)
if label != "" {
categories = append(categories, label)
} else {
term := strings.TrimSpace(category.Term)
if term != "" {
categories = append(categories, term)
}
}
}
return categories
}

50
internal/reader/atom/parser.go
View File

@ -4,7 +4,6 @@
package atom // import "miniflux.app/v2/internal/reader/atom"
import (
"encoding/xml"
"fmt"
"io"
@ -12,45 +11,20 @@ import (
xml_decoder "miniflux.app/v2/internal/reader/xml"
)
type atomFeed interface {
Transform(baseURL string) *model.Feed
}
// Parse returns a normalized feed struct from a Atom feed.
func Parse(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
var rawFeed atomFeed
if getAtomFeedVersion(r) == "0.3" {
rawFeed = new(atom03Feed)
} else {
rawFeed = new(atom10Feed)
}
r.Seek(0, io.SeekStart)
if err := xml_decoder.NewXMLDecoder(r).Decode(rawFeed); err != nil {
return nil, fmt.Errorf("atom: unable to parse feed: %w", err)
}
return rawFeed.Transform(baseURL), nil
}
func getAtomFeedVersion(data io.ReadSeeker) string {
decoder := xml_decoder.NewXMLDecoder(data)
for {
token, _ := decoder.Token()
if token == nil {
break
func Parse(baseURL string, r io.ReadSeeker, version string) (*model.Feed, error) {
switch version {
case "0.3":
atomFeed := new(Atom03Feed)
if err := xml_decoder.NewXMLDecoder(r).Decode(atomFeed); err != nil {
return nil, fmt.Errorf("atom: unable to parse Atom 0.3 feed: %w", err)
}
if element, ok := token.(xml.StartElement); ok {
if element.Name.Local == "feed" {
for _, attr := range element.Attr {
if attr.Name.Local == "version" && attr.Value == "0.3" {
return "0.3"
}
}
return "1.0"
}
return NewAtom03Adapter(atomFeed).BuildFeed(baseURL), nil
default:
atomFeed := new(Atom10Feed)
if err := xml_decoder.NewXMLDecoder(r).Decode(atomFeed); err != nil {
return nil, fmt.Errorf("atom: unable to parse Atom 1.0 feed: %w", err)
}
return NewAtom10Adapter(atomFeed).BuildFeed(baseURL), nil
}
return "1.0"
}

61
internal/reader/atom/parser_test.go
View File

@ -1,61 +0,0 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package atom // import "miniflux.app/v2/internal/reader/atom"
import (
"bytes"
"testing"
)
func TestDetectAtom10(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
<title>Example Feed</title>
<link href="http://example.org/"/>
<updated>2003-12-13T18:30:02Z</updated>
<author>
<name>John Doe</name>
</author>
<id>urn:uuid:60a76c80-d399-11d9-b93C-0003939e0af6</id>
<entry>
<title>Atom-Powered Robots Run Amok</title>
<link href="http://example.org/2003/12/13/atom03"/>
<id>urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a</id>
<updated>2003-12-13T18:30:02Z</updated>
<summary>Some text.</summary>
</entry>
</feed>`
version := getAtomFeedVersion(bytes.NewReader([]byte(data)))
if version != "1.0" {
t.Errorf(`Invalid Atom version detected: %s`, version)
}
}
func TestDetectAtom03(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed version="0.3" xmlns="http://purl.org/atom/ns#">
<title>dive into mark</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/"/>
<modified>2003-12-13T18:30:02Z</modified>
<author><name>Mark Pilgrim</name></author>
<entry>
<title>Atom 0.3 snapshot</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/2003/12/13/atom03"/>
<id>tag:diveintomark.org,2003:3.2397</id>
<issued>2003-12-13T08:29:29-04:00</issued>
<modified>2003-12-13T18:30:02Z</modified>
<summary type="text/plain">This is a test</summary>
<content type="text/html" mode="escaped"><![CDATA[<p>HTML content</p>]]></content>
</entry>
</feed>`
version := getAtomFeedVersion(bytes.NewReader([]byte(data)))
if version != "0.3" {
t.Errorf(`Invalid Atom version detected: %s`, version)
}
}

20
internal/reader/dublincore/dublincore.go
View File

@ -3,29 +3,13 @@
package dublincore // import "miniflux.app/v2/internal/reader/dublincore"
import (
"strings"
"miniflux.app/v2/internal/reader/sanitizer"
)
// DublinCoreFeedElement represents Dublin Core feed XML elements.
type DublinCoreFeedElement struct {
DublinCoreCreator string `xml:"http://purl.org/dc/elements/1.1/ channel>creator"`
type DublinCoreChannelElement struct {
DublinCoreCreator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
}
func (feed *DublinCoreFeedElement) GetSanitizedCreator() string {
return strings.TrimSpace(sanitizer.StripTags(feed.DublinCoreCreator))
}
// DublinCoreItemElement represents Dublin Core entry XML elements.
type DublinCoreItemElement struct {
DublinCoreTitle string `xml:"http://purl.org/dc/elements/1.1/ title"`
DublinCoreDate string `xml:"http://purl.org/dc/elements/1.1/ date"`
DublinCoreCreator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
DublinCoreContent string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
}
func (item *DublinCoreItemElement) GetSanitizedCreator() string {
return strings.TrimSpace(sanitizer.StripTags(item.DublinCoreCreator))
}

5
internal/reader/encoding/encoding.go
View File

@ -35,8 +35,3 @@ func CharsetReader(charsetLabel string, input io.Reader) (io.Reader, error) {
// Transform document to UTF-8 from the specified encoding in XML prolog.
return charset.NewReaderLabel(charsetLabel, r)
}
// CharsetReaderFromContentType is used when the encoding is not specified for the input document.
func CharsetReaderFromContentType(contentType string, input io.Reader) (io.Reader, error) {
return charset.NewReader(input, contentType)
}

52
internal/reader/fetcher/response_handler.go
View File

@ -10,6 +10,8 @@ import (
"io"
"net"
"net/http"
"net/url"
"os"
"miniflux.app/v2/internal/locale"
)
@ -94,23 +96,18 @@ func (r *ResponseHandler) ReadBody(maxBodySize int64) ([]byte, *locale.Localized
func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper {
if r.clientErr != nil {
switch r.clientErr.(type) {
case x509.CertificateInvalidError, x509.HostnameError:
switch {
case isSSLError(r.clientErr):
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.tls_error", r.clientErr)
case *net.OpError:
case isNetworkError(r.clientErr):
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_operation", r.clientErr)
case net.Error:
networkErr := r.clientErr.(net.Error)
if networkErr.Timeout() {
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr)
}
}
if errors.Is(r.clientErr, io.EOF) {
case os.IsTimeout(r.clientErr):
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.network_timeout", r.clientErr)
case errors.Is(r.clientErr, io.EOF):
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_empty_response")
default:
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr)
}
return locale.NewLocalizedErrorWrapper(fmt.Errorf("fetcher: %w", r.clientErr), "error.http_client_error", r.clientErr)
}
switch r.httpResponse.StatusCode {
@ -145,3 +142,32 @@ func (r *ResponseHandler) LocalizedError() *locale.LocalizedErrorWrapper {
return nil
}
func isNetworkError(err error) bool {
if _, ok := err.(*url.Error); ok {
return true
}
if err == io.EOF {
return true
}
var opErr *net.OpError
if ok := errors.As(err, &opErr); ok {
return true
}
return false
}
func isSSLError(err error) bool {
var certErr x509.UnknownAuthorityError
if errors.As(err, &certErr) {
return true
}
var hostErr x509.HostnameError
if errors.As(err, &hostErr) {
return true
}
var algErr x509.InsecureAlgorithmError
return errors.As(err, &algErr)
}

2
internal/reader/googleplay/googleplay.go
View File

@ -6,7 +6,7 @@ package googleplay // import "miniflux.app/v2/internal/reader/googleplay"
// Specs:
// https://support.google.com/googleplay/podcasts/answer/6260341
// https://www.google.com/schemas/play-podcasts/1.0/play-podcasts.xsd
type GooglePlayFeedElement struct {
type GooglePlayChannelElement struct {
GooglePlayAuthor string `xml:"http://www.google.com/schemas/play-podcasts/1.0 author"`
GooglePlayEmail string `xml:"http://www.google.com/schemas/play-podcasts/1.0 email"`
GooglePlayImage GooglePlayImageElement `xml:"http://www.google.com/schemas/play-podcasts/1.0 image"`

10
internal/reader/handler/handler.go
View File

@ -236,14 +236,18 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
requestBuilder.WithUsernameAndPassword(originalFeed.Username, originalFeed.Password)
requestBuilder.WithUserAgent(originalFeed.UserAgent, config.Opts.HTTPClientUserAgent())
requestBuilder.WithCookie(originalFeed.Cookie)
requestBuilder.WithETag(originalFeed.EtagHeader)
requestBuilder.WithLastModified(originalFeed.LastModifiedHeader)
requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
requestBuilder.UseProxy(originalFeed.FetchViaProxy)
requestBuilder.IgnoreTLSErrors(originalFeed.AllowSelfSignedCertificates)
requestBuilder.DisableHTTP2(originalFeed.DisableHTTP2)
ignoreHTTPCache := originalFeed.IgnoreHTTPCache || forceRefresh
if !ignoreHTTPCache {
requestBuilder.WithETag(originalFeed.EtagHeader)
requestBuilder.WithLastModified(originalFeed.LastModifiedHeader)
}
responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(originalFeed.FeedURL))
defer responseHandler.Close()
@ -261,7 +265,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
return localizedError
}
if originalFeed.IgnoreHTTPCache || responseHandler.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) {
if ignoreHTTPCache || responseHandler.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) {
slog.Debug("Feed modified",
slog.Int64("user_id", userID),
slog.Int64("feed_id", feedID),

4
internal/reader/icon/finder.go
View File

@ -15,11 +15,11 @@ import (
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/encoding"
"miniflux.app/v2/internal/reader/fetcher"
"miniflux.app/v2/internal/urllib"
"github.com/PuerkitoBio/goquery"
"golang.org/x/net/html/charset"
)
type IconFinder struct {
@ -191,7 +191,7 @@ func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string,
"link[rel='apple-touch-icon-precomposed.png']",
}
htmlDocumentReader, err := encoding.CharsetReaderFromContentType(contentType, body)
htmlDocumentReader, err := charset.NewReader(body, contentType)
if err != nil {
return nil, fmt.Errorf("icon: unable to create charset reader: %w", err)
}

4
internal/reader/itunes/itunes.go
View File

@ -6,7 +6,7 @@ package itunes // import "miniflux.app/v2/internal/reader/itunes"
import "strings"
// Specs: https://help.apple.com/itc/podcasts_connect/#/itcb54353390
type ItunesFeedElement struct {
type ItunesChannelElement struct {
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd author"`
ItunesBlock string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd block"`
ItunesCategories []ItunesCategoryElement `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd category"`
@ -22,7 +22,7 @@ type ItunesFeedElement struct {
ItunesType string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd type"`
}
func (i *ItunesFeedElement) GetItunesCategories() []string {
func (i *ItunesChannelElement) GetItunesCategories() []string {
var categories []string
for _, category := range i.ItunesCategories {
categories = append(categories, category.Text)

172
internal/reader/json/adapter.go Normal file
View File

@ -0,0 +1,172 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package json // import "miniflux.app/v2/internal/reader/json"
import (
"log/slog"
"slices"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
type JSONAdapter struct {
jsonFeed *JSONFeed
}
func NewJSONAdapter(jsonFeed *JSONFeed) *JSONAdapter {
return &JSONAdapter{jsonFeed}
}
func (j *JSONAdapter) BuildFeed(baseURL string) *model.Feed {
feed := &model.Feed{
Title: strings.TrimSpace(j.jsonFeed.Title),
FeedURL: strings.TrimSpace(j.jsonFeed.FeedURL),
SiteURL: strings.TrimSpace(j.jsonFeed.HomePageURL),
}
if feed.FeedURL == "" {
feed.FeedURL = strings.TrimSpace(baseURL)
}
// Fallback to the feed URL if the site URL is empty.
if feed.SiteURL == "" {
feed.SiteURL = feed.FeedURL
}
if feedURL, err := urllib.AbsoluteURL(baseURL, feed.FeedURL); err == nil {
feed.FeedURL = feedURL
}
if siteURL, err := urllib.AbsoluteURL(baseURL, feed.SiteURL); err == nil {
feed.SiteURL = siteURL
}
// Fallback to the feed URL if the title is empty.
if feed.Title == "" {
feed.Title = feed.SiteURL
}
// Populate the icon URL if present.
for _, iconURL := range []string{j.jsonFeed.FaviconURL, j.jsonFeed.IconURL} {
iconURL = strings.TrimSpace(iconURL)
if iconURL != "" {
if absoluteIconURL, err := urllib.AbsoluteURL(feed.SiteURL, iconURL); err == nil {
feed.IconURL = absoluteIconURL
break
}
}
}
for _, item := range j.jsonFeed.Items {
entry := model.NewEntry()
entry.Title = strings.TrimSpace(item.Title)
entry.URL = strings.TrimSpace(item.URL)
// Make sure the entry URL is absolute.
if entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL); err == nil {
entry.URL = entryURL
}
// The entry title is optional, so we need to find a fallback.
if entry.Title == "" {
for _, value := range []string{item.Summary, item.ContentText, item.ContentHTML} {
if value != "" {
entry.Title = sanitizer.TruncateHTML(value, 100)
}
}
}
// Fallback to the entry URL if the title is empty.
if entry.Title == "" {
entry.Title = entry.URL
}
// Populate the entry content.
for _, value := range []string{item.ContentHTML, item.ContentText, item.Summary} {
value = strings.TrimSpace(value)
if value != "" {
entry.Content = value
break
}
}
// Populate the entry date.
for _, value := range []string{item.DatePublished, item.DateModified} {
value = strings.TrimSpace(value)
if value != "" {
if date, err := date.Parse(value); err != nil {
slog.Debug("Unable to parse date from JSON feed",
slog.String("date", value),
slog.String("url", entry.URL),
slog.Any("error", err),
)
} else {
entry.Date = date
break
}
}
}
if entry.Date.IsZero() {
entry.Date = time.Now()
}
// Populate the entry author.
itemAuthors := j.jsonFeed.Authors
itemAuthors = append(itemAuthors, item.Authors...)
itemAuthors = append(itemAuthors, item.Author, j.jsonFeed.Author)
var authorNames []string
for _, author := range itemAuthors {
authorName := strings.TrimSpace(author.Name)
if authorName != "" {
authorNames = append(authorNames, authorName)
}
}
slices.Sort(authorNames)
authorNames = slices.Compact(authorNames)
entry.Author = strings.Join(authorNames, ", ")
// Populate the entry enclosures.
for _, attachment := range item.Attachments {
attachmentURL := strings.TrimSpace(attachment.URL)
if attachmentURL != "" {
if absoluteAttachmentURL, err := urllib.AbsoluteURL(feed.SiteURL, attachmentURL); err == nil {
entry.Enclosures = append(entry.Enclosures, &model.Enclosure{
URL: absoluteAttachmentURL,
MimeType: attachment.MimeType,
Size: attachment.Size,
})
}
}
}
// Populate the entry tags.
for _, tag := range item.Tags {
tag = strings.TrimSpace(tag)
if tag != "" {
entry.Tags = append(entry.Tags, tag)
}
}
// Generate a hash for the entry.
for _, value := range []string{item.ID, item.URL, item.ContentText + item.ContentHTML + item.Summary} {
value = strings.TrimSpace(value)
if value != "" {
entry.Hash = crypto.Hash(value)
break
}
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}

320
internal/reader/json/json.go
View File

@ -3,207 +3,141 @@
package json // import "miniflux.app/v2/internal/reader/json"
import (
"log/slog"
"strings"
"time"
// JSON Feed specs:
// https://www.jsonfeed.org/version/1.1/
// https://www.jsonfeed.org/version/1/
type JSONFeed struct {
// Version is the URL of the version of the format the feed uses.
// This should appear at the very top, though we recognize that not all JSON generators allow for ordering.
Version string `json:"version"`
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
// Title is the name of the feed, which will often correspond to the name of the website.
Title string `json:"title"`
type jsonFeed struct {
Version string `json:"version"`
Title string `json:"title"`
SiteURL string `json:"home_page_url"`
IconURL string `json:"icon"`
FaviconURL string `json:"favicon"`
FeedURL string `json:"feed_url"`
Authors []jsonAuthor `json:"authors"`
Author jsonAuthor `json:"author"`
Items []jsonItem `json:"items"`
// HomePageURL is the URL of the resource that the feed describes.
// This resource may or may not actually be a “home” page, but it should be an HTML page.
HomePageURL string `json:"home_page_url"`
// FeedURL is the URL of the feed, and serves as the unique identifier for the feed.
FeedURL string `json:"feed_url"`
// Description provides more detail, beyond the title, on what the feed is about.
Description string `json:"description"`
// IconURL is the URL of an image for the feed suitable to be used in a timeline, much the way an avatar might be used.
IconURL string `json:"icon"`
// FaviconURL is the URL of an image for the feed suitable to be used in a source list. It should be square and relatively small.
FaviconURL string `json:"favicon"`
// Authors specifies one or more feed authors. The author object has several members.
Authors []JSONAuthor `json:"authors"` // JSON Feed v1.1
// Author specifies the feed author. The author object has several members.
// JSON Feed v1 (deprecated)
Author JSONAuthor `json:"author"`
// Language is the primary language for the feed in the format specified in RFC 5646.
// The value is usually a 2-letter language tag from ISO 639-1, optionally followed by a region tag. (Examples: en or en-US.)
Language string `json:"language"`
// Expired is a boolean value that specifies whether or not the feed is finished.
Expired bool `json:"expired"`
// Items is an array, each representing an individual item in the feed.
Items []JSONItem `json:"items"`
// Hubs describes endpoints that can be used to subscribe to real-time notifications from the publisher of this feed.
Hubs []JSONHub `json:"hubs"`
}
type jsonAuthor struct {
type JSONAuthor struct {
// Author's name.
Name string `json:"name"`
URL string `json:"url"`
// Author's website URL (Blog or micro-blog).
WebsiteURL string `json:"url"`
// Author's avatar URL.
AvatarURL string `json:"avatar"`
}
type jsonItem struct {
ID string `json:"id"`
URL string `json:"url"`
Title string `json:"title"`
Summary string `json:"summary"`
Text string `json:"content_text"`
HTML string `json:"content_html"`
DatePublished string `json:"date_published"`
DateModified string `json:"date_modified"`
Authors []jsonAuthor `json:"authors"`
Author jsonAuthor `json:"author"`
Attachments []jsonAttachment `json:"attachments"`
Tags []string `json:"tags"`
type JSONHub struct {
// Type defines the protocol used to talk with the hub: "rssCloud" or "WebSub".
Type string `json:"type"`
// URL is the location of the hub.
URL string `json:"url"`
}
type jsonAttachment struct {
URL string `json:"url"`
type JSONItem struct {
// Unique identifier for the item.
// Ideally, the id is the full URL of the resource described by the item, since URLs make great unique identifiers.
ID string `json:"id"`
// URL of the resource described by the item.
URL string `json:"url"`
// ExternalURL is the URL of a page elsewhere.
// This is especially useful for linkblogs.
// If url links to where you’re talking about a thing, then external_url links to the thing you’re talking about.
ExternalURL string `json:"external_url"`
// Title of the item (optional).
// Microblog items in particular may omit titles.
Title string `json:"title"`
// ContentHTML is the HTML body of the item.
ContentHTML string `json:"content_html"`
// ContentText is the text body of the item.
ContentText string `json:"content_text"`
// Summary is a plain text sentence or two describing the item.
Summary string `json:"summary"`
// ImageURL is the URL of the main image for the item.
ImageURL string `json:"image"`
// BannerImageURL is the URL of an image to use as a banner.
BannerImageURL string `json:"banner_image"`
// DatePublished is the date the item was published.
DatePublished string `json:"date_published"`
// DateModified is the date the item was modified.
DateModified string `json:"date_modified"`
// Language is the language of the item.
Language string `json:"language"`
// Authors is an array of JSONAuthor.
Authors []JSONAuthor `json:"authors"`
// Author is a JSONAuthor.
// JSON Feed v1 (deprecated)
Author JSONAuthor `json:"author"`
// Tags is an array of strings.
Tags []string `json:"tags"`
// Attachments is an array of JSONAttachment.
Attachments []JSONAttachment `json:"attachments"`
}
type JSONAttachment struct {
// URL of the attachment.
URL string `json:"url"`
// MIME type of the attachment.
MimeType string `json:"mime_type"`
Title string `json:"title"`
Size int64 `json:"size_in_bytes"`
Duration int `json:"duration_in_seconds"`
}
func (j *jsonFeed) GetAuthor() string {
if len(j.Authors) > 0 {
return (getAuthor(j.Authors[0]))
}
return getAuthor(j.Author)
}
func (j *jsonFeed) Transform(baseURL string) *model.Feed {
var err error
feed := new(model.Feed)
feed.FeedURL, err = urllib.AbsoluteURL(baseURL, j.FeedURL)
if err != nil {
feed.FeedURL = j.FeedURL
}
feed.SiteURL, err = urllib.AbsoluteURL(baseURL, j.SiteURL)
if err != nil {
feed.SiteURL = j.SiteURL
}
feed.IconURL = strings.TrimSpace(j.IconURL)
if feed.IconURL == "" {
feed.IconURL = strings.TrimSpace(j.FaviconURL)
}
feed.Title = strings.TrimSpace(j.Title)
if feed.Title == "" {
feed.Title = feed.SiteURL
}
for _, item := range j.Items {
entry := item.Transform()
entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL)
if err == nil {
entry.URL = entryURL
}
if entry.Author == "" {
entry.Author = j.GetAuthor()
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}
func (j *jsonItem) GetDate() time.Time {
for _, value := range []string{j.DatePublished, j.DateModified} {
if value != "" {
d, err := date.Parse(value)
if err != nil {
slog.Debug("Unable to parse date from JSON feed",
slog.String("date", value),
slog.String("url", j.URL),
slog.Any("error", err),
)
return time.Now()
}
return d
}
}
return time.Now()
}
func (j *jsonItem) GetAuthor() string {
if len(j.Authors) > 0 {
return getAuthor(j.Authors[0])
}
return getAuthor(j.Author)
}
func (j *jsonItem) GetHash() string {
for _, value := range []string{j.ID, j.URL, j.Text + j.HTML + j.Summary} {
if value != "" {
return crypto.Hash(value)
}
}
return ""
}
func (j *jsonItem) GetTitle() string {
if j.Title != "" {
return j.Title
}
for _, value := range []string{j.Summary, j.Text, j.HTML} {
if value != "" {
return sanitizer.TruncateHTML(value, 100)
}
}
return j.URL
}
func (j *jsonItem) GetContent() string {
for _, value := range []string{j.HTML, j.Text, j.Summary} {
if value != "" {
return value
}
}
return ""
}
func (j *jsonItem) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
for _, attachment := range j.Attachments {
if attachment.URL == "" {
continue
}
enclosures = append(enclosures, &model.Enclosure{
URL: attachment.URL,
MimeType: attachment.MimeType,
Size: attachment.Size,
})
}
return enclosures
}
func (j *jsonItem) Transform() *model.Entry {
entry := model.NewEntry()
entry.URL = j.URL
entry.Date = j.GetDate()
entry.Author = j.GetAuthor()
entry.Hash = j.GetHash()
entry.Content = j.GetContent()
entry.Title = strings.TrimSpace(j.GetTitle())
entry.Enclosures = j.GetEnclosures()
if len(j.Tags) > 0 {
entry.Tags = j.Tags
}
return entry
}
func getAuthor(author jsonAuthor) string {
if author.Name != "" {
return strings.TrimSpace(author.Name)
}
return ""
// Title of the attachment.
Title string `json:"title"`
// Size of the attachment in bytes.
Size int64 `json:"size_in_bytes"`
// Duration of the attachment in seconds.
Duration int `json:"duration_in_seconds"`
}

6
internal/reader/json/parser.go
View File

@ -13,10 +13,10 @@ import (
// Parse returns a normalized feed struct from a JSON feed.
func Parse(baseURL string, data io.Reader) (*model.Feed, error) {
feed := new(jsonFeed)
if err := json.NewDecoder(data).Decode(&feed); err != nil {
jsonFeed := new(JSONFeed)
if err := json.NewDecoder(data).Decode(&jsonFeed); err != nil {
return nil, fmt.Errorf("json: unable to parse feed: %w", err)
}
return feed.Transform(baseURL), nil
return NewJSONAdapter(jsonFeed).BuildFeed(baseURL), nil
}

353
internal/reader/json/parser_test.go
View File

@ -10,7 +10,7 @@ import (
"time"
)
func TestParseJsonFeed(t *testing.T) {
func TestParseJsonFeedVersion1(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -49,7 +49,7 @@ func TestParseJsonFeed(t *testing.T) {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
if feed.IconURL != "https://micro.blog/jsonfeed/avatar.jpg" {
if feed.IconURL != "https://micro.blog/jsonfeed/favicon.png" {
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
}
@ -177,7 +177,157 @@ func TestParsePodcast(t *testing.T) {
}
}
func TestParseEntryWithoutAttachmentURL(t *testing.T) {
func TestParseFeedWithFeedURLWithTrailingSpace(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json ",
"items": []
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/feed.json" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
}
func TestParseFeedWithRelativeFeedURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "/feed.json",
"items": []
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/feed.json" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
}
func TestParseFeedSiteURLWithTrailingSpace(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/ ",
"feed_url": "https://example.org/feed.json",
"items": []
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "https://example.org/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
}
func TestParseFeedWithRelativeSiteURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "/home ",
"feed_url": "https://example.org/feed.json",
"items": []
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "https://example.org/home" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
}
func TestParseFeedWithoutTitle(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00"
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.Title != "https://example.org/" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseFeedWithoutHomePage(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"feed_url": "https://example.org/feed.json",
"title": "Some test",
"items": [
{
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00"
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "https://example.org/feed.json" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseFeedWithoutFeedURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "Some test",
"items": [
{
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00"
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "https://example.org/feed.json" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseItemWithoutAttachmentURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"user_comment": "This is a podcast feed. You can add this feed to your podcast client using the following URL: http://therecord.co/feed.json",
@ -216,7 +366,7 @@ func TestParseEntryWithoutAttachmentURL(t *testing.T) {
}
}
func TestParseFeedWithRelativeURL(t *testing.T) {
func TestParseItemWithRelativeURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "Example",
@ -241,7 +391,7 @@ func TestParseFeedWithRelativeURL(t *testing.T) {
}
}
func TestParseAuthor(t *testing.T) {
func TestParseItemWithLegacyAuthorField(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
@ -277,7 +427,7 @@ func TestParseAuthor(t *testing.T) {
}
}
func TestParseAuthors(t *testing.T) {
func TestParseItemWithMultipleAuthorFields(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1.1",
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
@ -285,11 +435,11 @@ func TestParseAuthors(t *testing.T) {
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"author": {
"name": "This field is deprecated, use authors",
"name": "Deprecated Author Field",
"url": "http://example.org/",
"avatar": "https://example.org/avatar.png"
},
"authors": [
"authors": [
{
"name": "Brent Simmons",
"url": "http://example.org/",
@ -315,14 +465,15 @@ func TestParseAuthors(t *testing.T) {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Author != "Brent Simmons" {
if feed.Entries[0].Author != "Brent Simmons, Deprecated Author Field" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseFeedWithoutTitle(t *testing.T) {
func TestParseItemWithMultipleDuplicateAuthors(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"version": "https://jsonfeed.org/version/1.1",
"title": "Example",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
@ -330,7 +481,24 @@ func TestParseFeedWithoutTitle(t *testing.T) {
"id": "2347259",
"url": "https://example.org/2347259",
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00"
"date_published": "2016-02-09T14:22:00-07:00",
"authors": [
{
"name": "Author B",
"url": "http://example.org/",
"avatar": "https://example.org/avatar.png"
},
{
"name": "Author A",
"url": "http://example.org/",
"avatar": "https://example.org/avatar.png"
},
{
"name": "Author B",
"url": "http://example.org/",
"avatar": "https://example.org/avatar.png"
}
]
}
]
}`
@ -340,12 +508,16 @@ func TestParseFeedWithoutTitle(t *testing.T) {
t.Fatal(err)
}
if feed.Title != "https://example.org/" {
t.Errorf("Incorrect title, got: %s", feed.Title)
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Author != "Author A, Author B" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseFeedItemWithInvalidDate(t *testing.T) {
func TestParseItemWithInvalidDate(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -376,34 +548,7 @@ func TestParseFeedItemWithInvalidDate(t *testing.T) {
}
}
func TestParseFeedItemWithoutID(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"content_text": "Some text."
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
}
func TestParseFeedItemWithoutTitleButWithURL(t *testing.T) {
func TestParseItemWithoutTitleButWithURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -430,7 +575,7 @@ func TestParseFeedItemWithoutTitleButWithURL(t *testing.T) {
}
}
func TestParseFeedItemWithoutTitleButWithSummary(t *testing.T) {
func TestParseItemWithoutTitleButWithSummary(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -457,7 +602,7 @@ func TestParseFeedItemWithoutTitleButWithSummary(t *testing.T) {
}
}
func TestParseFeedItemWithoutTitleButWithHTMLContent(t *testing.T) {
func TestParseItemWithoutTitleButWithHTMLContent(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -484,7 +629,7 @@ func TestParseFeedItemWithoutTitleButWithHTMLContent(t *testing.T) {
}
}
func TestParseFeedItemWithoutTitleButWithTextContent(t *testing.T) {
func TestParseItemWithoutTitleButWithTextContent(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -515,7 +660,7 @@ func TestParseFeedItemWithoutTitleButWithTextContent(t *testing.T) {
}
}
func TestParseTruncateItemTitleUnicode(t *testing.T) {
func TestParseItemWithTooLongUnicodeTitle(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
@ -573,15 +718,34 @@ func TestParseItemTitleWithXMLTags(t *testing.T) {
}
}
func TestParseInvalidJSON(t *testing.T) {
data := `garbage`
_, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err == nil {
t.Error("Parse should returns an error")
func TestParseItemWithoutID(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"content_text": "Some text."
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].Hash != "13b4c5aecd1b6d749afcee968fbf9c80f1ed1bbdbe1aaf25cb34ebd01144bbe9" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
}
func TestParseTags(t *testing.T) {
func TestParseItemTags(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"user_comment": "This is a microblog feed. You can add this to your feed reader using the following URL: https://example.org/feed.json",
@ -600,7 +764,8 @@ func TestParseTags(t *testing.T) {
"content_text": "Cats are neat. \n\nhttps://example.org/cats",
"date_published": "2016-02-09T14:22:00-07:00",
"tags": [
"tag 1",
" tag 1",
" ",
"tag 2"
]
}
@ -623,11 +788,11 @@ func TestParseTags(t *testing.T) {
}
}
func TestParseFavicon(t *testing.T) {
func TestParseFeedFavicon(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"favicon": "https://micro.blog/jsonfeed/favicon.png",
"favicon": "https://example.org/jsonfeed/favicon.png",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
@ -648,7 +813,81 @@ func TestParseFavicon(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if feed.IconURL != "https://micro.blog/jsonfeed/favicon.png" {
if feed.IconURL != "https://example.org/jsonfeed/favicon.png" {
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
}
}
func TestParseFeedIcon(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"icon": "https://example.org/jsonfeed/icon.png",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2",
"content_text": "This is a second item.",
"url": "https://example.org/second-item"
},
{
"id": "1",
"content_html": "<p>Hello, world!</p>",
"url": "https://example.org/initial-post"
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if feed.IconURL != "https://example.org/jsonfeed/icon.png" {
t.Errorf("Incorrect icon URL, got: %s", feed.IconURL)
}
}
func TestParseFeedWithRelativeAttachmentURL(t *testing.T) {
data := `{
"version": "https://jsonfeed.org/version/1",
"title": "My Example Feed",
"home_page_url": "https://example.org/",
"feed_url": "https://example.org/feed.json",
"items": [
{
"id": "2",
"content_text": "This is a second item.",
"url": "https://example.org/second-item",
"attachments": [
{
"url": " /attachment.mp3 ",
"mime_type": "audio/mpeg",
"size_in_bytes": 123456
}
]
}
]
}`
feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "https://example.org/attachment.mp3" {
t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL)
}
}
func TestParseInvalidJSON(t *testing.T) {
data := `garbage`
_, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data))
if err == nil {
t.Error("Parse should returns an error")
}
}

62
internal/reader/media/media.go
View File

@ -11,18 +11,18 @@ import (
var textLinkRegex = regexp.MustCompile(`(?mi)(\bhttps?:\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])`)
// Element represents XML media elements.
// Specs: https://www.rssboard.org/media-rss
type Element struct {
MediaGroups []Group `xml:"http://search.yahoo.com/mrss/ group"`
MediaContents []Content `xml:"http://search.yahoo.com/mrss/ content"`
MediaThumbnails []Thumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"`
MediaDescriptions DescriptionList `xml:"http://search.yahoo.com/mrss/ description"`
MediaPeerLinks []PeerLink `xml:"http://search.yahoo.com/mrss/ peerLink"`
type MediaItemElement struct {
MediaCategories MediaCategoryList `xml:"http://search.yahoo.com/mrss/ category"`
MediaGroups []Group `xml:"http://search.yahoo.com/mrss/ group"`
MediaContents []Content `xml:"http://search.yahoo.com/mrss/ content"`
MediaThumbnails []Thumbnail `xml:"http://search.yahoo.com/mrss/ thumbnail"`
MediaDescriptions DescriptionList `xml:"http://search.yahoo.com/mrss/ description"`
MediaPeerLinks []PeerLink `xml:"http://search.yahoo.com/mrss/ peerLink"`
}
// AllMediaThumbnails returns all thumbnail elements merged together.
func (e *Element) AllMediaThumbnails() []Thumbnail {
func (e *MediaItemElement) AllMediaThumbnails() []Thumbnail {
var items []Thumbnail
items = append(items, e.MediaThumbnails...)
for _, mediaGroup := range e.MediaGroups {
@ -32,7 +32,7 @@ func (e *Element) AllMediaThumbnails() []Thumbnail {
}
// AllMediaContents returns all content elements merged together.
func (e *Element) AllMediaContents() []Content {
func (e *MediaItemElement) AllMediaContents() []Content {
var items []Content
items = append(items, e.MediaContents...)
for _, mediaGroup := range e.MediaGroups {
@ -42,7 +42,7 @@ func (e *Element) AllMediaContents() []Content {
}
// AllMediaPeerLinks returns all peer link elements merged together.
func (e *Element) AllMediaPeerLinks() []PeerLink {
func (e *MediaItemElement) AllMediaPeerLinks() []PeerLink {
var items []PeerLink
items = append(items, e.MediaPeerLinks...)
for _, mediaGroup := range e.MediaGroups {
@ -52,7 +52,7 @@ func (e *Element) AllMediaPeerLinks() []PeerLink {
}
// FirstMediaDescription returns the first description element.
func (e *Element) FirstMediaDescription() string {
func (e *MediaItemElement) FirstMediaDescription() string {
description := e.MediaDescriptions.First()
if description != "" {
return description
@ -86,17 +86,17 @@ type Content struct {
// MimeType returns the attachment mime type.
func (mc *Content) MimeType() string {
switch {
case mc.Type == "" && mc.Medium == "image":
return "image/*"
case mc.Type == "" && mc.Medium == "video":
return "video/*"
case mc.Type == "" && mc.Medium == "audio":
return "audio/*"
case mc.Type == "" && mc.Medium == "video":
return "video/*"
case mc.Type != "":
if mc.Type != "" {
return mc.Type
}
switch mc.Medium {
case "image":
return "image/*"
case "video":
return "video/*"
case "audio":
return "audio/*"
default:
return "application/octet-stream"
}
@ -104,9 +104,6 @@ func (mc *Content) MimeType() string {
// Size returns the attachment size.
func (mc *Content) Size() int64 {
if mc.FileSize == "" {
return 0
}
size, _ := strconv.ParseInt(mc.FileSize, 10, 0)
return size
}
@ -174,3 +171,20 @@ func (dl DescriptionList) First() string {
}
return ""
}
type MediaCategoryList []MediaCategory
func (mcl MediaCategoryList) Labels() []string {
var labels []string
for _, category := range mcl {
label := strings.TrimSpace(category.Label)
if label != "" {
labels = append(labels, label)
}
}
return labels
}
type MediaCategory struct {
Label string `xml:"label,attr"`
}

17
internal/reader/parser/format.go
View File

@ -21,12 +21,12 @@ const (
)
// DetectFeedFormat tries to guess the feed format from input data.
func DetectFeedFormat(r io.ReadSeeker) string {
func DetectFeedFormat(r io.ReadSeeker) (string, string) {
data := make([]byte, 512)
r.Read(data)
if bytes.HasPrefix(bytes.TrimSpace(data), []byte("{")) {
return FormatJSON
return FormatJSON, ""
}
r.Seek(0, io.SeekStart)
@ -41,14 +41,19 @@ func DetectFeedFormat(r io.ReadSeeker) string {
if element, ok := token.(xml.StartElement); ok {
switch element.Name.Local {
case "rss":
return FormatRSS
return FormatRSS, ""
case "feed":
return FormatAtom
for _, attr := range element.Attr {
if attr.Name.Local == "version" && attr.Value == "0.3" {
return FormatAtom, "0.3"
}
}
return FormatAtom, "1.0"
case "RDF":
return FormatRDF
return FormatRDF, ""
}
}
}
return FormatUnknown
return FormatUnknown, ""
}

14
internal/reader/parser/format_test.go
View File

@ -10,7 +10,7 @@ import (
func TestDetectRDF(t *testing.T) {
data := `<?xml version="1.0"?><rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://my.netscape.com/rdf/simple/0.9/"></rdf:RDF>`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatRDF {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatRDF)
@ -19,7 +19,7 @@ func TestDetectRDF(t *testing.T) {
func TestDetectRSS(t *testing.T) {
data := `<?xml version="1.0"?><rss version="2.0"><channel></channel></rss>`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatRSS {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatRSS)
@ -28,7 +28,7 @@ func TestDetectRSS(t *testing.T) {
func TestDetectAtom10(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatAtom {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom)
@ -37,7 +37,7 @@ func TestDetectAtom10(t *testing.T) {
func TestDetectAtom03(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?><feed version="0.3" xmlns="http://purl.org/atom/ns#" xmlns:dc="http://purl.org/dc/elements/1.1/" xml:lang="en"></feed>`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatAtom {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom)
@ -46,7 +46,7 @@ func TestDetectAtom03(t *testing.T) {
func TestDetectAtomWithISOCharset(t *testing.T) {
data := `<?xml version="1.0" encoding="ISO-8859-15"?><feed xmlns="http://www.w3.org/2005/Atom"></feed>`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatAtom {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatAtom)
@ -60,7 +60,7 @@ func TestDetectJSON(t *testing.T) {
"title" : "Example"
}
`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatJSON {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatJSON)
@ -71,7 +71,7 @@ func TestDetectUnknown(t *testing.T) {
data := `
<!DOCTYPE html> <html> </html>
`
format := DetectFeedFormat(strings.NewReader(data))
format, _ := DetectFeedFormat(strings.NewReader(data))
if format != FormatUnknown {
t.Errorf(`Wrong format detected: %q instead of %q`, format, FormatUnknown)

5
internal/reader/parser/parser.go
View File

@ -19,10 +19,11 @@ var ErrFeedFormatNotDetected = errors.New("parser: unable to detect feed format"
// ParseFeed analyzes the input data and returns a normalized feed object.
func ParseFeed(baseURL string, r io.ReadSeeker) (*model.Feed, error) {
r.Seek(0, io.SeekStart)
switch DetectFeedFormat(r) {
format, version := DetectFeedFormat(r)
switch format {
case FormatAtom:
r.Seek(0, io.SeekStart)
return atom.Parse(baseURL, r)
return atom.Parse(baseURL, r, version)
case FormatRSS:
r.Seek(0, io.SeekStart)
return rss.Parse(baseURL, r)

51
internal/reader/parser/parser_test.go
View File

@ -4,10 +4,31 @@
package parser // import "miniflux.app/v2/internal/reader/parser"
import (
"os"
"strings"
"testing"
)
func BenchmarkParse(b *testing.B) {
var testCases = map[string][]string{
"large_atom.xml": {"https://dustri.org/b", ""},
"large_rss.xml": {"https://dustri.org/b", ""},
"small_atom.xml": {"https://github.com/miniflux/v2/commits/main", ""},
}
for filename := range testCases {
data, err := os.ReadFile("./testdata/" + filename)
if err != nil {
b.Fatalf(`Unable to read file %q: %v`, filename, err)
}
testCases[filename][1] = string(data)
}
for range b.N {
for _, v := range testCases {
ParseFeed(v[0], strings.NewReader(v[1]))
}
}
}
func FuzzParse(f *testing.F) {
f.Add("https://z.org", `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
@ -64,7 +85,35 @@ func FuzzParse(f *testing.F) {
})
}
func TestParseAtom(t *testing.T) {
func TestParseAtom03Feed(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed version="0.3" xmlns="http://purl.org/atom/ns#">
<title>dive into mark</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/"/>
<modified>2003-12-13T18:30:02Z</modified>
<author><name>Mark Pilgrim</name></author>
<entry>
<title>Atom 0.3 snapshot</title>
<link rel="alternate" type="text/html" href="http://diveintomark.org/2003/12/13/atom03"/>
<id>tag:diveintomark.org,2003:3.2397</id>
<issued>2003-12-13T08:29:29-04:00</issued>
<modified>2003-12-13T18:30:02Z</modified>
<summary type="text/plain">It&apos;s a test</summary>
<content type="text/html" mode="escaped"><![CDATA[<p>HTML content</p>]]></content>
</entry>
</feed>`
feed, err := ParseFeed("https://example.org/", strings.NewReader(data))
if err != nil {
t.Error(err)
}
if feed.Title != "dive into mark" {
t.Errorf("Incorrect title, got: %s", feed.Title)
}
}
func TestParseAtom10Feed(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">

1638
internal/reader/parser/testdata/large_atom.xml vendored Normal file
View File

File diff suppressed because it is too large Load Diff

1472
internal/reader/parser/testdata/large_rss.xml vendored Normal file
View File

File diff suppressed because it is too large Load Diff

396
internal/reader/parser/testdata/small_atom.xml vendored Normal file
View File

@ -0,0 +1,396 @@
<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/" xml:lang="en-US">
<id>tag:github.com,2008:/miniflux/v2/commits/main</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commits/main"/>
<link type="application/atom+xml" rel="self" href="https://github.com/miniflux/v2/commits/main.atom"/>
<title>Recent Commits to v2:main</title>
<updated>2024-03-12T05:30:27Z</updated>
<entry>
<id>tag:github.com,2008:Grit::Commit/6d97f8b4582414b6ce69467656824690057d4793</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/6d97f8b4582414b6ce69467656824690057d4793"/>
<title>
Parse podcast categories
</title>
<updated>2024-03-12T05:30:27Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Parse podcast categories&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/f8e50947f2885047155a8070dddab133a5c685c2</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/f8e50947f2885047155a8070dddab133a5c685c2"/>
<title>
Move iTunes and GooglePlay XML definitions to their own packages
</title>
<updated>2024-03-12T05:09:31Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Move iTunes and GooglePlay XML definitions to their own packages&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/9a637ce95e05459adc4712027e6a07eaabcfe657</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/9a637ce95e05459adc4712027e6a07eaabcfe657"/>
<title>
Refactor RSS parser to use default namespace
</title>
<updated>2024-03-12T04:07:13Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Refactor RSS parser to use default namespace
This change avoid some limitations of the Go XML parser regarding XML namespaces&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/d3a85b049b14d4a4ddd6b813134b2abd45fe5e8d</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/d3a85b049b14d4a4ddd6b813134b2abd45fe5e8d"/>
<title>
jsminifier: set JavaScript version
</title>
<updated>2024-03-12T02:02:52Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;jsminifier: set JavaScript version&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/5bcb37901c60463b27e1211e0f68295f213b19e6</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/5bcb37901c60463b27e1211e0f68295f213b19e6"/>
<title>
Use crypto.GenerateRandomBytes instead of doing it by hand
</title>
<updated>2024-03-11T23:31:43Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Use crypto.GenerateRandomBytes instead of doing it by hand
This makes the code a bit shorter, and properly handle
cryptographic error conditions.&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/9c8a7dfffe2f4596dcbde2c923a7539914bb252f</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/9c8a7dfffe2f4596dcbde2c923a7539914bb252f"/>
<title>
Make use of HashFromBytes everywhere
</title>
<updated>2024-03-11T22:22:22Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Make use of HashFromBytes everywhere
It feels a bit silly to have a function and to not make use of it.&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/74e4032ffc9faad4fec602f283a32d2af8dec47e</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/74e4032ffc9faad4fec602f283a32d2af8dec47e"/>
<title>
Small refactor of app.js
</title>
<updated>2024-03-11T22:18:57Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Small refactor of app.js
- replace a lot of `let` with `const`
- inline some `querySelectorAll` calls
- reduce the scope of some variables
- use some ternaries where it makes sense
- inline one-line functions&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/fd1fee852cb35fa0f5b0ed6dc0c23b4a6ce368c3</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/fd1fee852cb35fa0f5b0ed6dc0c23b4a6ce368c3"/>
<title>
Simplify DomHelper.getVisibleElements
</title>
<updated>2024-03-11T22:03:00Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Simplify DomHelper.getVisibleElements
Use a `filter` instead of a loop with an index.&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/c51a3270da1f6af796b7d23fa4b434ccf11818e7</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/c51a3270da1f6af796b7d23fa4b434ccf11818e7"/>
<title>
GitHub Actions: Add basic ESLinter checks
</title>
<updated>2024-03-11T03:57:27Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;GitHub Actions: Add basic ESLinter checks&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/45fa641d26a5f68e663aa9af72e97523d8d63c1e</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/45fa641d26a5f68e663aa9af72e97523d8d63c1e"/>
<title>
Fix JavaScript linter path in GitHub Actions
</title>
<updated>2024-03-11T03:37:18Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Fix JavaScript linter path in GitHub Actions&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/fd8f25916b025a92b1b8349ef9d0acdb832a9e8e</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/fd8f25916b025a92b1b8349ef9d0acdb832a9e8e"/>
<title>
First steps towards trusted-types support
</title>
<updated>2024-03-11T03:14:30Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;First steps towards trusted-types support
Refactor away some trival usages of `.innerHTML`. Unfortunately, there is no way to
enabled trusted-types in report-only mode via `&amp;lt;meta&amp;gt;` tags, see
https://github.com/w3c/webappsec-csp/issues/277&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/826e4d654f511ea8d1d385bdc09cbed69ff6a70f</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/826e4d654f511ea8d1d385bdc09cbed69ff6a70f"/>
<title>
Replace DomHelper.findParent with .closest
</title>
<updated>2024-03-11T03:06:54Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Replace DomHelper.findParent with .closest
See https://developer.mozilla.org/en-US/docs/Web/API/Element/closest&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/d9d17f0d69d1dafb3bd9d81bf9fc27df3def4f4c</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/d9d17f0d69d1dafb3bd9d81bf9fc27df3def4f4c"/>
<title>
Use a `Set` instead of an array in a KeyboardHandler&#39;s member
</title>
<updated>2024-03-11T02:41:13Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Use a `Set` instead of an array in a KeyboardHandler&amp;#39;s member
The variable `triggers` is only used to check if in contains a particular
value. Given that the number of keyboard shortcuts is starting to be
significant, let&amp;#39;s future-proof the performances and use a `Set` instead of an
`Array` instead.&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/eaaeb68474ff194f682e9521a848d7ab2c89348e</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/eaaeb68474ff194f682e9521a848d7ab2c89348e"/>
<title>
Fix conditions to publish packages in GitHub workflows
</title>
<updated>2024-03-10T19:25:13Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Fix conditions to publish packages in GitHub workflows&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/382885f14403526adfa6c303927889c76fd5a1eb</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/382885f14403526adfa6c303927889c76fd5a1eb"/>
<title>
Update changeLog
</title>
<updated>2024-03-10T17:50:47Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/323546?s=30&amp;v=4"/>
<author>
<name>fguillot</name>
<uri>https://github.com/fguillot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Update changeLog&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/0f7b047b0a81253b6d146e05d561545303016b74</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/0f7b047b0a81253b6d146e05d561545303016b74"/>
<title>
Bump github.com/go-jose/go-jose/v3 from 3.0.1 to 3.0.3
</title>
<updated>2024-03-08T04:59:42Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/in/29110?s=30&amp;v=4"/>
<author>
<name>dependabot</name>
<uri>https://github.com/dependabot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Bump github.com/go-jose/go-jose/v3 from 3.0.1 to 3.0.3
Bumps [github.com/go-jose/go-jose/v3](https://github.com/go-jose/go-jose) from 3.0.1 to 3.0.3.
- [Release notes](https://github.com/go-jose/go-jose/releases)
- [Changelog](https://github.com/go-jose/go-jose/blob/v3.0.3/CHANGELOG.md)
- [Commits](https://github.com/go-jose/go-jose/compare/v3.0.1...v3.0.3)
---
updated-dependencies:
- dependency-name: github.com/go-jose/go-jose/v3
dependency-type: indirect
...
Signed-off-by: dependabot[bot] &amp;lt;support@github.com&amp;gt;&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/a074773e6c5d3b2066094cbac0502094aa364713</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/a074773e6c5d3b2066094cbac0502094aa364713"/>
<title>
Use an io.ReadSeeker instead of an io.Reader to parse feeds
</title>
<updated>2024-03-07T04:13:39Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Use an io.ReadSeeker instead of an io.Reader to parse feeds
This will allow to make use of func (*Reader) Seek, instead of re-recreating a
new reader. It&amp;#39;s a large commit for a small change, but anything to simply the
reader/buffer/ReadAll/… mess is a step in the right direction I think, and it
should enable more follow-up simplifications.&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/3d0126be0b8a603401b7593250f80b0a8042b995</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/3d0126be0b8a603401b7593250f80b0a8042b995"/>
<title>
Speed the sanitizer up a bit, again
</title>
<updated>2024-03-06T03:31:50Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Speed the sanitizer up a bit, again
- allow youtube urls to start with `www`
- use `strings.Builder` instead of a `bytes.Buffer`
- use a `strings.NewReader` instead of a `bytes.NewBufferString`
- sprinkles a couple of `continue` to make the code-flow more obvious
- inline calls to `inList`, and put their parameters in the right order
- simplify isPixelTracker
- simplify `isValidIframeSource`, by extracting the hostname and comparing it
directly, instead of using the full url and checking if it starts with
multiple variations of the same one (`//`, `http:`, `https://` multiplied by
``/`www.`)
- add a benchmark&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/eda2e2f3f5c278e44e2def72caedc33667a0fb6c</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/eda2e2f3f5c278e44e2def72caedc33667a0fb6c"/>
<title>
Bump golang.org/x/oauth2 from 0.17.0 to 0.18.0
</title>
<updated>2024-03-05T23:39:07Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/in/29110?s=30&amp;v=4"/>
<author>
<name>dependabot</name>
<uri>https://github.com/dependabot</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Bump golang.org/x/oauth2 from 0.17.0 to 0.18.0
Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.17.0 to 0.18.0.
- [Commits](https://github.com/golang/oauth2/compare/v0.17.0...v0.18.0)
---
updated-dependencies:
- dependency-name: golang.org/x/oauth2
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] &amp;lt;support@github.com&amp;gt;&lt;/pre&gt;
</content>
</entry>
<entry>
<id>tag:github.com,2008:Grit::Commit/111e3f2106646cd29f7f74c0102f2a570c598e2e</id>
<link type="text/html" rel="alternate" href="https://github.com/miniflux/v2/commit/111e3f2106646cd29f7f74c0102f2a570c598e2e"/>
<title>
Reuse a Reader instead of copying to a buffer when parsing an atom feed
</title>
<updated>2024-03-05T01:36:10Z</updated>
<media:thumbnail height="30" width="30" url="https://avatars.githubusercontent.com/u/325724?s=30&amp;v=4"/>
<author>
<name>jvoisin</name>
<uri>https://github.com/jvoisin</uri>
</author>
<content type="html">
&lt;pre style=&#39;white-space:pre-wrap;width:81ex&#39;&gt;Reuse a Reader instead of copying to a buffer when parsing an atom feed&lt;/pre&gt;
</content>
</entry>
</feed>

147
internal/reader/processor/processor.go
View File

@ -23,6 +23,8 @@ import (
"miniflux.app/v2/internal/storage"
"github.com/PuerkitoBio/goquery"
"github.com/tdewolff/minify/v2"
"github.com/tdewolff/minify/v2/html"
)
var (
@ -36,31 +38,38 @@ var (
func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.User, forceRefresh bool) {
var filteredEntries model.Entries
minifier := minify.New()
minifier.AddFunc("text/html", html.Minify)
// Process older entries first
for i := len(feed.Entries) - 1; i >= 0; i-- {
entry := feed.Entries[i]
slog.Debug("Processing entry",
slog.Int64("user_id", user.ID),
slog.Int64("entry_id", entry.ID),
slog.String("entry_url", entry.URL),
slog.String("entry_hash", entry.Hash),
slog.String("entry_title", entry.Title),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
)
if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) {
if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) || !isRecentEntry(entry) {
continue
}
websiteURL := getUrlFromEntry(feed, entry)
entryIsNew := !store.EntryURLExists(feed.ID, entry.URL)
entryIsNew := store.IsNewEntry(feed.ID, entry.Hash)
if feed.Crawler && (entryIsNew || forceRefresh) {
slog.Debug("Scraping entry",
slog.Int64("user_id", user.ID),
slog.Int64("entry_id", entry.ID),
slog.String("entry_url", entry.URL),
slog.String("entry_hash", entry.Hash),
slog.String("entry_title", entry.Title),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
slog.Bool("entry_is_new", entryIsNew),
slog.Bool("force_refresh", forceRefresh),
slog.String("website_url", websiteURL),
)
startTime := time.Now()
@ -91,7 +100,6 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
if scraperErr != nil {
slog.Warn("Unable to scrape entry",
slog.Int64("user_id", user.ID),
slog.Int64("entry_id", entry.ID),
slog.String("entry_url", entry.URL),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
@ -99,7 +107,11 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
)
} else if content != "" {
// We replace the entry content only if the scraper doesn't return any error.
entry.Content = content
if minifiedHTML, err := minifier.String("text/html", content); err == nil {
entry.Content = minifiedHTML
} else {
entry.Content = content
}
}
}
@ -116,62 +128,70 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
}
func isBlockedEntry(feed *model.Feed, entry *model.Entry) bool {
if feed.BlocklistRules != "" {
containsBlockedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool {
return matchField(feed.BlocklistRules, tag)
})
if feed.BlocklistRules == "" {
return false
}
if matchField(feed.BlocklistRules, entry.URL) || matchField(feed.BlocklistRules, entry.Title) || matchField(feed.BlocklistRules, entry.Author) || containsBlockedTag {
slog.Debug("Blocking entry based on rule",
slog.Int64("entry_id", entry.ID),
slog.String("entry_url", entry.URL),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
slog.String("rule", feed.BlocklistRules),
)
return true
}
compiledBlocklist, err := regexp.Compile(feed.BlocklistRules)
if err != nil {
slog.Debug("Failed on regexp compilation",
slog.String("pattern", feed.BlocklistRules),
slog.Any("error", err),
)
return false
}
containsBlockedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool {
return compiledBlocklist.MatchString(tag)
})
if compiledBlocklist.MatchString(entry.URL) || compiledBlocklist.MatchString(entry.Title) || compiledBlocklist.MatchString(entry.Author) || containsBlockedTag {
slog.Debug("Blocking entry based on rule",
slog.String("entry_url", entry.URL),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
slog.String("rule", feed.BlocklistRules),
)
return true
}
return false
}
func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool {
if feed.KeeplistRules != "" {
containsAllowedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool {
return matchField(feed.KeeplistRules, tag)
})
if matchField(feed.KeeplistRules, entry.URL) || matchField(feed.KeeplistRules, entry.Title) || matchField(feed.KeeplistRules, entry.Author) || containsAllowedTag {
slog.Debug("Allow entry based on rule",
slog.Int64("entry_id", entry.ID),
slog.String("entry_url", entry.URL),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
slog.String("rule", feed.KeeplistRules),
)
return true
}
return false
if feed.KeeplistRules == "" {
return true
}
return true
}
func matchField(pattern, value string) bool {
match, err := regexp.MatchString(pattern, value)
compiledKeeplist, err := regexp.Compile(feed.KeeplistRules)
if err != nil {
slog.Debug("Failed on regexp match",
slog.String("pattern", pattern),
slog.String("value", value),
slog.Bool("match", match),
slog.Debug("Failed on regexp compilation",
slog.String("pattern", feed.KeeplistRules),
slog.Any("error", err),
)
return false
}
return match
containsAllowedTag := slices.ContainsFunc(entry.Tags, func(tag string) bool {
return compiledKeeplist.MatchString(tag)
})
if compiledKeeplist.MatchString(entry.URL) || compiledKeeplist.MatchString(entry.Title) || compiledKeeplist.MatchString(entry.Author) || containsAllowedTag {
slog.Debug("Allow entry based on rule",
slog.String("entry_url", entry.URL),
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
slog.String("rule", feed.KeeplistRules),
)
return true
}
return false
}
// ProcessEntryWebPage downloads the entry web page and apply rewrite rules.
func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User) error {
minifier := minify.New()
minifier.AddFunc("text/html", html.Minify)
startTime := time.Now()
websiteURL := getUrlFromEntry(feed, entry)
@ -203,7 +223,11 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry, user *model.User)
}
if content != "" {
entry.Content = content
if minifiedHTML, err := minifier.String("text/html", content); err == nil {
entry.Content = minifiedHTML
} else {
entry.Content = content
}
if user.ShowReadingTime {
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
}
@ -224,7 +248,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
re := regexp.MustCompile(parts[1])
url = re.ReplaceAllString(entry.URL, parts[2])
slog.Debug("Rewriting entry URL",
slog.Int64("entry_id", entry.ID),
slog.String("original_entry_url", entry.URL),
slog.String("rewritten_entry_url", url),
slog.Int64("feed_id", feed.ID),
@ -232,7 +255,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
)
} else {
slog.Debug("Cannot find search and replace terms for replace rule",
slog.Int64("entry_id", entry.ID),
slog.String("original_entry_url", entry.URL),
slog.String("rewritten_entry_url", url),
slog.Int64("feed_id", feed.ID),
@ -245,6 +267,11 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
}
func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *model.Entry, entryIsNew bool, user *model.User) {
if !user.ShowReadingTime {
slog.Debug("Skip reading time estimation for this user", slog.Int64("user_id", user.ID))
return
}
if shouldFetchYouTubeWatchTime(entry) {
if entryIsNew {
watchTime, err := fetchYouTubeWatchTime(entry.URL)
@ -260,7 +287,7 @@ func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *mod
}
entry.ReadingTime = watchTime
} else {
entry.ReadingTime = store.GetReadTime(entry, feed)
entry.ReadingTime = store.GetReadTime(feed.ID, entry.Hash)
}
}
@ -279,14 +306,13 @@ func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *mod
}
entry.ReadingTime = watchTime
} else {
entry.ReadingTime = store.GetReadTime(entry, feed)
entry.ReadingTime = store.GetReadTime(feed.ID, entry.Hash)
}
}
// Handle YT error case and non-YT entries.
if entry.ReadingTime == 0 {
if user.ShowReadingTime {
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
}
entry.ReadingTime = readingtime.EstimateReadingTime(entry.Content, user.DefaultReadingSpeed, user.CJKReadingSpeed)
}
}
@ -394,11 +420,11 @@ func parseISO8601(from string) (time.Duration, error) {
switch name {
case "hour":
d = d + (time.Duration(val) * time.Hour)
d += (time.Duration(val) * time.Hour)
case "minute":
d = d + (time.Duration(val) * time.Minute)
d += (time.Duration(val) * time.Minute)
case "second":
d = d + (time.Duration(val) * time.Second)
d += (time.Duration(val) * time.Second)
default:
return 0, fmt.Errorf("unknown field %s", name)
}
@ -406,3 +432,10 @@ func parseISO8601(from string) (time.Duration, error) {
return d, nil
}
func isRecentEntry(entry *model.Entry) bool {
if config.Opts.FilterEntryMaxAgeDays() == 0 || entry.Date.After(time.Now().AddDate(0, 0, -config.Opts.FilterEntryMaxAgeDays())) {
return true
}
return false
}

25
internal/reader/processor/processor_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
)
@ -92,3 +93,27 @@ func TestParseISO8601(t *testing.T) {
}
}
}
func TestIsRecentEntry(t *testing.T) {
parser := config.NewParser()
var err error
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
var scenarios = []struct {
entry *model.Entry
expected bool
}{
{&model.Entry{Title: "Example1", Date: time.Date(2005, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
{&model.Entry{Title: "Example2", Date: time.Date(2010, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
{&model.Entry{Title: "Example3", Date: time.Date(2020, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
{&model.Entry{Title: "Example4", Date: time.Date(2024, 3, 15, 05, 05, 05, 05, time.UTC)}, true},
}
for _, tc := range scenarios {
result := isRecentEntry(tc.entry)
if tc.expected != result {
t.Errorf(`Unexpected result, got %v for entry %q`, result, tc.entry.Title)
}
}
}

114
internal/reader/rdf/adapter.go Normal file
View File

@ -0,0 +1,114 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package rdf // import "miniflux.app/v2/internal/reader/rdf"
import (
"html"
"log/slog"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
type RDFAdapter struct {
rdf *RDF
}
func NewRDFAdapter(rdf *RDF) *RDFAdapter {
return &RDFAdapter{rdf}
}
func (r *RDFAdapter) BuildFeed(baseURL string) *model.Feed {
feed := &model.Feed{
Title: stripTags(r.rdf.Channel.Title),
FeedURL: strings.TrimSpace(baseURL),
SiteURL: strings.TrimSpace(r.rdf.Channel.Link),
}
if feed.Title == "" {
feed.Title = baseURL
}
if siteURL, err := urllib.AbsoluteURL(feed.FeedURL, feed.SiteURL); err == nil {
feed.SiteURL = siteURL
}
for _, item := range r.rdf.Items {
entry := model.NewEntry()
itemLink := strings.TrimSpace(item.Link)
// Populate the entry URL.
if itemLink == "" {
entry.URL = feed.SiteURL // Fallback to the feed URL if the entry URL is empty.
} else if entryURL, err := urllib.AbsoluteURL(feed.SiteURL, itemLink); err == nil {
entry.URL = entryURL
} else {
entry.URL = itemLink
}
// Populate the entry title.
for _, title := range []string{item.Title, item.DublinCoreTitle} {
title = strings.TrimSpace(title)
if title != "" {
entry.Title = html.UnescapeString(title)
break
}
}
// If the entry title is empty, we use the entry URL as a fallback.
if entry.Title == "" {
entry.Title = entry.URL
}
// Populate the entry content.
if item.DublinCoreContent != "" {
entry.Content = item.DublinCoreContent
} else {
entry.Content = item.Description
}
// Generate the entry hash.
hashValue := itemLink
if hashValue == "" {
hashValue = item.Title + item.Description // Fallback to the title and description if the link is empty.
}
entry.Hash = crypto.Hash(hashValue)
// Populate the entry date.
entry.Date = time.Now()
if item.DublinCoreDate != "" {
if itemDate, err := date.Parse(item.DublinCoreDate); err != nil {
slog.Debug("Unable to parse date from RDF feed",
slog.String("date", item.DublinCoreDate),
slog.String("link", itemLink),
slog.Any("error", err),
)
} else {
entry.Date = itemDate
}
}
// Populate the entry author.
switch {
case item.DublinCoreCreator != "":
entry.Author = stripTags(item.DublinCoreCreator)
case r.rdf.Channel.DublinCoreCreator != "":
entry.Author = stripTags(r.rdf.Channel.DublinCoreCreator)
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}
func stripTags(value string) string {
return strings.TrimSpace(sanitizer.StripTags(value))
}

6
internal/reader/rdf/parser.go
View File

@ -13,10 +13,10 @@ import (
// Parse returns a normalized feed struct from a RDF feed.
func Parse(baseURL string, data io.ReadSeeker) (*model.Feed, error) {
feed := new(rdfFeed)
if err := xml.NewXMLDecoder(data).Decode(feed); err != nil {
xmlFeed := new(RDF)
if err := xml.NewXMLDecoder(data).Decode(xmlFeed); err != nil {
return nil, fmt.Errorf("rdf: unable to parse feed: %w", err)
}
return feed.Transform(baseURL), nil
return NewRDFAdapter(xmlFeed).BuildFeed(baseURL), nil
}

616
internal/reader/rdf/parser_test.go
View File

@ -228,63 +228,117 @@ func TestParseRDFSampleWithDublinCore(t *testing.T) {
}
}
func TestParseItemWithOnlyFeedAuthor(t *testing.T) {
func TestParseRDFFeedWithEmptyTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns="http://purl.org/rss/1.0/"
>
<channel rdf:about="http://meerkat.oreillynet.com/?_fl=rss1.0">
<title>Meerkat</title>
<link>http://meerkat.oreillynet.com</link>
<dc:creator>Rael Dornfest (mailto:rael@oreilly.com)</dc:creator>
</channel>
<item rdf:about="http://c.moreover.com/click/here.pl?r123">
<title>XML: A Disruptive Technology</title>
<link>http://c.moreover.com/click/here.pl?r123</link>
<dc:description>
XML is placing increasingly heavy loads on the existing technical
infrastructure of the Internet.
</dc:description>
</item>
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/">
<channel>
<link>http://example.org/item</link>
</channel>
<item>
<title>Example</title>
<link>http://example.org/item</link>
<description>Test</description>
</item>
</rdf:RDF>`
feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data)))
feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Author != "Rael Dornfest (mailto:rael@oreilly.com)" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
if feed.Title != "http://example.org/feed" {
t.Errorf(`Incorrect title, got: %q`, feed.Title)
}
}
func TestParseItemRelativeURL(t *testing.T) {
func TestParseRDFFeedWithEmptyLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
</channel>
<item>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<link>http://example.org/item</link>
<description>Test</description>
<link>something.html</link>
</item>
</item>
</rdf:RDF>`
feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data)))
feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].URL != "http://example.org/something.html" {
t.Errorf("Incorrect entry url, got: %s", feed.Entries[0].URL)
if feed.SiteURL != "http://example.org/feed" {
t.Errorf(`Incorrect SiteURL, got: %q`, feed.SiteURL)
}
if feed.FeedURL != "http://example.org/feed" {
t.Errorf(`Incorrect FeedURL, got: %q`, feed.FeedURL)
}
}
func TestParseRDFFeedWithRelativeLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
<link>/test/index.html </link>
</channel>
<item>
<title>Example</title>
<link>http://example.org/item</link>
<description>Test</description>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "http://example.org/test/index.html" {
t.Errorf(`Incorrect SiteURL, got: %q`, feed.SiteURL)
}
if feed.FeedURL != "http://example.org/feed" {
t.Errorf(`Incorrect FeedURL, got: %q`, feed.FeedURL)
}
}
func TestParseRDFFeedSiteURLWithTrailingSpace(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
<link>http://example.org/test/index.html </link>
</channel>
<item>
<title>Example</title>
<link>http://example.org/item</link>
<description>Test</description>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/feed", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "http://example.org/test/index.html" {
t.Errorf(`Incorrect SiteURL, got: %q`, feed.SiteURL)
}
if feed.FeedURL != "http://example.org/feed" {
t.Errorf(`Incorrect FeedURL, got: %q`, feed.FeedURL)
}
}
@ -321,63 +375,7 @@ func TestParseItemWithoutLink(t *testing.T) {
}
}
func TestParseItemWithDublicCoreDate(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<description>Test</description>
<link>http://example.org/test.html</link>
<dc:creator>Tester</dc:creator>
<dc:date>2018-04-10T05:00:00+00:00</dc:date>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
expectedDate := time.Date(2018, time.April, 10, 5, 0, 0, 0, time.UTC)
if !feed.Entries[0].Date.Equal(expectedDate) {
t.Errorf("Incorrect entry date, got: %v, want: %v", feed.Entries[0].Date, expectedDate)
}
}
func TestParseItemWithEncodedHTMLInDCCreatorField(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<description>Test</description>
<link>http://example.org/test.html</link>
<dc:creator>&lt;a href=&quot;http://example.org/author1&quot;>Author 1&lt;/a&gt; (University 1), &lt;a href=&quot;http://example.org/author2&quot;>Author 2&lt;/a&gt; (University 2)</dc:creator>
<dc:date>2018-04-10T05:00:00+00:00</dc:date>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
expectedAuthor := "Author 1 (University 1), Author 2 (University 2)"
if feed.Entries[0].Author != expectedAuthor {
t.Errorf("Incorrect entry author, got: %s, want: %s", feed.Entries[0].Author, expectedAuthor)
}
}
func TestParseItemWithoutDate(t *testing.T) {
func TestParseItemRelativeURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
@ -388,90 +386,17 @@ func TestParseItemWithoutDate(t *testing.T) {
<item>
<title>Title</title>
<description>Test</description>
<link>http://example.org/test.html</link>
<link>something.html</link>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
expectedDate := time.Now().In(time.Local)
diff := expectedDate.Sub(feed.Entries[0].Date)
if diff > time.Second {
t.Errorf("Incorrect entry date, got: %v", diff)
}
}
func TestParseItemWithEncodedHTMLTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>AT&amp;amp;T</title>
<description>Test</description>
<link>http://example.org/test.html</link>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Title != `AT&T` {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title)
}
}
func TestParseInvalidXml(t *testing.T) {
data := `garbage`
_, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err == nil {
t.Fatal("Parse should returns an error")
}
}
func TestParseFeedWithHTMLEntity(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example &nbsp; Feed</title>
<link>http://example.org</link>
</channel>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Title != "Example \u00a0 Feed" {
t.Errorf(`Incorrect title, got: %q`, feed.Title)
}
}
func TestParseFeedWithInvalidCharacterEntity(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
<link>http://example.org/a&b</link>
</channel>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "http://example.org/a&b" {
t.Errorf(`Incorrect URL, got: %q`, feed.SiteURL)
if feed.Entries[0].URL != "http://example.org/something.html" {
t.Errorf("Incorrect entry url, got: %s", feed.Entries[0].URL)
}
}
@ -539,6 +464,130 @@ func TestParseFeedWithURLWrappedInSpaces(t *testing.T) {
}
}
func TestParseRDFItemWitEmptyTitleElement(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
<link>http://example.org/</link>
</channel>
<item>
<title> </title>
<link>http://example.org/item</link>
<description>Test</description>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries))
}
expected := `http://example.org/item`
result := feed.Entries[0].Title
if result != expected {
t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected)
}
}
func TestParseRDFItemWithDublinCoreTitleElement(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Example Feed</title>
<link>http://example.org/</link>
</channel>
<item>
<dc:title>Dublin Core Title</dc:title>
<link>http://example.org/</link>
<description>Test</description>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries))
}
expected := `Dublin Core Title`
result := feed.Entries[0].Title
if result != expected {
t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected)
}
}
func TestParseRDFItemWithDuplicateTitleElement(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Example Feed</title>
<link>http://example.org/</link>
</channel>
<item>
<title>Item Title</title>
<dc:title/>
<link>http://example.org/</link>
<description>Test</description>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries))
}
expected := `Item Title`
result := feed.Entries[0].Title
if result != expected {
t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected)
}
}
func TestParseItemWithEncodedHTMLTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>AT&amp;amp;T</title>
<description>Test</description>
<link>http://example.org/test.html</link>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Title != `AT&T` {
t.Errorf("Incorrect entry title, got: %q", feed.Entries[0].Title)
}
}
func TestParseRDFWithContentEncoded(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
@ -605,101 +654,194 @@ func TestParseRDFWithEncodedHTMLDescription(t *testing.T) {
}
}
func TestParseRDFItemWithDuplicateTitleElement(t *testing.T) {
func TestParseItemWithoutDate(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Example Feed</title>
<link>http://example.org/</link>
</channel>
<item>
<title>Item Title</title>
<dc:title/>
<link>http://example.org/</link>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<description>Test</description>
</item>
<link>http://example.org/test.html</link>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries))
}
expected := `Item Title`
result := feed.Entries[0].Title
if result != expected {
t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected)
expectedDate := time.Now().In(time.Local)
diff := expectedDate.Sub(feed.Entries[0].Date)
if diff > time.Second {
t.Errorf("Incorrect entry date, got: %v", diff)
}
}
func TestParseRDFItemWithDublinCoreTitleElement(t *testing.T) {
func TestParseItemWithDublicCoreDate(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/"
xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Example Feed</title>
<link>http://example.org/</link>
</channel>
<item>
<dc:title>Dublin Core Title</dc:title>
<link>http://example.org/</link>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<description>Test</description>
</item>
<link>http://example.org/test.html</link>
<dc:creator>Tester</dc:creator>
<dc:date>2018-04-10T05:00:00+00:00</dc:date>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries))
}
expected := `Dublin Core Title`
result := feed.Entries[0].Title
if result != expected {
t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected)
expectedDate := time.Date(2018, time.April, 10, 5, 0, 0, 0, time.UTC)
if !feed.Entries[0].Date.Equal(expectedDate) {
t.Errorf("Incorrect entry date, got: %v, want: %v", feed.Entries[0].Date, expectedDate)
}
}
func TestParseRDFItemWitEmptyTitleElement(t *testing.T) {
func TestParseItemWithInvalidDublicCoreDate(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
<link>http://example.org/</link>
</channel>
<item>
<title> </title>
<link>http://example.org/item</link>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<description>Test</description>
</item>
<link>http://example.org/test.html</link>
<dc:creator>Tester</dc:creator>
<dc:date>20-04-10T05:00:00+00:00</dc:date>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org/", bytes.NewReader([]byte(data)))
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf(`Unexpected number of entries, got %d`, len(feed.Entries))
}
expected := `http://example.org/item`
result := feed.Entries[0].Title
if result != expected {
t.Errorf(`Unexpected entry title, got %q instead of %q`, result, expected)
expectedDate := time.Now().In(time.Local)
diff := expectedDate.Sub(feed.Entries[0].Date)
if diff > time.Second {
t.Errorf("Incorrect entry date, got: %v", diff)
}
}
func TestParseItemWithEncodedHTMLInDCCreatorField(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:slash="http://purl.org/rss/1.0/modules/slash/">
<channel>
<title>Example</title>
<link>http://example.org</link>
</channel>
<item>
<title>Title</title>
<description>Test</description>
<link>http://example.org/test.html</link>
<dc:creator>&lt;a href=&quot;http://example.org/author1&quot;>Author 1&lt;/a&gt; (University 1), &lt;a href=&quot;http://example.org/author2&quot;>Author 2&lt;/a&gt; (University 2)</dc:creator>
<dc:date>2018-04-10T05:00:00+00:00</dc:date>
</item>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
expectedAuthor := "Author 1 (University 1), Author 2 (University 2)"
if feed.Entries[0].Author != expectedAuthor {
t.Errorf("Incorrect entry author, got: %s, want: %s", feed.Entries[0].Author, expectedAuthor)
}
}
func TestParseItemWithOnlyFeedAuthor(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns="http://purl.org/rss/1.0/"
>
<channel rdf:about="http://meerkat.oreillynet.com/?_fl=rss1.0">
<title>Meerkat</title>
<link>http://meerkat.oreillynet.com</link>
<dc:creator>Rael Dornfest (mailto:rael@oreilly.com)</dc:creator>
</channel>
<item rdf:about="http://c.moreover.com/click/here.pl?r123">
<title>XML: A Disruptive Technology</title>
<link>http://c.moreover.com/click/here.pl?r123</link>
<dc:description>
XML is placing increasingly heavy loads on the existing technical
infrastructure of the Internet.
</dc:description>
</item>
</rdf:RDF>`
feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Author != "Rael Dornfest (mailto:rael@oreilly.com)" {
t.Errorf("Incorrect entry author, got: %s", feed.Entries[0].Author)
}
}
func TestParseInvalidXml(t *testing.T) {
data := `garbage`
_, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err == nil {
t.Fatal("Parse should returns an error")
}
}
func TestParseFeedWithHTMLEntity(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example &nbsp; Feed</title>
<link>http://example.org</link>
</channel>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Title != "Example \u00a0 Feed" {
t.Errorf(`Incorrect title, got: %q`, feed.Title)
}
}
func TestParseFeedWithInvalidCharacterEntity(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns="http://purl.org/rss/1.0/">
<channel>
<title>Example Feed</title>
<link>http://example.org/a&b</link>
</channel>
</rdf:RDF>`
feed, err := Parse("http://example.org", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "http://example.org/a&b" {
t.Errorf(`Incorrect URL, got: %q`, feed.SiteURL)
}
}

125
internal/reader/rdf/rdf.go
View File

@ -5,130 +5,27 @@ package rdf // import "miniflux.app/v2/internal/reader/rdf"
import (
"encoding/xml"
"html"
"log/slog"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/dublincore"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
type rdfFeed struct {
XMLName xml.Name `xml:"RDF"`
Title string `xml:"channel>title"`
Link string `xml:"channel>link"`
Items []rdfItem `xml:"item"`
dublincore.DublinCoreFeedElement
// RDF sepcs: https://web.resource.org/rss/1.0/spec
type RDF struct {
XMLName xml.Name `xml:"http://www.w3.org/1999/02/22-rdf-syntax-ns# RDF"`
Channel RDFChannel `xml:"channel"`
Items []RDFItem `xml:"item"`
}
func (r *rdfFeed) Transform(baseURL string) *model.Feed {
var err error
feed := new(model.Feed)
feed.Title = sanitizer.StripTags(r.Title)
feed.FeedURL = baseURL
feed.SiteURL, err = urllib.AbsoluteURL(baseURL, r.Link)
if err != nil {
feed.SiteURL = r.Link
}
for _, item := range r.Items {
entry := item.Transform()
if entry.Author == "" && r.DublinCoreCreator != "" {
entry.Author = r.GetSanitizedCreator()
}
if entry.URL == "" {
entry.URL = feed.SiteURL
} else {
entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL)
if err == nil {
entry.URL = entryURL
}
}
feed.Entries = append(feed.Entries, entry)
}
return feed
type RDFChannel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
dublincore.DublinCoreChannelElement
}
type rdfItem struct {
type RDFItem struct {
Title string `xml:"http://purl.org/rss/1.0/ title"`
Link string `xml:"link"`
Description string `xml:"description"`
dublincore.DublinCoreItemElement
}
func (r *rdfItem) Transform() *model.Entry {
entry := model.NewEntry()
entry.Title = r.entryTitle()
entry.Author = r.entryAuthor()
entry.URL = r.entryURL()
entry.Content = r.entryContent()
entry.Hash = r.entryHash()
entry.Date = r.entryDate()
if entry.Title == "" {
entry.Title = entry.URL
}
return entry
}
func (r *rdfItem) entryTitle() string {
for _, title := range []string{r.Title, r.DublinCoreTitle} {
title = strings.TrimSpace(title)
if title != "" {
return html.UnescapeString(title)
}
}
return ""
}
func (r *rdfItem) entryContent() string {
switch {
case r.DublinCoreContent != "":
return r.DublinCoreContent
default:
return r.Description
}
}
func (r *rdfItem) entryAuthor() string {
return r.GetSanitizedCreator()
}
func (r *rdfItem) entryURL() string {
return strings.TrimSpace(r.Link)
}
func (r *rdfItem) entryDate() time.Time {
if r.DublinCoreDate != "" {
result, err := date.Parse(r.DublinCoreDate)
if err != nil {
slog.Debug("Unable to parse date from RDF feed",
slog.String("date", r.DublinCoreDate),
slog.String("link", r.Link),
slog.Any("error", err),
)
return time.Now()
}
return result
}
return time.Now()
}
func (r *rdfItem) entryHash() string {
value := r.Link
if value == "" {
value = r.Title + r.Description
}
return crypto.Hash(value)
}

9
internal/reader/readability/readability.go
View File

@ -45,11 +45,12 @@ func (c *candidate) String() string {
id, _ := c.selection.Attr("id")
class, _ := c.selection.Attr("class")
if id != "" && class != "" {
switch {
case id != "" && class != "":
return fmt.Sprintf("%s#%s.%s => %f", c.Node().DataAtom, id, class, c.score)
} else if id != "" {
case id != "":
return fmt.Sprintf("%s#%s => %f", c.Node().DataAtom, id, c.score)
} else if class != "" {
case class != "":
return fmt.Sprintf("%s.%s => %f", c.Node().DataAtom, class, c.score)
}
@ -222,7 +223,7 @@ func getCandidates(document *goquery.Document) candidateList {
// should have a relatively small link density (5% or less) and be mostly
// unaffected by this operation
for _, candidate := range candidates {
candidate.score = candidate.score * (1 - getLinkDensity(candidate.selection))
candidate.score *= (1 - getLinkDensity(candidate.selection))
}
return candidates

29
internal/reader/rewrite/rewrite_functions.go
View File

@ -14,6 +14,8 @@ import (
"miniflux.app/v2/internal/config"
nethtml "golang.org/x/net/html"
"github.com/PuerkitoBio/goquery"
"github.com/yuin/goldmark"
goldmarkhtml "github.com/yuin/goldmark/renderer/html"
@ -301,10 +303,6 @@ func replaceTextLinks(input string) string {
return textLinkRegex.ReplaceAllString(input, `<a href="${1}">${1}</a>`)
}
func replaceLineFeeds(input string) string {
return strings.ReplaceAll(input, "\n", "<br>")
}
func replaceCustom(entryContent string, searchTerm string, replaceTerm string) string {
re, err := regexp.Compile(searchTerm)
if err == nil {
@ -334,7 +332,7 @@ func addCastopodEpisode(entryURL, entryContent string) string {
func applyFuncOnTextContent(entryContent string, selector string, repl func(string) string) string {
var treatChildren func(i int, s *goquery.Selection)
treatChildren = func(i int, s *goquery.Selection) {
if s.Nodes[0].Type == 1 {
if s.Nodes[0].Type == nethtml.TextNode {
s.ReplaceWithHtml(repl(s.Nodes[0].Data))
} else {
s.Contents().Each(treatChildren)
@ -378,7 +376,8 @@ func addHackerNewsLinksUsing(entryContent, app string) string {
return
}
if app == "opener" {
switch app {
case "opener":
params := url.Values{}
params.Add("url", hn_uri.String())
@ -391,12 +390,12 @@ func addHackerNewsLinksUsing(entryContent, app string) string {
open_with_opener := `<a href="` + url.String() + `">Open with Opener</a>`
a.Parent().AppendHtml(" " + open_with_opener)
} else if app == "hack" {
case "hack":
url := strings.Replace(hn_uri.String(), hn_prefix, "hack://", 1)
open_with_hack := `<a href="` + url + `">Open with HACK</a>`
a.Parent().AppendHtml(" " + open_with_hack)
} else {
default:
slog.Warn("Unknown app provided for openHackerNewsLinksWith rewrite rule",
slog.String("app", app),
)
@ -457,17 +456,3 @@ func removeTables(entryContent string) string {
output, _ := doc.Find("body").First().Html()
return output
}
func removeClickbait(entryTitle string) string {
titleWords := []string{}
for _, word := range strings.Fields(entryTitle) {
runes := []rune(word)
if len(runes) > 1 {
// keep first rune as is to keep the first capital letter
titleWords = append(titleWords, string([]rune{runes[0]})+strings.ToLower(string(runes[1:])))
} else {
titleWords = append(titleWords, word)
}
}
return strings.Join(titleWords, " ")
}

97
internal/reader/rewrite/rewriter.go
View File

@ -11,6 +11,9 @@ import (
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/urllib"
"golang.org/x/text/cases"
"golang.org/x/text/language"
)
type rule struct {
@ -18,50 +21,7 @@ type rule struct {
args []string
}
// Rewriter modify item contents with a set of rewriting rules.
func Rewriter(entryURL string, entry *model.Entry, customRewriteRules string) {
rulesList := getPredefinedRewriteRules(entryURL)
if customRewriteRules != "" {
rulesList = customRewriteRules
}
rules := parseRules(rulesList)
rules = append(rules, rule{name: "add_pdf_download_link"})
slog.Debug("Rewrite rules applied",
slog.Any("rules", rules),
slog.String("entry_url", entryURL),
)
for _, rule := range rules {
applyRule(entryURL, entry, rule)
}
}
func parseRules(rulesText string) (rules []rule) {
scan := scanner.Scanner{Mode: scanner.ScanIdents | scanner.ScanStrings}
scan.Init(strings.NewReader(rulesText))
for {
switch scan.Scan() {
case scanner.Ident:
rules = append(rules, rule{name: scan.TokenText()})
case scanner.String:
if l := len(rules) - 1; l >= 0 {
text := scan.TokenText()
text, _ = strconv.Unquote(text)
rules[l].args = append(rules[l].args, text)
}
case scanner.EOF:
return
}
}
}
func applyRule(entryURL string, entry *model.Entry, rule rule) {
func (rule rule) applyRule(entryURL string, entry *model.Entry) {
switch rule.name {
case "add_image_title":
entry.Content = addImageTitle(entryURL, entry.Content)
@ -82,7 +42,7 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) {
case "add_pdf_download_link":
entry.Content = addPDFLink(entryURL, entry.Content)
case "nl2br":
entry.Content = replaceLineFeeds(entry.Content)
entry.Content = strings.ReplaceAll(entry.Content, "\n", "<br>")
case "convert_text_link", "convert_text_links":
entry.Content = replaceTextLinks(entry.Content)
case "fix_medium_images":
@ -122,11 +82,11 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) {
case "add_castopod_episode":
entry.Content = addCastopodEpisode(entryURL, entry.Content)
case "base64_decode":
selector := "body"
if len(rule.args) >= 1 {
entry.Content = applyFuncOnTextContent(entry.Content, rule.args[0], decodeBase64Content)
} else {
entry.Content = applyFuncOnTextContent(entry.Content, "body", decodeBase64Content)
selector = rule.args[0]
}
entry.Content = applyFuncOnTextContent(entry.Content, selector, decodeBase64Content)
case "add_hn_links_using_hack":
entry.Content = addHackerNewsLinksUsing(entry.Content, "hack")
case "add_hn_links_using_opener":
@ -136,7 +96,46 @@ func applyRule(entryURL string, entry *model.Entry, rule rule) {
case "remove_tables":
entry.Content = removeTables(entry.Content)
case "remove_clickbait":
entry.Title = removeClickbait(entry.Title)
entry.Title = cases.Title(language.English).String(strings.ToLower(entry.Title))
}
}
// Rewriter modify item contents with a set of rewriting rules.
func Rewriter(entryURL string, entry *model.Entry, customRewriteRules string) {
rulesList := getPredefinedRewriteRules(entryURL)
if customRewriteRules != "" {
rulesList = customRewriteRules
}
rules := parseRules(rulesList)
rules = append(rules, rule{name: "add_pdf_download_link"})
slog.Debug("Rewrite rules applied",
slog.Any("rules", rules),
slog.String("entry_url", entryURL),
)
for _, rule := range rules {
rule.applyRule(entryURL, entry)
}
}
func parseRules(rulesText string) (rules []rule) {
scan := scanner.Scanner{Mode: scanner.ScanIdents | scanner.ScanStrings}
scan.Init(strings.NewReader(rulesText))
for {
switch scan.Scan() {
case scanner.Ident:
rules = append(rules, rule{name: scan.TokenText()})
case scanner.String:
if l := len(rules) - 1; l >= 0 {
text, _ := strconv.Unquote(scan.TokenText())
rules[l].args = append(rules[l].args, text)
}
case scanner.EOF:
return
}
}
}

2
internal/reader/rewrite/rules.go
View File

@ -24,7 +24,7 @@ var predefinedRules = map[string]string{
"monkeyuser.com": "add_image_title",
"mrlovenstein.com": "add_image_title",
"nedroid.com": "add_image_title",
"oglaf.com": "add_image_title",
"oglaf.com": `replace("media.oglaf.com/story/tt(.+).gif"|"media.oglaf.com/comic/$1.jpg"),add_image_title`,
"optipess.com": "add_image_title",
"peebleslab.com": "add_image_title",
"quantamagazine.org": `add_youtube_video_from_id, remove("h6:not(.byline,.post__title__kicker), #comments, .next-post__content, .footer__section, figure .outer--content, script")`,

370
internal/reader/rss/adapter.go Normal file
View File

@ -0,0 +1,370 @@
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package rss // import "miniflux.app/v2/internal/reader/rss"
import (
"html"
"log/slog"
"path"
"strconv"
"strings"
"time"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/reader/date"
"miniflux.app/v2/internal/reader/sanitizer"
"miniflux.app/v2/internal/urllib"
)
type RSSAdapter struct {
rss *RSS
}
func NewRSSAdapter(rss *RSS) *RSSAdapter {
return &RSSAdapter{rss}
}
func (r *RSSAdapter) BuildFeed(baseURL string) *model.Feed {
feed := &model.Feed{
Title: html.UnescapeString(strings.TrimSpace(r.rss.Channel.Title)),
FeedURL: strings.TrimSpace(baseURL),
SiteURL: strings.TrimSpace(r.rss.Channel.Link),
}
// Ensure the Site URL is absolute.
if siteURL, err := urllib.AbsoluteURL(baseURL, feed.SiteURL); err == nil {
feed.SiteURL = siteURL
}
// Try to find the feed URL from the Atom links.
for _, atomLink := range r.rss.Channel.AtomLinks.Links {
atomLinkHref := strings.TrimSpace(atomLink.Href)
if atomLinkHref != "" && atomLink.Rel == "self" {
if absoluteFeedURL, err := urllib.AbsoluteURL(feed.FeedURL, atomLinkHref); err == nil {
feed.FeedURL = absoluteFeedURL
break
}
}
}
// Fallback to the site URL if the title is empty.
if feed.Title == "" {
feed.Title = feed.SiteURL
}
// Get TTL if defined.
if r.rss.Channel.TTL != "" {
if ttl, err := strconv.Atoi(r.rss.Channel.TTL); err == nil {
feed.TTL = ttl
}
}
// Get the feed icon URL if defined.
if r.rss.Channel.Image != nil {
if absoluteIconURL, err := urllib.AbsoluteURL(feed.SiteURL, r.rss.Channel.Image.URL); err == nil {
feed.IconURL = absoluteIconURL
}
}
for _, item := range r.rss.Channel.Items {
entry := model.NewEntry()
entry.Date = findEntryDate(&item)
entry.Content = findEntryContent(&item)
entry.Enclosures = findEntryEnclosures(&item, feed.SiteURL)
// Populate the entry URL.
entryURL := findEntryURL(&item)
if entryURL == "" {
entry.URL = feed.SiteURL
} else {
if absoluteEntryURL, err := urllib.AbsoluteURL(feed.SiteURL, entryURL); err == nil {
entry.URL = absoluteEntryURL
} else {
entry.URL = entryURL
}
}
// Populate the entry title.
entry.Title = findEntryTitle(&item)
if entry.Title == "" {
entry.Title = sanitizer.TruncateHTML(entry.Content, 100)
if entry.Title == "" {
entry.Title = entry.URL
}
}
entry.Author = findEntryAuthor(&item)
if entry.Author == "" {
entry.Author = findFeedAuthor(&r.rss.Channel)
}
// Generate the entry hash.
if item.GUID.Data != "" {
entry.Hash = crypto.Hash(item.GUID.Data)
} else if entryURL != "" {
entry.Hash = crypto.Hash(entryURL)
}
// Find CommentsURL if defined.
if absoluteCommentsURL := strings.TrimSpace(item.CommentsURL); absoluteCommentsURL != "" && urllib.IsAbsoluteURL(absoluteCommentsURL) {
entry.CommentsURL = absoluteCommentsURL
}
// Set podcast listening time.
if item.ItunesDuration != "" {
if duration, err := getDurationInMinutes(item.ItunesDuration); err == nil {
entry.ReadingTime = duration
}
}
// Populate entry categories.
for _, tag := range item.Categories {
if tag != "" {
entry.Tags = append(entry.Tags, tag)
}
}
for _, tag := range item.MediaCategories.Labels() {
if tag != "" {
entry.Tags = append(entry.Tags, tag)
}
}
if len(entry.Tags) == 0 {
for _, tag := range r.rss.Channel.Categories {
if tag != "" {
entry.Tags = append(entry.Tags, tag)
}
}
for _, tag := range r.rss.Channel.GetItunesCategories() {
if tag != "" {
entry.Tags = append(entry.Tags, tag)
}
}
if r.rss.Channel.GooglePlayCategory.Text != "" {
entry.Tags = append(entry.Tags, r.rss.Channel.GooglePlayCategory.Text)
}
}
feed.Entries = append(feed.Entries, entry)
}
return feed
}
func findFeedAuthor(rssChannel *RSSChannel) string {
var author string
switch {
case rssChannel.ItunesAuthor != "":
author = rssChannel.ItunesAuthor
case rssChannel.GooglePlayAuthor != "":
author = rssChannel.GooglePlayAuthor
case rssChannel.ItunesOwner.String() != "":
author = rssChannel.ItunesOwner.String()
case rssChannel.ManagingEditor != "":
author = rssChannel.ManagingEditor
case rssChannel.Webmaster != "":
author = rssChannel.Webmaster
}
return sanitizer.StripTags(strings.TrimSpace(author))
}
func findEntryTitle(rssItem *RSSItem) string {
title := rssItem.Title
if rssItem.DublinCoreTitle != "" {
title = rssItem.DublinCoreTitle
}
return html.UnescapeString(strings.TrimSpace(title))
}
func findEntryURL(rssItem *RSSItem) string {
for _, link := range []string{rssItem.FeedBurnerLink, rssItem.Link} {
if link != "" {
return strings.TrimSpace(link)
}
}
for _, atomLink := range rssItem.AtomLinks.Links {
if atomLink.Href != "" && (strings.EqualFold(atomLink.Rel, "alternate") || atomLink.Rel == "") {
return strings.TrimSpace(atomLink.Href)
}
}
// Specs: https://cyber.harvard.edu/rss/rss.html#ltguidgtSubelementOfLtitemgt
// isPermaLink is optional, its default value is true.
// If its value is false, the guid may not be assumed to be a url, or a url to anything in particular.
if rssItem.GUID.IsPermaLink == "true" || rssItem.GUID.IsPermaLink == "" {
return strings.TrimSpace(rssItem.GUID.Data)
}
return ""
}
func findEntryContent(rssItem *RSSItem) string {
for _, value := range []string{
rssItem.DublinCoreContent,
rssItem.Description,
rssItem.GooglePlayDescription,
rssItem.ItunesSummary,
rssItem.ItunesSubtitle,
} {
if value != "" {
return value
}
}
return ""
}
func findEntryDate(rssItem *RSSItem) time.Time {
value := rssItem.PubDate
if rssItem.DublinCoreDate != "" {
value = rssItem.DublinCoreDate
}
if value != "" {
result, err := date.Parse(value)
if err != nil {
slog.Debug("Unable to parse date from RSS feed",
slog.String("date", value),
slog.String("guid", rssItem.GUID.Data),
slog.Any("error", err),
)
return time.Now()
}
return result
}
return time.Now()
}
func findEntryAuthor(rssItem *RSSItem) string {
var author string
switch {
case rssItem.GooglePlayAuthor != "":
author = rssItem.GooglePlayAuthor
case rssItem.ItunesAuthor != "":
author = rssItem.ItunesAuthor
case rssItem.DublinCoreCreator != "":
author = rssItem.DublinCoreCreator
case rssItem.AtomAuthor.PersonName() != "":
author = rssItem.AtomAuthor.PersonName()
case strings.Contains(rssItem.Author.Inner, "<![CDATA["):
author = rssItem.Author.Data
default:
author = rssItem.Author.Inner
}
return strings.TrimSpace(sanitizer.StripTags(author))
}
func findEntryEnclosures(rssItem *RSSItem, siteURL string) model.EnclosureList {
enclosures := make(model.EnclosureList, 0)
duplicates := make(map[string]bool)
for _, mediaThumbnail := range rssItem.AllMediaThumbnails() {
mediaURL := strings.TrimSpace(mediaThumbnail.URL)
if mediaURL == "" {
continue
}
if _, found := duplicates[mediaURL]; !found {
if mediaAbsoluteURL, err := urllib.AbsoluteURL(siteURL, mediaURL); err != nil {
slog.Debug("Unable to build absolute URL for media thumbnail",
slog.String("url", mediaThumbnail.URL),
slog.String("site_url", siteURL),
slog.Any("error", err),
)
} else {
duplicates[mediaAbsoluteURL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: mediaAbsoluteURL,
MimeType: mediaThumbnail.MimeType(),
Size: mediaThumbnail.Size(),
})
}
}
}
for _, enclosure := range rssItem.Enclosures {
enclosureURL := enclosure.URL
if rssItem.FeedBurnerEnclosureLink != "" {
filename := path.Base(rssItem.FeedBurnerEnclosureLink)
if strings.HasSuffix(enclosureURL, filename) {
enclosureURL = rssItem.FeedBurnerEnclosureLink
}
}
enclosureURL = strings.TrimSpace(enclosureURL)
if enclosureURL == "" {
continue
}
if absoluteEnclosureURL, err := urllib.AbsoluteURL(siteURL, enclosureURL); err == nil {
enclosureURL = absoluteEnclosureURL
}
if _, found := duplicates[enclosureURL]; !found {
duplicates[enclosureURL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: enclosureURL,
MimeType: enclosure.Type,
Size: enclosure.Size(),
})
}
}
for _, mediaContent := range rssItem.AllMediaContents() {
mediaURL := strings.TrimSpace(mediaContent.URL)
if mediaURL == "" {
continue
}
if _, found := duplicates[mediaURL]; !found {
mediaURL := strings.TrimSpace(mediaContent.URL)
if mediaAbsoluteURL, err := urllib.AbsoluteURL(siteURL, mediaURL); err != nil {
slog.Debug("Unable to build absolute URL for media content",
slog.String("url", mediaContent.URL),
slog.String("site_url", siteURL),
slog.Any("error", err),
)
} else {
duplicates[mediaAbsoluteURL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: mediaAbsoluteURL,
MimeType: mediaContent.MimeType(),
Size: mediaContent.Size(),
})
}
}
}
for _, mediaPeerLink := range rssItem.AllMediaPeerLinks() {
mediaURL := strings.TrimSpace(mediaPeerLink.URL)
if mediaURL == "" {
continue
}
if _, found := duplicates[mediaURL]; !found {
mediaURL := strings.TrimSpace(mediaPeerLink.URL)
if mediaAbsoluteURL, err := urllib.AbsoluteURL(siteURL, mediaURL); err != nil {
slog.Debug("Unable to build absolute URL for media peer link",
slog.String("url", mediaPeerLink.URL),
slog.String("site_url", siteURL),
slog.Any("error", err),
)
} else {
duplicates[mediaAbsoluteURL] = true
enclosures = append(enclosures, &model.Enclosure{
URL: mediaAbsoluteURL,
MimeType: mediaPeerLink.MimeType(),
Size: mediaPeerLink.Size(),
})
}
}
}
return enclosures
}

37
internal/reader/rss/atom.go
View File

@ -3,41 +3,18 @@
package rss // import "miniflux.app/v2/internal/reader/rss"
import "strings"
import (
"miniflux.app/v2/internal/reader/atom"
)
type AtomAuthor struct {
Author AtomPerson `xml:"http://www.w3.org/2005/Atom author"`
Author atom.AtomPerson `xml:"http://www.w3.org/2005/Atom author"`
}
func (a *AtomAuthor) String() string {
return a.Author.String()
}
type AtomPerson struct {
Name string `xml:"name"`
Email string `xml:"email"`
}
func (a *AtomPerson) String() string {
var name string
switch {
case a.Name != "":
name = a.Name
case a.Email != "":
name = a.Email
}
return strings.TrimSpace(name)
}
type AtomLink struct {
URL string `xml:"href,attr"`
Type string `xml:"type,attr"`
Rel string `xml:"rel,attr"`
Length string `xml:"length,attr"`
func (a *AtomAuthor) PersonName() string {
return a.Author.PersonName()
}
type AtomLinks struct {
Links []*AtomLink `xml:"http://www.w3.org/2005/Atom link"`
Links []*atom.AtomLink `xml:"http://www.w3.org/2005/Atom link"`
}

4
internal/reader/rss/feedburner.go
View File

@ -3,8 +3,8 @@
package rss // import "miniflux.app/v2/internal/reader/rss"
// FeedBurnerElement represents FeedBurner XML elements.
type FeedBurnerElement struct {
// FeedBurnerItemElement represents FeedBurner XML elements.
type FeedBurnerItemElement struct {
FeedBurnerLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"`
FeedBurnerEnclosureLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origEnclosureLink"`
}

6
internal/reader/rss/parser.go
View File

@ -13,11 +13,11 @@ import (
// Parse returns a normalized feed struct from a RSS feed.
func Parse(baseURL string, data io.ReadSeeker) (*model.Feed, error) {
feed := new(rssFeed)
rssFeed := new(RSS)
decoder := xml.NewXMLDecoder(data)
decoder.DefaultSpace = "rss"
if err := decoder.Decode(feed); err != nil {
if err := decoder.Decode(rssFeed); err != nil {
return nil, fmt.Errorf("rss: unable to parse feed: %w", err)
}
return feed.Transform(baseURL), nil
return NewRSSAdapter(rssFeed).BuildFeed(baseURL), nil
}

503
internal/reader/rss/parser_test.go
View File

@ -109,6 +109,100 @@ func TestParseRss2Sample(t *testing.T) {
}
}
func TestParseFeedWithFeedURLWithTrailingSpace(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<atom:link href="https://example.org/rss " type="application/rss+xml" rel="self"></atom:link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/ ", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/rss" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
}
func TestParseFeedWithRelativeFeedURL(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<atom:link href="/rss" type="application/rss+xml" rel="self"></atom:link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.FeedURL != "https://example.org/rss" {
t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL)
}
}
func TestParseFeedSiteURLWithTrailingSpace(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>Example</title>
<link>https://example.org/ </link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "https://example.org/" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
}
func TestParseFeedWithRelativeSiteURL(t *testing.T) {
data := `<?xml version="1.0"?>
<rss version="2.0">
<channel>
<title>Example</title>
<link>/example </link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.SiteURL != "https://example.org/example" {
t.Errorf("Incorrect site URL, got: %s", feed.SiteURL)
}
}
func TestParseFeedWithoutTitle(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
@ -746,6 +840,106 @@ func TestParseEntryWithContentEncoded(t *testing.T) {
}
}
// https://www.rssboard.org/rss-encoding-examples
func TestParseEntryDescriptionWithEncodedHTMLTags(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<description>this is &lt;b&gt;bold&lt;/b&gt;</description>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Content != `this is <b>bold</b>` {
t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content)
}
}
// https://www.rssboard.org/rss-encoding-examples
func TestParseEntryWithDescriptionWithHTMLCDATA(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<description><![CDATA[this is <b>bold</b>]]></description>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Content != `this is <b>bold</b>` {
t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content)
}
}
// https://www.rssboard.org/rss-encoding-examples
func TestParseEntryDescriptionWithEncodingAngleBracketsInText(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<description>5 &amp;lt; 8, ticker symbol &amp;lt;BIGCO&amp;gt;</description>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Content != `5 &lt; 8, ticker symbol &lt;BIGCO&gt;` {
t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content)
}
}
// https://www.rssboard.org/rss-encoding-examples
func TestParseEntryDescriptionWithEncodingAngleBracketsWithinCDATASection(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/">
<channel>
<title>Example</title>
<link>http://example.org/</link>
<item>
<title>Item 1</title>
<link>http://example.org/item1</link>
<description><![CDATA[5 &lt; 8, ticker symbol &lt;BIGCO&gt;]]></description>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if feed.Entries[0].Content != `5 &lt; 8, ticker symbol &lt;BIGCO&gt;` {
t.Errorf("Incorrect entry content, got: %q", feed.Entries[0].Content)
}
}
func TestParseEntryWithFeedBurnerLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
@ -822,15 +1016,11 @@ func TestParseEntryWithEnclosures(t *testing.T) {
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
@ -846,6 +1036,88 @@ func TestParseEntryWithEnclosures(t *testing.T) {
}
}
func TestParseEntryWithIncorrectEnclosureLength(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<title>My Podcast Feed</title>
<link>http://example.org</link>
<author>some.email@example.org</author>
<item>
<title>Podcasting with RSS</title>
<link>http://www.example.org/entries/1</link>
<description>An overview of RSS podcasting</description>
<pubDate>Fri, 15 Jul 2005 00:00:00 -0500</pubDate>
<guid isPermaLink="true">http://www.example.org/entries/1</guid>
<enclosure url="http://www.example.org/myaudiofile.mp3" length="invalid" type="audio/mpeg" />
<enclosure url="http://www.example.org/myaudiofile.wav" length=" " type="audio" />
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 2 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
if feed.Entries[0].Enclosures[0].MimeType != "audio/mpeg" {
t.Errorf("Incorrect enclosure type, got: %s", feed.Entries[0].Enclosures[0].MimeType)
}
if feed.Entries[0].Enclosures[0].Size != 0 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
if feed.Entries[0].Enclosures[1].Size != 0 {
t.Errorf("Incorrect enclosure length, got: %d", feed.Entries[0].Enclosures[0].Size)
}
}
func TestParseEntryWithDuplicatedEnclosureURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<title>My Podcast Feed</title>
<link>http://example.org</link>
<item>
<title>Podcasting with RSS</title>
<link>http://www.example.org/entries/1</link>
<enclosure url="http://www.example.org/myaudiofile.mp3" type="audio/mpeg" />
<enclosure url=" http://www.example.org/myaudiofile.mp3 " type="audio/mpeg" />
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://www.example.org/myaudiofile.mp3" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
}
func TestParseEntryWithEmptyEnclosureURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
@ -859,7 +1131,7 @@ func TestParseEntryWithEmptyEnclosureURL(t *testing.T) {
<description>An overview of RSS podcasting</description>
<pubDate>Fri, 15 Jul 2005 00:00:00 -0500</pubDate>
<guid isPermaLink="true">http://www.example.org/entries/1</guid>
<enclosure url="" length="0"/>
<enclosure url=" " length="0"/>
</item>
</channel>
</rss>`
@ -870,15 +1142,47 @@ func TestParseEntryWithEmptyEnclosureURL(t *testing.T) {
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 0 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
}
func TestParseEntryWithRelativeEnclosureURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<title>My Podcast Feed</title>
<link>http://example.org</link>
<author>some.email@example.org</author>
<item>
<title>Podcasting with RSS</title>
<link>http://www.example.org/entries/1</link>
<description>An overview of RSS podcasting</description>
<pubDate>Fri, 15 Jul 2005 00:00:00 -0500</pubDate>
<guid isPermaLink="true">http://www.example.org/entries/1</guid>
<enclosure url=" /files/file.mp3 "/>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://example.org/files/file.mp3" {
t.Errorf("Incorrect enclosure URL, got: %q", feed.Entries[0].Enclosures[0].URL)
}
}
@ -907,15 +1211,11 @@ func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) {
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if feed.Entries[0].URL != "http://www.example.org/entries/1" {
t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL)
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Errorf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://example.org/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3" {
@ -931,6 +1231,42 @@ func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) {
}
}
func TestParseEntryWithFeedBurnerEnclosuresAndRelativeURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0">
<channel>
<title>My Example Feed</title>
<link>http://example.org</link>
<item>
<title>Example Item</title>
<link>http://www.example.org/entries/1</link>
<enclosure
url="http://feedproxy.google.com/~r/example/~5/lpMyFSCvubs/File.mp3"
length="76192460"
type="audio/mpeg" />
<feedburner:origEnclosureLink>/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3</feedburner:origEnclosureLink>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
if feed.Entries[0].Enclosures[0].URL != "http://example.org/67ca416c-f22a-4228-a681-68fc9998ec10/File.mp3" {
t.Errorf("Incorrect enclosure URL, got: %s", feed.Entries[0].Enclosures[0].URL)
}
}
func TestParseEntryWithRelativeURL(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
@ -1142,7 +1478,7 @@ func TestParseEntryWithMediaGroup(t *testing.T) {
<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<title>My Example Feed</title>
<link>http://example.org</link>
<link>https://example.org</link>
<item>
<title>Example Item</title>
<link>http://www.example.org/entries/1</link>
@ -1153,7 +1489,9 @@ func TestParseEntryWithMediaGroup(t *testing.T) {
<media:content type="application/x-bittorrent" url="https://example.org/file2.torrent" isDefault="true"></media:content>
<media:content type="application/x-bittorrent" url="https://example.org/file3.torrent"></media:content>
<media:content type="application/x-bittorrent" url="https://example.org/file4.torrent"></media:content>
<media:content type="application/x-bittorrent" url="https://example.org/file5.torrent" fileSize="42"></media:content>
<media:content type="application/x-bittorrent" url="https://example.org/file4.torrent"></media:content>
<media:content type="application/x-bittorrent" url=" file5.torrent " fileSize="42"></media:content>
<media:content type="application/x-bittorrent" url=" " fileSize="42"></media:content>
<media:rating>nonadult</media:rating>
</media:group>
<media:thumbnail url="https://example.org/image.jpg" height="122" width="223"></media:thumbnail>
@ -1206,15 +1544,19 @@ func TestParseEntryWithMediaContent(t *testing.T) {
<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<title>My Example Feed</title>
<link>http://example.org</link>
<link>https://example.org</link>
<item>
<title>Example Item</title>
<link>http://www.example.org/entries/1</link>
<media:thumbnail url="https://example.org/thumbnail.jpg" />
<media:thumbnail url="https://example.org/thumbnail.jpg" />
<media:thumbnail url=" thumbnail.jpg " />
<media:thumbnail url=" " />
<media:content url="https://example.org/media1.jpg" medium="image">
<media:title type="html">Some Title for Media 1</media:title>
</media:content>
<media:content url="https://example.org/media2.jpg" medium="image" />
<media:content url=" /media2.jpg " medium="image" />
<media:content url=" " medium="image" />
</item>
</channel>
</rss>`
@ -1225,9 +1567,9 @@ func TestParseEntryWithMediaContent(t *testing.T) {
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 3 {
if len(feed.Entries[0].Enclosures) != 4 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
@ -1236,6 +1578,7 @@ func TestParseEntryWithMediaContent(t *testing.T) {
mimeType string
size int64
}{
{"https://example.org/thumbnail.jpg", "image/*", 0},
{"https://example.org/thumbnail.jpg", "image/*", 0},
{"https://example.org/media1.jpg", "image/*", 0},
{"https://example.org/media2.jpg", "image/*", 0},
@ -1261,11 +1604,14 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) {
<rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<title>My Example Feed</title>
<link>http://example.org</link>
<link>https://website.example.org</link>
<item>
<title>Example Item</title>
<link>http://www.example.org/entries/1</link>
<media:peerLink type="application/x-bittorrent" href="http://www.example.org/file.torrent" />
<media:peerLink type="application/x-bittorrent" href="https://www.example.org/file.torrent" />
<media:peerLink type="application/x-bittorrent" href="https://www.example.org/file.torrent" />
<media:peerLink type="application/x-bittorrent" href=" file2.torrent " />
<media:peerLink type="application/x-bittorrent" href=" " />
</item>
</channel>
</rss>`
@ -1276,10 +1622,10 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) {
}
if len(feed.Entries) != 1 {
t.Errorf("Incorrect number of entries, got: %d", len(feed.Entries))
t.Fatalf("Incorrect number of entries, got: %d", len(feed.Entries))
}
if len(feed.Entries[0].Enclosures) != 1 {
if len(feed.Entries[0].Enclosures) != 2 {
t.Fatalf("Incorrect number of enclosures, got: %d", len(feed.Entries[0].Enclosures))
}
@ -1288,7 +1634,8 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) {
mimeType string
size int64
}{
{"http://www.example.org/file.torrent", "application/x-bittorrent", 0},
{"https://www.example.org/file.torrent", "application/x-bittorrent", 0},
{"https://website.example.org/file2.torrent", "application/x-bittorrent", 0},
}
for index, enclosure := range feed.Entries[0].Enclosures {
@ -1306,6 +1653,60 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) {
}
}
func TestParseItunesDuration(t *testing.T) {
data := `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>Podcast Example</title>
<link>http://www.example.com/index.html</link>
<item>
<title>Podcast Episode</title>
<guid>http://example.com/episode.m4a</guid>
<pubDate>Tue, 08 Mar 2016 12:00:00 GMT</pubDate>
<itunes:duration>1:23:45</itunes:duration>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
expected := 83
result := feed.Entries[0].ReadingTime
if expected != result {
t.Errorf(`Unexpected podcast duration, got %d instead of %d`, result, expected)
}
}
func TestParseIncorrectItunesDuration(t *testing.T) {
data := `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
<channel>
<title>Podcast Example</title>
<link>http://www.example.com/index.html</link>
<item>
<title>Podcast Episode</title>
<guid>http://example.com/episode.m4a</guid>
<pubDate>Tue, 08 Mar 2016 12:00:00 GMT</pubDate>
<itunes:duration>invalid</itunes:duration>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
expected := 0
result := feed.Entries[0].ReadingTime
if expected != result {
t.Errorf(`Unexpected podcast duration, got %d instead of %d`, result, expected)
}
}
func TestEntryDescriptionFromItunesSummary(t *testing.T) {
data := `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
@ -1489,11 +1890,11 @@ func TestParseEntryWithCategories(t *testing.T) {
t.Fatal(err)
}
if len(feed.Entries[0].Tags) != 3 {
t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags))
if len(feed.Entries[0].Tags) != 2 {
t.Fatalf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags))
}
expected := []string{"Category 1", "Category 2", "Category 3"}
expected := []string{"Category 1", "Category 2"}
result := feed.Entries[0].Tags
for i, tag := range result {
@ -1574,6 +1975,42 @@ func TestParseFeedWithGooglePlayCategory(t *testing.T) {
}
}
func TestParseEntryWithMediaCategories(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:media="http://search.yahoo.com/mrss/" version="2.0">
<channel>
<title>Example</title>
<link>https://example.org/</link>
<item>
<title>Test</title>
<link>https://example.org/item</link>
<media:category label="Visual Art">visual_art</media:category>
<media:category scheme="http://search.yahoo.com/mrss/category_ schema">music/artist/album/song</media:category>
<media:category scheme="urn:flickr:tags">ycantpark mobile</media:category>
<media:category scheme="http://dmoz.org" label="Ace Ventura - Pet Detective">Arts/Movies/Titles/A/Ace_Ventura_Series/Ace_Ventura_ -_Pet_Detective</media:category>
</item>
</channel>
</rss>`
feed, err := Parse("https://example.org/", bytes.NewReader([]byte(data)))
if err != nil {
t.Fatal(err)
}
if len(feed.Entries[0].Tags) != 2 {
t.Errorf("Incorrect number of tags, got: %d", len(feed.Entries[0].Tags))
}
expected := []string{"Visual Art", "Ace Ventura - Pet Detective"}
result := feed.Entries[0].Tags
for i, tag := range result {
if tag != expected[i] {
t.Errorf("Incorrect tag, got: %q", tag)
}
}
}
func TestParseFeedWithTTLField(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">

Some files were not shown because too many files have changed in this diff Show More